Spaces:
Build error
Build error
File size: 1,081 Bytes
9aba307 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 |
"""sudachi.py
Data processing script for sudachi dictionary
"""
import warnings
from pathlib import Path
import pandas as pd
from config import config
from config.config import logger
warnings.filterwarnings("ignore")
def sudachi_data():
sudachi_file = list(Path(config.RAW_DATA_DIR, "sudachi").glob("*.csv"))
df = pd.DataFrame()
for file in sudachi_file:
logger.info(file.name)
# Load file
df = pd.concat(
[
df,
pd.read_csv(
file,
header=None,
),
]
)
df["surface"] = df[0].astype(str).str.strip()
df["kana"] = df[11].astype(str).str.strip()
df["type"] = df[5].astype(str).str.strip()
df = df[df["kana"] != "*"]
df = df[df["surface"] != df["kana"]]
df = df[df["type"] != "補助記号"]
df = df[["surface", "kana"]]
df.to_csv(Path(config.READING_DATA_DIR, "sudachi.csv"), index=False)
logger.info("✅ Processed sudachi data!")
if __name__ == "__main__":
sudachi_data()
|