#wget http://downloads.tatoeba.org/exports/sentences.tar.bz2
#bunzip2 sentences.tar.bz2
#tar xvf sentences.tar
#awk -F"\t" '{print"__label__"$2" "$3}' < sentences.csv | shuf > all.txt
#head -3 all.txt
#head -n 10000 all.txt > validation_tatoeba.txt
#tail -n +10001 all.txt > train_tatoeba.txt

python create_fasttext_data.py
python create_tatoeba_data.py

cat train*.txt | shuf > train_all.txt
cat validation*.txt | shuf > validation_all.txt
cat test*.txt | shuf > test_all.txt
python <<EOF
from pathlib import Path
import pandas as pd

pd.DataFrame.from_records([line[9:].split(" ", 1) for line in Path("train_all.txt").read_text().split("\n") if line], columns=["lang", "text"]).to_csv("train_all.csv", index=False)
pd.DataFrame.from_records([line[9:].split(" ", 1) for line in Path("validation_all.txt").read_text().split("\n") if line], columns=["lang", "text"]).to_csv("validation_all.csv", index=False)
pd.DataFrame.from_records([line[9:].split(" ", 1) for line in Path("test_all.txt").read_text().split("\n") if line], columns=["lang", "text"]).to_csv("test_all.csv", index=False)

EOF