Add instructions for scripts

scripts/README.md

@@ -10,4 +10,23 @@
 8. Put CV files into dataset files folder
 9. Put dev.csv and test.csv into folder
 
+Note: you can also specify dataset with "," e.g. dataset1/train.csv,dataset2/train.csv.
+
 You have a reproducible dataset!
+
+
+# Scorer
+
+1. Refer to DeepSpeech guide for further explanations.
+
+2. Generate scorer package.
+```
+python3 generate_lm.py --input_txt ../../../voice-recognition-ua/data/all_text.txt --output_dir . \
+  --top_k 500000 --kenlm_bins ../../../voice-recognition-ua/kenlm/build/bin \
+  --arpa_order 5 --max_arpa_memory "85%" --arpa_prune "0|0|1" \
+  --binary_a_bits 255 --binary_q_bits 8 --binary_type trie
+```
+3. Run lm_optimizer to find the best scorer value.
+4. Rerun step 2 to generate new scorer.
+
+Caution: scorer is very model-dependant, so you'll likely need to adjust it to each model.

scripts/extract_text_corpus.py

@@ -1,3 +1,4 @@
+# this script is used for importing random texts from folder and converting it for scorer
 import os
 import nltk
 import re
@@ -32,10 +33,19 @@ for subdir, dirs, files in os.walk(FOLDER):
             text = text.strip()
 
             words = tokenizer.tokenize(text)
-            words = [i for i in words if i.isalnum()]
             words = [i for i in words if not i.isdigit()]
-            words = [i for i in words if len(i) > 1]
-            if any([any(j not in allowed_chars for j in i) for i in words]):
+            new_words = []
+            for word in words:
+                include = True
+                for letter in word:
+                    if word.startswith("-"):
+                        word = word[1:]
+                    if letter not in allowed_chars:
+                        include = False
+                if include:
+                    new_words.append(word)
+            words = new_words
+            if all([len(i) <= 1 for i in words]):
                 continue
             if len(words) == 0:
                 continue

scripts/wiki_import.py

@@ -1,12 +1,16 @@
+# this script is used for importing wiki text into scorer format
 from wiki_dump_reader import Cleaner, iterate
 from os import remove
+from os.path import exists
 import nltk
 import re
 nltk.download("punkt")
 
+OUT_PATH = "../data/wiki_text.txt"
 
-remove("../data/wiki_text.txt")
-text_file = open("../data/wiki_text.txt", mode="a")
+if exists(OUT_PATH):
+    remove(OUT_PATH)
+text_file = open(OUT_PATH, mode="a")
 
 tokenizer = nltk.SpaceTokenizer()
 paranthesis_regex = re.compile(r'\(.*\)')
@@ -14,6 +18,7 @@ allowed_chars = ["а", "б", "в", "г", "ґ", "д", "е", "є", "ж", "з", "и
                  "м", "н", "о", "п", "р", "с", "т", "у", "ф", "х", "ц", "ч", "ш", "щ", "ь", "ю", "я", "-", "'"]
 
 cleaner = Cleaner()
+# iter = 0
 for title, text in iterate('../data/ukwiki-20210320-pages-articles-multistream.xml'):
     text = cleaner.clean_text(text)
     cleaned_text, _ = cleaner.build_links(text)
@@ -34,10 +39,19 @@ for title, text in iterate('../data/ukwiki-20210320-pages-articles-multistream.x
             continue
 
         words = tokenizer.tokenize(text)
-        words = [i for i in words if i.isalnum()]
         words = [i for i in words if not i.isdigit()]
-        words = [i for i in words if len(i) > 1]
-        if any([any(j not in allowed_chars for j in i) for i in words]):
+        new_words = []
+        for word in words:
+            include = True
+            for letter in word:
+                if word.startswith("-"):
+                    word = word[1:]
+                if letter not in allowed_chars:
+                    include = False
+            if include:
+                new_words.append(word)
+        words = new_words
+        if all([len(i) <= 1 for i in words]):
             continue
         if len(words) == 0:
             continue
@@ -47,5 +61,8 @@ for title, text in iterate('../data/ukwiki-20210320-pages-articles-multistream.x
     if cleaned_text == "":
         continue
     text_file.write(cleaned_text + "\n")
+    # iter += 1
+    # if iter > 5:
+    #    break
 
 text_file.close()