fix training lm script

train_n_gram_lm_with_KenLM.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 34,
    "metadata": {
     "id": "YP3vVkqYUpLx"
    },
@@ -14,7 +14,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 35,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
@@ -22,22 +22,14 @@
     "id": "AWly9SmkgSwE",
     "outputId": "8af190ed-5037-4e3b-b91b-b5286d8e0888"
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "/bin/bash: sudo: command not found\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "!sudo apt-get install git-lfs tree"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 36,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
@@ -54,42 +46,42 @@
       "\u001b[33mWARNING: Ignoring invalid distribution -atasets (/opt/conda/lib/python3.8/site-packages)\u001b[0m\n",
       "Requirement already satisfied: datasets in /workspace/.local/lib/python3.8/site-packages (2.2.2)\n",
       "Requirement already satisfied: transformers in /opt/conda/lib/python3.8/site-packages (4.17.0.dev0)\n",
-      "Requirement already satisfied: packaging in /opt/conda/lib/python3.8/site-packages (from datasets) (21.3)\n",
-      "Requirement already satisfied: responses<0.19 in /opt/conda/lib/python3.8/site-packages (from datasets) (0.18.0)\n",
-      "Requirement already satisfied: fsspec[http]>=2021.05.0 in /opt/conda/lib/python3.8/site-packages (from datasets) (2022.1.0)\n",
-      "Requirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.8/site-packages (from datasets) (2.24.0)\n",
-      "Requirement already satisfied: pyarrow>=6.0.0 in /opt/conda/lib/python3.8/site-packages (from datasets) (6.0.1)\n",
+      "Requirement already satisfied: aiohttp in /opt/conda/lib/python3.8/site-packages (from datasets) (3.8.1)\n",
+      "Requirement already satisfied: dill<0.3.5 in /opt/conda/lib/python3.8/site-packages (from datasets) (0.3.4)\n",
       "Requirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.8/site-packages (from datasets) (4.62.3)\n",
-      "Requirement already satisfied: pandas in /opt/conda/lib/python3.8/site-packages (from datasets) (1.4.0)\n",
+      "Requirement already satisfied: xxhash in /opt/conda/lib/python3.8/site-packages (from datasets) (2.0.2)\n",
+      "Requirement already satisfied: responses<0.19 in /opt/conda/lib/python3.8/site-packages (from datasets) (0.18.0)\n",
       "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.8/site-packages (from datasets) (1.19.2)\n",
-      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.8/site-packages (from datasets) (0.70.12.2)\n",
       "Requirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /opt/conda/lib/python3.8/site-packages (from datasets) (0.4.0)\n",
-      "Requirement already satisfied: xxhash in /opt/conda/lib/python3.8/site-packages (from datasets) (2.0.2)\n",
-      "Requirement already satisfied: dill<0.3.5 in /opt/conda/lib/python3.8/site-packages (from datasets) (0.3.4)\n",
-      "Requirement already satisfied: aiohttp in /opt/conda/lib/python3.8/site-packages (from datasets) (3.8.1)\n",
+      "Requirement already satisfied: packaging in /opt/conda/lib/python3.8/site-packages (from datasets) (21.3)\n",
+      "Requirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.8/site-packages (from datasets) (2.24.0)\n",
+      "Requirement already satisfied: fsspec[http]>=2021.05.0 in /opt/conda/lib/python3.8/site-packages (from datasets) (2022.1.0)\n",
+      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.8/site-packages (from datasets) (0.70.12.2)\n",
+      "Requirement already satisfied: pandas in /opt/conda/lib/python3.8/site-packages (from datasets) (1.4.0)\n",
+      "Requirement already satisfied: pyarrow>=6.0.0 in /opt/conda/lib/python3.8/site-packages (from datasets) (6.0.1)\n",
       "Requirement already satisfied: sacremoses in /opt/conda/lib/python3.8/site-packages (from transformers) (0.0.47)\n",
-      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.8/site-packages (from transformers) (5.4.1)\n",
       "Requirement already satisfied: filelock in /opt/conda/lib/python3.8/site-packages (from transformers) (3.0.12)\n",
       "Requirement already satisfied: tokenizers!=0.11.3,>=0.10.1 in /opt/conda/lib/python3.8/site-packages (from transformers) (0.11.4)\n",
       "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.8/site-packages (from transformers) (2022.1.18)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.8/site-packages (from transformers) (5.4.1)\n",
       "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (4.0.1)\n",
       "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.8/site-packages (from packaging->datasets) (3.0.7)\n",
-      "Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
-      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2020.12.5)\n",
       "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (1.25.11)\n",
+      "Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
       "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2.10)\n",
-      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets) (1.3.0)\n",
-      "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets) (2.0.10)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2020.12.5)\n",
       "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets) (4.0.2)\n",
-      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets) (6.0.2)\n",
-      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets) (1.2.0)\n",
       "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets) (21.4.0)\n",
+      "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets) (2.0.10)\n",
       "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets) (1.7.2)\n",
-      "Requirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.8/site-packages (from pandas->datasets) (2.8.2)\n",
+      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets) (1.3.0)\n",
+      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets) (6.0.2)\n",
+      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets) (1.2.0)\n",
       "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.8/site-packages (from pandas->datasets) (2021.1)\n",
-      "Requirement already satisfied: click in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers) (8.0.3)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.8/site-packages (from pandas->datasets) (2.8.2)\n",
       "Requirement already satisfied: six in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers) (1.15.0)\n",
       "Requirement already satisfied: joblib in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers) (1.1.0)\n",
+      "Requirement already satisfied: click in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers) (8.0.3)\n",
       "\u001b[33mWARNING: Ignoring invalid distribution -atasets (/opt/conda/lib/python3.8/site-packages)\u001b[0m\n",
       "\u001b[33mWARNING: Ignoring invalid distribution -atasets (/opt/conda/lib/python3.8/site-packages)\u001b[0m\n",
       "\u001b[33mWARNING: Ignoring invalid distribution -atasets (/opt/conda/lib/python3.8/site-packages)\u001b[0m\n",
@@ -105,7 +97,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 37,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
@@ -121,24 +113,15 @@
       "\u001b[33mWARNING: Ignoring invalid distribution -atasets (/opt/conda/lib/python3.8/site-packages)\u001b[0m\n",
       "\u001b[33mWARNING: Ignoring invalid distribution -atasets (/opt/conda/lib/python3.8/site-packages)\u001b[0m\n",
       "Collecting https://github.com/kpu/kenlm/archive/master.zip\n",
-      "  Downloading https://github.com/kpu/kenlm/archive/master.zip (542 kB)\n",
-      "     |████████████████████████████████| 542 kB 3.8 MB/s            \n",
-      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
+      "  Using cached https://github.com/kpu/kenlm/archive/master.zip (542 kB)\n",
+      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
       "\u001b[?25hRequirement already satisfied: pyctcdecode in /opt/conda/lib/python3.8/site-packages (0.3.0)\n",
-      "Requirement already satisfied: hypothesis<7,>=6.14 in /opt/conda/lib/python3.8/site-packages (from pyctcdecode) (6.46.9)\n",
       "Requirement already satisfied: numpy<2.0.0,>=1.15.0 in /opt/conda/lib/python3.8/site-packages (from pyctcdecode) (1.19.2)\n",
+      "Requirement already satisfied: hypothesis<7,>=6.14 in /opt/conda/lib/python3.8/site-packages (from pyctcdecode) (6.46.9)\n",
       "Requirement already satisfied: pygtrie<3.0,>=2.1 in /opt/conda/lib/python3.8/site-packages (from pyctcdecode) (2.4.2)\n",
       "Requirement already satisfied: attrs>=19.2.0 in /opt/conda/lib/python3.8/site-packages (from hypothesis<7,>=6.14->pyctcdecode) (21.4.0)\n",
       "Requirement already satisfied: sortedcontainers<3.0.0,>=2.1.0 in /opt/conda/lib/python3.8/site-packages (from hypothesis<7,>=6.14->pyctcdecode) (2.4.0)\n",
-      "Building wheels for collected packages: kenlm\n",
-      "  Building wheel for kenlm (setup.py) ... \u001b[?25ldone\n",
-      "\u001b[?25h  Created wheel for kenlm: filename=kenlm-0.0.0-cp38-cp38-linux_x86_64.whl size=2341844 sha256=7389c3819998781002180209fa8ff1711b65630ca5dc282cff4b128a9db2c0bd\n",
-      "  Stored in directory: /tmp/pip-ephem-wheel-cache-yk63c6mt/wheels/ff/08/4e/a3ddc0e786e0f3c1fcd2e7a82c4324c02fc3ae2638471406d2\n",
-      "Successfully built kenlm\n",
-      "\u001b[33mWARNING: Ignoring invalid distribution -atasets (/opt/conda/lib/python3.8/site-packages)\u001b[0m\n",
-      "Installing collected packages: kenlm\n",
       "\u001b[33mWARNING: Ignoring invalid distribution -atasets (/opt/conda/lib/python3.8/site-packages)\u001b[0m\n",
-      "Successfully installed kenlm-0.0.0\n",
       "\u001b[33mWARNING: Ignoring invalid distribution -atasets (/opt/conda/lib/python3.8/site-packages)\u001b[0m\n",
       "\u001b[33mWARNING: Ignoring invalid distribution -atasets (/opt/conda/lib/python3.8/site-packages)\u001b[0m\n",
       "\u001b[33mWARNING: Ignoring invalid distribution -atasets (/opt/conda/lib/python3.8/site-packages)\u001b[0m\n",
@@ -153,7 +136,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 38,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -185,7 +168,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "c3eef7b5d70d46feaa5d3d7f1281eb82",
+       "model_id": "fb1fe87003eb4d6b936693d8dce9066e",
        "version_major": 2,
        "version_minor": 0
       },
@@ -205,7 +188,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 39,
    "metadata": {
     "id": "fsrpUSEBYH7g"
    },
@@ -216,7 +199,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 40,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
@@ -229,7 +212,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "/bin/bash: sudo: command not found\n"
+      "E: Could not open lock file /var/lib/dpkg/lock-frontend - open (13: Permission denied)\n",
+      "E: Unable to acquire the dpkg frontend lock (/var/lib/dpkg/lock-frontend), are you root?\n"
      ]
     }
    ],
@@ -239,7 +223,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 28,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
@@ -252,16 +236,16 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "--2022-05-26 11:49:56--  https://kheafield.com/code/kenlm.tar.gz\n",
+      "--2022-05-26 13:39:11--  https://kheafield.com/code/kenlm.tar.gz\n",
       "Resolving kheafield.com (kheafield.com)... 35.196.63.85\n",
       "Connecting to kheafield.com (kheafield.com)|35.196.63.85|:443... connected.\n",
       "HTTP request sent, awaiting response... 200 OK\n",
       "Length: 491888 (480K) [application/x-gzip]\n",
       "Saving to: ‘STDOUT’\n",
       "\n",
-      "-                   100%[===================>] 480.36K   799KB/s    in 0.6s    \n",
+      "-                   100%[===================>] 480.36K   845KB/s    in 0.6s    \n",
       "\n",
-      "2022-05-26 11:49:57 (799 KB/s) - written to stdout [491888/491888]\n",
+      "2022-05-26 13:39:12 (845 KB/s) - written to stdout [491888/491888]\n",
       "\n"
      ]
     }
@@ -757,7 +741,6 @@
     }
    ],
    "source": [
-    "\n",
     "!kenlm/build/bin/lmplz -o 5 <\"text.txt\" > \"5gram.arpa\""
    ]
   },
@@ -815,7 +798,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -892,134 +875,7 @@
     "id": "paV71gdAtkDC",
     "outputId": "c2df6859-db57-4d4a-92b0-41b54a4215bf"
    },
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "8f7c10edbad644688af3cd4e4674eac7",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/260 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "e2a9c7fbf0c143e3a80565e429d10095",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/335 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d038c5533a9f4167a48eb3e70ebd156a",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/301 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "cdb0e7da895b4f449599544001e57d12",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/23.0 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ada56f3555ec48a8a22d1d5a2ae81f5f",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/5.20k [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "f8314566f4154fa990c72e90d2ea7d9a",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/223 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "30104479e44c4532b7dab3babcece67a",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/2.19G [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "57e1b10e91024d00af578d9e175b3ae8",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/78.0 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "e48e0eb6e25c4dbb9a53d45124c80eeb",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/6.03M [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "from transformers import AutoProcessor\n",
     "\n",
@@ -1028,7 +884,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 18,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -1088,92 +944,8 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Cloning https://huggingface.co/marinone94/xls-r-300m-sv-robust into local empty directory.\n"
+      "/workspace/xls-r-300m-sv-robust/xls-r-300m-sv-robust is already a clone of https://huggingface.co/marinone94/xls-r-300m-sv-robust. Make sure you pull the latest changes with `repo.git_pull()`.\n"
      ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "4d79ace826ea4922a278dc33e8362513",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Download file language_model/5gram.bin:   0%|          | 15.6k/2.04G [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "caae4b13e94d491587e027c710efe8fb",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Download file training_args.bin:  62%|######1   | 1.84k/2.98k [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "58d1adfe09944bfe8e0e2a8588abed90",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Download file pytorch_model.bin:   0%|          | 3.58k/1.18G [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a209b62ddcea4cd9968172391ac53d59",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Clean file training_args.bin:  34%|###3      | 1.00k/2.98k [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ecf7be42d7c14242bb70bfacb59e7d1c",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Clean file pytorch_model.bin:   0%|          | 1.00k/1.18G [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "aec000d36ed74e008c56f924e8d07d34",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Clean file language_model/5gram.bin:   0%|          | 1.00k/2.04G [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
     }
    ],
    "source": [
@@ -1184,7 +956,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 19,
    "metadata": {
     "id": "ZKwKxMoitoGS"
    },
@@ -1196,7 +968,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 20,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
@@ -1209,8 +981,26 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Found entries of length > 1 in alphabet. This is unusual unless style is BPE, but the alphabet was not recognized as BPE type. Is this correct?\n",
-      "Unigrams and labels don't seem to agree.\n"
+      "Loading the LM will be faster if you build a binary file.\n",
+      "Reading /workspace/xls-r-300m-sv-robust/5gram_correct.arpa\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n"
+     ]
+    },
+    {
+     "ename": "OSError",
+     "evalue": "Cannot read model '5gram_correct.arpa' (End of file Byte: 0)",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
+      "File \u001b[0;32mkenlm.pyx:139\u001b[0m, in \u001b[0;36mkenlm.Model.__init__\u001b[0;34m()\u001b[0m\n",
+      "\u001b[0;31mRuntimeError\u001b[0m: End of file Byte: 0",
+      "\nThe above exception was the direct cause of the following exception:\n",
+      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
+      "Input \u001b[0;32mIn [20]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpyctcdecode\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m build_ctcdecoder\n\u001b[0;32m----> 3\u001b[0m decoder \u001b[38;5;241m=\u001b[39m \u001b[43mbuild_ctcdecoder\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlabels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43msorted_vocab_dict\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkeys\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      5\u001b[0m \u001b[43m    \u001b[49m\u001b[43mkenlm_model_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m5gram_correct.arpa\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      6\u001b[0m \u001b[43m    \u001b[49m\u001b[43malpha\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.5\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbeta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1.5\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      8\u001b[0m \u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/pyctcdecode/decoder.py:790\u001b[0m, in \u001b[0;36mbuild_ctcdecoder\u001b[0;34m(labels, kenlm_model_path, unigrams, alpha, beta, unk_score_offset, lm_score_boundary)\u001b[0m\n\u001b[1;32m    767\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mbuild_ctcdecoder\u001b[39m(\n\u001b[1;32m    768\u001b[0m     labels: List[\u001b[38;5;28mstr\u001b[39m],\n\u001b[1;32m    769\u001b[0m     kenlm_model_path: Optional[\u001b[38;5;28mstr\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    774\u001b[0m     lm_score_boundary: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m DEFAULT_SCORE_LM_BOUNDARY,\n\u001b[1;32m    775\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m BeamSearchDecoderCTC:\n\u001b[1;32m    776\u001b[0m     \u001b[38;5;124;03m\"\"\"Build a BeamSearchDecoderCTC instance with main functionality.\u001b[39;00m\n\u001b[1;32m    777\u001b[0m \n\u001b[1;32m    778\u001b[0m \u001b[38;5;124;03m    Args:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    788\u001b[0m \u001b[38;5;124;03m        instance of BeamSearchDecoderCTC\u001b[39;00m\n\u001b[1;32m    789\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 790\u001b[0m     kenlm_model \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01mif\u001b[39;00m kenlm_model_path \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[43mkenlm\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mModel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkenlm_model_path\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    791\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m kenlm_model_path \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m kenlm_model_path\u001b[38;5;241m.\u001b[39mendswith(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.arpa\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[1;32m    792\u001b[0m         logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUsing arpa instead of binary LM file, decoder instantiation might be slow.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "File \u001b[0;32mkenlm.pyx:142\u001b[0m, in \u001b[0;36mkenlm.Model.__init__\u001b[0;34m()\u001b[0m\n",
+      "\u001b[0;31mOSError\u001b[0m: Cannot read model '5gram_correct.arpa' (End of file Byte: 0)"
      ]
     }
    ],