{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setting Up the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "source_language = \"fr\"\n",
    "target_language = \"swc\" # ln is the language code of lingala \n",
    "lc = False  # If True, lowercase the data.\n",
    "seed = 42  # Random seed for shuffling.\n",
    "tag = \"baseline_v2\" # Give a unique name to your folder - this is to ensure you don't rewrite any models you've already submitted\n",
    "\n",
    "os.environ[\"src\"] = source_language # Sets them in bash as well, since we often use bash scripts\n",
    "os.environ[\"tgt\"] = target_language\n",
    "os.environ[\"tag\"] = tag\n",
    "\n",
    "# No need to use gdrive since we are training on gcp\n",
    "!mkdir -p \"$src-$tgt-$tag\"\n",
    "os.environ[\"gdrive_path\"] = \"%s-%s-%s\" % (source_language, target_language, tag) # saving directly on the vm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fr-swc-baseline_v2\r\n"
     ]
    }
   ],
   "source": [
    "!echo $gdrive_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Downloading the corpus data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "remove the old data and redownload them for verification puprose"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -f w300.$src jw300.$tgt JW300_latest_xml_$src-$tgt.xml.gz JW300_latest_xml_$src-$tgt.xml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Alignment file /proj/nlpl/data/OPUS/JW300/latest/xml/fr-swc.xml.gz not found. The following files are available for downloading:\n",
      "\n",
      "   5 MB https://object.pouta.csc.fi/OPUS-JW300/v1/xml/fr-swc.xml.gz\n",
      " 278 MB https://object.pouta.csc.fi/OPUS-JW300/v1/xml/fr.zip\n",
      "  54 MB https://object.pouta.csc.fi/OPUS-JW300/v1/xml/swc.zip\n",
      "\n",
      " 338 MB Total size\n",
      "./JW300_latest_xml_fr-swc.xml.gz ... 100% of 5 MB\n",
      "./JW300_latest_xml_fr.zip ... 100% of 278 MB\n",
      "./JW300_latest_xml_swc.zip ... 100% of 54 MBwc.zip ... 37% of 54 MB\n"
     ]
    }
   ],
   "source": [
    "# Downloading our corpus\n",
    "! opus_read -d JW300 -s $src -t $tgt -wm moses -w jw300.$src jw300.$tgt -q\n",
    "\n",
    "# extract the corpus file\n",
    "! gunzip JW300_latest_xml_$src-$tgt.xml.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! wget https://raw.githubusercontent.com/espoirMur/masakhane/add-french-global-test-set/jw300_utils/test/test.fr-any.fr\n",
    "\n",
    "os.environ[\"trg\"] = target_language \n",
    "os.environ[\"src\"] = source_language \n",
    "\n",
    "#! wget https://raw.githubusercontent.com/juliakreutzer/masakhane/master/jw300_utils/test/test.en-$trg.en \n",
    "#! mv test.en-$trg.en test.en\n",
    "#! wget https://raw.githubusercontent.com/juliakreutzer/masakhane/master/jw300_utils/test/test.en-$trg.$trg \n",
    "#! mv test.en-$trg.$trg test.$trg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "read the french test data generate in [this notebook](./buiding_french_global_test_set.ipynb) and save the data in a set for quick retrieval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generating test French Swahili Congo dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First we get the global english swahili congo test set "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-02-13 08:27:54--  https://raw.githubusercontent.com/juliakreutzer/masakhane/master/jw300_utils/test/test.en-swc.en\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.192.133, 151.101.128.133, 151.101.64.133, ...\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.192.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 205351 (201K) [text/plain]\n",
      "Saving to: ‘test.en-swc.en’\n",
      "\n",
      "test.en-swc.en      100%[===================>] 200.54K  --.-KB/s    in 0.04s   \n",
      "\n",
      "2020-02-13 08:27:55 (4.81 MB/s) - ‘test.en-swc.en’ saved [205351/205351]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "! wget https://raw.githubusercontent.com/juliakreutzer/masakhane/master/jw300_utils/test/test.en-swc.en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-02-13 08:34:07--  https://raw.githubusercontent.com/juliakreutzer/masakhane/master/jw300_utils/test/test.en-swc.swc\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.192.133, 151.101.128.133, 151.101.64.133, ...\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.192.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 231897 (226K) [text/plain]\n",
      "Saving to: ‘test.en-swc.swc’\n",
      "\n",
      "test.en-swc.swc     100%[===================>] 226.46K  --.-KB/s    in 0.04s   \n",
      "\n",
      "2020-02-13 08:34:07 (5.55 MB/s) - ‘test.en-swc.swc’ saved [231897/231897]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "! wget https://raw.githubusercontent.com/juliakreutzer/masakhane/master/jw300_utils/test/test.en-swc.swc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The we check if we have alogned data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sommaire\r\n",
      "8 janvier 2000\r\n",
      "Médecine et chirurgie sans transfusion : une discipline en plein essor\r\n",
      "De plus en plus de patients optent pour la chirurgie sans transfusion .\r\n",
      "Pourquoi , et quels sont les résultats ?\r\n",
      "3 Pionniers de la médecine\r\n",
      "4 Transfusion sanguine : une histoire riche en revirements\r\n",
      "7 Un intérêt croissant pour la médecine et la chirurgie sans transfusion\r\n",
      "12 Voulez - ​ vous apprendre une langue étrangère ?\r\n",
      "14 Le café : mauvais pour le cholestérol ?\r\n"
     ]
    }
   ],
   "source": [
    "!head -10 jw300.fr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ukurasa wa pili\r\n",
      "Mwezi wa 8 , 2000\r\n",
      "Tiba na Upasuaji Bila Damu Uhitaji Unaoongezeka 3 - 11\r\n",
      "Sasa tiba na upasuaji bila damu ni wa kawaida zaidi kuliko wakati mwingine wowote .\r\n",
      "Kwa nini unahitajiwa sana namna hiyo ?\r\n",
      "\r\n",
      "\r\n",
      "\r\n",
      "Je , ni njia badala iliyo salama ya utiaji - damu mishipani ?\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!head -10 jw300.swc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "source_file = \"jw300.swc\"\n",
    "target_file = \"jw300.fr\"\n",
    "test_file = \"test.en-swc.swc\"\n",
    "with open(test_file) as swc_test_file, open(target_file) as fr_full_file, open(source_file) as swc_full_file:\n",
    "    swc_test_sentences = swc_test_file.readlines()\n",
    "    fr_full_sentences = fr_full_file.readlines()\n",
    "    swc_full_sentences = swc_full_file.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "swc_test_sentences = [sentence.strip() for sentence in swc_test_sentences]\n",
    "fr_full_sentences = [sentence.strip() for sentence in fr_full_sentences]\n",
    "swc_full_sentences = [sentence.strip() for sentence in swc_full_sentences]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Ukurasa wa pili',\n",
       " 'Mwezi wa 8 , 2000',\n",
       " 'Tiba na Upasuaji Bila Damu Uhitaji Unaoongezeka 3 - 11',\n",
       " 'Sasa tiba na upasuaji bila damu ni wa kawaida zaidi kuliko wakati mwingine wowote .',\n",
       " 'Kwa nini unahitajiwa sana namna hiyo ?',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " 'Je , ni njia badala iliyo salama ya utiaji - damu mishipani ?',\n",
       " '']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "swc_full_sentences[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Sommaire',\n",
       " '8 janvier 2000',\n",
       " 'Médecine et chirurgie sans transfusion : une discipline en plein essor',\n",
       " 'De plus en plus de patients optent pour la chirurgie sans transfusion .',\n",
       " 'Pourquoi , et quels sont les résultats ?',\n",
       " '3 Pionniers de la médecine',\n",
       " '4 Transfusion sanguine : une histoire riche en revirements',\n",
       " '7 Un intérêt croissant pour la médecine et la chirurgie sans transfusion',\n",
       " '12 Voulez - \\u200b vous apprendre une langue étrangère ?',\n",
       " '14 Le café : mauvais pour le cholestérol ?']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fr_full_sentences[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For any swahili sentence in the global swahili test set, get the french equivalent "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "matching_fr_test_sentences = []\n",
    "matching_swc_test_sentences = []\n",
    "\n",
    "for index, swahili_line in enumerate(swc_full_sentences):\n",
    "    if swahili_line in swc_test_sentences and swahili_line:\n",
    "        matching_fr_test_sentences.append(fr_full_sentences[index])\n",
    "        matching_swc_test_sentences.append(swahili_line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "french_swc_test_dataset = pd.DataFrame(zip(matching_fr_test_sentences, matching_swc_test_sentences), columns=['french_sentence', 'swahili_congo_sentence'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>french_sentence</th>\n",
       "      <th>swahili_congo_sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Par souci d’anonymat , certains noms ont été c...</td>\n",
       "      <td>Baadhi ya majina katika makala haya yamebadili...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Publié par les Témoins de Jéhovah , mais épuisé .</td>\n",
       "      <td>Kilichapishwa na Mashahidi wa Yehova lakini ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Vous ne pouvez travailler comme des esclaves p...</td>\n",
       "      <td>Hamwezi kutumikia kama watumwa Mungu na Utajir...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Vous ne pouvez travailler comme des esclaves p...</td>\n",
       "      <td>Hamwezi kutumikia kama watumwa Mungu na Utajir...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Il n’appartient pas à l’homme qui marche de di...</td>\n",
       "      <td>Ama kwa hakika , mwanadamu anahitaji msaada wa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>“ Le monde entier se trouve au pouvoir du méch...</td>\n",
       "      <td>“ Ulimwengu mzima unakaa katika nguvu za yule ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Ne regarde pas tout autour , car je suis ton D...</td>\n",
       "      <td>Usitazame huku na huku , kwa maana mimi ni Mun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Ne regarde pas tout autour , car je suis ton D...</td>\n",
       "      <td>Usitazame huku na huku , kwa maana mimi ni Mun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>“ Un véritable compagnon aime tout le temps et...</td>\n",
       "      <td>“ Rafiki wa kweli anapenda nyakati zote , naye...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>En outre , n’appelez personne votre père sur l...</td>\n",
       "      <td>Zaidi ya hayo , msimwite mtu yeyote baba yenu ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     french_sentence  \\\n",
       "0  Par souci d’anonymat , certains noms ont été c...   \n",
       "1  Publié par les Témoins de Jéhovah , mais épuisé .   \n",
       "2  Vous ne pouvez travailler comme des esclaves p...   \n",
       "3  Vous ne pouvez travailler comme des esclaves p...   \n",
       "4  Il n’appartient pas à l’homme qui marche de di...   \n",
       "5  “ Le monde entier se trouve au pouvoir du méch...   \n",
       "6  Ne regarde pas tout autour , car je suis ton D...   \n",
       "7  Ne regarde pas tout autour , car je suis ton D...   \n",
       "8  “ Un véritable compagnon aime tout le temps et...   \n",
       "9  En outre , n’appelez personne votre père sur l...   \n",
       "\n",
       "                              swahili_congo_sentence  \n",
       "0  Baadhi ya majina katika makala haya yamebadili...  \n",
       "1  Kilichapishwa na Mashahidi wa Yehova lakini ha...  \n",
       "2  Hamwezi kutumikia kama watumwa Mungu na Utajir...  \n",
       "3  Hamwezi kutumikia kama watumwa Mungu na Utajir...  \n",
       "4  Ama kwa hakika , mwanadamu anahitaji msaada wa...  \n",
       "5  “ Ulimwengu mzima unakaa katika nguvu za yule ...  \n",
       "6  Usitazame huku na huku , kwa maana mimi ni Mun...  \n",
       "7  Usitazame huku na huku , kwa maana mimi ni Mun...  \n",
       "8  “ Rafiki wa kweli anapenda nyakati zote , naye...  \n",
       "9  Zaidi ya hayo , msimwite mtu yeyote baba yenu ...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "french_swc_test_dataset.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>french_sentence</th>\n",
       "      <th>swahili_congo_sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2840</th>\n",
       "      <td>Et qu’est - ​ ce qui est plus joli : le bruit ...</td>\n",
       "      <td>Na namna gani unaweza kulinganisha mulio wa av...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2841</th>\n",
       "      <td>Alors , qui est le plus intelligent : le créat...</td>\n",
       "      <td>Ni nani mwenye kuwa na akili sana ; mutu mweny...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2842</th>\n",
       "      <td>Un père conseille : « Ne vous fatiguez jamais ...</td>\n",
       "      <td>Baba mumoja alisema hivi : “ Usichoke hata kid...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2843</th>\n",
       "      <td>Quand ils étaient tout petits , je faisais 15 ...</td>\n",
       "      <td>Tangu wakati walikuwa wadogo sana , nilijifunz...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2844</th>\n",
       "      <td>Avec le temps , j’ai obtenu beaucoup de répons...</td>\n",
       "      <td>Kisha wakati fulani , nilipata majibu mengi ya...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2845</th>\n",
       "      <td>C’est pour cela que c’est important que les pa...</td>\n",
       "      <td>Ndiyo sababu ni jambo la maana wazazi wasiache...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2846</th>\n",
       "      <td>Montre - ​ leur que Jéhovah est vraiment réel ...</td>\n",
       "      <td>Acha watoto wako watambue kama unaona Yehova k...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2847</th>\n",
       "      <td>« Nous disons à la plus grande : “ Fais totale...</td>\n",
       "      <td>Wazazi hao wanasema hivi : “ Tuliambia pia mut...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2848</th>\n",
       "      <td>Lorsqu’elle voit comment les choses s’arrangen...</td>\n",
       "      <td>Wakati anaona matokeo , anaelewa kama Yehova a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2849</th>\n",
       "      <td>C’est excellent pour sa foi en Dieu et en la B...</td>\n",
       "      <td>Hilo limemusaidia sana akuwe na imani yenye ng...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        french_sentence  \\\n",
       "2840  Et qu’est - ​ ce qui est plus joli : le bruit ...   \n",
       "2841  Alors , qui est le plus intelligent : le créat...   \n",
       "2842  Un père conseille : « Ne vous fatiguez jamais ...   \n",
       "2843  Quand ils étaient tout petits , je faisais 15 ...   \n",
       "2844  Avec le temps , j’ai obtenu beaucoup de répons...   \n",
       "2845  C’est pour cela que c’est important que les pa...   \n",
       "2846  Montre - ​ leur que Jéhovah est vraiment réel ...   \n",
       "2847  « Nous disons à la plus grande : “ Fais totale...   \n",
       "2848  Lorsqu’elle voit comment les choses s’arrangen...   \n",
       "2849  C’est excellent pour sa foi en Dieu et en la B...   \n",
       "\n",
       "                                 swahili_congo_sentence  \n",
       "2840  Na namna gani unaweza kulinganisha mulio wa av...  \n",
       "2841  Ni nani mwenye kuwa na akili sana ; mutu mweny...  \n",
       "2842  Baba mumoja alisema hivi : “ Usichoke hata kid...  \n",
       "2843  Tangu wakati walikuwa wadogo sana , nilijifunz...  \n",
       "2844  Kisha wakati fulani , nilipata majibu mengi ya...  \n",
       "2845  Ndiyo sababu ni jambo la maana wazazi wasiache...  \n",
       "2846  Acha watoto wako watambue kama unaona Yehova k...  \n",
       "2847  Wazazi hao wanasema hivi : “ Tuliambia pia mut...  \n",
       "2848  Wakati anaona matokeo , anaelewa kama Yehova a...  \n",
       "2849  Hilo limemusaidia sana akuwe na imani yenye ng...  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "french_swc_test_dataset.tail(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove duplicates from both sides"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2478, 2)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "french_swc_test_dataset = french_swc_test_dataset.drop_duplicates(subset='french_sentence')\n",
    "french_swc_test_dataset = french_swc_test_dataset.drop_duplicates(subset='swahili_congo_sentence')\n",
    "french_swc_test_dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 3332 global test sentences to filter from the training/dev data.\n"
     ]
    }
   ],
   "source": [
    "fr_test_sents = set()\n",
    "filter_test_sents = \"test.fr-any.fr\"\n",
    "j = 0\n",
    "with open(filter_test_sents) as f:\n",
    "    for line in f:\n",
    "        fr_test_sents.add(line.strip())\n",
    "        j += 1\n",
    "print('Loaded {} global test sentences to filter from the training/dev data.'.format(j))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(643, 2)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "french_swc_test_dataset.loc[~french_swc_test_dataset.french_sentence.isin(fr_test_sents)].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>french_sentence</th>\n",
       "      <th>swahili_congo_sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Il n’appartient pas à l’homme qui marche de di...</td>\n",
       "      <td>Ama kwa hakika , mwanadamu anahitaji msaada wa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>“ Le monde entier se trouve au pouvoir du méch...</td>\n",
       "      <td>“ Ulimwengu mzima unakaa katika nguvu za yule ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>“ Un véritable compagnon aime tout le temps et...</td>\n",
       "      <td>“ Rafiki wa kweli anapenda nyakati zote , naye...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>En outre , n’appelez personne votre père sur l...</td>\n",
       "      <td>Zaidi ya hayo , msimwite mtu yeyote baba yenu ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>“ Les justes posséderont la terre , et sur ell...</td>\n",
       "      <td>“ Waadilifu wenyewe wataimiliki dunia , nao wa...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      french_sentence  \\\n",
       "4   Il n’appartient pas à l’homme qui marche de di...   \n",
       "5   “ Le monde entier se trouve au pouvoir du méch...   \n",
       "8   “ Un véritable compagnon aime tout le temps et...   \n",
       "9   En outre , n’appelez personne votre père sur l...   \n",
       "12  “ Les justes posséderont la terre , et sur ell...   \n",
       "\n",
       "                               swahili_congo_sentence  \n",
       "4   Ama kwa hakika , mwanadamu anahitaji msaada wa...  \n",
       "5   “ Ulimwengu mzima unakaa katika nguvu za yule ...  \n",
       "8   “ Rafiki wa kweli anapenda nyakati zote , naye...  \n",
       "9   Zaidi ya hayo , msimwite mtu yeyote baba yenu ...  \n",
       "12  “ Waadilifu wenyewe wataimiliki dunia , nao wa...  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "french_swc_test_dataset.loc[~french_swc_test_dataset.french_sentence.isin(fr_test_sents)].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Only 643 sentences are in the french swc test set but not in the global test set!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add those file to the global test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "french_swc_test_dataset.loc[~french_swc_test_dataset.french_sentence.isin(fr_test_sents)].french_sentence.to_csv('test.fr-any.fr', mode='a', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2837    Elle raconte : « Ils l’ont fait avec beaucoup ...\n",
       "2838    Quand je leur ai demandé pourquoi ils avaient ...\n",
       "2842    Un père conseille : « Ne vous fatiguez jamais ...\n",
       "2843    Quand ils étaient tout petits , je faisais 15 ...\n",
       "2844    Avec le temps , j’ai obtenu beaucoup de répons...\n",
       "2845    C’est pour cela que c’est important que les pa...\n",
       "2846    Montre - ​ leur que Jéhovah est vraiment réel ...\n",
       "2847    « Nous disons à la plus grande : “ Fais totale...\n",
       "2848    Lorsqu’elle voit comment les choses s’arrangen...\n",
       "2849    C’est excellent pour sa foi en Dieu et en la B...\n",
       "Name: french_sentence, dtype: object"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "french_swc_test_dataset.loc[~french_swc_test_dataset.french_sentence.isin(fr_test_sents)].french_sentence.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elle raconte : « Ils l’ont fait avec beaucoup de soin .\r\n",
      "\"Quand je leur ai demandé pourquoi ils avaient fait autant attention , ils ont répondu qu’ils voulaient que le café soit exactement comme je l’aime .\"\r\n",
      "Un père conseille : « Ne vous fatiguez jamais d’essayer de nouvelles méthodes pour reparler de vieux sujets .\r\n",
      "\"Quand ils étaient tout petits , je faisais 15 minutes d’étude avec eux tous les jours , sauf les jours de réunion .\"\r\n",
      "\"Avec le temps , j’ai obtenu beaucoup de réponses aux réunions , ou durant l’étude familiale ou individuelle .\"\r\n",
      "C’est pour cela que c’est important que les parents n’arrêtent jamais d’enseigner .\r\n",
      "Montre - ​ leur que Jéhovah est vraiment réel pour toi .\r\n",
      "\"« Nous disons à la plus grande : “ Fais totalement confiance à Jéhovah , reste active dans le service du Royaume et ne t’inquiète pas trop . ”\"\r\n",
      "\"Lorsqu’elle voit comment les choses s’arrangent , elle se rend compte que Jéhovah nous aide .\"\r\n",
      "C’est excellent pour sa foi en Dieu et en la Bible . »\r\n"
     ]
    }
   ],
   "source": [
    "!tail -10 test.fr-any.fr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the test set with to new files "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "french_swc_test_dataset.swahili_congo_sentence.to_csv(\"test.swc\",sep='\\n', header=False, index=False)\n",
    "french_swc_test_dataset.french_sentence.to_csv(\"test.fr\",sep='\\n', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Building the model dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Par souci d’anonymat , certains noms ont été changés .\r\n",
      "Publié par les Témoins de Jéhovah , mais épuisé .\r\n",
      "Vous ne pouvez travailler comme des esclaves pour Dieu et pour la Richesse .\r\n",
      "Il n’appartient pas à l’homme qui marche de diriger son pas .\r\n",
      "“ Le monde entier se trouve au pouvoir du méchant .\r\n"
     ]
    }
   ],
   "source": [
    "!head -5 test.$src"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baadhi ya majina katika makala haya yamebadilishwa .\r\n",
      "Kilichapishwa na Mashahidi wa Yehova lakini hakichapwi tena .\r\n",
      "Hamwezi kutumikia kama watumwa Mungu na Utajiri . ”\r\n",
      "Ama kwa hakika , mwanadamu anahitaji msaada wa Mungu .\r\n",
      "“ Ulimwengu mzima unakaa katika nguvu za yule mwovu . ”\r\n"
     ]
    }
   ],
   "source": [
    "!head -5 test.$tgt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_lines_to_ignore = fr_test_sents.union(set(french_swc_test_dataset.french_sentence.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded data and skipped 13705/576109 lines since contained in test set.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_sentence</th>\n",
       "      <th>target_sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8 janvier 2000</td>\n",
       "      <td>Mwezi wa 8 , 2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Médecine et chirurgie sans transfusion : une d...</td>\n",
       "      <td>Tiba na Upasuaji Bila Damu Uhitaji Unaoongezek...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>De plus en plus de patients optent pour la chi...</td>\n",
       "      <td>Sasa tiba na upasuaji bila damu ni wa kawaida ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Pourquoi , et quels sont les résultats ?</td>\n",
       "      <td>Kwa nini unahitajiwa sana namna hiyo ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3 Pionniers de la médecine</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4 Transfusion sanguine : une histoire riche en...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7 Un intérêt croissant pour la médecine et la ...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>12 Voulez - ​ vous apprendre une langue étrang...</td>\n",
       "      <td>Je , ni njia badala iliyo salama ya utiaji - d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>14 Le café : mauvais pour le cholestérol ?</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>20 Le dilemme des mères atteintes du sida</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     source_sentence  \\\n",
       "0                                     8 janvier 2000   \n",
       "1  Médecine et chirurgie sans transfusion : une d...   \n",
       "2  De plus en plus de patients optent pour la chi...   \n",
       "3           Pourquoi , et quels sont les résultats ?   \n",
       "4                         3 Pionniers de la médecine   \n",
       "5  4 Transfusion sanguine : une histoire riche en...   \n",
       "6  7 Un intérêt croissant pour la médecine et la ...   \n",
       "7  12 Voulez - ​ vous apprendre une langue étrang...   \n",
       "8         14 Le café : mauvais pour le cholestérol ?   \n",
       "9          20 Le dilemme des mères atteintes du sida   \n",
       "\n",
       "                                     target_sentence  \n",
       "0                                  Mwezi wa 8 , 2000  \n",
       "1  Tiba na Upasuaji Bila Damu Uhitaji Unaoongezek...  \n",
       "2  Sasa tiba na upasuaji bila damu ni wa kawaida ...  \n",
       "3             Kwa nini unahitajiwa sana namna hiyo ?  \n",
       "4                                                     \n",
       "5                                                     \n",
       "6                                                     \n",
       "7  Je , ni njia badala iliyo salama ya utiaji - d...  \n",
       "8                                                     \n",
       "9                                                     "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TMX file to dataframe\n",
    "source_file = 'jw300.' + source_language\n",
    "target_file = 'jw300.' + target_language\n",
    "\n",
    "source = []\n",
    "target = []\n",
    "skip_lines = []  # Collect the line numbers of the source portion to skip the same lines for the target portion.\n",
    "with open(source_file) as f:\n",
    "    for i, line in enumerate(f):\n",
    "        # Skip sentences that are contained in the test set or in the frc_test_set\n",
    "        if line.strip() not in test_lines_to_ignore:\n",
    "            source.append(line.strip())\n",
    "        else:\n",
    "            skip_lines.append(i)             \n",
    "with open(target_file) as f:\n",
    "    for j, line in enumerate(f):\n",
    "        # Only add to corpus if corresponding source was not skipped.\n",
    "        if j not in skip_lines:\n",
    "            target.append(line.strip())\n",
    "    \n",
    "print('Loaded data and skipped {}/{} lines since contained in test set.'.format(len(skip_lines), i))\n",
    "    \n",
    "df = pd.DataFrame(zip(source, target), columns=['source_sentence', 'target_sentence'])\n",
    "# if you get TypeError: data argument can't be an iterator is because of your zip version run this below\n",
    "#df = pd.DataFrame(list(zip(source, target)), columns=['source_sentence', 'target_sentence'])\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some tests :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_sentence</th>\n",
       "      <th>target_sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>562395</th>\n",
       "      <td>Comme les chrétiens hébreux , nous pouvons étu...</td>\n",
       "      <td>Kama vile Wakristo Waebrania , tunaweza kusoma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562396</th>\n",
       "      <td>Pour montrer que cette promesse est biblique ,...</td>\n",
       "      <td>Ili kukazia kama ahadi hiyo inategemea Maandik...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562397</th>\n",
       "      <td>Nous sommes touchés de savoir que « la promess...</td>\n",
       "      <td>Inagusa sana mioyo yetu kujua kama ‘ ahadi ime...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562398</th>\n",
       "      <td>Nous sommes convaincus que c’est possible d’en...</td>\n",
       "      <td>Tunatumaini kama inawezekana kabisa kuingia ka...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562399</th>\n",
       "      <td>Pas en obéissant à la Loi de Moïse , ni en fai...</td>\n",
       "      <td>Hatufanye hivyo kwa kujaribu kufuata Sheria ya...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562400</th>\n",
       "      <td>Mais c’est plutôt en travaillant avec foi et d...</td>\n",
       "      <td>Lakini , kwa sababu ya imani yetu kwa Mungu , ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562401</th>\n",
       "      <td>De plus , des milliers de personnes dans le mo...</td>\n",
       "      <td>Mamilioni ya watu katika dunia yote wameanza p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562402</th>\n",
       "      <td>Cette étude a motivé beaucoup d’entre elles à ...</td>\n",
       "      <td>Hilo limechochea wengi kati yao wafanye mabadi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562403</th>\n",
       "      <td>L’effet que « la parole de Dieu » a sur ces pe...</td>\n",
       "      <td>Namna wanaendelea kufanya mabadiliko inaonyesh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562404</th>\n",
       "      <td>Les déclarations de Jéhovah sur son projet qui...</td>\n",
       "      <td>Mambo yenye Mungu amefunua katika Biblia juu y...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          source_sentence  \\\n",
       "562395  Comme les chrétiens hébreux , nous pouvons étu...   \n",
       "562396  Pour montrer que cette promesse est biblique ,...   \n",
       "562397  Nous sommes touchés de savoir que « la promess...   \n",
       "562398  Nous sommes convaincus que c’est possible d’en...   \n",
       "562399  Pas en obéissant à la Loi de Moïse , ni en fai...   \n",
       "562400  Mais c’est plutôt en travaillant avec foi et d...   \n",
       "562401  De plus , des milliers de personnes dans le mo...   \n",
       "562402  Cette étude a motivé beaucoup d’entre elles à ...   \n",
       "562403  L’effet que « la parole de Dieu » a sur ces pe...   \n",
       "562404  Les déclarations de Jéhovah sur son projet qui...   \n",
       "\n",
       "                                          target_sentence  \n",
       "562395  Kama vile Wakristo Waebrania , tunaweza kusoma...  \n",
       "562396  Ili kukazia kama ahadi hiyo inategemea Maandik...  \n",
       "562397  Inagusa sana mioyo yetu kujua kama ‘ ahadi ime...  \n",
       "562398  Tunatumaini kama inawezekana kabisa kuingia ka...  \n",
       "562399  Hatufanye hivyo kwa kujaribu kufuata Sheria ya...  \n",
       "562400  Lakini , kwa sababu ya imani yetu kwa Mungu , ...  \n",
       "562401  Mamilioni ya watu katika dunia yote wameanza p...  \n",
       "562402  Hilo limechochea wengi kati yao wafanye mabadi...  \n",
       "562403  Namna wanaendelea kufanya mabadiliko inaonyesh...  \n",
       "562404  Mambo yenye Mungu amefunua katika Biblia juu y...  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Pour toutes ces raisons , la médecine sans transfusion a suscité , ces dernières années , un intérêt croissant .'"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.source_sentence[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Basi yaeleweka ni kwa nini katika miaka ya karibuni watu wengi wamependezwa zaidi na tiba na upasuaji bila damu .'"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.target_sentence[100]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pre-processing and export\n",
    "\n",
    "It is generally a good idea to remove duplicate translations and conflicting translations from the corpus. In practice, these public corpora include some number of these that need to be cleaned.\n",
    "\n",
    "In addition we will split our data into dev/test/train and export to the filesystem."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_without_duplicates = df.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_without_duplicates = df_without_duplicates.drop_duplicates(subset='source_sentence', inplace=False)\n",
    "df_without_duplicates = df_without_duplicates.drop_duplicates(subset='target_sentence', inplace=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove all sentences that can be found in the swahili test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_without_duplicates = df_without_duplicates.loc[~df_without_duplicates.target_sentence.isin(french_swc_test_dataset.swahili_congo_sentence)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(503504, 2)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_without_duplicates.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following lines am checking that all sentences we have kept in the main dataset are not in the test set anymore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert  df_without_duplicates.loc[df_without_duplicates.target_sentence.isna()].shape[0] == 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also check if the splitting was done corectly "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert df_without_duplicates.loc[df_without_duplicates.source_sentence.isin(fr_test_sents)].shape[0] == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert df_without_duplicates.loc[df_without_duplicates.source_sentence.isin(french_swc_test_dataset.french_sentence)].shape[0] == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert df_without_duplicates.loc[df_without_duplicates.target_sentence.isin(french_swc_test_dataset.swahili_congo_sentence)].shape[0] == 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If any of these test fails please check preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_without_duplicates = df_without_duplicates.sample(frac=1, random_state=seed).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_sentence</th>\n",
       "      <th>target_sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Comme l’endroit vous est inconnu , vous êtes q...</td>\n",
       "      <td>Kwa kuwa uko ugenini , unaanza kuwa na wasiwasi .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Enfin , Paul se montrait remarquablement inven...</td>\n",
       "      <td>Paulo alionyesha ubunifu wa hali ya juu alipos...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Touchés par ces pensées , les membres de cette...</td>\n",
       "      <td>Kwa kuwa waliheshimu Neno la Mungu na kufurahi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ils ne disposent pas non plus des milliers de ...</td>\n",
       "      <td>Na hawana maelfu ya dola zinazohitajika ili wa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Parfois , on brisait les os pour recueillir la...</td>\n",
       "      <td>Nyakati nyingine mifupa huvunjwa ili uboho uto...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Ils partagent les sentiments de frère Alexande...</td>\n",
       "      <td>H . Macmillan , ambaye baada ya karibu miaka 6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>La Bible est le livre le plus fiable sur ce su...</td>\n",
       "      <td>Biblia ndicho kitabu kinachotegemeka zaidi kuh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>D’autres ont accepté d’étudier la Bible avec n...</td>\n",
       "      <td>Wengine wamekubali tujifunze nao Biblia lakini...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>” Il est important de noter qu’un seul symptôm...</td>\n",
       "      <td>Ni muhimu kung’amua kwamba dalili moja tu hait...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Jésus pensait - ​ il à cela ?</td>\n",
       "      <td>Je , Yesu alikuwa anarejelea lango hilo ?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     source_sentence  \\\n",
       "0  Comme l’endroit vous est inconnu , vous êtes q...   \n",
       "1  Enfin , Paul se montrait remarquablement inven...   \n",
       "2  Touchés par ces pensées , les membres de cette...   \n",
       "3  Ils ne disposent pas non plus des milliers de ...   \n",
       "4  Parfois , on brisait les os pour recueillir la...   \n",
       "5  Ils partagent les sentiments de frère Alexande...   \n",
       "6  La Bible est le livre le plus fiable sur ce su...   \n",
       "7  D’autres ont accepté d’étudier la Bible avec n...   \n",
       "8  ” Il est important de noter qu’un seul symptôm...   \n",
       "9                      Jésus pensait - ​ il à cela ?   \n",
       "\n",
       "                                     target_sentence  \n",
       "0  Kwa kuwa uko ugenini , unaanza kuwa na wasiwasi .  \n",
       "1  Paulo alionyesha ubunifu wa hali ya juu alipos...  \n",
       "2  Kwa kuwa waliheshimu Neno la Mungu na kufurahi...  \n",
       "3  Na hawana maelfu ya dola zinazohitajika ili wa...  \n",
       "4  Nyakati nyingine mifupa huvunjwa ili uboho uto...  \n",
       "5  H . Macmillan , ambaye baada ya karibu miaka 6...  \n",
       "6  Biblia ndicho kitabu kinachotegemeka zaidi kuh...  \n",
       "7  Wengine wamekubali tujifunze nao Biblia lakini...  \n",
       "8  Ni muhimu kung’amua kwamba dalili moja tu hait...  \n",
       "9          Je , Yesu alikuwa anarejelea lango hilo ?  "
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_without_duplicates.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(503504, 2)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_without_duplicates.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from fuzzywuzzy import process\n",
    "import numpy as np\n",
    "from os import cpu_count\n",
    "from functools import partial\n",
    "from multiprocessing import Pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "## reset the index after\n",
    "df_without_duplicates.reset_index(inplace=True, drop=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filtering function. Adjust pad to narrow down the candidate matches to\n",
    "# within a certain length of characters of the given sample.\n",
    "def fuzzfilter(sample, candidates, pad):\n",
    "    candidates = [x for x in candidates if len(x) <= len(sample)+pad and len(x) >= len(sample)-pad] \n",
    "    if len(candidates) > 0:\n",
    "        return process.extractOne(sample, candidates)[1]\n",
    "    else:\n",
    "        return np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '→ → →']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '․']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '’ ) .']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '↓ ↓ ↓']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '→']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '’ ”']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '- -']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '» .']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '↓']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '€ € €']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '’ .']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '⇩']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '— ― ― ― ― ― ― ―']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '․ ․ ․ ․ ․ €']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '← ◯']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '\\']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '●']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '” )']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '← ←']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '↑']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '[ . . . ]']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '” * .']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '’ ” .']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '*']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '» ? .']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '. .']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '→ →']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '⇧']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '․ ․ ․']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '​ — — ​']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '’']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '” ) .']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '● ● ● ● ●']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '”']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '- - ​ — — ​']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '“ . . .']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '←']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '» ) .']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '← ← ← ← ← ← ← ← ← ←']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․ ․']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '□ ․ ․ ․ ․ ․']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '” » .']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done in 0.0h:43.0min:55.992006063461304seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "### iterating over pandas dataframe rows is not recomended, let use multi processing to apply the function\n",
    "\n",
    "with Pool(cpu_count()-1) as pool:\n",
    "    scores = pool.map(partial(fuzzfilter, candidates=list(fr_test_sents), pad=5), df_without_duplicates['source_sentence'])\n",
    "hours, rem = divmod(time.time() - start_time, 3600)\n",
    "minutes, seconds = divmod(rem, 60)\n",
    "print(\"done in {}h:{}min:{}seconds\".format(hours, minutes, seconds))\n",
    "\n",
    "# Filter out \"almost overlapping samples\"\n",
    "df_without_duplicates = df_without_duplicates.assign(scores=scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_without_duplicates = df_without_duplicates[df_without_duplicates['scores'] < 95]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df_without_duplicates.to_csv('data_process_without_duplicates.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>source_sentence</th>\n",
       "      <th>target_sentence</th>\n",
       "      <th>scores</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Comme l’endroit vous est inconnu , vous êtes q...</td>\n",
       "      <td>Kwa kuwa uko ugenini , unaanza kuwa na wasiwasi .</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Enfin , Paul se montrait remarquablement inven...</td>\n",
       "      <td>Paulo alionyesha ubunifu wa hali ya juu alipos...</td>\n",
       "      <td>51.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Touchés par ces pensées , les membres de cette...</td>\n",
       "      <td>Kwa kuwa waliheshimu Neno la Mungu na kufurahi...</td>\n",
       "      <td>53.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Ils ne disposent pas non plus des milliers de ...</td>\n",
       "      <td>Na hawana maelfu ya dola zinazohitajika ili wa...</td>\n",
       "      <td>54.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Parfois , on brisait les os pour recueillir la...</td>\n",
       "      <td>Nyakati nyingine mifupa huvunjwa ili uboho uto...</td>\n",
       "      <td>55.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index                                    source_sentence  \\\n",
       "0      0  Comme l’endroit vous est inconnu , vous êtes q...   \n",
       "1      1  Enfin , Paul se montrait remarquablement inven...   \n",
       "2      2  Touchés par ces pensées , les membres de cette...   \n",
       "3      3  Ils ne disposent pas non plus des milliers de ...   \n",
       "4      4  Parfois , on brisait les os pour recueillir la...   \n",
       "\n",
       "                                     target_sentence  scores  \n",
       "0  Kwa kuwa uko ugenini , unaanza kuwa na wasiwasi .    56.0  \n",
       "1  Paulo alionyesha ubunifu wa hali ya juu alipos...    51.0  \n",
       "2  Kwa kuwa waliheshimu Neno la Mungu na kufurahi...    53.0  \n",
       "3  Na hawana maelfu ya dola zinazohitajika ili wa...    54.0  \n",
       "4  Nyakati nyingine mifupa huvunjwa ili uboho uto...    55.0  "
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_without_duplicates.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "splitting the data into train/test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==> train.fr <==\n",
      "Comme l’endroit vous est inconnu , vous êtes quelque peu angoissé .\n",
      "Enfin , Paul se montrait remarquablement inventif pour adapter son enseignement à différents auditoires .\n",
      "Touchés par ces pensées , les membres de cette famille , qui avaient un grand respect pour la Parole de Dieu , ont accepté de l’étudier régulièrement en compagnie des Témoins de Jéhovah .\n",
      "Ils ne disposent pas non plus des milliers de dollars qu’il faut pour s’en débarrasser correctement . ”\n",
      "Parfois , on brisait les os pour recueillir la moelle .\n",
      "Ils partagent les sentiments de frère Alexander Macmillan qui , après avoir servi Dieu avec dévouement pendant plus de 60 ans , a dit : “ Je suis plus déterminé que jamais à persévérer dans ma foi .\n",
      "La Bible est le livre le plus fiable sur ce sujet ; elle contient de nombreuses prières d’hommes et de femmes fidèles .\n",
      "D’autres ont accepté d’étudier la Bible avec nous mais ne veulent pas s’engager .\n",
      "” Il est important de noter qu’un seul symptôme n’est pas le signe d’une dépression ou d’un trouble bipolaire .\n",
      "Jésus pensait - ​ il à cela ?\n",
      "\n",
      "==> train.swc <==\n",
      "Kwa kuwa uko ugenini , unaanza kuwa na wasiwasi .\n",
      "Paulo alionyesha ubunifu wa hali ya juu aliposhughulika na watu mbalimbali .\n",
      "Kwa kuwa waliheshimu Neno la Mungu na kufurahia waliyosikia , washiriki wa familia hiyo walikubali kujifunza Biblia kwa ukawaida pamoja na Mashahidi wa Yehova .\n",
      "Na hawana maelfu ya dola zinazohitajika ili wapate hati zinazoonyesha kuwa mashua hizo si mali yao tena . ”\n",
      "Nyakati nyingine mifupa huvunjwa ili uboho utoke .\n",
      "H . Macmillan , ambaye baada ya karibu miaka 60 ya utumishi wa kujitoa wakfu kwa Mungu , alisema hivi : “ Nimeazimia kuliko wakati mwingine wowote kuendelea katika imani yangu .\n",
      "Biblia ndicho kitabu kinachotegemeka zaidi kuhusiana na suala hili , na kina sala za wanaume na wanawake wengi waaminifu .\n",
      "Wengine wamekubali tujifunze nao Biblia lakini hawako tayari kutoa ahadi ya kumutumikia Yehova .\n",
      "Ni muhimu kung’amua kwamba dalili moja tu haitoshi kuonyesha kwamba mtu ameshuka moyo au ana ugonjwa wa hisia zinazobadilika - badilika .\n",
      "Je , Yesu alikuwa anarejelea lango hilo ?\n",
      "==> dev.fr <==\n",
      "Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "Les simulies véhiculent la cécité des rivières .\n",
      "À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "Immersion même en cas de gros handicap ou d’une très mauvaise santé ?\n",
      "” Évidemment , notre cœur peut nous tromper quand notre endurance est mise à l’épreuve .\n",
      "” Une “ situation de crise ” en voiture peut rapidement dégénérer , notamment parce que l’espace clos du véhicule ne permet pas de s’en aller .\n",
      "Pendant des siècles , il a été le cadre de sanglants combats de gladiateurs .\n",
      "L’abrasion produit davantage de sable .\n",
      "Rigueur et méticulosité\n",
      "\n",
      "==> dev.swc <==\n",
      "Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "Nzi anayeeneza upofu wa mtoni\n",
      "Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "Mti Unaotoa “ Machozi , ” 1 / 15\n",
      "Bila shaka , moyo unaweza kutupotosha tunapokabili jaribu la ustahimilivu .\n",
      "Gazeti hilo linasema kwamba “ ugomvi ” unapotokea ndani ya gari unaweza kuwa mbaya sana , hasa kwa sababu hakuna mahali pa kuuondokea .\n",
      "Kwa muda wa karne nyingi mapigano ya kikatili kati ya watu au kati ya mtu na mnyama yalionyeshwa humo .\n",
      "Vipande hivyo hukwaruza miamba na kufanyiza mchanga zaidi .\n",
      "Mwangalifu Sana na Mwenye Nidhamu\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# We use 1000 dev test and the given test set.\n",
    "import csv\n",
    "\n",
    "# Do the split between dev/train and create parallel corpora\n",
    "num_dev_patterns = 1000\n",
    "\n",
    "# Optional: lower case the corpora - this will make it easier to generalize, but without proper casing.\n",
    "if lc:  # Julia: making lowercasing optional\n",
    "    df_without_duplicates[\"source_sentence\"] = df_without_duplicates[\"source_sentence\"].str.lower()\n",
    "    df_without_duplicates[\"target_sentence\"] = df_without_duplicates[\"target_sentence\"].str.lower()\n",
    "\n",
    "# Julia: test sets are already generated\n",
    "dev = df_without_duplicates.tail(num_dev_patterns) # Herman: Error in original\n",
    "stripped = df_without_duplicates.drop(df_without_duplicates.tail(num_dev_patterns).index)\n",
    "\n",
    "with open(\"train.\"+source_language, \"w\") as src_file, open(\"train.\"+target_language, \"w\") as trg_file:\n",
    "    for index, row in stripped.iterrows():\n",
    "        src_file.write(row[\"source_sentence\"]+\"\\n\")\n",
    "        trg_file.write(row[\"target_sentence\"]+\"\\n\")\n",
    "    \n",
    "with open(\"dev.\"+source_language, \"w\") as src_file, open(\"dev.\"+target_language, \"w\") as trg_file:\n",
    "    for index, row in dev.iterrows():\n",
    "        src_file.write(row[\"source_sentence\"]+\"\\n\")\n",
    "        trg_file.write(row[\"target_sentence\"]+\"\\n\")\n",
    "\n",
    "#stripped[[\"source_sentence\"]].to_csv(\"train.\"+source_language, header=False, index=False)  # Herman: Added `header=False` everywhere\n",
    "#stripped[[\"target_sentence\"]].to_csv(\"train.\"+target_language, header=False, index=False)  # Julia: Problematic handling of quotation marks.\n",
    "\n",
    "#dev[[\"source_sentence\"]].to_csv(\"dev.\"+source_language, header=False, index=False)\n",
    "#dev[[\"target_sentence\"]].to_csv(\"dev.\"+target_language, header=False, index=False)\n",
    "\n",
    "# Doublecheck the format below. There should be no extra quotation marks or weird characters.\n",
    "! head train.*\n",
    "! head dev.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "# checking if joeymnt is installed and skip the installation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "joeynmt==0.0.1\r\n"
     ]
    }
   ],
   "source": [
    "!pip3 freeze | grep joeynmt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "subword-nmt==0.3.6\r\n"
     ]
    }
   ],
   "source": [
    "!pip3 freeze | grep subword-nmt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/usr/local/bin/subword-nmt\r\n"
     ]
    }
   ],
   "source": [
    "!which subword-nmt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing the Data into Subword BPE Tokens\n",
    "\n",
    "- One of the most powerful improvements for agglutinative languages (a feature of most Bantu languages) is using BPE tokenization (Sennrich, 2015) .\n",
    "\n",
    "- It was also shown that by optimizing the umber of BPE codes we significantly improve results for low-resourced languages (Sennrich, 2019) (Martinus, 2019)\n",
    "\n",
    "- Below we have the scripts for doing BPE tokenization of our data. We use 4000 tokens as recommended by (Sennrich, 2019). You do not need to change anything. Simply running the below will be suitable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "create the new data folder from the working directory since joeymnt is already installed we are using another data folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘data’: File exists\r\n"
     ]
    }
   ],
   "source": [
    "!mkdir data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Learn BPEs on the training data.\n",
    "os.environ[\"data_path\"] = path.join(\"data\", source_language + target_language) # Herma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "! subword-nmt learn-joint-bpe-and-vocab --input train.$src train.$tgt -s 4000 -o bpe.codes.4000 --write-vocabulary vocab.$src vocab.$tgt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply BPE splits to the development and test data.\n",
    "! subword-nmt apply-bpe -c bpe.codes.4000 --vocabulary vocab.$src < train.$src > train.bpe.$src\n",
    "! subword-nmt apply-bpe -c bpe.codes.4000 --vocabulary vocab.$tgt < train.$tgt > train.bpe.$tgt\n",
    "\n",
    "! subword-nmt apply-bpe -c bpe.codes.4000 --vocabulary vocab.$src < dev.$src > dev.bpe.$src\n",
    "! subword-nmt apply-bpe -c bpe.codes.4000 --vocabulary vocab.$tgt < dev.$tgt > dev.bpe.$tgt\n",
    "! subword-nmt apply-bpe -c bpe.codes.4000 --vocabulary vocab.$src < test.$src > test.bpe.$src\n",
    "! subword-nmt apply-bpe -c bpe.codes.4000 --vocabulary vocab.$tgt < test.$tgt > test.bpe.$tgt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "creating the data folder from my language and copying everything to it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bpe.codes.4000\ttest.bpe.fr\t  test.en-any.en.2  test.fr-swc.fr   train.fr\r\n",
      "dev.bpe.fr\ttest.bpe.swc\t  test.en-swc.en    test.fr-swc.swc  train.swc\r\n",
      "dev.bpe.swc\ttest.en\t\t  test.en-swc.swc   test.swc\t     vocab.txt\r\n",
      "dev.fr\t\ttest.en-any.en\t  test.fr\t    train.bpe.fr\r\n",
      "dev.swc\t\ttest.en-any.en.1  test.fr-any.fr    train.bpe.swc\r\n"
     ]
    }
   ],
   "source": [
    "! mkdir -p $data_path\n",
    "! cp train.* $data_path\n",
    "! cp test.* $data_path\n",
    "! cp dev.* $data_path\n",
    "! cp bpe.codes.4000 $data_path\n",
    "! ls $data_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating the vocabulary using joeymnt "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "getting the script manually and checking  and run it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget https://raw.githubusercontent.com/joeynmt/joeynmt/master/scripts/build_vocab.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python3 build_vocab.py data/$src$tgt/train.bpe.$src data/$src$tgt/train.bpe.$tgt --output_path data/$src$tgt/vocab.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPE Swahili congo Sentences\n",
      "Ndiyo sababu ni jambo la maana wazazi wasi@@ a@@ che ku@@ fundisha watoto wao . ”\n",
      "A@@ cha watoto wako wat@@ amb@@ ue kama una@@ ona Yehova kuwa mutu wa kweli kabisa .\n",
      "Wazazi hao wan@@ asema hivi : “ Tuli@@ ambia pia mut@@ oto wetu mu@@ kubwa mwanam@@ uke kama am@@ ut@@ egem@@ ee kabisa Yehova , a@@ endelee kufanya mengi katika kazi ya Ufalme , na asi@@ hangai@@ ke sana .\n",
      "Wakati ana@@ ona matokeo , ana@@ elewa kama Yehova ana@@ endelea kutusaidia .\n",
      "Hilo li@@ m@@ em@@ usaidia sana a@@ kuwe na imani yenye nguvu kwa Mungu na katika Biblia . ”\n",
      "Combined BPE Vocab\n",
      "і@@\n",
      "д@@\n",
      "і\n",
      "<\n",
      "Ō@@\n",
      "ù@@\n",
      "~\n",
      "£\n",
      "γ\n",
      "ז@@\n"
     ]
    }
   ],
   "source": [
    "# Some output\n",
    "! echo \"BPE Swahili congo Sentences\"\n",
    "! tail -n 5 test.bpe.$tgt\n",
    "! echo \"Combined BPE Vocab\"\n",
    "! tail -n 10 data/$src$tgt/vocab.txt "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating the JoeyNMT Config\n",
    "\n",
    "JoeyNMT requires a yaml config. We provide a template below. We've also set a number of defaults with it, that you may play with!\n",
    "\n",
    "- We used Transformer architecture\n",
    "- We set our dropout to reasonably high: 0.3 (recommended in (Sennrich, 2019))\n",
    "Things worth playing with:\n",
    "\n",
    "- The batch size (also recommended to change for low-resourced languages)\n",
    "- The number of epochs (we've set it at 30 just so it runs in about an hour, for testing purposes)\n",
    "- The decoder options (beam_size, alpha)\n",
    "Evaluation metrics (BLEU versus Crhf4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "JW300_latest_xml_fr-swc.xml\t\t     fr-swc-baseline_v2\r\n",
      "JW300_latest_xml_fr.zip\t\t\t     jw300.fr\r\n",
      "JW300_latest_xml_swc.zip\t\t     jw300.swc\r\n",
      "baseline.ipynb\t\t\t\t     models\r\n",
      "bpe.codes.4000\t\t\t\t     test.bpe.fr\r\n",
      "buiding_french_global_test_set.ipynb\t     test.bpe.swc\r\n",
      "buiding_swahili_congo_french_test_set.ipynb  test.en-swc.en\r\n",
      "build_vocab.py\t\t\t\t     test.en-swc.swc\r\n",
      "config\t\t\t\t\t     test.fr\r\n",
      "data\t\t\t\t\t     test.fr-any.fr\r\n",
      "data_process_without_duplicates.csv\t     test.swc\r\n",
      "dev.bpe.fr\t\t\t\t     train.bpe.fr\r\n",
      "dev.bpe.swc\t\t\t\t     train.bpe.swc\r\n",
      "dev.fr\t\t\t\t\t     train.fr\r\n",
      "dev.swc\t\t\t\t\t     train.swc\r\n",
      "en-fr-baseline\t\t\t\t     vocab.fr\r\n",
      "fr-swc-baseline\t\t\t\t     vocab.swc\r\n"
     ]
    }
   ],
   "source": [
    "!ls "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "remove the old model folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "! rm -fr models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘config’: File exists\r\n"
     ]
    }
   ],
   "source": [
    "!mkdir config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This creates the config file for our JoeyNMT system. It might seem overwhelming so we've provided a couple of useful parameters you'll need to update\n",
    "# (You can of course play with all the parameters if you'd like!)\n",
    "\n",
    "name = '{}{}'.format(source_language, target_language)\n",
    "gdrive_path = os.environ[\"gdrive_path\"]\n",
    "\n",
    "# Create the config\n",
    "config = \"\"\"\n",
    "name: \"{name}_transformer\"\n",
    "\n",
    "data:\n",
    "    src: \"{source_language}\"\n",
    "    trg: \"{target_language}\"\n",
    "    train: \"data/{name}/train.bpe\"\n",
    "    dev:   \"data/{name}/dev.bpe\"\n",
    "    test:  \"data/{name}/test.bpe\"\n",
    "    level: \"bpe\"\n",
    "    lowercase: False\n",
    "    max_sent_length: 100\n",
    "    src_vocab: \"data/{name}/vocab.txt\"\n",
    "    trg_vocab: \"data/{name}/vocab.txt\"\n",
    "\n",
    "testing:\n",
    "    beam_size: 5\n",
    "    alpha: 1.0\n",
    "\n",
    "training:\n",
    "    #load_model: \"{gdrive_path}/models/{name}_transformer/1.ckpt\" # if uncommented, load a pre-trained model from this checkpoint\n",
    "    random_seed: 42\n",
    "    optimizer: \"adam\"\n",
    "    normalization: \"tokens\"\n",
    "    adam_betas: [0.9, 0.999] \n",
    "    scheduling: \"plateau\"           # TODO: try switching from plateau to Noam scheduling\n",
    "    patience: 5                     # For plateau: decrease learning rate by decrease_factor if validation score has not improved for this many validation rounds.\n",
    "    learning_rate_factor: 0.5       # factor for Noam scheduler (used with Transformer)\n",
    "    learning_rate_warmup: 1000      # warmup steps for Noam scheduler (used with Transformer)\n",
    "    decrease_factor: 0.7\n",
    "    loss: \"crossentropy\"\n",
    "    learning_rate: 0.0003\n",
    "    learning_rate_min: 0.00000001\n",
    "    weight_decay: 0.0\n",
    "    label_smoothing: 0.1\n",
    "    batch_size: 4096\n",
    "    batch_type: \"token\"\n",
    "    eval_batch_size: 3600\n",
    "    eval_batch_type: \"token\"\n",
    "    batch_multiplier: 1\n",
    "    early_stopping_metric: \"ppl\"\n",
    "    epochs: 50                   # TODO: Decrease for when playing around and checking of working. Around 30 is sufficient to check if its working at all\n",
    "    validation_freq: 1000          # TODO: Set to at least once per epoch.\n",
    "    logging_freq: 100\n",
    "    eval_metric: \"bleu\"\n",
    "    model_dir: \"models/{name}_transformer\"\n",
    "    overwrite: False               # TODO: Set to True if you want to overwrite possibly existing models. \n",
    "    shuffle: True\n",
    "    use_cuda: True\n",
    "    max_output_length: 100\n",
    "    print_valid_sents: [0, 1, 2, 3, 10, 15, 20, 30, 35]\n",
    "    keep_last_ckpts: 3\n",
    "\n",
    "model:\n",
    "    initializer: \"xavier\"\n",
    "    bias_initializer: \"zeros\"\n",
    "    init_gain: 1.0\n",
    "    embed_initializer: \"xavier\"\n",
    "    embed_init_gain: 1.0\n",
    "    tied_embeddings: True\n",
    "    tied_softmax: True\n",
    "    encoder:\n",
    "        type: \"transformer\"\n",
    "        num_layers: 6\n",
    "        num_heads: 8            # TODO: Increase to 8 for larger data.\n",
    "        embeddings:\n",
    "            embedding_dim: 512   # TODO: Increase to 512 for larger data.\n",
    "            scale: True\n",
    "            dropout: 0.2\n",
    "        # typically ff_size = 4 x hidden_size\n",
    "        hidden_size: 512        # TODO: Increase to 512 for larger data.\n",
    "        ff_size: 2048           # TODO: Increase to 2048 for larger data.\n",
    "        dropout: 0.3\n",
    "    decoder:\n",
    "        type: \"transformer\"\n",
    "        num_layers: 6\n",
    "        num_heads: 8            # TODO: Increase to 8 for larger data.\n",
    "        embeddings:\n",
    "            embedding_dim: 512    # TODO: Increase to 512 for larger data.\n",
    "            scale: True\n",
    "            dropout: 0.2\n",
    "        # typically ff_size = 4 x hidden_size\n",
    "        hidden_size: 512         # TODO: Increase to 512 for larger data.\n",
    "        ff_size: 2048            # TODO: Increase to 2048 for larger data.\n",
    "        dropout: 0.3\n",
    "\"\"\".format(name=name, gdrive_path=os.environ[\"gdrive_path\"], source_language=source_language, target_language=target_language)\n",
    "with open(\"config/transformer_{name}.yaml\".format(name=name),'w') as f:\n",
    "    f.write(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "transformer_frswc.yaml\r\n"
     ]
    }
   ],
   "source": [
    "!ls config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Trying to fix the issue with tensorboard not found"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip3 freeze | grep tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip3 install tensorboard==1.14.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip3 install --upgrade torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This single line of joeynmt runs the training using the config we made above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/espoir_mur_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/espoir_mur_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/espoir_mur_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/espoir_mur_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/espoir_mur_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/espoir_mur_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "2020-02-13 10:27:39,250 Hello! This is Joey-NMT.\n",
      "2020-02-13 10:27:39,256 Total params: 46426624\n",
      "2020-02-13 10:27:39,258 Trainable parameters: ['decoder.layer_norm.bias', 'decoder.layer_norm.weight', 'decoder.layers.0.dec_layer_norm.bias', 'decoder.layers.0.dec_layer_norm.weight', 'decoder.layers.0.feed_forward.layer_norm.bias', 'decoder.layers.0.feed_forward.layer_norm.weight', 'decoder.layers.0.feed_forward.pwff_layer.0.bias', 'decoder.layers.0.feed_forward.pwff_layer.0.weight', 'decoder.layers.0.feed_forward.pwff_layer.3.bias', 'decoder.layers.0.feed_forward.pwff_layer.3.weight', 'decoder.layers.0.src_trg_att.k_layer.bias', 'decoder.layers.0.src_trg_att.k_layer.weight', 'decoder.layers.0.src_trg_att.output_layer.bias', 'decoder.layers.0.src_trg_att.output_layer.weight', 'decoder.layers.0.src_trg_att.q_layer.bias', 'decoder.layers.0.src_trg_att.q_layer.weight', 'decoder.layers.0.src_trg_att.v_layer.bias', 'decoder.layers.0.src_trg_att.v_layer.weight', 'decoder.layers.0.trg_trg_att.k_layer.bias', 'decoder.layers.0.trg_trg_att.k_layer.weight', 'decoder.layers.0.trg_trg_att.output_layer.bias', 'decoder.layers.0.trg_trg_att.output_layer.weight', 'decoder.layers.0.trg_trg_att.q_layer.bias', 'decoder.layers.0.trg_trg_att.q_layer.weight', 'decoder.layers.0.trg_trg_att.v_layer.bias', 'decoder.layers.0.trg_trg_att.v_layer.weight', 'decoder.layers.0.x_layer_norm.bias', 'decoder.layers.0.x_layer_norm.weight', 'decoder.layers.1.dec_layer_norm.bias', 'decoder.layers.1.dec_layer_norm.weight', 'decoder.layers.1.feed_forward.layer_norm.bias', 'decoder.layers.1.feed_forward.layer_norm.weight', 'decoder.layers.1.feed_forward.pwff_layer.0.bias', 'decoder.layers.1.feed_forward.pwff_layer.0.weight', 'decoder.layers.1.feed_forward.pwff_layer.3.bias', 'decoder.layers.1.feed_forward.pwff_layer.3.weight', 'decoder.layers.1.src_trg_att.k_layer.bias', 'decoder.layers.1.src_trg_att.k_layer.weight', 'decoder.layers.1.src_trg_att.output_layer.bias', 'decoder.layers.1.src_trg_att.output_layer.weight', 'decoder.layers.1.src_trg_att.q_layer.bias', 'decoder.layers.1.src_trg_att.q_layer.weight', 'decoder.layers.1.src_trg_att.v_layer.bias', 'decoder.layers.1.src_trg_att.v_layer.weight', 'decoder.layers.1.trg_trg_att.k_layer.bias', 'decoder.layers.1.trg_trg_att.k_layer.weight', 'decoder.layers.1.trg_trg_att.output_layer.bias', 'decoder.layers.1.trg_trg_att.output_layer.weight', 'decoder.layers.1.trg_trg_att.q_layer.bias', 'decoder.layers.1.trg_trg_att.q_layer.weight', 'decoder.layers.1.trg_trg_att.v_layer.bias', 'decoder.layers.1.trg_trg_att.v_layer.weight', 'decoder.layers.1.x_layer_norm.bias', 'decoder.layers.1.x_layer_norm.weight', 'decoder.layers.2.dec_layer_norm.bias', 'decoder.layers.2.dec_layer_norm.weight', 'decoder.layers.2.feed_forward.layer_norm.bias', 'decoder.layers.2.feed_forward.layer_norm.weight', 'decoder.layers.2.feed_forward.pwff_layer.0.bias', 'decoder.layers.2.feed_forward.pwff_layer.0.weight', 'decoder.layers.2.feed_forward.pwff_layer.3.bias', 'decoder.layers.2.feed_forward.pwff_layer.3.weight', 'decoder.layers.2.src_trg_att.k_layer.bias', 'decoder.layers.2.src_trg_att.k_layer.weight', 'decoder.layers.2.src_trg_att.output_layer.bias', 'decoder.layers.2.src_trg_att.output_layer.weight', 'decoder.layers.2.src_trg_att.q_layer.bias', 'decoder.layers.2.src_trg_att.q_layer.weight', 'decoder.layers.2.src_trg_att.v_layer.bias', 'decoder.layers.2.src_trg_att.v_layer.weight', 'decoder.layers.2.trg_trg_att.k_layer.bias', 'decoder.layers.2.trg_trg_att.k_layer.weight', 'decoder.layers.2.trg_trg_att.output_layer.bias', 'decoder.layers.2.trg_trg_att.output_layer.weight', 'decoder.layers.2.trg_trg_att.q_layer.bias', 'decoder.layers.2.trg_trg_att.q_layer.weight', 'decoder.layers.2.trg_trg_att.v_layer.bias', 'decoder.layers.2.trg_trg_att.v_layer.weight', 'decoder.layers.2.x_layer_norm.bias', 'decoder.layers.2.x_layer_norm.weight', 'decoder.layers.3.dec_layer_norm.bias', 'decoder.layers.3.dec_layer_norm.weight', 'decoder.layers.3.feed_forward.layer_norm.bias', 'decoder.layers.3.feed_forward.layer_norm.weight', 'decoder.layers.3.feed_forward.pwff_layer.0.bias', 'decoder.layers.3.feed_forward.pwff_layer.0.weight', 'decoder.layers.3.feed_forward.pwff_layer.3.bias', 'decoder.layers.3.feed_forward.pwff_layer.3.weight', 'decoder.layers.3.src_trg_att.k_layer.bias', 'decoder.layers.3.src_trg_att.k_layer.weight', 'decoder.layers.3.src_trg_att.output_layer.bias', 'decoder.layers.3.src_trg_att.output_layer.weight', 'decoder.layers.3.src_trg_att.q_layer.bias', 'decoder.layers.3.src_trg_att.q_layer.weight', 'decoder.layers.3.src_trg_att.v_layer.bias', 'decoder.layers.3.src_trg_att.v_layer.weight', 'decoder.layers.3.trg_trg_att.k_layer.bias', 'decoder.layers.3.trg_trg_att.k_layer.weight', 'decoder.layers.3.trg_trg_att.output_layer.bias', 'decoder.layers.3.trg_trg_att.output_layer.weight', 'decoder.layers.3.trg_trg_att.q_layer.bias', 'decoder.layers.3.trg_trg_att.q_layer.weight', 'decoder.layers.3.trg_trg_att.v_layer.bias', 'decoder.layers.3.trg_trg_att.v_layer.weight', 'decoder.layers.3.x_layer_norm.bias', 'decoder.layers.3.x_layer_norm.weight', 'decoder.layers.4.dec_layer_norm.bias', 'decoder.layers.4.dec_layer_norm.weight', 'decoder.layers.4.feed_forward.layer_norm.bias', 'decoder.layers.4.feed_forward.layer_norm.weight', 'decoder.layers.4.feed_forward.pwff_layer.0.bias', 'decoder.layers.4.feed_forward.pwff_layer.0.weight', 'decoder.layers.4.feed_forward.pwff_layer.3.bias', 'decoder.layers.4.feed_forward.pwff_layer.3.weight', 'decoder.layers.4.src_trg_att.k_layer.bias', 'decoder.layers.4.src_trg_att.k_layer.weight', 'decoder.layers.4.src_trg_att.output_layer.bias', 'decoder.layers.4.src_trg_att.output_layer.weight', 'decoder.layers.4.src_trg_att.q_layer.bias', 'decoder.layers.4.src_trg_att.q_layer.weight', 'decoder.layers.4.src_trg_att.v_layer.bias', 'decoder.layers.4.src_trg_att.v_layer.weight', 'decoder.layers.4.trg_trg_att.k_layer.bias', 'decoder.layers.4.trg_trg_att.k_layer.weight', 'decoder.layers.4.trg_trg_att.output_layer.bias', 'decoder.layers.4.trg_trg_att.output_layer.weight', 'decoder.layers.4.trg_trg_att.q_layer.bias', 'decoder.layers.4.trg_trg_att.q_layer.weight', 'decoder.layers.4.trg_trg_att.v_layer.bias', 'decoder.layers.4.trg_trg_att.v_layer.weight', 'decoder.layers.4.x_layer_norm.bias', 'decoder.layers.4.x_layer_norm.weight', 'decoder.layers.5.dec_layer_norm.bias', 'decoder.layers.5.dec_layer_norm.weight', 'decoder.layers.5.feed_forward.layer_norm.bias', 'decoder.layers.5.feed_forward.layer_norm.weight', 'decoder.layers.5.feed_forward.pwff_layer.0.bias', 'decoder.layers.5.feed_forward.pwff_layer.0.weight', 'decoder.layers.5.feed_forward.pwff_layer.3.bias', 'decoder.layers.5.feed_forward.pwff_layer.3.weight', 'decoder.layers.5.src_trg_att.k_layer.bias', 'decoder.layers.5.src_trg_att.k_layer.weight', 'decoder.layers.5.src_trg_att.output_layer.bias', 'decoder.layers.5.src_trg_att.output_layer.weight', 'decoder.layers.5.src_trg_att.q_layer.bias', 'decoder.layers.5.src_trg_att.q_layer.weight', 'decoder.layers.5.src_trg_att.v_layer.bias', 'decoder.layers.5.src_trg_att.v_layer.weight', 'decoder.layers.5.trg_trg_att.k_layer.bias', 'decoder.layers.5.trg_trg_att.k_layer.weight', 'decoder.layers.5.trg_trg_att.output_layer.bias', 'decoder.layers.5.trg_trg_att.output_layer.weight', 'decoder.layers.5.trg_trg_att.q_layer.bias', 'decoder.layers.5.trg_trg_att.q_layer.weight', 'decoder.layers.5.trg_trg_att.v_layer.bias', 'decoder.layers.5.trg_trg_att.v_layer.weight', 'decoder.layers.5.x_layer_norm.bias', 'decoder.layers.5.x_layer_norm.weight', 'encoder.layer_norm.bias', 'encoder.layer_norm.weight', 'encoder.layers.0.feed_forward.layer_norm.bias', 'encoder.layers.0.feed_forward.layer_norm.weight', 'encoder.layers.0.feed_forward.pwff_layer.0.bias', 'encoder.layers.0.feed_forward.pwff_layer.0.weight', 'encoder.layers.0.feed_forward.pwff_layer.3.bias', 'encoder.layers.0.feed_forward.pwff_layer.3.weight', 'encoder.layers.0.layer_norm.bias', 'encoder.layers.0.layer_norm.weight', 'encoder.layers.0.src_src_att.k_layer.bias', 'encoder.layers.0.src_src_att.k_layer.weight', 'encoder.layers.0.src_src_att.output_layer.bias', 'encoder.layers.0.src_src_att.output_layer.weight', 'encoder.layers.0.src_src_att.q_layer.bias', 'encoder.layers.0.src_src_att.q_layer.weight', 'encoder.layers.0.src_src_att.v_layer.bias', 'encoder.layers.0.src_src_att.v_layer.weight', 'encoder.layers.1.feed_forward.layer_norm.bias', 'encoder.layers.1.feed_forward.layer_norm.weight', 'encoder.layers.1.feed_forward.pwff_layer.0.bias', 'encoder.layers.1.feed_forward.pwff_layer.0.weight', 'encoder.layers.1.feed_forward.pwff_layer.3.bias', 'encoder.layers.1.feed_forward.pwff_layer.3.weight', 'encoder.layers.1.layer_norm.bias', 'encoder.layers.1.layer_norm.weight', 'encoder.layers.1.src_src_att.k_layer.bias', 'encoder.layers.1.src_src_att.k_layer.weight', 'encoder.layers.1.src_src_att.output_layer.bias', 'encoder.layers.1.src_src_att.output_layer.weight', 'encoder.layers.1.src_src_att.q_layer.bias', 'encoder.layers.1.src_src_att.q_layer.weight', 'encoder.layers.1.src_src_att.v_layer.bias', 'encoder.layers.1.src_src_att.v_layer.weight', 'encoder.layers.2.feed_forward.layer_norm.bias', 'encoder.layers.2.feed_forward.layer_norm.weight', 'encoder.layers.2.feed_forward.pwff_layer.0.bias', 'encoder.layers.2.feed_forward.pwff_layer.0.weight', 'encoder.layers.2.feed_forward.pwff_layer.3.bias', 'encoder.layers.2.feed_forward.pwff_layer.3.weight', 'encoder.layers.2.layer_norm.bias', 'encoder.layers.2.layer_norm.weight', 'encoder.layers.2.src_src_att.k_layer.bias', 'encoder.layers.2.src_src_att.k_layer.weight', 'encoder.layers.2.src_src_att.output_layer.bias', 'encoder.layers.2.src_src_att.output_layer.weight', 'encoder.layers.2.src_src_att.q_layer.bias', 'encoder.layers.2.src_src_att.q_layer.weight', 'encoder.layers.2.src_src_att.v_layer.bias', 'encoder.layers.2.src_src_att.v_layer.weight', 'encoder.layers.3.feed_forward.layer_norm.bias', 'encoder.layers.3.feed_forward.layer_norm.weight', 'encoder.layers.3.feed_forward.pwff_layer.0.bias', 'encoder.layers.3.feed_forward.pwff_layer.0.weight', 'encoder.layers.3.feed_forward.pwff_layer.3.bias', 'encoder.layers.3.feed_forward.pwff_layer.3.weight', 'encoder.layers.3.layer_norm.bias', 'encoder.layers.3.layer_norm.weight', 'encoder.layers.3.src_src_att.k_layer.bias', 'encoder.layers.3.src_src_att.k_layer.weight', 'encoder.layers.3.src_src_att.output_layer.bias', 'encoder.layers.3.src_src_att.output_layer.weight', 'encoder.layers.3.src_src_att.q_layer.bias', 'encoder.layers.3.src_src_att.q_layer.weight', 'encoder.layers.3.src_src_att.v_layer.bias', 'encoder.layers.3.src_src_att.v_layer.weight', 'encoder.layers.4.feed_forward.layer_norm.bias', 'encoder.layers.4.feed_forward.layer_norm.weight', 'encoder.layers.4.feed_forward.pwff_layer.0.bias', 'encoder.layers.4.feed_forward.pwff_layer.0.weight', 'encoder.layers.4.feed_forward.pwff_layer.3.bias', 'encoder.layers.4.feed_forward.pwff_layer.3.weight', 'encoder.layers.4.layer_norm.bias', 'encoder.layers.4.layer_norm.weight', 'encoder.layers.4.src_src_att.k_layer.bias', 'encoder.layers.4.src_src_att.k_layer.weight', 'encoder.layers.4.src_src_att.output_layer.bias', 'encoder.layers.4.src_src_att.output_layer.weight', 'encoder.layers.4.src_src_att.q_layer.bias', 'encoder.layers.4.src_src_att.q_layer.weight', 'encoder.layers.4.src_src_att.v_layer.bias', 'encoder.layers.4.src_src_att.v_layer.weight', 'encoder.layers.5.feed_forward.layer_norm.bias', 'encoder.layers.5.feed_forward.layer_norm.weight', 'encoder.layers.5.feed_forward.pwff_layer.0.bias', 'encoder.layers.5.feed_forward.pwff_layer.0.weight', 'encoder.layers.5.feed_forward.pwff_layer.3.bias', 'encoder.layers.5.feed_forward.pwff_layer.3.weight', 'encoder.layers.5.layer_norm.bias', 'encoder.layers.5.layer_norm.weight', 'encoder.layers.5.src_src_att.k_layer.bias', 'encoder.layers.5.src_src_att.k_layer.weight', 'encoder.layers.5.src_src_att.output_layer.bias', 'encoder.layers.5.src_src_att.output_layer.weight', 'encoder.layers.5.src_src_att.q_layer.bias', 'encoder.layers.5.src_src_att.q_layer.weight', 'encoder.layers.5.src_src_att.v_layer.bias', 'encoder.layers.5.src_src_att.v_layer.weight', 'src_embed.lut.weight']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-13 10:27:44,469 cfg.name                           : frswc_transformer\n",
      "2020-02-13 10:27:44,469 cfg.data.src                       : fr\n",
      "2020-02-13 10:27:44,469 cfg.data.trg                       : swc\n",
      "2020-02-13 10:27:44,469 cfg.data.train                     : data/frswc/train.bpe\n",
      "2020-02-13 10:27:44,469 cfg.data.dev                       : data/frswc/dev.bpe\n",
      "2020-02-13 10:27:44,469 cfg.data.test                      : data/frswc/test.bpe\n",
      "2020-02-13 10:27:44,469 cfg.data.level                     : bpe\n",
      "2020-02-13 10:27:44,469 cfg.data.lowercase                 : False\n",
      "2020-02-13 10:27:44,469 cfg.data.max_sent_length           : 100\n",
      "2020-02-13 10:27:44,470 cfg.data.src_vocab                 : data/frswc/vocab.txt\n",
      "2020-02-13 10:27:44,470 cfg.data.trg_vocab                 : data/frswc/vocab.txt\n",
      "2020-02-13 10:27:44,470 cfg.testing.beam_size              : 5\n",
      "2020-02-13 10:27:44,470 cfg.testing.alpha                  : 1.0\n",
      "2020-02-13 10:27:44,470 cfg.training.random_seed           : 42\n",
      "2020-02-13 10:27:44,470 cfg.training.optimizer             : adam\n",
      "2020-02-13 10:27:44,470 cfg.training.normalization         : tokens\n",
      "2020-02-13 10:27:44,470 cfg.training.adam_betas            : [0.9, 0.999]\n",
      "2020-02-13 10:27:44,470 cfg.training.scheduling            : plateau\n",
      "2020-02-13 10:27:44,470 cfg.training.patience              : 5\n",
      "2020-02-13 10:27:44,470 cfg.training.learning_rate_factor  : 0.5\n",
      "2020-02-13 10:27:44,470 cfg.training.learning_rate_warmup  : 1000\n",
      "2020-02-13 10:27:44,470 cfg.training.decrease_factor       : 0.7\n",
      "2020-02-13 10:27:44,470 cfg.training.loss                  : crossentropy\n",
      "2020-02-13 10:27:44,470 cfg.training.learning_rate         : 0.0003\n",
      "2020-02-13 10:27:44,471 cfg.training.learning_rate_min     : 1e-08\n",
      "2020-02-13 10:27:44,471 cfg.training.weight_decay          : 0.0\n",
      "2020-02-13 10:27:44,471 cfg.training.label_smoothing       : 0.1\n",
      "2020-02-13 10:27:44,471 cfg.training.batch_size            : 4096\n",
      "2020-02-13 10:27:44,471 cfg.training.batch_type            : token\n",
      "2020-02-13 10:27:44,471 cfg.training.eval_batch_size       : 3600\n",
      "2020-02-13 10:27:44,471 cfg.training.eval_batch_type       : token\n",
      "2020-02-13 10:27:44,471 cfg.training.batch_multiplier      : 1\n",
      "2020-02-13 10:27:44,471 cfg.training.early_stopping_metric : ppl\n",
      "2020-02-13 10:27:44,471 cfg.training.epochs                : 50\n",
      "2020-02-13 10:27:44,471 cfg.training.validation_freq       : 1000\n",
      "2020-02-13 10:27:44,471 cfg.training.logging_freq          : 100\n",
      "2020-02-13 10:27:44,471 cfg.training.eval_metric           : bleu\n",
      "2020-02-13 10:27:44,471 cfg.training.model_dir             : models/frswc_transformer\n",
      "2020-02-13 10:27:44,471 cfg.training.overwrite             : False\n",
      "2020-02-13 10:27:44,472 cfg.training.shuffle               : True\n",
      "2020-02-13 10:27:44,472 cfg.training.use_cuda              : True\n",
      "2020-02-13 10:27:44,472 cfg.training.max_output_length     : 100\n",
      "2020-02-13 10:27:44,472 cfg.training.print_valid_sents     : [0, 1, 2, 3, 10, 15, 20, 30, 35]\n",
      "2020-02-13 10:27:44,472 cfg.training.keep_last_ckpts       : 3\n",
      "2020-02-13 10:27:44,472 cfg.model.initializer              : xavier\n",
      "2020-02-13 10:27:44,472 cfg.model.bias_initializer         : zeros\n",
      "2020-02-13 10:27:44,472 cfg.model.init_gain                : 1.0\n",
      "2020-02-13 10:27:44,472 cfg.model.embed_initializer        : xavier\n",
      "2020-02-13 10:27:44,472 cfg.model.embed_init_gain          : 1.0\n",
      "2020-02-13 10:27:44,472 cfg.model.tied_embeddings          : True\n",
      "2020-02-13 10:27:44,472 cfg.model.tied_softmax             : True\n",
      "2020-02-13 10:27:44,472 cfg.model.encoder.type             : transformer\n",
      "2020-02-13 10:27:44,472 cfg.model.encoder.num_layers       : 6\n",
      "2020-02-13 10:27:44,472 cfg.model.encoder.num_heads        : 8\n",
      "2020-02-13 10:27:44,473 cfg.model.encoder.embeddings.embedding_dim : 512\n",
      "2020-02-13 10:27:44,473 cfg.model.encoder.embeddings.scale : True\n",
      "2020-02-13 10:27:44,473 cfg.model.encoder.embeddings.dropout : 0.2\n",
      "2020-02-13 10:27:44,473 cfg.model.encoder.hidden_size      : 512\n",
      "2020-02-13 10:27:44,473 cfg.model.encoder.ff_size          : 2048\n",
      "2020-02-13 10:27:44,473 cfg.model.encoder.dropout          : 0.3\n",
      "2020-02-13 10:27:44,473 cfg.model.decoder.type             : transformer\n",
      "2020-02-13 10:27:44,473 cfg.model.decoder.num_layers       : 6\n",
      "2020-02-13 10:27:44,473 cfg.model.decoder.num_heads        : 8\n",
      "2020-02-13 10:27:44,473 cfg.model.decoder.embeddings.embedding_dim : 512\n",
      "2020-02-13 10:27:44,473 cfg.model.decoder.embeddings.scale : True\n",
      "2020-02-13 10:27:44,473 cfg.model.decoder.embeddings.dropout : 0.2\n",
      "2020-02-13 10:27:44,473 cfg.model.decoder.hidden_size      : 512\n",
      "2020-02-13 10:27:44,473 cfg.model.decoder.ff_size          : 2048\n",
      "2020-02-13 10:27:44,474 cfg.model.decoder.dropout          : 0.3\n",
      "2020-02-13 10:27:44,474 Data set sizes: \n",
      "\ttrain 497882,\n",
      "\tvalid 1000,\n",
      "\ttest 2478\n",
      "2020-02-13 10:27:44,474 First training example:\n",
      "\t[SRC] Comme l’@@ end@@ r@@ oit vous est in@@ connu , vous êtes quelque peu ang@@ o@@ iss@@ é .\n",
      "\t[TRG] Kwa kuwa uko ug@@ en@@ ini , una@@ anza kuwa na wasi@@ wasi .\n",
      "2020-02-13 10:27:44,474 First 10 words (src): (0) <unk> (1) <pad> (2) <s> (3) </s> (4) . (5) , (6) de (7) na (8) ya (9) la\n",
      "2020-02-13 10:27:44,474 First 10 words (trg): (0) <unk> (1) <pad> (2) <s> (3) </s> (4) . (5) , (6) de (7) na (8) ya (9) la\n",
      "2020-02-13 10:27:44,474 Number of Src words (types): 4465\n",
      "2020-02-13 10:27:44,475 Number of Trg words (types): 4465\n",
      "2020-02-13 10:27:44,475 Model(\n",
      "\tencoder=TransformerEncoder(num_layers=6, num_heads=8),\n",
      "\tdecoder=TransformerDecoder(num_layers=6, num_heads=8),\n",
      "\tsrc_embed=Embeddings(embedding_dim=512, vocab_size=4465),\n",
      "\ttrg_embed=Embeddings(embedding_dim=512, vocab_size=4465))\n",
      "2020-02-13 10:27:44,479 EPOCH 1\n",
      "2020-02-13 10:28:51,400 Epoch   1 Step:      100 Batch Loss:     5.740164 Tokens per Sec:     3111, Lr: 0.000300\n",
      "2020-02-13 10:29:57,141 Epoch   1 Step:      200 Batch Loss:     5.477104 Tokens per Sec:     3052, Lr: 0.000300\n",
      "2020-02-13 10:31:04,290 Epoch   1 Step:      300 Batch Loss:     5.093490 Tokens per Sec:     3057, Lr: 0.000300\n",
      "2020-02-13 10:32:11,116 Epoch   1 Step:      400 Batch Loss:     4.858027 Tokens per Sec:     3070, Lr: 0.000300\n",
      "2020-02-13 10:33:17,567 Epoch   1 Step:      500 Batch Loss:     4.987360 Tokens per Sec:     3056, Lr: 0.000300\n",
      "2020-02-13 10:34:25,008 Epoch   1 Step:      600 Batch Loss:     4.810420 Tokens per Sec:     3099, Lr: 0.000300\n",
      "2020-02-13 10:35:32,204 Epoch   1 Step:      700 Batch Loss:     4.639736 Tokens per Sec:     3136, Lr: 0.000300\n",
      "2020-02-13 10:36:38,534 Epoch   1 Step:      800 Batch Loss:     4.673367 Tokens per Sec:     3024, Lr: 0.000300\n",
      "2020-02-13 10:37:44,740 Epoch   1 Step:      900 Batch Loss:     4.657762 Tokens per Sec:     3074, Lr: 0.000300\n",
      "2020-02-13 10:38:51,729 Epoch   1 Step:     1000 Batch Loss:     4.257544 Tokens per Sec:     3088, Lr: 0.000300\n",
      "2020-02-13 10:42:49,626 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 10:42:49,626 Saving new checkpoint.\n",
      "2020-02-13 10:42:50,279 Example #0\n",
      "2020-02-13 10:42:50,279 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 10:42:50,279 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 10:42:50,279 \tHypothesis: Mnamo mwaka wa miaka mingi , nilianza kuchemwa miaka mingi , na umri wa miaka mingi .\n",
      "2020-02-13 10:42:50,279 Example #1\n",
      "2020-02-13 10:42:50,279 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 10:42:50,279 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 10:42:50,280 \tHypothesis: Mchaka ya kupua kwamba nilikuwa na umri wa miaka mingi .\n",
      "2020-02-13 10:42:50,280 Example #2\n",
      "2020-02-13 10:42:50,280 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 10:42:50,280 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 10:42:50,280 \tHypothesis: Mnamo mwaka wa 1900 , nilianza kuchemwa miaka mingi , na umri wa miaka mingi .\n",
      "2020-02-13 10:42:50,280 Example #3\n",
      "2020-02-13 10:42:50,280 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 10:42:50,280 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 10:42:50,280 \tHypothesis: Kwa sababu ya kufanya hivyo , na kufanya hivyo , na kufanya hivyo , na kufanya hivyo , na kufanya hivyo .\n",
      "2020-02-13 10:42:50,280 Example #10\n",
      "2020-02-13 10:42:50,281 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 10:42:50,281 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 10:42:50,281 \tHypothesis: “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “\n",
      "2020-02-13 10:42:50,281 Example #15\n",
      "2020-02-13 10:42:50,281 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 10:42:50,281 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 10:42:50,281 \tHypothesis: Kwa sababu ya Mungu , Yesu alionyesha kwamba Mungu alikuwa na kufanya hivyo , na Mungu .\n",
      "2020-02-13 10:42:50,281 Example #20\n",
      "2020-02-13 10:42:50,281 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 10:42:50,281 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 10:42:50,282 \tHypothesis: Kwa kweli , Yesu alikuwa na kufanya hivyo , na kufanya hivyo , na kufanya hivyo .\n",
      "2020-02-13 10:42:50,282 Example #30\n",
      "2020-02-13 10:42:50,282 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 10:42:50,282 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 10:42:50,282 \tHypothesis: “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “ “\n",
      "2020-02-13 10:42:50,282 Example #35\n",
      "2020-02-13 10:42:50,282 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 10:42:50,282 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 10:42:50,282 \tHypothesis: Ni nini ni nini kwa nini ?\n",
      "2020-02-13 10:42:50,282 Validation result at epoch   1, step     1000: bleu:   0.31, loss: 117640.8516, ppl:  72.9677, duration: 238.5525s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-13 10:43:56,720 Epoch   1 Step:     1100 Batch Loss:     4.558829 Tokens per Sec:     3043, Lr: 0.000300\n",
      "2020-02-13 10:45:03,917 Epoch   1 Step:     1200 Batch Loss:     4.192190 Tokens per Sec:     3063, Lr: 0.000300\n",
      "2020-02-13 10:46:10,380 Epoch   1 Step:     1300 Batch Loss:     4.008200 Tokens per Sec:     3050, Lr: 0.000300\n",
      "2020-02-13 10:47:15,705 Epoch   1 Step:     1400 Batch Loss:     4.137323 Tokens per Sec:     3088, Lr: 0.000300\n",
      "2020-02-13 10:48:21,979 Epoch   1 Step:     1500 Batch Loss:     3.771139 Tokens per Sec:     3060, Lr: 0.000300\n",
      "2020-02-13 10:49:28,269 Epoch   1 Step:     1600 Batch Loss:     3.627386 Tokens per Sec:     3090, Lr: 0.000300\n",
      "2020-02-13 10:50:35,448 Epoch   1 Step:     1700 Batch Loss:     3.456552 Tokens per Sec:     3048, Lr: 0.000300\n",
      "2020-02-13 10:51:42,802 Epoch   1 Step:     1800 Batch Loss:     3.673240 Tokens per Sec:     3140, Lr: 0.000300\n",
      "2020-02-13 10:52:48,498 Epoch   1 Step:     1900 Batch Loss:     3.750333 Tokens per Sec:     3045, Lr: 0.000300\n",
      "2020-02-13 10:53:56,004 Epoch   1 Step:     2000 Batch Loss:     3.749588 Tokens per Sec:     3121, Lr: 0.000300\n",
      "2020-02-13 10:57:53,878 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 10:57:53,879 Saving new checkpoint.\n",
      "2020-02-13 10:57:54,482 Example #0\n",
      "2020-02-13 10:57:54,483 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 10:57:54,483 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 10:57:54,483 \tHypothesis: Mnamo mwaka wa 1966 , nilianza kuhudhuria mikutano ya K.W.K .\n",
      "2020-02-13 10:57:54,483 Example #1\n",
      "2020-02-13 10:57:54,483 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 10:57:54,483 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 10:57:54,483 \tHypothesis: Mahali ya watu wa kawaida hutumia mazungumzo ya watu .\n",
      "2020-02-13 10:57:54,483 Example #2\n",
      "2020-02-13 10:57:54,483 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 10:57:54,483 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 10:57:54,484 \tHypothesis: Kwa sababu ya kua , watu wengi waliopita , na walipiga , na kua , na kuingia katika nchi za kigiriki .\n",
      "2020-02-13 10:57:54,484 Example #3\n",
      "2020-02-13 10:57:54,484 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 10:57:54,484 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 10:57:54,484 \tHypothesis: Lakini , nilianza kukumbuka kwamba nilipokuwa na umri wa miaka mingi , na nilianza kuhudhuria mikutano ya Kiloni .\n",
      "2020-02-13 10:57:54,484 Example #10\n",
      "2020-02-13 10:57:54,484 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 10:57:54,484 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 10:57:54,484 \tHypothesis: ▪ Uchunguzi wa kutosha , na kua , na kulevya , na “ wafanyakazi wa kawaida , ” na “ wafanyakazi wa kawaida .\n",
      "2020-02-13 10:57:54,484 Example #15\n",
      "2020-02-13 10:57:54,485 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 10:57:54,485 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 10:57:54,485 \tHypothesis: Kwa kweli , ‘ Mutume wako wako wako , ’ na kwa sababu ya kupenda Mungu , ’ na ‘ Mutume wake .\n",
      "2020-02-13 10:57:54,485 Example #20\n",
      "2020-02-13 10:57:54,485 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 10:57:54,485 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 10:57:54,485 \tHypothesis: Kwa hiyo , tunahitaji kupenda Yesu na Yesu na Yesu .\n",
      "2020-02-13 10:57:54,485 Example #30\n",
      "2020-02-13 10:57:54,485 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 10:57:54,485 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 10:57:54,486 \tHypothesis: Mahali ya Kazi ya Kazi ya Kazi ya Kazi ya Kazi ya Kazi ya Kazi ya Kazi ya Kazi ya Kazi ya Kazi ya Kazi ya Kazi ya Kazi ya Kazi ya Kumla Mungu\n",
      "2020-02-13 10:57:54,486 Example #35\n",
      "2020-02-13 10:57:54,486 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 10:57:54,486 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 10:57:54,486 \tHypothesis: Tunaonyesha kwamba tunahitaji kumtumikia Mungu ?\n",
      "2020-02-13 10:57:54,486 Validation result at epoch   1, step     2000: bleu:   1.45, loss: 96072.3359, ppl:  33.2308, duration: 238.4808s\n",
      "2020-02-13 10:59:00,206 Epoch   1 Step:     2100 Batch Loss:     3.608715 Tokens per Sec:     3060, Lr: 0.000300\n",
      "2020-02-13 11:00:05,624 Epoch   1 Step:     2200 Batch Loss:     3.504120 Tokens per Sec:     3062, Lr: 0.000300\n",
      "2020-02-13 11:01:12,396 Epoch   1 Step:     2300 Batch Loss:     3.496947 Tokens per Sec:     3103, Lr: 0.000300\n",
      "2020-02-13 11:02:19,119 Epoch   1 Step:     2400 Batch Loss:     3.215005 Tokens per Sec:     3099, Lr: 0.000300\n",
      "2020-02-13 11:03:25,650 Epoch   1 Step:     2500 Batch Loss:     3.477642 Tokens per Sec:     3088, Lr: 0.000300\n",
      "2020-02-13 11:04:31,427 Epoch   1 Step:     2600 Batch Loss:     3.447724 Tokens per Sec:     3038, Lr: 0.000300\n",
      "2020-02-13 11:05:37,811 Epoch   1 Step:     2700 Batch Loss:     3.344538 Tokens per Sec:     3086, Lr: 0.000300\n",
      "2020-02-13 11:06:45,112 Epoch   1 Step:     2800 Batch Loss:     3.273542 Tokens per Sec:     3088, Lr: 0.000300\n",
      "2020-02-13 11:07:51,434 Epoch   1 Step:     2900 Batch Loss:     3.279399 Tokens per Sec:     3072, Lr: 0.000300\n",
      "2020-02-13 11:08:57,215 Epoch   1 Step:     3000 Batch Loss:     3.312492 Tokens per Sec:     3000, Lr: 0.000300\n",
      "2020-02-13 11:12:54,779 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 11:12:54,780 Saving new checkpoint.\n",
      "2020-02-13 11:12:55,382 Example #0\n",
      "2020-02-13 11:12:55,382 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 11:12:55,382 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 11:12:55,382 \tHypothesis: Nilikuwa na miaka mingi , nilianza kuhudhuria mji wa Sovieti , na nikaanza kuhamia Bahari ya Pato .\n",
      "2020-02-13 11:12:55,382 Example #1\n",
      "2020-02-13 11:12:55,382 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 11:12:55,383 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 11:12:55,383 \tHypothesis: Maisha ya watu wa kale ni ya kawaida .\n",
      "2020-02-13 11:12:55,383 Example #2\n",
      "2020-02-13 11:12:55,383 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 11:12:55,383 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 11:12:55,383 \tHypothesis: Katika karne ya kwanza , watu wa kwanza walipata wafalme wa Sulemani , walipewa na wafanyakazi wa Sovieti , na kuhani wa Sovieti .\n",
      "2020-02-13 11:12:55,383 Example #3\n",
      "2020-02-13 11:12:55,383 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 11:12:55,384 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 11:12:55,384 \tHypothesis: Lakini nilikuwa na uhakika kwamba nilikuwa na uhakika wa kwamba nilikuwa na uhakika wa kwamba nilikuwa na miaka mingi . ” — Mwezi wa 3 .\n",
      "2020-02-13 11:12:55,384 Example #10\n",
      "2020-02-13 11:12:55,384 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 11:12:55,384 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 11:12:55,384 \tHypothesis: ● “ Maisha ya watu wengi ni kwamba watu wengi wanaompenda sana , ” anasema hivi : “ Wale wanaoishi katika hali yao ya kihisia za kihisia .\n",
      "2020-02-13 11:12:55,384 Example #15\n",
      "2020-02-13 11:12:55,385 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 11:12:55,385 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 11:12:55,385 \tHypothesis: Ikiwa unapaswa kukumbuka kwamba ‘ Mungu atawapa watu wake ’ kwa sababu ya ‘ kupenda .\n",
      "2020-02-13 11:12:55,385 Example #20\n",
      "2020-02-13 11:12:55,385 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 11:12:55,385 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 11:12:55,385 \tHypothesis: Kwa hiyo , tunajua kwamba Yesu alituambia kwamba Yesu alitupa nafasi ya kuhubiri .\n",
      "2020-02-13 11:12:55,385 Example #30\n",
      "2020-02-13 11:12:55,386 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 11:12:55,386 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 11:12:55,386 \tHypothesis: Maisha ya Mungu ya Kimaandiko la Mungu , 1 / 1\n",
      "2020-02-13 11:12:55,386 Example #35\n",
      "2020-02-13 11:12:55,386 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 11:12:55,386 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 11:12:55,386 \tHypothesis: Je , tunajua Neno la Mungu ?\n",
      "2020-02-13 11:12:55,386 Validation result at epoch   1, step     3000: bleu:   2.57, loss: 86643.2422, ppl:  23.5618, duration: 238.1702s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-13 11:14:01,989 Epoch   1 Step:     3100 Batch Loss:     3.326180 Tokens per Sec:     3061, Lr: 0.000300\n",
      "2020-02-13 11:15:08,836 Epoch   1 Step:     3200 Batch Loss:     3.256266 Tokens per Sec:     3093, Lr: 0.000300\n",
      "2020-02-13 11:16:15,226 Epoch   1 Step:     3300 Batch Loss:     2.836759 Tokens per Sec:     3071, Lr: 0.000300\n",
      "2020-02-13 11:17:20,907 Epoch   1 Step:     3400 Batch Loss:     3.258413 Tokens per Sec:     3071, Lr: 0.000300\n",
      "2020-02-13 11:18:27,093 Epoch   1 Step:     3500 Batch Loss:     3.192358 Tokens per Sec:     3034, Lr: 0.000300\n",
      "2020-02-13 11:19:32,799 Epoch   1 Step:     3600 Batch Loss:     2.874379 Tokens per Sec:     3064, Lr: 0.000300\n",
      "2020-02-13 11:20:39,387 Epoch   1 Step:     3700 Batch Loss:     3.178644 Tokens per Sec:     3125, Lr: 0.000300\n",
      "2020-02-13 11:21:46,154 Epoch   1 Step:     3800 Batch Loss:     2.896233 Tokens per Sec:     3029, Lr: 0.000300\n",
      "2020-02-13 11:22:52,895 Epoch   1 Step:     3900 Batch Loss:     2.805963 Tokens per Sec:     3088, Lr: 0.000300\n",
      "2020-02-13 11:23:59,332 Epoch   1 Step:     4000 Batch Loss:     2.975837 Tokens per Sec:     3060, Lr: 0.000300\n",
      "2020-02-13 11:27:56,978 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 11:27:56,978 Saving new checkpoint.\n",
      "2020-02-13 11:27:57,652 Example #0\n",
      "2020-02-13 11:27:57,653 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 11:27:57,653 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 11:27:57,653 \tHypothesis: Mama aliniambia kwamba nilipokuwa na umri wa miaka mingi , nilianza kuhamia kwenye mji wa Sovieti .\n",
      "2020-02-13 11:27:57,653 Example #1\n",
      "2020-02-13 11:27:57,653 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 11:27:57,653 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 11:27:57,653 \tHypothesis: Mabadiliko hayo yana uwezo wa kufanyiza .\n",
      "2020-02-13 11:27:57,654 Example #2\n",
      "2020-02-13 11:27:57,654 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 11:27:57,654 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 11:27:57,654 \tHypothesis: Kwa hiyo , watu wengi waliishi katika nchi ya Bahari ya Porland , na walipewa vita vya kale .\n",
      "2020-02-13 11:27:57,654 Example #3\n",
      "2020-02-13 11:27:57,654 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 11:27:57,654 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 11:27:57,654 \tHypothesis: Lakini nilipokuwa na maoni yangu na nilipokuwa na umri wa miaka 20 , nilianza kuandika mambo mengi ya kihistoria . ” — Mt .\n",
      "2020-02-13 11:27:57,654 Example #10\n",
      "2020-02-13 11:27:57,655 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 11:27:57,655 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 11:27:57,655 \tHypothesis: ● Vijana wengi wanasema hivi : “ Wewe ni wachache tu tu tu wanaofanya kazi yao . ”\n",
      "2020-02-13 11:27:57,655 Example #15\n",
      "2020-02-13 11:27:57,655 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 11:27:57,655 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 11:27:57,655 \tHypothesis: Uwe na hakika kwamba unahitaji kuwa na amani ya Mungu , lakini unaweza kuwa na furaha ya kiroho .\n",
      "2020-02-13 11:27:57,655 Example #20\n",
      "2020-02-13 11:27:57,655 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 11:27:57,655 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 11:27:57,655 \tHypothesis: Kwa nini tunaweza kuwa na uhakika kwamba Yesu alituambia kwamba Yesu alitupa nafasi ya kuhubiri habari njema .\n",
      "2020-02-13 11:27:57,656 Example #30\n",
      "2020-02-13 11:27:57,656 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 11:27:57,656 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 11:27:57,656 \tHypothesis: Maisha Yanayofanya Kumufaisha , 2 / 15\n",
      "2020-02-13 11:27:57,656 Example #35\n",
      "2020-02-13 11:27:57,656 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 11:27:57,656 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 11:27:57,656 \tHypothesis: Je , tunajua Neno la Mungu kwa ajili ya siku za mwisho ?\n",
      "2020-02-13 11:27:57,656 Validation result at epoch   1, step     4000: bleu:   3.81, loss: 80421.3594, ppl:  18.7789, duration: 238.3239s\n",
      "2020-02-13 11:29:03,126 Epoch   1 Step:     4100 Batch Loss:     2.997982 Tokens per Sec:     3023, Lr: 0.000300\n",
      "2020-02-13 11:30:09,575 Epoch   1 Step:     4200 Batch Loss:     2.736851 Tokens per Sec:     3032, Lr: 0.000300\n",
      "2020-02-13 11:31:16,945 Epoch   1 Step:     4300 Batch Loss:     3.080659 Tokens per Sec:     3101, Lr: 0.000300\n",
      "2020-02-13 11:32:23,718 Epoch   1 Step:     4400 Batch Loss:     2.721848 Tokens per Sec:     3036, Lr: 0.000300\n",
      "2020-02-13 11:33:31,001 Epoch   1 Step:     4500 Batch Loss:     3.055081 Tokens per Sec:     3157, Lr: 0.000300\n",
      "2020-02-13 11:34:37,336 Epoch   1 Step:     4600 Batch Loss:     2.722386 Tokens per Sec:     3033, Lr: 0.000300\n",
      "2020-02-13 11:35:44,298 Epoch   1 Step:     4700 Batch Loss:     2.737727 Tokens per Sec:     3103, Lr: 0.000300\n",
      "2020-02-13 11:36:50,876 Epoch   1 Step:     4800 Batch Loss:     3.036048 Tokens per Sec:     3086, Lr: 0.000300\n",
      "2020-02-13 11:37:56,590 Epoch   1 Step:     4900 Batch Loss:     2.763457 Tokens per Sec:     3050, Lr: 0.000300\n",
      "2020-02-13 11:39:03,503 Epoch   1 Step:     5000 Batch Loss:     2.833720 Tokens per Sec:     3060, Lr: 0.000300\n",
      "2020-02-13 11:43:01,019 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 11:43:01,019 Saving new checkpoint.\n",
      "2020-02-13 11:43:01,684 Example #0\n",
      "2020-02-13 11:43:01,684 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 11:43:01,684 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 11:43:01,684 \tHypothesis: Nilipokuwa nikienda kwenye kituo cha kulia , nilitaka kuingia katika eneo hilo .\n",
      "2020-02-13 11:43:01,684 Example #1\n",
      "2020-02-13 11:43:01,685 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 11:43:01,685 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 11:43:01,685 \tHypothesis: Mabadiliko hayo ya kiuchumi yanapatikana katika maeneo ya kigeni .\n",
      "2020-02-13 11:43:01,685 Example #2\n",
      "2020-02-13 11:43:01,685 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 11:43:01,685 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 11:43:01,685 \tHypothesis: Katika siku za Noa , watu wengi waliotoka kwenye mji wa Sovieti , walijiunga na kuingia katika jiji hilo .\n",
      "2020-02-13 11:43:01,685 Example #3\n",
      "2020-02-13 11:43:01,685 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 11:43:01,686 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 11:43:01,686 \tHypothesis: Lakini , nilitaka kumusikiliza na kuomba nifanye hivyo . ” — Mt .\n",
      "2020-02-13 11:43:01,686 Example #10\n",
      "2020-02-13 11:43:01,686 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 11:43:01,686 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 11:43:01,686 \tHypothesis: ● Kujifunza mambo hayo kunaweza kukusaidia watu wengi wafanye hivyo , “ huenda wengine wakajaribu kukabiliana na hali yao . ”\n",
      "2020-02-13 11:43:01,686 Example #15\n",
      "2020-02-13 11:43:01,687 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 11:43:01,687 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 11:43:01,687 \tHypothesis: Unaweza kukumbuka kwamba Mungu anaweza ‘ kukusaidia watu wafanye amani ’ ili wapate furaha .\n",
      "2020-02-13 11:43:01,687 Example #20\n",
      "2020-02-13 11:43:01,687 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 11:43:01,687 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 11:43:01,687 \tHypothesis: Kwa hiyo , tunaweza kuwa na uhakika kwamba Yesu alijua kwamba alikuwa amekufa na alikuwa amekufa .\n",
      "2020-02-13 11:43:01,687 Example #30\n",
      "2020-02-13 11:43:01,687 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 11:43:01,688 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 11:43:01,688 \tHypothesis: Kuhusu “ Msiba wa Mungu ” ( Mwezi wa 3 ) , 3 / 1\n",
      "2020-02-13 11:43:01,688 Example #35\n",
      "2020-02-13 11:43:01,688 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 11:43:01,688 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 11:43:01,688 \tHypothesis: Je , tunajua Neno la Mungu ?\n",
      "2020-02-13 11:43:01,688 Validation result at epoch   1, step     5000: bleu:   5.58, loss: 75786.3438, ppl:  15.8585, duration: 238.1841s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-13 11:44:08,689 Epoch   1 Step:     5100 Batch Loss:     3.052882 Tokens per Sec:     3139, Lr: 0.000300\n",
      "2020-02-13 11:45:14,484 Epoch   1 Step:     5200 Batch Loss:     2.692811 Tokens per Sec:     3056, Lr: 0.000300\n",
      "2020-02-13 11:46:21,032 Epoch   1 Step:     5300 Batch Loss:     2.750236 Tokens per Sec:     3117, Lr: 0.000300\n",
      "2020-02-13 11:47:28,525 Epoch   1 Step:     5400 Batch Loss:     2.762382 Tokens per Sec:     3093, Lr: 0.000300\n",
      "2020-02-13 11:48:34,377 Epoch   1 Step:     5500 Batch Loss:     2.917958 Tokens per Sec:     3053, Lr: 0.000300\n",
      "2020-02-13 11:49:40,216 Epoch   1 Step:     5600 Batch Loss:     2.864330 Tokens per Sec:     3069, Lr: 0.000300\n",
      "2020-02-13 11:50:47,365 Epoch   1 Step:     5700 Batch Loss:     2.948759 Tokens per Sec:     3091, Lr: 0.000300\n",
      "2020-02-13 11:51:54,050 Epoch   1 Step:     5800 Batch Loss:     2.580101 Tokens per Sec:     3060, Lr: 0.000300\n",
      "2020-02-13 11:53:01,419 Epoch   1 Step:     5900 Batch Loss:     2.778599 Tokens per Sec:     3096, Lr: 0.000300\n",
      "2020-02-13 11:58:05,176 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 11:58:05,176 Saving new checkpoint.\n",
      "2020-02-13 11:58:05,807 Example #0\n",
      "2020-02-13 11:58:05,807 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 11:58:05,807 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 11:58:05,807 \tHypothesis: Baada ya kuondoka kwenye kituo cha kulia , nilianza kuenda kwenye masomo ya kigeni .\n",
      "2020-02-13 11:58:05,807 Example #1\n",
      "2020-02-13 11:58:05,808 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 11:58:05,808 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 11:58:05,808 \tHypothesis: Mabadiliko ya kiuchumi yalitokea .\n",
      "2020-02-13 11:58:05,808 Example #2\n",
      "2020-02-13 11:58:05,808 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 11:58:05,808 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 11:58:05,808 \tHypothesis: Wakati huo , watu wa ukoo wa Sodoma walipewa mji wa Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari .\n",
      "2020-02-13 11:58:05,808 Example #3\n",
      "2020-02-13 11:58:05,808 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 11:58:05,808 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 11:58:05,809 \tHypothesis: Lakini nilitaka nijifunze jinsi nilivyokuwa nikizungumza na watu wa ukoo . ” — Mt .\n",
      "2020-02-13 11:58:05,809 Example #10\n",
      "2020-02-13 11:58:05,809 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 11:58:05,809 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 11:58:05,809 \tHypothesis: ● Kujua kwamba huenda wengine wakasema kwamba “ wafanyakazi wenzao hawawezi kurekebisha . ”\n",
      "2020-02-13 11:58:05,809 Example #15\n",
      "2020-02-13 11:58:05,809 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 11:58:05,810 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 11:58:05,810 \tHypothesis: Pia , unaweza kuwa na uhakika kwamba ‘ Mungu anaweza kukusaidia ’ ao kukabiliana na matatizo .\n",
      "2020-02-13 11:58:05,810 Example #20\n",
      "2020-02-13 11:58:05,810 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 11:58:05,810 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 11:58:05,810 \tHypothesis: Kwa hiyo , tunapaswa kuwa na uhakika kwamba Yesu alikufa na kuondolewa .\n",
      "2020-02-13 11:58:05,810 Example #30\n",
      "2020-02-13 11:58:05,810 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 11:58:05,810 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 11:58:05,810 \tHypothesis: Kuwaza Wanyama wa Kale , 3 / 1\n",
      "2020-02-13 11:58:05,810 Example #35\n",
      "2020-02-13 11:58:05,811 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 11:58:05,811 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 11:58:05,811 \tHypothesis: Je , tuna Neno la Mungu ?\n",
      "2020-02-13 11:58:05,811 Validation result at epoch   1, step     6000: bleu:   6.41, loss: 72432.7344, ppl:  14.0330, duration: 238.1490s\n",
      "2020-02-13 11:59:12,353 Epoch   1 Step:     6100 Batch Loss:     2.751053 Tokens per Sec:     3104, Lr: 0.000300\n",
      "2020-02-13 12:00:18,546 Epoch   1 Step:     6200 Batch Loss:     2.723702 Tokens per Sec:     3066, Lr: 0.000300\n",
      "2020-02-13 12:01:25,804 Epoch   1 Step:     6300 Batch Loss:     2.843826 Tokens per Sec:     3137, Lr: 0.000300\n",
      "2020-02-13 12:02:32,549 Epoch   1 Step:     6400 Batch Loss:     2.380005 Tokens per Sec:     3164, Lr: 0.000300\n",
      "2020-02-13 12:03:39,569 Epoch   1 Step:     6500 Batch Loss:     2.508609 Tokens per Sec:     3087, Lr: 0.000300\n",
      "2020-02-13 12:04:36,867 Epoch   1: total training loss 22692.47\n",
      "2020-02-13 12:04:36,868 EPOCH 2\n",
      "2020-02-13 12:04:46,280 Epoch   2 Step:     6600 Batch Loss:     2.694419 Tokens per Sec:     2891, Lr: 0.000300\n",
      "2020-02-13 12:05:51,922 Epoch   2 Step:     6700 Batch Loss:     2.407087 Tokens per Sec:     3051, Lr: 0.000300\n",
      "2020-02-13 12:06:58,072 Epoch   2 Step:     6800 Batch Loss:     2.720469 Tokens per Sec:     3098, Lr: 0.000300\n",
      "2020-02-13 12:08:04,654 Epoch   2 Step:     6900 Batch Loss:     2.626773 Tokens per Sec:     3054, Lr: 0.000300\n",
      "2020-02-13 12:09:10,474 Epoch   2 Step:     7000 Batch Loss:     2.691118 Tokens per Sec:     3075, Lr: 0.000300\n",
      "2020-02-13 12:13:07,868 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 12:13:07,869 Saving new checkpoint.\n",
      "2020-02-13 12:13:08,570 Example #0\n",
      "2020-02-13 12:13:08,571 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 12:13:08,571 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 12:13:08,571 \tHypothesis: Baada ya kukaa kwenye meli hiyo , nilianza kukaa kwenye masomo ya magharibi mwa Afrika .\n",
      "2020-02-13 12:13:08,571 Example #1\n",
      "2020-02-13 12:13:08,571 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 12:13:08,571 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 12:13:08,571 \tHypothesis: Mabadiliko ya kiuchumi yanatokana na vifaa vya kuhifadhi .\n",
      "2020-02-13 12:13:08,572 Example #2\n",
      "2020-02-13 12:13:08,572 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 12:13:08,572 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 12:13:08,572 \tHypothesis: Wakati huo , watu wa Nebukadneza walitumia mbinu hiyo ya kulia , na kuanzia mwaka wa 500 .\n",
      "2020-02-13 12:13:08,572 Example #3\n",
      "2020-02-13 12:13:08,572 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 12:13:08,573 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 12:13:08,573 \tHypothesis: Lakini , nilitaka niwe na imani na usawaziko na imani yangu . ” — Mei 19 , 1919 .\n",
      "2020-02-13 12:13:08,573 Example #10\n",
      "2020-02-13 12:13:08,573 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 12:13:08,573 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 12:13:08,573 \tHypothesis: ● Hata hivyo , televisheni hususa kwa sababu ya “ kungano ” na watu fulani ambao huenda wakawa na uwezo wa kuharibu .\n",
      "2020-02-13 12:13:08,573 Example #15\n",
      "2020-02-13 12:13:08,574 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 12:13:08,574 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 12:13:08,574 \tHypothesis: Pia , unaweza pia kukumbuka kwamba ‘ amani ya Mungu ’ inaweza kukusaidia .\n",
      "2020-02-13 12:13:08,574 Example #20\n",
      "2020-02-13 12:13:08,574 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 12:13:08,574 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 12:13:08,574 \tHypothesis: Kwa hiyo , tunaweza kuwa na uhakika kwamba Yesu alifufuliwa na kifo chake .\n",
      "2020-02-13 12:13:08,574 Example #30\n",
      "2020-02-13 12:13:08,575 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 12:13:08,575 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 12:13:08,575 \tHypothesis: Mabadiliko ya Kimaandiko Katika Ulimwengu wa Kale , 2 / 15\n",
      "2020-02-13 12:13:08,575 Example #35\n",
      "2020-02-13 12:13:08,575 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 12:13:08,575 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 12:13:08,576 \tHypothesis: Je , tunajua Neno la Mungu kila siku ?\n",
      "2020-02-13 12:13:08,576 Validation result at epoch   2, step     7000: bleu:   7.38, loss: 70071.1875, ppl:  12.8750, duration: 238.1014s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-13 12:14:14,814 Epoch   2 Step:     7100 Batch Loss:     2.750644 Tokens per Sec:     3053, Lr: 0.000300\n",
      "2020-02-13 12:15:21,438 Epoch   2 Step:     7200 Batch Loss:     2.655128 Tokens per Sec:     3078, Lr: 0.000300\n",
      "2020-02-13 12:16:28,129 Epoch   2 Step:     7300 Batch Loss:     2.415816 Tokens per Sec:     3069, Lr: 0.000300\n",
      "2020-02-13 12:17:35,079 Epoch   2 Step:     7400 Batch Loss:     2.332900 Tokens per Sec:     3085, Lr: 0.000300\n",
      "2020-02-13 12:18:40,823 Epoch   2 Step:     7500 Batch Loss:     2.408705 Tokens per Sec:     3032, Lr: 0.000300\n",
      "2020-02-13 12:19:46,980 Epoch   2 Step:     7600 Batch Loss:     2.394734 Tokens per Sec:     3093, Lr: 0.000300\n",
      "2020-02-13 12:20:53,997 Epoch   2 Step:     7700 Batch Loss:     2.406074 Tokens per Sec:     3105, Lr: 0.000300\n",
      "2020-02-13 12:22:00,645 Epoch   2 Step:     7800 Batch Loss:     2.686860 Tokens per Sec:     3050, Lr: 0.000300\n",
      "2020-02-13 12:23:06,346 Epoch   2 Step:     7900 Batch Loss:     2.542987 Tokens per Sec:     3084, Lr: 0.000300\n",
      "2020-02-13 12:24:13,210 Epoch   2 Step:     8000 Batch Loss:     2.594875 Tokens per Sec:     3131, Lr: 0.000300\n",
      "2020-02-13 12:28:10,645 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 12:28:10,646 Saving new checkpoint.\n",
      "2020-02-13 12:28:11,321 Example #0\n",
      "2020-02-13 12:28:11,321 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 12:28:11,321 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 12:28:11,321 \tHypothesis: Kabla ya kuondoka kwenye kambi ya kulia , nilipeleka kwenye kambi ya Hispania , Craia , Cartydney , City , City .\n",
      "2020-02-13 12:28:11,321 Example #1\n",
      "2020-02-13 12:28:11,321 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 12:28:11,321 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 12:28:11,322 \tHypothesis: Mbinu za mimea hutumiwa katika sehemu mbalimbali za mawe .\n",
      "2020-02-13 12:28:11,322 Example #2\n",
      "2020-02-13 12:28:11,322 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 12:28:11,322 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 12:28:11,322 \tHypothesis: Wakati huo , watu wa ukoo wa Nathani walipeleka kwenye Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari .\n",
      "2020-02-13 12:28:11,322 Example #3\n",
      "2020-02-13 12:28:11,322 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 12:28:11,322 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 12:28:11,322 \tHypothesis: Lakini , nilitaka kuwasaidia watu wajifunze na kuandika imani yangu . ” — Jessica , mwenye umri wa miaka 19 .\n",
      "2020-02-13 12:28:11,322 Example #10\n",
      "2020-02-13 12:28:11,323 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 12:28:11,323 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 12:28:11,323 \tHypothesis: ● Kujua kwamba huenda mtu akajifunza kwamba huenda mtu akajifunza “ maadili ” au “ kuuawa . ”\n",
      "2020-02-13 12:28:11,323 Example #15\n",
      "2020-02-13 12:28:11,323 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 12:28:11,323 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 12:28:11,323 \tHypothesis: Pia , unaweza kuwa na amani ya Mungu ya kusema : ‘ Ukuwe na amani ’ na wenye kupendeza .\n",
      "2020-02-13 12:28:11,323 Example #20\n",
      "2020-02-13 12:28:11,323 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 12:28:11,323 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 12:28:11,323 \tHypothesis: Kwa hiyo , tunaweza kuwa na hakika kwamba Yesu alifufuliwa na Yesu Kristo na kwamba sasa sasa ana tumaini .\n",
      "2020-02-13 12:28:11,324 Example #30\n",
      "2020-02-13 12:28:11,324 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 12:28:11,324 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 12:28:11,324 \tHypothesis: Mbona wa “ Taifa la Mungu , ” 5 / 15\n",
      "2020-02-13 12:28:11,324 Example #35\n",
      "2020-02-13 12:28:11,324 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 12:28:11,324 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 12:28:11,324 \tHypothesis: Je , tunatumia Neno la Mungu kila siku ?\n",
      "2020-02-13 12:28:11,324 Validation result at epoch   2, step     8000: bleu:   8.53, loss: 66923.0391, ppl:  11.4786, duration: 238.1132s\n",
      "2020-02-13 12:29:17,615 Epoch   2 Step:     8100 Batch Loss:     2.678753 Tokens per Sec:     3088, Lr: 0.000300\n",
      "2020-02-13 12:30:22,999 Epoch   2 Step:     8200 Batch Loss:     2.714362 Tokens per Sec:     3034, Lr: 0.000300\n",
      "2020-02-13 12:31:29,155 Epoch   2 Step:     8300 Batch Loss:     2.510592 Tokens per Sec:     3091, Lr: 0.000300\n",
      "2020-02-13 12:32:35,238 Epoch   2 Step:     8400 Batch Loss:     3.061571 Tokens per Sec:     3108, Lr: 0.000300\n",
      "2020-02-13 12:33:41,743 Epoch   2 Step:     8500 Batch Loss:     2.354805 Tokens per Sec:     3080, Lr: 0.000300\n",
      "2020-02-13 12:37:01,324 Epoch   2 Step:     8800 Batch Loss:     2.362063 Tokens per Sec:     3092, Lr: 0.000300\n",
      "2020-02-13 12:38:08,424 Epoch   2 Step:     8900 Batch Loss:     2.518574 Tokens per Sec:     3127, Lr: 0.000300\n",
      "2020-02-13 12:39:13,844 Epoch   2 Step:     9000 Batch Loss:     2.579125 Tokens per Sec:     3075, Lr: 0.000300\n",
      "2020-02-13 12:43:11,288 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 12:43:11,288 Saving new checkpoint.\n",
      "2020-02-13 12:43:11,951 Example #0\n",
      "2020-02-13 12:43:11,951 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 12:43:11,951 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 12:43:11,951 \tHypothesis: Kabla ya kuingia katika mji wa Chuo Kikuu cha Horea , nilianza kuhamia katika mji wa Chuo cha Corland .\n",
      "2020-02-13 12:43:11,951 Example #1\n",
      "2020-02-13 12:43:11,952 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 12:43:11,952 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 12:43:11,952 \tHypothesis: Mabomu ya kiangazi hupatikana katika sehemu za mbali .\n",
      "2020-02-13 12:43:11,952 Example #2\n",
      "2020-02-13 12:43:11,952 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 12:43:11,952 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 12:43:11,952 \tHypothesis: Wakati huo , watu wa Nabali walianza kuitwa Nabali , walipelekwa kwenye Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya B@@\n",
      "2020-02-13 12:43:11,952 Example #3\n",
      "2020-02-13 12:43:11,952 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 12:43:11,952 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 12:43:11,953 \tHypothesis: Lakini nilitaka niwaeleze kwa uangalifu na niliposikia imani yangu . ” — Jan , mwenye umri wa miaka 19 .\n",
      "2020-02-13 12:43:11,953 Example #10\n",
      "2020-02-13 12:43:11,953 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 12:43:11,953 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 12:43:11,953 \tHypothesis: ● Kujua kwamba huenda watu fulani wakasema kwamba “ mazungumzo ya uchunguzi ” yanaweza kuathiri uwezo wao wa kuondoa nguvu .\n",
      "2020-02-13 12:43:11,953 Example #15\n",
      "2020-02-13 12:43:11,953 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 12:43:11,953 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 12:43:11,953 \tHypothesis: Ukifanya hivyo , unaweza ‘ kukaza akili ya Mungu ’ ili kukabiliana na matatizo .\n",
      "2020-02-13 12:43:11,953 Example #20\n",
      "2020-02-13 12:43:11,954 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 12:43:11,954 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 12:43:11,954 \tHypothesis: Kwa hiyo , tunaweza kuwa na hakika kwamba Yesu alifufuliwa na kwamba alifufuliwa na kuwa hai .\n",
      "2020-02-13 12:43:11,954 Example #30\n",
      "2020-02-13 12:43:11,954 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 12:43:11,954 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 12:43:11,954 \tHypothesis: Ugonjwa wa Kuwa Mwenye Uvumilivu , 1 / 1\n",
      "2020-02-13 12:43:11,954 Example #35\n",
      "2020-02-13 12:43:11,954 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 12:43:11,954 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 12:43:11,954 \tHypothesis: Je , tuna Neno la Mungu kila siku ?\n",
      "2020-02-13 12:43:11,955 Validation result at epoch   2, step     9000: bleu:   9.22, loss: 64534.8008, ppl:  10.5212, duration: 238.1102s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-13 12:44:18,942 Epoch   2 Step:     9100 Batch Loss:     2.424658 Tokens per Sec:     3097, Lr: 0.000300\n",
      "2020-02-13 12:45:25,782 Epoch   2 Step:     9200 Batch Loss:     2.756666 Tokens per Sec:     3104, Lr: 0.000300\n",
      "2020-02-13 12:46:32,193 Epoch   2 Step:     9300 Batch Loss:     2.483221 Tokens per Sec:     3126, Lr: 0.000300\n",
      "2020-02-13 12:47:37,240 Epoch   2 Step:     9400 Batch Loss:     2.309789 Tokens per Sec:     3035, Lr: 0.000300\n",
      "2020-02-13 12:48:44,456 Epoch   2 Step:     9500 Batch Loss:     2.617984 Tokens per Sec:     3147, Lr: 0.000300\n",
      "2020-02-13 12:49:50,397 Epoch   2 Step:     9600 Batch Loss:     2.087964 Tokens per Sec:     3030, Lr: 0.000300\n",
      "2020-02-13 12:50:56,997 Epoch   2 Step:     9700 Batch Loss:     2.237980 Tokens per Sec:     3066, Lr: 0.000300\n",
      "2020-02-13 12:52:02,967 Epoch   2 Step:     9800 Batch Loss:     2.613254 Tokens per Sec:     3032, Lr: 0.000300\n",
      "2020-02-13 12:53:09,564 Epoch   2 Step:     9900 Batch Loss:     2.342013 Tokens per Sec:     3146, Lr: 0.000300\n",
      "2020-02-13 12:54:16,313 Epoch   2 Step:    10000 Batch Loss:     2.506524 Tokens per Sec:     3079, Lr: 0.000300\n",
      "2020-02-13 12:58:13,521 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 12:58:13,522 Saving new checkpoint.\n",
      "2020-02-13 12:58:14,172 Example #0\n",
      "2020-02-13 12:58:14,172 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 12:58:14,173 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 12:58:14,173 \tHypothesis: Kabla ya kukaa kwenye mlango wa muzunguko , nilianza kuhamia katika muji wa Hispania , Crrry .\n",
      "2020-02-13 12:58:14,173 Example #1\n",
      "2020-02-13 12:58:14,173 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 12:58:14,173 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 12:58:14,173 \tHypothesis: Maji ya mimea huvutia sana .\n",
      "2020-02-13 12:58:14,173 Example #2\n",
      "2020-02-13 12:58:14,173 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 12:58:14,173 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 12:58:14,173 \tHypothesis: Wakati huo , Nathan , watu wa ukoo waliotembelea Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Kaskazini .\n",
      "2020-02-13 12:58:14,174 Example #3\n",
      "2020-02-13 12:58:14,174 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 12:58:14,174 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 12:58:14,174 \tHypothesis: Lakini , nilitia moyo nieleze kwa uangalifu na nikaanza kueleza imani yangu . ” — Jan , mwenye umri wa miaka 19 , Uingereza .\n",
      "2020-02-13 12:58:14,174 Example #10\n",
      "2020-02-13 12:58:14,174 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 12:58:14,174 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 12:58:14,174 \tHypothesis: ● Kujua kwamba kuna uwezekano wa kuondoa “ maagizo ” ambayo huenda yakawa “ maarufu ” ambayo huenda yakawa na nguvu .\n",
      "2020-02-13 12:58:14,174 Example #15\n",
      "2020-02-13 12:58:14,175 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 12:58:14,175 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 12:58:14,175 \tHypothesis: Pia , unaweza kufanya hivyo kwa sababu ya “ amani ya Mungu ” ambayo inaweza kutatua matatizo ya afya .\n",
      "2020-02-13 12:58:14,175 Example #20\n",
      "2020-02-13 12:58:14,175 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 12:58:14,175 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 12:58:14,175 \tHypothesis: Tunaweza kujifunza nini kutokana na sababu gani tunaweza kuwa hakika kwamba Yesu alifufuliwa na kwamba sasa ni hai .\n",
      "2020-02-13 12:58:14,175 Example #30\n",
      "2020-02-13 12:58:14,176 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 12:58:14,176 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 12:58:14,176 \tHypothesis: “ Uso wa Kimungu ” ( 1 / 1\n",
      "2020-02-13 12:58:14,176 Example #35\n",
      "2020-02-13 12:58:14,176 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 12:58:14,176 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 12:58:14,176 \tHypothesis: Je , tunategemea Neno la Mungu kila siku ?\n",
      "2020-02-13 12:58:14,176 Validation result at epoch   2, step    10000: bleu:  10.65, loss: 62539.7344, ppl:   9.7829, duration: 237.8627s\n",
      "2020-02-13 12:59:20,813 Epoch   2 Step:    10100 Batch Loss:     2.250509 Tokens per Sec:     3093, Lr: 0.000300\n",
      "2020-02-13 13:00:27,503 Epoch   2 Step:    10200 Batch Loss:     2.637855 Tokens per Sec:     3086, Lr: 0.000300\n",
      "2020-02-13 13:01:33,694 Epoch   2 Step:    10300 Batch Loss:     2.252343 Tokens per Sec:     3038, Lr: 0.000300\n",
      "2020-02-13 13:02:40,361 Epoch   2 Step:    10400 Batch Loss:     2.171839 Tokens per Sec:     3122, Lr: 0.000300\n",
      "2020-02-13 13:03:47,884 Epoch   2 Step:    10500 Batch Loss:     2.410120 Tokens per Sec:     3122, Lr: 0.000300\n",
      "2020-02-13 13:04:54,652 Epoch   2 Step:    10600 Batch Loss:     2.351223 Tokens per Sec:     3088, Lr: 0.000300\n",
      "2020-02-13 13:06:01,213 Epoch   2 Step:    10700 Batch Loss:     2.332752 Tokens per Sec:     3097, Lr: 0.000300\n",
      "2020-02-13 13:07:07,903 Epoch   2 Step:    10800 Batch Loss:     2.375034 Tokens per Sec:     3075, Lr: 0.000300\n",
      "2020-02-13 13:08:13,731 Epoch   2 Step:    10900 Batch Loss:     2.470683 Tokens per Sec:     3012, Lr: 0.000300\n",
      "2020-02-13 13:09:20,920 Epoch   2 Step:    11000 Batch Loss:     2.049070 Tokens per Sec:     3066, Lr: 0.000300\n",
      "2020-02-13 13:13:18,237 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 13:13:18,237 Saving new checkpoint.\n",
      "2020-02-13 13:13:18,889 Example #0\n",
      "2020-02-13 13:13:18,889 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 13:13:18,889 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 13:13:18,889 \tHypothesis: Kabla ya kuingia katika kambi ya Herode , nilipewa mgawo wa kuenda kwenye kambi ya Herode .\n",
      "2020-02-13 13:13:18,889 Example #1\n",
      "2020-02-13 13:13:18,889 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 13:13:18,890 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 13:13:18,890 \tHypothesis: Mimea ya kiasili huathiriwa na viungo vya watu .\n",
      "2020-02-13 13:13:18,890 Example #2\n",
      "2020-02-13 13:13:18,890 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 13:13:18,890 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 13:13:18,890 \tHypothesis: Wakati huo , Nabali , watu waliotembelea Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Kati walipewa mkate wa bahari .\n",
      "2020-02-13 13:13:18,890 Example #3\n",
      "2020-02-13 13:13:18,890 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 13:13:18,890 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 13:13:18,890 \tHypothesis: Lakini , niliendelea kueleza kwa uangalifu na kueleza jinsi nilivyozungumza kuhusu imani yangu . ” — Jan , mwenye umri wa miaka 19 , Uingereza .\n",
      "2020-02-13 13:13:18,890 Example #10\n",
      "2020-02-13 13:13:18,891 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 13:13:18,891 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 13:13:18,891 \tHypothesis: ● Matatizo ya kihisia yanapendekeza kwamba dawa fulani zinaweza kuathiri uwezo wao wa kuondoa nguvu zao .\n",
      "2020-02-13 13:13:18,891 Example #15\n",
      "2020-02-13 13:13:18,891 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 13:13:18,891 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 13:13:18,891 \tHypothesis: Pia , unaweza kufanya hivyo kwa sababu ya “ amani ya Mungu ” ya Mungu .\n",
      "2020-02-13 13:13:18,891 Example #20\n",
      "2020-02-13 13:13:18,891 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 13:13:18,891 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 13:13:18,892 \tHypothesis: Kwa hiyo , tunaweza kuwa na hakika kwamba Yesu alifufuliwa na kwamba sasa ni hai .\n",
      "2020-02-13 13:13:18,892 Example #30\n",
      "2020-02-13 13:13:18,892 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 13:13:18,892 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 13:13:18,892 \tHypothesis: “ Mbona ” ( 1 / 1\n",
      "2020-02-13 13:13:18,892 Example #35\n",
      "2020-02-13 13:13:18,892 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 13:13:18,892 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 13:13:18,892 \tHypothesis: Je , tuna Neno la Mungu kila siku ?\n",
      "2020-02-13 13:13:18,892 Validation result at epoch   2, step    11000: bleu:  11.02, loss: 60636.7266, ppl:   9.1271, duration: 237.9714s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-13 13:14:25,292 Epoch   2 Step:    11100 Batch Loss:     2.285390 Tokens per Sec:     3099, Lr: 0.000300\n",
      "2020-02-13 13:15:31,624 Epoch   2 Step:    11200 Batch Loss:     2.292525 Tokens per Sec:     3086, Lr: 0.000300\n",
      "2020-02-13 13:17:44,938 Epoch   2 Step:    11400 Batch Loss:     2.375286 Tokens per Sec:     3074, Lr: 0.000300\n",
      "2020-02-13 13:18:51,282 Epoch   2 Step:    11500 Batch Loss:     2.447987 Tokens per Sec:     3046, Lr: 0.000300\n",
      "2020-02-13 13:19:58,140 Epoch   2 Step:    11600 Batch Loss:     2.218121 Tokens per Sec:     3072, Lr: 0.000300\n",
      "2020-02-13 13:21:04,425 Epoch   2 Step:    11700 Batch Loss:     2.157409 Tokens per Sec:     3063, Lr: 0.000300\n",
      "2020-02-13 13:22:11,588 Epoch   2 Step:    11800 Batch Loss:     2.219285 Tokens per Sec:     3114, Lr: 0.000300\n",
      "2020-02-13 13:23:17,401 Epoch   2 Step:    11900 Batch Loss:     2.222605 Tokens per Sec:     3098, Lr: 0.000300\n",
      "2020-02-13 13:24:24,188 Epoch   2 Step:    12000 Batch Loss:     2.433710 Tokens per Sec:     3055, Lr: 0.000300\n",
      "2020-02-13 13:28:21,584 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 13:28:21,585 Saving new checkpoint.\n",
      "2020-02-13 13:28:22,246 Example #0\n",
      "2020-02-13 13:28:22,247 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 13:28:22,247 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 13:28:22,247 \tHypothesis: Kabla ya kuingia katika kambi ya mateso , nilihamia katika kambi ya Herode , Craracus , Crary .\n",
      "2020-02-13 13:28:22,247 Example #1\n",
      "2020-02-13 13:28:22,247 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 13:28:22,247 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 13:28:22,247 \tHypothesis: Mabomu ya nyota hutumiwa kuenea kwa njia mbalimbali .\n",
      "2020-02-13 13:28:22,247 Example #2\n",
      "2020-02-13 13:28:22,248 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 13:28:22,248 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 13:28:22,248 \tHypothesis: Wakati huo , Nabali , watu wa inchi ya Nabali walipelekwa kwenye Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Kati .\n",
      "2020-02-13 13:28:22,248 Example #3\n",
      "2020-02-13 13:28:22,248 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 13:28:22,249 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 13:28:22,249 \tHypothesis: Lakini , nilianza kueleza kwa uangalifu na kueleza imani yangu wakati nilipozungumza na watu hao . ” — Jan , mwenye umri wa miaka 19 .\n",
      "2020-02-13 13:28:22,249 Example #10\n",
      "2020-02-13 13:28:22,249 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 13:28:22,249 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 13:28:22,249 \tHypothesis: ● Matatizo ya kitiba yanaonyesha kwamba dawa fulani za kuua “ miujiza ” zinaweza kuharibu nguvu zao .\n",
      "2020-02-13 13:28:22,249 Example #15\n",
      "2020-02-13 13:28:22,250 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 13:28:22,250 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 13:28:22,250 \tHypothesis: Pia , unaweza kufanya hivyo kwa sababu ya “ amani ya Mungu ” ya Mungu yenye kuumiza katika hali ya juu ya matatizo .\n",
      "2020-02-13 13:28:22,250 Example #20\n",
      "2020-02-13 13:28:22,250 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 13:28:22,250 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 13:28:22,250 \tHypothesis: Kwa nini tunaweza kuwa hakika kwamba Yesu alifufuliwa na kwamba yeye ndiye hai leo .\n",
      "2020-02-13 13:28:22,251 Example #30\n",
      "2020-02-13 13:28:22,251 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 13:28:22,251 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 13:28:22,251 \tHypothesis: “ Mbona Uwe Mwenye Kuvunja Moyo ” ( M .\n",
      "2020-02-13 13:28:22,251 Example #35\n",
      "2020-02-13 13:28:22,251 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 13:28:22,251 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 13:28:22,251 \tHypothesis: Je , tunategemea Neno la Mungu kila siku ?\n",
      "2020-02-13 13:28:22,252 Validation result at epoch   2, step    12000: bleu:  11.87, loss: 59227.6016, ppl:   8.6699, duration: 238.0629s\n",
      "2020-02-13 13:29:29,094 Epoch   2 Step:    12100 Batch Loss:     2.281516 Tokens per Sec:     3126, Lr: 0.000300\n",
      "2020-02-13 13:30:35,652 Epoch   2 Step:    12200 Batch Loss:     2.155887 Tokens per Sec:     3061, Lr: 0.000300\n",
      "2020-02-13 13:31:42,404 Epoch   2 Step:    12300 Batch Loss:     2.487732 Tokens per Sec:     3057, Lr: 0.000300\n",
      "2020-02-13 13:32:48,654 Epoch   2 Step:    12400 Batch Loss:     2.117725 Tokens per Sec:     3071, Lr: 0.000300\n",
      "2020-02-13 13:33:55,109 Epoch   2 Step:    12500 Batch Loss:     1.980806 Tokens per Sec:     3081, Lr: 0.000300\n",
      "2020-02-13 13:35:00,571 Epoch   2 Step:    12600 Batch Loss:     2.535526 Tokens per Sec:     3020, Lr: 0.000300\n",
      "2020-02-13 13:36:06,063 Epoch   2 Step:    12700 Batch Loss:     2.387476 Tokens per Sec:     3062, Lr: 0.000300\n",
      "2020-02-13 13:37:12,160 Epoch   2 Step:    12800 Batch Loss:     2.310774 Tokens per Sec:     3077, Lr: 0.000300\n",
      "2020-02-13 13:38:18,568 Epoch   2 Step:    12900 Batch Loss:     2.440764 Tokens per Sec:     3061, Lr: 0.000300\n",
      "2020-02-13 13:39:23,976 Epoch   2 Step:    13000 Batch Loss:     2.290435 Tokens per Sec:     3031, Lr: 0.000300\n",
      "2020-02-13 13:43:21,282 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 13:43:21,282 Saving new checkpoint.\n",
      "2020-02-13 13:43:21,935 Example #0\n",
      "2020-02-13 13:43:21,936 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 13:43:21,936 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 13:43:21,936 \tHypothesis: Kabla ya kuhamia katika kambi ya Herode , nilihamia katika kambi ya Herode , Crrary .\n",
      "2020-02-13 13:43:21,936 Example #1\n",
      "2020-02-13 13:43:21,936 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 13:43:21,936 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 13:43:21,936 \tHypothesis: Maji ya nyuki hutumiwa katika vijiji vya nyuklia .\n",
      "2020-02-13 13:43:21,936 Example #2\n",
      "2020-02-13 13:43:21,936 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 13:43:21,936 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 13:43:21,937 \tHypothesis: Wakati huo , Nabali , watu wa ukoo waliokuwa wamewekwa rasmi kutoka Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari ya Bahari\n",
      "2020-02-13 13:43:21,937 Example #3\n",
      "2020-02-13 13:43:21,937 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 13:43:21,937 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 13:43:21,937 \tHypothesis: Lakini , nilisaidiwa na kueleza kwa uangalifu mambo niliyoamini . ” — Jane , mwenye umri wa miaka 19 , Uingereza .\n",
      "2020-02-13 13:43:21,937 Example #10\n",
      "2020-02-13 13:43:21,937 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 13:43:21,937 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 13:43:21,937 \tHypothesis: ● Kutafakari kuhusu dawa fulani , huenda baadhi ya “ miujiza ” zaweza kuharibu uwezo wao wa kutatua .\n",
      "2020-02-13 13:43:21,937 Example #15\n",
      "2020-02-13 13:43:21,938 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 13:43:21,938 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 13:43:21,938 \tHypothesis: Unafanya hivyo pia ili kufanya amani ya Mungu , unaweza kuwa na matatizo gani ?\n",
      "2020-02-13 13:43:21,938 Example #20\n",
      "2020-02-13 13:43:21,938 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 13:43:21,938 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 13:43:21,938 \tHypothesis: Tunaweza kujifunza sababu gani tunaweza kuwa hakika kwamba Yesu alifufuliwa na kwamba sasa yeye ndiye aliye hai .\n",
      "2020-02-13 13:43:21,938 Example #30\n",
      "2020-02-13 13:43:21,938 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 13:43:21,938 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 13:43:21,938 \tHypothesis: Kuwa Mwenye Kuwa na Ubuni wa Kimungu , 3 / 15\n",
      "2020-02-13 13:43:21,938 Example #35\n",
      "2020-02-13 13:43:21,939 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 13:43:21,939 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 13:43:21,939 \tHypothesis: Je , Neno la Mungu linatutia moyo kila siku ?\n",
      "2020-02-13 13:43:21,939 Validation result at epoch   2, step    13000: bleu:  12.08, loss: 58186.7734, ppl:   8.3470, duration: 237.9620s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-13 13:44:29,273 Epoch   2 Step:    13100 Batch Loss:     2.325762 Tokens per Sec:     3134, Lr: 0.000300\n",
      "2020-02-13 13:45:23,353 Epoch   2: total training loss 15970.08\n",
      "2020-02-13 13:45:23,353 EPOCH 3\n",
      "2020-02-13 13:45:36,088 Epoch   3 Step:    13200 Batch Loss:     2.586302 Tokens per Sec:     2889, Lr: 0.000300\n",
      "2020-02-13 13:46:42,030 Epoch   3 Step:    13300 Batch Loss:     2.261342 Tokens per Sec:     3042, Lr: 0.000300\n",
      "2020-02-13 13:47:48,610 Epoch   3 Step:    13400 Batch Loss:     2.160784 Tokens per Sec:     3124, Lr: 0.000300\n",
      "2020-02-13 13:48:55,039 Epoch   3 Step:    13500 Batch Loss:     2.480360 Tokens per Sec:     3099, Lr: 0.000300\n",
      "2020-02-13 13:51:09,349 Epoch   3 Step:    13700 Batch Loss:     1.981722 Tokens per Sec:     3093, Lr: 0.000300\n",
      "2020-02-13 13:52:15,976 Epoch   3 Step:    13800 Batch Loss:     2.083113 Tokens per Sec:     3156, Lr: 0.000300\n",
      "2020-02-13 13:53:21,723 Epoch   3 Step:    13900 Batch Loss:     2.070894 Tokens per Sec:     3054, Lr: 0.000300\n",
      "2020-02-13 13:54:27,656 Epoch   3 Step:    14000 Batch Loss:     2.039570 Tokens per Sec:     3077, Lr: 0.000300\n",
      "2020-02-13 13:58:24,863 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 13:58:24,863 Saving new checkpoint.\n",
      "2020-02-13 13:58:25,510 Example #0\n",
      "2020-02-13 13:58:25,510 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 13:58:25,510 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 13:58:25,510 \tHypothesis: Kabla ya kwenda kwenye mashua , nilihamia katika kambi ya Herode , Crrame .\n",
      "2020-02-13 13:58:25,511 Example #1\n",
      "2020-02-13 13:58:25,511 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 13:58:25,511 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 13:58:25,511 \tHypothesis: Mbinu za kuua nyuki huvutia sana .\n",
      "2020-02-13 13:58:25,511 Example #2\n",
      "2020-02-13 13:58:25,511 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 13:58:25,511 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 13:58:25,511 \tHypothesis: Wakati huo , Nabali , watu wa ukoo waliwekwa kuwa watu wa baharini , walipelekwa kwenye bahari ya Bahari ya Bahari ya Kaskazini .\n",
      "2020-02-13 13:58:25,511 Example #3\n",
      "2020-02-13 13:58:25,512 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 13:58:25,512 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 13:58:25,512 \tHypothesis: Lakini nilitaka nielewe kwa njia ya mfano na nilipozungumza na watu wa dini hizo . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 13:58:25,512 Example #10\n",
      "2020-02-13 13:58:25,512 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 13:58:25,512 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 13:58:25,512 \tHypothesis: ● Matibabu fulani ya kiuchumi yanatambua kwamba dawa fulani “ miujiza ” inaweza kuharibu uwezo wao .\n",
      "2020-02-13 13:58:25,512 Example #15\n",
      "2020-02-13 13:58:25,512 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 13:58:25,513 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 13:58:25,513 \tHypothesis: Unaweza pia kufanya hivyo kwa sababu ya ‘ amani ya Mungu ’ inaweza kukusaidia kuepuka magumu yenye unajisikia .\n",
      "2020-02-13 13:58:25,513 Example #20\n",
      "2020-02-13 13:58:25,513 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 13:58:25,513 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 13:58:25,513 \tHypothesis: Kwa nini tunaweza kujifunza kwamba Yesu alifufuliwa na kwamba sasa ni hai .\n",
      "2020-02-13 13:58:25,513 Example #30\n",
      "2020-02-13 13:58:25,513 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 13:58:25,513 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 13:58:25,513 \tHypothesis: Mahali pa Mungu , 12 / 1\n",
      "2020-02-13 13:58:25,513 Example #35\n",
      "2020-02-13 13:58:25,514 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 13:58:25,514 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 13:58:25,514 \tHypothesis: Je , tunasoma Neno la Mungu kila siku ?\n",
      "2020-02-13 13:58:25,514 Validation result at epoch   3, step    14000: bleu:  12.99, loss: 56896.0625, ppl:   7.9632, duration: 237.8569s\n",
      "2020-02-13 13:59:32,042 Epoch   3 Step:    14100 Batch Loss:     2.146423 Tokens per Sec:     3064, Lr: 0.000300\n",
      "2020-02-13 14:00:37,534 Epoch   3 Step:    14200 Batch Loss:     2.419403 Tokens per Sec:     3052, Lr: 0.000300\n",
      "2020-02-13 14:01:44,291 Epoch   3 Step:    14300 Batch Loss:     2.863248 Tokens per Sec:     3095, Lr: 0.000300\n",
      "2020-02-13 14:02:50,843 Epoch   3 Step:    14400 Batch Loss:     2.310760 Tokens per Sec:     3042, Lr: 0.000300\n",
      "2020-02-13 14:03:57,480 Epoch   3 Step:    14500 Batch Loss:     2.368809 Tokens per Sec:     3109, Lr: 0.000300\n",
      "2020-02-13 14:05:04,642 Epoch   3 Step:    14600 Batch Loss:     2.127818 Tokens per Sec:     3160, Lr: 0.000300\n",
      "2020-02-13 14:06:09,572 Epoch   3 Step:    14700 Batch Loss:     2.811626 Tokens per Sec:     3070, Lr: 0.000300\n",
      "2020-02-13 14:07:15,211 Epoch   3 Step:    14800 Batch Loss:     2.152117 Tokens per Sec:     3076, Lr: 0.000300\n",
      "2020-02-13 14:08:21,534 Epoch   3 Step:    14900 Batch Loss:     2.223515 Tokens per Sec:     3114, Lr: 0.000300\n",
      "2020-02-13 14:09:27,666 Epoch   3 Step:    15000 Batch Loss:     2.273570 Tokens per Sec:     3060, Lr: 0.000300\n",
      "2020-02-13 14:13:24,800 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 14:13:24,801 Saving new checkpoint.\n",
      "2020-02-13 14:13:25,495 Example #0\n",
      "2020-02-13 14:13:25,495 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 14:13:25,495 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 14:13:25,495 \tHypothesis: Kabla ya kukaa kwenye kambi ya mateso , nilihamia katika kambi ya mateso ya Herlion , Crama .\n",
      "2020-02-13 14:13:25,495 Example #1\n",
      "2020-02-13 14:13:25,495 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 14:13:25,495 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 14:13:25,495 \tHypothesis: Mabawa ya kiuchumi huathiriwa na viini vya nyuklia .\n",
      "2020-02-13 14:13:25,496 Example #2\n",
      "2020-02-13 14:13:25,496 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 14:13:25,496 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 14:13:25,496 \tHypothesis: Wakati huo , watu wa Nabali , ambao waliwekwa rasmi kuwa baharini kutoka Bahari ya Bahari ya Bahari ya Atlantiki walipelekwa kwenye meli hiyo .\n",
      "2020-02-13 14:13:25,496 Example #3\n",
      "2020-02-13 14:13:25,496 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 14:13:25,496 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 14:13:25,496 \tHypothesis: Lakini nilisaidiwa nijifunze na kueleza kuhusu imani yangu . ” — Jane , mwenye umri wa miaka 19 , Uingereza .\n",
      "2020-02-13 14:13:25,496 Example #10\n",
      "2020-02-13 14:13:25,496 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 14:13:25,497 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 14:13:25,497 \tHypothesis: ● Matibabu fulani yanatambua kwamba dawa fulani za kulevya zinaweza kuathiri uwezo wao wa kutatua tatizo hilo .\n",
      "2020-02-13 14:13:25,497 Example #15\n",
      "2020-02-13 14:13:25,497 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 14:13:25,497 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 14:13:25,497 \tHypothesis: Unafanya hivyo pia ili uwe na hakika kwamba ‘ amani ya Mungu ’ inaweza kukusaidia ufanye mabadiliko makubwa .\n",
      "2020-02-13 14:13:25,497 Example #20\n",
      "2020-02-13 14:13:25,497 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 14:13:25,497 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 14:13:25,498 \tHypothesis: Fikiria sababu gani tunaweza kuwa hakika kwamba Yesu alifufuliwa na kwamba sasa yeye ndiye hai .\n",
      "2020-02-13 14:13:25,498 Example #30\n",
      "2020-02-13 14:13:25,498 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 14:13:25,498 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 14:13:25,498 \tHypothesis: Kufanya Maendeleo ya Kimaandiko , 3 / 1\n",
      "2020-02-13 14:13:25,498 Example #35\n",
      "2020-02-13 14:13:25,498 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 14:13:25,498 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 14:13:25,498 \tHypothesis: Je , tunatumia Neno la Mungu kila siku ?\n",
      "2020-02-13 14:13:25,498 Validation result at epoch   3, step    15000: bleu:  13.11, loss: 55913.6836, ppl:   7.6830, duration: 237.8316s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-13 14:14:32,789 Epoch   3 Step:    15100 Batch Loss:     2.222872 Tokens per Sec:     3132, Lr: 0.000300\n",
      "2020-02-13 14:15:38,904 Epoch   3 Step:    15200 Batch Loss:     2.195426 Tokens per Sec:     3044, Lr: 0.000300\n",
      "2020-02-13 14:16:44,537 Epoch   3 Step:    15300 Batch Loss:     2.243739 Tokens per Sec:     3053, Lr: 0.000300\n",
      "2020-02-13 14:17:50,546 Epoch   3 Step:    15400 Batch Loss:     2.060385 Tokens per Sec:     3081, Lr: 0.000300\n",
      "2020-02-13 14:18:56,952 Epoch   3 Step:    15500 Batch Loss:     2.117823 Tokens per Sec:     3069, Lr: 0.000300\n",
      "2020-02-13 14:20:02,499 Epoch   3 Step:    15600 Batch Loss:     2.067538 Tokens per Sec:     3034, Lr: 0.000300\n",
      "2020-02-13 14:21:08,341 Epoch   3 Step:    15700 Batch Loss:     2.123886 Tokens per Sec:     3089, Lr: 0.000300\n",
      "2020-02-13 14:22:15,031 Epoch   3 Step:    15800 Batch Loss:     2.252592 Tokens per Sec:     3090, Lr: 0.000300\n",
      "2020-02-13 14:23:21,822 Epoch   3 Step:    15900 Batch Loss:     1.752465 Tokens per Sec:     3090, Lr: 0.000300\n",
      "2020-02-13 14:24:28,547 Epoch   3 Step:    16000 Batch Loss:     2.174019 Tokens per Sec:     3103, Lr: 0.000300\n",
      "2020-02-13 14:28:25,770 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 14:28:25,771 Saving new checkpoint.\n",
      "2020-02-13 14:28:26,424 Example #0\n",
      "2020-02-13 14:28:26,424 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 14:28:26,424 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 14:28:26,424 \tHypothesis: Kabla ya kuhitimu , nilihamia kambi ya kuenda kwenye kambi ya Herlion , Crina .\n",
      "2020-02-13 14:28:26,424 Example #1\n",
      "2020-02-13 14:28:26,425 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 14:28:26,425 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 14:28:26,425 \tHypothesis: Mapambo ya kemikali hutumiwa katika njia mbalimbali .\n",
      "2020-02-13 14:28:26,425 Example #2\n",
      "2020-02-13 14:28:26,425 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 14:28:26,425 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 14:28:26,426 \tHypothesis: Wakati huo , watu wa Nabali , ambao walikuwa wamewekwa rasmi kuwa mtawala wa Bahari ya Chumvi , walipelekwa kwenye meli hiyo .\n",
      "2020-02-13 14:28:26,426 Example #3\n",
      "2020-02-13 14:28:26,426 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 14:28:26,426 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 14:28:26,426 \tHypothesis: Lakini nilisaidiwa nieleze kwa uangalifu na kwa uangalifu nilipozungumza kuhusu imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 14:28:26,426 Example #10\n",
      "2020-02-13 14:28:26,426 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 14:28:26,427 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 14:28:26,427 \tHypothesis: ● Matibabu ya kitiba yanatambua kwamba dawa fulani za kulevya zinaweza kuharibu uwezo wao .\n",
      "2020-02-13 14:28:26,427 Example #15\n",
      "2020-02-13 14:28:26,427 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 14:28:26,427 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 14:28:26,427 \tHypothesis: Unaweza pia kufanya hivyo kwa sababu ya hali hiyo yenye kuhuzunisha : “ Amani ya Mungu ” inaweza kukusaidia kuepuka matatizo .\n",
      "2020-02-13 14:28:26,427 Example #20\n",
      "2020-02-13 14:28:26,428 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 14:28:26,428 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 14:28:26,428 \tHypothesis: Tunaweza kujifunza sababu gani tunaweza kuwa hakika kwamba Yesu alifufuliwa na kwamba sasa ni hai .\n",
      "2020-02-13 14:28:26,428 Example #30\n",
      "2020-02-13 14:28:26,428 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 14:28:26,428 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 14:28:26,428 \tHypothesis: “ Uwe Mwenye Kujivunia ” ( M .\n",
      "2020-02-13 14:28:26,429 Example #35\n",
      "2020-02-13 14:28:26,429 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 14:28:26,429 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 14:28:26,429 \tHypothesis: Je , tuna Neno la Mungu kila siku ?\n",
      "2020-02-13 14:28:26,429 Validation result at epoch   3, step    16000: bleu:  13.97, loss: 54779.1562, ppl:   7.3716, duration: 237.8817s\n",
      "2020-02-13 14:29:33,303 Epoch   3 Step:    16100 Batch Loss:     1.950021 Tokens per Sec:     3081, Lr: 0.000300\n",
      "2020-02-13 14:30:38,697 Epoch   3 Step:    16200 Batch Loss:     2.037808 Tokens per Sec:     2970, Lr: 0.000300\n",
      "2020-02-13 14:31:44,610 Epoch   3 Step:    16300 Batch Loss:     2.308502 Tokens per Sec:     3075, Lr: 0.000300\n",
      "2020-02-13 14:32:50,414 Epoch   3 Step:    16400 Batch Loss:     2.061450 Tokens per Sec:     3012, Lr: 0.000300\n",
      "2020-02-13 14:33:56,885 Epoch   3 Step:    16500 Batch Loss:     2.035269 Tokens per Sec:     3092, Lr: 0.000300\n",
      "2020-02-13 14:35:03,685 Epoch   3 Step:    16600 Batch Loss:     2.019110 Tokens per Sec:     3105, Lr: 0.000300\n",
      "2020-02-13 14:36:10,915 Epoch   3 Step:    16700 Batch Loss:     2.142120 Tokens per Sec:     3121, Lr: 0.000300\n",
      "2020-02-13 14:37:17,016 Epoch   3 Step:    16800 Batch Loss:     2.091386 Tokens per Sec:     3055, Lr: 0.000300\n",
      "2020-02-13 14:38:23,039 Epoch   3 Step:    16900 Batch Loss:     1.921609 Tokens per Sec:     3044, Lr: 0.000300\n",
      "2020-02-13 14:39:28,902 Epoch   3 Step:    17000 Batch Loss:     2.194227 Tokens per Sec:     3068, Lr: 0.000300\n",
      "2020-02-13 14:43:26,131 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 14:43:26,132 Saving new checkpoint.\n",
      "2020-02-13 14:43:26,785 Example #0\n",
      "2020-02-13 14:43:26,786 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 14:43:26,786 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 14:43:26,786 \tHypothesis: Kabla ya kuhitimu , nilihamia kambi ya kambi ya Herlion , Crence .\n",
      "2020-02-13 14:43:26,786 Example #1\n",
      "2020-02-13 14:43:26,786 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 14:43:26,786 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 14:43:26,786 \tHypothesis: Mapambo ya nyota hutumiwa katika viwanda vya nyota .\n",
      "2020-02-13 14:43:26,786 Example #2\n",
      "2020-02-13 14:43:26,786 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 14:43:26,786 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 14:43:26,787 \tHypothesis: Wakati huo , watu wa kale waliotumikia katika Bahari ya Chumvi walipata maliki wa Bahari ya Chumvi , walifanya kazi ya kuajiriwa .\n",
      "2020-02-13 14:43:26,787 Example #3\n",
      "2020-02-13 14:43:26,787 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 14:43:26,787 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 14:43:26,787 \tHypothesis: Lakini nilisaidiwa nielewe na watu wengine na nilipozungumza kuhusu imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 14:43:26,787 Example #10\n",
      "2020-02-13 14:43:26,787 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 14:43:26,787 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 14:43:26,787 \tHypothesis: ● Kujua kwamba madaktari wanajua kwamba dawa fulani “ miujiza ” inaweza kuharibu mamlaka yao .\n",
      "2020-02-13 14:43:26,787 Example #15\n",
      "2020-02-13 14:43:26,788 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 14:43:26,788 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 14:43:26,788 \tHypothesis: Pia , unaweza kufanya hivyo kwa sababu ‘ amani ya Mungu ’ inaweza kukusaidia uwe na matatizo yenye kuhuzunisha .\n",
      "2020-02-13 14:43:26,788 Example #20\n",
      "2020-02-13 14:43:26,788 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 14:43:26,788 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 14:43:26,788 \tHypothesis: Kwa nini tunaweza kuwa na hakika kwamba Yesu alifufuliwa na kwamba sasa yeye ni hai .\n",
      "2020-02-13 14:43:26,788 Example #30\n",
      "2020-02-13 14:43:26,789 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 14:43:26,789 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 14:43:26,789 \tHypothesis: “ Mbona Mwenye Nguvu za Kujipendea ” ( Linganisha na Mkono wa Mulinzi ) , 11 / 1\n",
      "2020-02-13 14:43:26,789 Example #35\n",
      "2020-02-13 14:43:26,789 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 14:43:26,789 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 14:43:26,789 \tHypothesis: Je , tunatumia Neno la Mungu kila siku ?\n",
      "2020-02-13 14:43:26,790 Validation result at epoch   3, step    17000: bleu:  14.30, loss: 54069.1641, ppl:   7.1832, duration: 237.8869s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-13 14:44:32,223 Epoch   3 Step:    17100 Batch Loss:     2.063174 Tokens per Sec:     3097, Lr: 0.000300\n",
      "2020-02-13 14:45:38,621 Epoch   3 Step:    17200 Batch Loss:     2.196168 Tokens per Sec:     3034, Lr: 0.000300\n",
      "2020-02-13 14:46:45,951 Epoch   3 Step:    17300 Batch Loss:     1.837460 Tokens per Sec:     3159, Lr: 0.000300\n",
      "2020-02-13 14:47:51,476 Epoch   3 Step:    17400 Batch Loss:     2.279250 Tokens per Sec:     3033, Lr: 0.000300\n",
      "2020-02-13 14:48:58,028 Epoch   3 Step:    17500 Batch Loss:     1.921667 Tokens per Sec:     3094, Lr: 0.000300\n",
      "2020-02-13 14:50:04,624 Epoch   3 Step:    17600 Batch Loss:     2.051232 Tokens per Sec:     3091, Lr: 0.000300\n",
      "2020-02-13 14:51:10,133 Epoch   3 Step:    17700 Batch Loss:     2.162503 Tokens per Sec:     3067, Lr: 0.000300\n",
      "2020-02-13 14:52:15,613 Epoch   3 Step:    17800 Batch Loss:     2.190103 Tokens per Sec:     3047, Lr: 0.000300\n",
      "2020-02-13 14:53:22,612 Epoch   3 Step:    17900 Batch Loss:     1.962997 Tokens per Sec:     3128, Lr: 0.000300\n",
      "2020-02-13 14:54:28,582 Epoch   3 Step:    18000 Batch Loss:     1.952743 Tokens per Sec:     3046, Lr: 0.000300\n",
      "2020-02-13 14:58:25,716 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 14:58:25,716 Saving new checkpoint.\n",
      "2020-02-13 14:58:26,378 Example #0\n",
      "2020-02-13 14:58:26,378 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 14:58:26,378 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 14:58:26,378 \tHypothesis: Kabla ya kwenda kwenye mashua , niliwekwa kwenye kambi ya mawe huko Herlie , Krstantino .\n",
      "2020-02-13 14:58:26,378 Example #1\n",
      "2020-02-13 14:58:26,378 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 14:58:26,378 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 14:58:26,379 \tHypothesis: Mbinu za kuhifadhi maji huchangia kuongezeka kwa mawimbi .\n",
      "2020-02-13 14:58:26,379 Example #2\n",
      "2020-02-13 14:58:26,379 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 14:58:26,379 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 14:58:26,379 \tHypothesis: Wakati huo , watu wa Nabali , ambao waliwekwa rasmi kuwa wakipeleka bahari ya Mori , walipelekwa kwenye mstari wa mstari wa mstari wa nondo .\n",
      "2020-02-13 14:58:26,379 Example #3\n",
      "2020-02-13 14:58:26,379 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 14:58:26,379 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 14:58:26,380 \tHypothesis: Lakini nilitaka kuwaeleza wengine kuhusu imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 14:58:26,380 Example #10\n",
      "2020-02-13 14:58:26,380 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 14:58:26,380 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 14:58:26,380 \tHypothesis: ● Tatizo la kitiba linaweza kuathiri sana afya ya watu , yaani , dawa za kulevya , na “ miujiza ” zinaweza kuwa na nguvu zao .\n",
      "2020-02-13 14:58:26,380 Example #15\n",
      "2020-02-13 14:58:26,380 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 14:58:26,380 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 14:58:26,380 \tHypothesis: Unaweza pia kufanya hivyo kwa sababu ya “ amani ya Mungu ” inaweza kukusaidia kuepuka matatizo .\n",
      "2020-02-13 14:58:26,380 Example #20\n",
      "2020-02-13 14:58:26,381 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 14:58:26,381 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 14:58:26,381 \tHypothesis: Tunaweza kujifunza sababu gani tunaweza kuwa hakika kwamba Yesu alifufuliwa na kwamba sasa ni hai .\n",
      "2020-02-13 14:58:26,381 Example #30\n",
      "2020-02-13 14:58:26,381 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 14:58:26,381 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 14:58:26,381 \tHypothesis: Kuwa Mwenye Upendo , 6 / 1\n",
      "2020-02-13 14:58:26,381 Example #35\n",
      "2020-02-13 14:58:26,381 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 14:58:26,381 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 14:58:26,382 \tHypothesis: Je , tunatumia Neno la Mungu kila siku ?\n",
      "2020-02-13 14:58:26,382 Validation result at epoch   3, step    18000: bleu:  14.82, loss: 53412.5117, ppl:   7.0132, duration: 237.7990s\n",
      "2020-02-13 14:59:32,835 Epoch   3 Step:    18100 Batch Loss:     1.951807 Tokens per Sec:     3127, Lr: 0.000300\n",
      "2020-02-13 15:00:39,667 Epoch   3 Step:    18200 Batch Loss:     2.159610 Tokens per Sec:     3128, Lr: 0.000300\n",
      "2020-02-13 15:01:45,470 Epoch   3 Step:    18300 Batch Loss:     2.038137 Tokens per Sec:     3008, Lr: 0.000300\n",
      "2020-02-13 15:02:52,384 Epoch   3 Step:    18400 Batch Loss:     1.980756 Tokens per Sec:     3147, Lr: 0.000300\n",
      "2020-02-13 15:03:59,331 Epoch   3 Step:    18500 Batch Loss:     1.997968 Tokens per Sec:     3099, Lr: 0.000300\n",
      "2020-02-13 15:05:05,998 Epoch   3 Step:    18600 Batch Loss:     1.924604 Tokens per Sec:     3055, Lr: 0.000300\n",
      "2020-02-13 15:06:12,082 Epoch   3 Step:    18700 Batch Loss:     1.982773 Tokens per Sec:     3105, Lr: 0.000300\n",
      "2020-02-13 15:07:18,732 Epoch   3 Step:    18800 Batch Loss:     1.980728 Tokens per Sec:     3090, Lr: 0.000300\n",
      "2020-02-13 15:08:24,739 Epoch   3 Step:    18900 Batch Loss:     2.258067 Tokens per Sec:     3065, Lr: 0.000300\n",
      "2020-02-13 15:09:30,756 Epoch   3 Step:    19000 Batch Loss:     2.222753 Tokens per Sec:     3063, Lr: 0.000300\n",
      "2020-02-13 15:13:27,959 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 15:13:27,960 Saving new checkpoint.\n",
      "2020-02-13 15:13:28,619 Example #0\n",
      "2020-02-13 15:13:28,619 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 15:13:28,619 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 15:13:28,619 \tHypothesis: Kabla ya kukaa katika kambi ya mateso , nilihamishwa katika kambi ya Herlion , Crsal .\n",
      "2020-02-13 15:13:28,619 Example #1\n",
      "2020-02-13 15:13:28,619 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 15:13:28,620 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 15:13:28,620 \tHypothesis: Mbinu za kuhifadhi mimea hutumiwa katika njia za maisha .\n",
      "2020-02-13 15:13:28,620 Example #2\n",
      "2020-02-13 15:13:28,620 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 15:13:28,620 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 15:13:28,620 \tHypothesis: Wakati huo , watu wa Nabali , ambao waliwekwa rasmi kuwa wakimtembelea Bahari ya Bahari ya Chumvi , walipewa maliki wa baharini .\n",
      "2020-02-13 15:13:28,620 Example #3\n",
      "2020-02-13 15:13:28,620 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 15:13:28,620 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 15:13:28,620 \tHypothesis: Lakini nilisaidiwa nieleze kwa bidii na kwa ustadi niliposema kuhusu imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 15:13:28,621 Example #10\n",
      "2020-02-13 15:13:28,621 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 15:13:28,621 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 15:13:28,621 \tHypothesis: ● Matibabu ya kitiba yanaonyesha kwamba dawa fulani za kulevya zinaweza kuathiri uwezo wao wa kufanya hivyo .\n",
      "2020-02-13 15:13:28,621 Example #15\n",
      "2020-02-13 15:13:28,621 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 15:13:28,621 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 15:13:28,621 \tHypothesis: Unaweza pia kufanya hivyo kwa sababu ya ‘ amani ya Mungu ’ ya Mungu .\n",
      "2020-02-13 15:13:28,621 Example #20\n",
      "2020-02-13 15:13:28,622 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 15:13:28,622 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 15:13:28,622 \tHypothesis: Tunaweza kujifunza sababu gani tunaweza kuwa na hakika kwamba Yesu alifufuliwa na kuwa hai leo .\n",
      "2020-02-13 15:13:28,622 Example #30\n",
      "2020-02-13 15:13:28,622 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 15:13:28,622 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 15:13:28,622 \tHypothesis: “ Mbona Uwe Mwenye Upendo wa Mungu ? ”\n",
      "2020-02-13 15:13:28,622 Example #35\n",
      "2020-02-13 15:13:28,622 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 15:13:28,622 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 15:13:28,622 \tHypothesis: Je , sisi hutumia Neno la Mungu kila siku ?\n",
      "2020-02-13 15:13:28,622 Validation result at epoch   3, step    19000: bleu:  15.10, loss: 52716.4297, ppl:   6.8374, duration: 237.8657s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-13 15:14:33,696 Epoch   3 Step:    19100 Batch Loss:     1.976501 Tokens per Sec:     3075, Lr: 0.000300\n",
      "2020-02-13 15:15:40,421 Epoch   3 Step:    19200 Batch Loss:     2.078254 Tokens per Sec:     3104, Lr: 0.000300\n",
      "2020-02-13 15:16:48,080 Epoch   3 Step:    19300 Batch Loss:     1.978653 Tokens per Sec:     3130, Lr: 0.000300\n",
      "2020-02-13 15:17:53,928 Epoch   3 Step:    19400 Batch Loss:     2.059787 Tokens per Sec:     3065, Lr: 0.000300\n",
      "2020-02-13 15:19:00,368 Epoch   3 Step:    19500 Batch Loss:     2.308555 Tokens per Sec:     3051, Lr: 0.000300\n",
      "2020-02-13 15:20:06,941 Epoch   3 Step:    19600 Batch Loss:     1.970548 Tokens per Sec:     3094, Lr: 0.000300\n",
      "2020-02-13 15:21:14,402 Epoch   3 Step:    19700 Batch Loss:     2.170464 Tokens per Sec:     3132, Lr: 0.000300\n",
      "2020-02-13 15:22:07,390 Epoch   3: total training loss 13917.49\n",
      "2020-02-13 15:22:07,390 EPOCH 4\n",
      "2020-02-13 15:22:21,982 Epoch   4 Step:    19800 Batch Loss:     2.074150 Tokens per Sec:     3073, Lr: 0.000300\n",
      "2020-02-13 15:23:28,596 Epoch   4 Step:    19900 Batch Loss:     2.042379 Tokens per Sec:     3072, Lr: 0.000300\n",
      "2020-02-13 15:24:34,832 Epoch   4 Step:    20000 Batch Loss:     2.013269 Tokens per Sec:     3097, Lr: 0.000300\n",
      "2020-02-13 15:28:32,242 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 15:28:32,242 Saving new checkpoint.\n",
      "2020-02-13 15:28:32,914 Example #0\n",
      "2020-02-13 15:28:32,914 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 15:28:32,914 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 15:28:32,914 \tHypothesis: Kabla ya kutembelea mashua hiyo , nilihamia katika kambi ya Heraclion , Cril .\n",
      "2020-02-13 15:28:32,914 Example #1\n",
      "2020-02-13 15:28:32,914 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 15:28:32,914 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 15:28:32,915 \tHypothesis: Miti ya nyama hutumiwa kuhifadhi wanyama .\n",
      "2020-02-13 15:28:32,915 Example #2\n",
      "2020-02-13 15:28:32,915 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 15:28:32,915 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 15:28:32,915 \tHypothesis: Wakati huo , watu wa Nabali , ambao waliwekwa rasmi kuwa wametembelea Bahari ya Morte , walipewa mgawo wa kuuza benki .\n",
      "2020-02-13 15:28:32,915 Example #3\n",
      "2020-02-13 15:28:32,915 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 15:28:32,915 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 15:28:32,915 \tHypothesis: Lakini nilisaidiwa nielewe kwa uchangamfu na kwa ustadi nilipozungumza kuhusu imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 15:28:32,916 Example #10\n",
      "2020-02-13 15:28:32,916 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 15:28:32,916 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 15:28:32,916 \tHypothesis: ● Madaktari fulani hujiuliza hivi : “ Mbele ya dawa za kulevya , ” huenda wengine wakafikiri kwamba wanaweza kuweza kuficha mamlaka yao .\n",
      "2020-02-13 15:28:32,916 Example #15\n",
      "2020-02-13 15:28:32,916 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 15:28:32,916 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 15:28:32,916 \tHypothesis: Unafanya pia uone kama ‘ amani ya Mungu ’ inaweza kuleta matatizo ambayo wewe pia unaweza kufanya .\n",
      "2020-02-13 15:28:32,916 Example #20\n",
      "2020-02-13 15:28:32,917 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 15:28:32,917 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 15:28:32,917 \tHypothesis: Tunaweza kujifunza kwa nini tunaweza kuwa hakika kwamba Yesu alifufuliwa na kuwa hai leo .\n",
      "2020-02-13 15:28:32,917 Example #30\n",
      "2020-02-13 15:28:32,917 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 15:28:32,917 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 15:28:32,917 \tHypothesis: “ Mbona Tuwe Mwenye Kujivunia ” ?\n",
      "2020-02-13 15:28:32,917 Example #35\n",
      "2020-02-13 15:28:32,917 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 15:28:32,918 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 15:28:32,918 \tHypothesis: Je , tunatumia Neno la Mungu kila siku ?\n",
      "2020-02-13 15:28:32,918 Validation result at epoch   4, step    20000: bleu:  15.80, loss: 51864.3359, ppl:   6.6282, duration: 238.0852s\n",
      "2020-02-13 15:29:39,524 Epoch   4 Step:    20100 Batch Loss:     2.042109 Tokens per Sec:     3072, Lr: 0.000300\n",
      "2020-02-13 15:30:45,471 Epoch   4 Step:    20200 Batch Loss:     1.764562 Tokens per Sec:     3067, Lr: 0.000300\n",
      "2020-02-13 15:31:51,765 Epoch   4 Step:    20300 Batch Loss:     1.967745 Tokens per Sec:     3078, Lr: 0.000300\n",
      "2020-02-13 15:32:58,162 Epoch   4 Step:    20400 Batch Loss:     1.848688 Tokens per Sec:     3037, Lr: 0.000300\n",
      "2020-02-13 15:34:03,673 Epoch   4 Step:    20500 Batch Loss:     1.815820 Tokens per Sec:     3025, Lr: 0.000300\n",
      "2020-02-13 15:35:09,783 Epoch   4 Step:    20600 Batch Loss:     1.939749 Tokens per Sec:     3080, Lr: 0.000300\n",
      "2020-02-13 15:36:16,595 Epoch   4 Step:    20700 Batch Loss:     2.129430 Tokens per Sec:     3104, Lr: 0.000300\n",
      "2020-02-13 15:37:21,205 Epoch   4 Step:    20800 Batch Loss:     2.002142 Tokens per Sec:     3006, Lr: 0.000300\n",
      "2020-02-13 15:38:26,753 Epoch   4 Step:    20900 Batch Loss:     2.024477 Tokens per Sec:     3058, Lr: 0.000300\n",
      "2020-02-13 15:39:33,955 Epoch   4 Step:    21000 Batch Loss:     2.071982 Tokens per Sec:     3156, Lr: 0.000300\n",
      "2020-02-13 15:43:31,180 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 15:43:31,180 Saving new checkpoint.\n",
      "2020-02-13 15:43:31,864 Example #0\n",
      "2020-02-13 15:43:31,864 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 15:43:31,865 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 15:43:31,865 \tHypothesis: Kabla ya kuhamia eneo la mbali , nilihamia katika kambi ya Heraclion , Crama .\n",
      "2020-02-13 15:43:31,865 Example #1\n",
      "2020-02-13 15:43:31,865 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 15:43:31,865 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 15:43:31,865 \tHypothesis: Mimea hiyo hufanyiza vifaa vya kupikia .\n",
      "2020-02-13 15:43:31,865 Example #2\n",
      "2020-02-13 15:43:31,865 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 15:43:31,865 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 15:43:31,865 \tHypothesis: Wakati huo , watu wa Nabathi waliotembelea Bahari ya Bahari ya Chumvi walishambulia meli hiyo .\n",
      "2020-02-13 15:43:31,866 Example #3\n",
      "2020-02-13 15:43:31,866 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 15:43:31,866 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 15:43:31,866 \tHypothesis: Lakini hilo linanisaidia nielewe kwa njia yenye kustaajabisha na yenye kueleweka wakati ninazungumuza na wengine juu ya imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 15:43:31,866 Example #10\n",
      "2020-02-13 15:43:31,866 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 15:43:31,866 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 15:43:31,866 \tHypothesis: ● Daktari anajua kwamba dawa fulani “ za kuua ” zinaweza kuathiri uwezo wao .\n",
      "2020-02-13 15:43:31,866 Example #15\n",
      "2020-02-13 15:43:31,867 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 15:43:31,867 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 15:43:31,867 \tHypothesis: Unafanya pia ili uwe na “ amani ya Mungu ” inaweza kukusaidia kupata magumu yoyote yenye unaweza kufanya .\n",
      "2020-02-13 15:43:31,867 Example #20\n",
      "2020-02-13 15:43:31,867 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 15:43:31,867 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 15:43:31,867 \tHypothesis: Tunaweza kujifunza kwa nini tunaweza kuwa na uhakika kwamba Yesu alifufuliwa na kuwa hai leo .\n",
      "2020-02-13 15:43:31,867 Example #30\n",
      "2020-02-13 15:43:31,867 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 15:43:31,867 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 15:43:31,867 \tHypothesis: Unawaza , 3 / 15\n",
      "2020-02-13 15:43:31,868 Example #35\n",
      "2020-02-13 15:43:31,868 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 15:43:31,868 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 15:43:31,868 \tHypothesis: Je , tuna Neno la Mungu kila siku ?\n",
      "2020-02-13 15:43:31,868 Validation result at epoch   4, step    21000: bleu:  15.93, loss: 51605.4844, ppl:   6.5660, duration: 237.9122s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-13 15:44:37,968 Epoch   4 Step:    21100 Batch Loss:     2.030633 Tokens per Sec:     3093, Lr: 0.000300\n",
      "2020-02-13 15:45:43,896 Epoch   4 Step:    21200 Batch Loss:     2.071697 Tokens per Sec:     3072, Lr: 0.000300\n",
      "2020-02-13 15:46:49,520 Epoch   4 Step:    21300 Batch Loss:     1.853978 Tokens per Sec:     3046, Lr: 0.000300\n",
      "2020-02-13 15:47:56,876 Epoch   4 Step:    21400 Batch Loss:     1.915925 Tokens per Sec:     3132, Lr: 0.000300\n",
      "2020-02-13 15:49:02,406 Epoch   4 Step:    21500 Batch Loss:     1.919946 Tokens per Sec:     3094, Lr: 0.000300\n",
      "2020-02-13 15:50:09,138 Epoch   4 Step:    21600 Batch Loss:     1.969241 Tokens per Sec:     3095, Lr: 0.000300\n",
      "2020-02-13 15:51:15,559 Epoch   4 Step:    21700 Batch Loss:     2.111245 Tokens per Sec:     3088, Lr: 0.000300\n",
      "2020-02-13 15:52:22,926 Epoch   4 Step:    21800 Batch Loss:     1.973348 Tokens per Sec:     3073, Lr: 0.000300\n",
      "2020-02-13 15:53:29,445 Epoch   4 Step:    21900 Batch Loss:     2.006958 Tokens per Sec:     3109, Lr: 0.000300\n",
      "2020-02-13 15:54:36,037 Epoch   4 Step:    22000 Batch Loss:     1.894109 Tokens per Sec:     3040, Lr: 0.000300\n",
      "2020-02-13 15:58:33,165 Hooray! New best validation result [ppl]!\n",
      "2020-02-13 15:58:33,166 Saving new checkpoint.\n",
      "2020-02-13 15:58:33,826 Example #0\n",
      "2020-02-13 15:58:33,826 \tSource:     Avant de passer en cour martiale , j’ai été transféré dans un camp militaire à Héraclion , en Crète .\n",
      "2020-02-13 15:58:33,826 \tReference:  Kabla ya kupelekwa kortini , nilihamishiwa kwenye kambi ya kijeshi huko Iráklion , Krete .\n",
      "2020-02-13 15:58:33,826 \tHypothesis: Kabla ya kuhamia kwenye kambi ya kijeshi huko Heraclion , Kroste .\n",
      "2020-02-13 15:58:33,827 Example #1\n",
      "2020-02-13 15:58:33,827 \tSource:     Les simulies véhiculent la cécité des rivières .\n",
      "2020-02-13 15:58:33,827 \tReference:  Nzi anayeeneza upofu wa mtoni\n",
      "2020-02-13 15:58:33,827 \tHypothesis: Mimea ya kale hutumia vifaa vya kupima joto .\n",
      "2020-02-13 15:58:33,827 Example #2\n",
      "2020-02-13 15:58:33,827 \tSource:     À cette époque , les Nabatéens , peuple nomade qui s’était établi autour de la mer Morte , ont monopolisé le commerce du bitume .\n",
      "2020-02-13 15:58:33,827 \tReference:  Inafikiriwa kwamba katikati ya karne ya nne hivi K.W.K . , Wamisri walianza kutumia sana lami katika kuhifadhi maiti , ingawa maoni hayo yanapingwa na wataalamu fulani .\n",
      "2020-02-13 15:58:33,827 \tHypothesis: Wakati huo , watu wa Nabathani waliwekwa rasmi kuwa wakimtembelea Bahari ya Morea , walipanda meli ya kifalme ya Bahari ya Atlantiki .\n",
      "2020-02-13 15:58:33,827 Example #3\n",
      "2020-02-13 15:58:33,828 \tSource:     Mais ça m’aide à m’exprimer avec aisance et clarté quand je parle de mes croyances . ” . — Jane , 19 ans , Angleterre .\n",
      "2020-02-13 15:58:33,828 \tReference:  Lakini kuisoma kumenisaidia kuzungumza kwa uhakika na waziwazi ninapowaeleza wengine imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 15:58:33,828 \tHypothesis: Lakini hilo lilinisaidia nielewe kwa njia yenye kueleweka na kwa njia ya kawaida niliposema kuhusu imani yangu . ” — Jane , 19 , Uingereza .\n",
      "2020-02-13 15:58:33,828 Example #10\n",
      "2020-02-13 15:58:33,828 \tSource:     ● La médecine connaît un revirement funeste : certains médicaments “ miracles ” pourraient avoir perdu leur pouvoir .\n",
      "2020-02-13 15:58:33,828 \tReference:  Magonjwa ya kuambukiza yaliyokuwa yakiwaua mamilioni ya watu miaka mingi iliyopita kama vile , ukoma na kifua kikuu , yalidhibitiwa kwa dawa za kuua viini .\n",
      "2020-02-13 15:58:33,828 \tHypothesis: ● Madaktari wanajua kwamba dawa fulani “ miujiza ” inaweza kuharibu uwezo wao .\n",
      "2020-02-13 15:58:33,829 Example #15\n",
      "2020-02-13 15:58:33,829 \tSource:     Tu peux toi aussi en faire l’expérience : la « paix de Dieu » peut triompher de n’importe lequel de tes problèmes .\n",
      "2020-02-13 15:58:33,829 \tReference:  Ndugu na dada wengi wamekwisha kupata amani hiyo . Wewe pia unaweza kuipata . “ Amani ya Mungu ” inaweza kukusaidia upate ushindi juu ya magumu yoyote !\n",
      "2020-02-13 15:58:33,829 \tHypothesis: Unaweza pia kukumbuka kwamba “ amani ya Mungu ” inaweza kuharibu kabisa matatizo yako .\n",
      "2020-02-13 15:58:33,829 Example #20\n",
      "2020-02-13 15:58:33,829 \tSource:     Apprends pourquoi nous pouvons être sûrs que Jésus a été ressuscité et qu’il est aujourd’hui vivant .\n",
      "2020-02-13 15:58:33,829 \tReference:  Katika habari hii tutaona sababu gani tunapaswa kuwa hakika kwamba Yesu alifufuliwa na kwamba leo anaishi .\n",
      "2020-02-13 15:58:33,829 \tHypothesis: Tafazali , tunaweza kujua sababu gani tunaweza kuwa hakika kwamba Yesu alifufuliwa na kwamba sasa iko hai .\n",
      "2020-02-13 15:58:33,829 Example #30\n",
      "2020-02-13 15:58:33,830 \tSource:     Assemblées de district “ Guidés par l’esprit de Dieu ” , 1 / 3\n",
      "2020-02-13 15:58:33,830 \tReference:  Yaremchuk ) , 6 / 1\n",
      "2020-02-13 15:58:33,830 \tHypothesis: “ Msiwe Mwenye Upendo , ” 15 / 09\n",
      "2020-02-13 15:58:33,830 Example #35\n",
      "2020-02-13 15:58:33,830 \tSource:     Lisons - ​ nous la Parole de Dieu chaque jour ?\n",
      "2020-02-13 15:58:33,830 \tReference:  Je , tunapendezwa na sheria ya Mungu na kusoma Neno lake kila siku ?\n",
      "2020-02-13 15:58:33,830 \tHypothesis: Je , Neno la Mungu Kila Siku ?\n",
      "2020-02-13 15:58:33,830 Validation result at epoch   4, step    22000: bleu:  15.38, loss: 50841.4453, ppl:   6.3856, duration: 237.7921s\n"
     ]
    }
   ],
   "source": [
    "# Train the model\n",
    "# You can press Ctrl-C to stop. And then run the next cell to save your checkpoints! \n",
    "!python3 -m joeynmt train config/transformer_$src$tgt.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-16 20:58:59,082 Epoch  50 Step:   329500 Batch Loss:     1.093634 Tokens per Sec:     2922, Lr: 0.000008\r\n",
      "2020-02-16 21:00:09,071 Epoch  50 Step:   329600 Batch Loss:     1.067766 Tokens per Sec:     3013, Lr: 0.000008\r\n",
      "2020-02-16 21:01:18,867 Epoch  50 Step:   329700 Batch Loss:     1.089992 Tokens per Sec:     2963, Lr: 0.000008\r\n",
      "2020-02-16 21:01:20,211 Epoch  50: total training loss 7458.01\r\n",
      "2020-02-16 21:01:20,212 Training ended after  50 epochs.\r\n",
      "2020-02-16 21:01:20,212 Best validation result at step   329000:   3.61 ppl.\r\n",
      "2020-02-16 21:03:49,905  dev bleu:  26.73 [Beam search decoding with beam size = 5 and alpha = 1.0]\r\n",
      "2020-02-16 21:03:49,907 Translations saved to: models/frswc_transformer/00329000.hyps.dev\r\n",
      "2020-02-16 21:07:32,639 test bleu:  33.73 [Beam search decoding with beam size = 5 and alpha = 1.0]\r\n",
      "2020-02-16 21:07:32,643 Translations saved to: models/frswc_transformer/00329000.hyps.test\r\n"
     ]
    }
   ],
   "source": [
    "! tail -10 models/frswc_transformer/train.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00329000.hyps.dev   16000.hyps\t 221000.hyps  284000.hyps  47000.hyps\r\n",
      "00329000.hyps.test  160000.hyps  222000.hyps  285000.hyps  48000.hyps\r\n",
      "1000.hyps\t    161000.hyps  223000.hyps  286000.hyps  49000.hyps\r\n",
      "10000.hyps\t    162000.hyps  224000.hyps  287000.hyps  5000.hyps\r\n",
      "100000.hyps\t    163000.hyps  225000.hyps  288000.hyps  50000.hyps\r\n",
      "101000.hyps\t    164000.hyps  226000.hyps  289000.hyps  51000.hyps\r\n",
      "102000.hyps\t    165000.hyps  227000.hyps  29000.hyps   52000.hyps\r\n",
      "103000.hyps\t    166000.hyps  228000.hyps  290000.hyps  53000.hyps\r\n",
      "104000.hyps\t    167000.hyps  229000.hyps  291000.hyps  54000.hyps\r\n",
      "105000.hyps\t    168000.hyps  23000.hyps   292000.hyps  55000.hyps\r\n",
      "106000.hyps\t    169000.hyps  230000.hyps  293000.hyps  56000.hyps\r\n",
      "107000.hyps\t    17000.hyps\t 231000.hyps  294000.hyps  57000.hyps\r\n",
      "108000.hyps\t    170000.hyps  232000.hyps  295000.hyps  58000.hyps\r\n",
      "109000.hyps\t    171000.hyps  233000.hyps  296000.hyps  59000.hyps\r\n",
      "11000.hyps\t    172000.hyps  234000.hyps  297000.hyps  6000.hyps\r\n",
      "110000.hyps\t    173000.hyps  235000.hyps  298000.hyps  60000.hyps\r\n",
      "111000.hyps\t    174000.hyps  236000.hyps  299000.hyps  61000.hyps\r\n",
      "112000.hyps\t    175000.hyps  237000.hyps  3000.hyps    62000.hyps\r\n",
      "113000.hyps\t    176000.hyps  238000.hyps  30000.hyps   63000.hyps\r\n",
      "114000.hyps\t    177000.hyps  239000.hyps  300000.hyps  64000.hyps\r\n",
      "115000.hyps\t    178000.hyps  24000.hyps   301000.hyps  65000.hyps\r\n",
      "116000.hyps\t    179000.hyps  240000.hyps  302000.hyps  66000.hyps\r\n",
      "117000.hyps\t    18000.hyps\t 241000.hyps  303000.hyps  67000.hyps\r\n",
      "118000.hyps\t    180000.hyps  242000.hyps  304000.hyps  68000.hyps\r\n",
      "119000.hyps\t    181000.hyps  243000.hyps  305000.hyps  69000.hyps\r\n",
      "12000.hyps\t    182000.hyps  244000.hyps  306000.hyps  7000.hyps\r\n",
      "120000.hyps\t    183000.hyps  245000.hyps  307000.hyps  70000.hyps\r\n",
      "121000.hyps\t    184000.hyps  246000.hyps  308000.hyps  71000.hyps\r\n",
      "122000.hyps\t    185000.hyps  247000.hyps  309000.hyps  72000.hyps\r\n",
      "123000.hyps\t    186000.hyps  248000.hyps  31000.hyps   73000.hyps\r\n",
      "124000.hyps\t    187000.hyps  249000.hyps  310000.hyps  74000.hyps\r\n",
      "125000.hyps\t    188000.hyps  25000.hyps   311000.hyps  75000.hyps\r\n",
      "126000.hyps\t    189000.hyps  250000.hyps  312000.hyps  76000.hyps\r\n",
      "127000.hyps\t    19000.hyps\t 251000.hyps  313000.hyps  77000.hyps\r\n",
      "128000.hyps\t    190000.hyps  252000.hyps  314000.hyps  78000.hyps\r\n",
      "129000.hyps\t    191000.hyps  253000.hyps  315000.hyps  79000.hyps\r\n",
      "13000.hyps\t    192000.hyps  254000.hyps  316000.hyps  8000.hyps\r\n",
      "130000.hyps\t    193000.hyps  255000.hyps  317000.hyps  80000.hyps\r\n",
      "131000.hyps\t    194000.hyps  256000.hyps  318000.hyps  81000.hyps\r\n",
      "132000.hyps\t    195000.hyps  257000.hyps  319000.hyps  82000.hyps\r\n",
      "133000.hyps\t    196000.hyps  258000.hyps  32000.hyps   83000.hyps\r\n",
      "134000.hyps\t    197000.hyps  259000.hyps  320000.ckpt  84000.hyps\r\n",
      "135000.hyps\t    198000.hyps  26000.hyps   320000.hyps  85000.hyps\r\n",
      "136000.hyps\t    199000.hyps  260000.hyps  321000.hyps  86000.hyps\r\n",
      "137000.hyps\t    2000.hyps\t 261000.hyps  322000.ckpt  87000.hyps\r\n",
      "138000.hyps\t    20000.hyps\t 262000.hyps  322000.hyps  88000.hyps\r\n",
      "139000.hyps\t    200000.hyps  263000.hyps  323000.hyps  89000.hyps\r\n",
      "14000.hyps\t    201000.hyps  264000.hyps  324000.hyps  9000.hyps\r\n",
      "140000.hyps\t    202000.hyps  265000.hyps  325000.hyps  90000.hyps\r\n",
      "141000.hyps\t    203000.hyps  266000.hyps  326000.hyps  91000.hyps\r\n",
      "142000.hyps\t    204000.hyps  267000.hyps  327000.hyps  92000.hyps\r\n",
      "143000.hyps\t    205000.hyps  268000.hyps  328000.hyps  93000.hyps\r\n",
      "144000.hyps\t    206000.hyps  269000.hyps  329000.ckpt  94000.hyps\r\n",
      "145000.hyps\t    207000.hyps  27000.hyps   329000.hyps  95000.hyps\r\n",
      "146000.hyps\t    208000.hyps  270000.hyps  33000.hyps   96000.hyps\r\n",
      "147000.hyps\t    209000.hyps  271000.hyps  34000.hyps   97000.hyps\r\n",
      "148000.hyps\t    21000.hyps\t 272000.hyps  35000.hyps   98000.hyps\r\n",
      "149000.hyps\t    210000.hyps  273000.hyps  36000.hyps   99000.hyps\r\n",
      "15000.hyps\t    211000.hyps  274000.hyps  37000.hyps   best.ckpt\r\n",
      "150000.hyps\t    212000.hyps  275000.hyps  38000.hyps   config.yaml\r\n",
      "151000.hyps\t    213000.hyps  276000.hyps  39000.hyps   src_vocab.txt\r\n",
      "152000.hyps\t    214000.hyps  277000.hyps  4000.hyps    tensorboard\r\n",
      "153000.hyps\t    215000.hyps  278000.hyps  40000.hyps   train.log\r\n",
      "154000.hyps\t    216000.hyps  279000.hyps  41000.hyps   trg_vocab.txt\r\n",
      "155000.hyps\t    217000.hyps  28000.hyps   42000.hyps   validations.txt\r\n",
      "156000.hyps\t    218000.hyps  280000.hyps  43000.hyps\r\n",
      "157000.hyps\t    219000.hyps  281000.hyps  44000.hyps\r\n",
      "158000.hyps\t    22000.hyps\t 282000.hyps  45000.hyps\r\n",
      "159000.hyps\t    220000.hyps  283000.hyps  46000.hyps\r\n"
     ]
    }
   ],
   "source": [
    "!ls models/${src}${tgt}_transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-16 20:58:59,082 Epoch  50 Step:   329500 Batch Loss:     1.093634 Tokens per Sec:     2922, Lr: 0.000008\r\n",
      "2020-02-16 21:00:09,071 Epoch  50 Step:   329600 Batch Loss:     1.067766 Tokens per Sec:     3013, Lr: 0.000008\r\n",
      "2020-02-16 21:01:18,867 Epoch  50 Step:   329700 Batch Loss:     1.089992 Tokens per Sec:     2963, Lr: 0.000008\r\n",
      "2020-02-16 21:01:20,211 Epoch  50: total training loss 7458.01\r\n",
      "2020-02-16 21:01:20,212 Training ended after  50 epochs.\r\n",
      "2020-02-16 21:01:20,212 Best validation result at step   329000:   3.61 ppl.\r\n",
      "2020-02-16 21:03:49,905  dev bleu:  26.73 [Beam search decoding with beam size = 5 and alpha = 1.0]\r\n",
      "2020-02-16 21:03:49,907 Translations saved to: models/frswc_transformer/00329000.hyps.dev\r\n",
      "2020-02-16 21:07:32,639 test bleu:  33.73 [Beam search decoding with beam size = 5 and alpha = 1.0]\r\n",
      "2020-02-16 21:07:32,643 Translations saved to: models/frswc_transformer/00329000.hyps.test\r\n"
     ]
    }
   ],
   "source": [
    "! tail models/${src}${tgt}_transformer/train.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 1000\tLoss: 117640.85156\tPPL: 72.96771\tbleu: 0.30677\tLR: 0.00030000\t*\r\n",
      "Steps: 2000\tLoss: 96072.33594\tPPL: 33.23078\tbleu: 1.44532\tLR: 0.00030000\t*\r\n",
      "Steps: 3000\tLoss: 86643.24219\tPPL: 23.56177\tbleu: 2.57306\tLR: 0.00030000\t*\r\n",
      "Steps: 4000\tLoss: 80421.35938\tPPL: 18.77885\tbleu: 3.81392\tLR: 0.00030000\t*\r\n",
      "Steps: 5000\tLoss: 75786.34375\tPPL: 15.85851\tbleu: 5.58060\tLR: 0.00030000\t*\r\n",
      "Steps: 6000\tLoss: 72432.73438\tPPL: 14.03297\tbleu: 6.41285\tLR: 0.00030000\t*\r\n",
      "Steps: 7000\tLoss: 70071.18750\tPPL: 12.87504\tbleu: 7.37862\tLR: 0.00030000\t*\r\n",
      "Steps: 8000\tLoss: 66923.03906\tPPL: 11.47863\tbleu: 8.53179\tLR: 0.00030000\t*\r\n",
      "Steps: 9000\tLoss: 64534.80078\tPPL: 10.52123\tbleu: 9.22430\tLR: 0.00030000\t*\r\n",
      "Steps: 10000\tLoss: 62539.73438\tPPL: 9.78295\tbleu: 10.64979\tLR: 0.00030000\t*\r\n",
      "Steps: 11000\tLoss: 60636.72656\tPPL: 9.12706\tbleu: 11.01825\tLR: 0.00030000\t*\r\n",
      "Steps: 12000\tLoss: 59227.60156\tPPL: 8.66990\tbleu: 11.86854\tLR: 0.00030000\t*\r\n",
      "Steps: 13000\tLoss: 58186.77344\tPPL: 8.34699\tbleu: 12.08454\tLR: 0.00030000\t*\r\n",
      "Steps: 14000\tLoss: 56896.06250\tPPL: 7.96321\tbleu: 12.99344\tLR: 0.00030000\t*\r\n",
      "Steps: 15000\tLoss: 55913.68359\tPPL: 7.68298\tbleu: 13.11308\tLR: 0.00030000\t*\r\n",
      "Steps: 16000\tLoss: 54779.15625\tPPL: 7.37160\tbleu: 13.97093\tLR: 0.00030000\t*\r\n",
      "Steps: 17000\tLoss: 54069.16406\tPPL: 7.18319\tbleu: 14.29817\tLR: 0.00030000\t*\r\n",
      "Steps: 18000\tLoss: 53412.51172\tPPL: 7.01323\tbleu: 14.81524\tLR: 0.00030000\t*\r\n",
      "Steps: 19000\tLoss: 52716.42969\tPPL: 6.83744\tbleu: 15.10384\tLR: 0.00030000\t*\r\n",
      "Steps: 20000\tLoss: 51864.33594\tPPL: 6.62825\tbleu: 15.79898\tLR: 0.00030000\t*\r\n",
      "Steps: 21000\tLoss: 51605.48438\tPPL: 6.56597\tbleu: 15.92662\tLR: 0.00030000\t*\r\n",
      "Steps: 22000\tLoss: 50841.44531\tPPL: 6.38556\tbleu: 15.37756\tLR: 0.00030000\t*\r\n",
      "Steps: 23000\tLoss: 50231.08203\tPPL: 6.24500\tbleu: 15.72780\tLR: 0.00030000\t*\r\n",
      "Steps: 24000\tLoss: 50051.07031\tPPL: 6.20413\tbleu: 16.73068\tLR: 0.00030000\t*\r\n",
      "Steps: 25000\tLoss: 49529.12109\tPPL: 6.08716\tbleu: 16.83326\tLR: 0.00030000\t*\r\n",
      "Steps: 26000\tLoss: 48766.23047\tPPL: 5.92015\tbleu: 16.67082\tLR: 0.00030000\t*\r\n",
      "Steps: 27000\tLoss: 48758.48047\tPPL: 5.91848\tbleu: 16.42519\tLR: 0.00030000\t*\r\n",
      "Steps: 28000\tLoss: 48517.66406\tPPL: 5.86673\tbleu: 17.09880\tLR: 0.00030000\t*\r\n",
      "Steps: 29000\tLoss: 47935.69141\tPPL: 5.74353\tbleu: 17.26956\tLR: 0.00030000\t*\r\n",
      "Steps: 30000\tLoss: 47536.41016\tPPL: 5.66051\tbleu: 18.17331\tLR: 0.00030000\t*\r\n",
      "Steps: 31000\tLoss: 47233.03125\tPPL: 5.59823\tbleu: 17.86630\tLR: 0.00030000\t*\r\n",
      "Steps: 32000\tLoss: 46845.16016\tPPL: 5.51960\tbleu: 17.55114\tLR: 0.00030000\t*\r\n",
      "Steps: 33000\tLoss: 46555.50391\tPPL: 5.46161\tbleu: 18.08572\tLR: 0.00030000\t*\r\n",
      "Steps: 34000\tLoss: 46314.35156\tPPL: 5.41379\tbleu: 18.53433\tLR: 0.00030000\t*\r\n",
      "Steps: 35000\tLoss: 46390.54297\tPPL: 5.42885\tbleu: 18.58724\tLR: 0.00030000\t\r\n",
      "Steps: 36000\tLoss: 45857.84375\tPPL: 5.32441\tbleu: 18.35147\tLR: 0.00030000\t*\r\n",
      "Steps: 37000\tLoss: 45609.14062\tPPL: 5.27634\tbleu: 19.16506\tLR: 0.00030000\t*\r\n",
      "Steps: 38000\tLoss: 45250.14062\tPPL: 5.20771\tbleu: 19.12660\tLR: 0.00030000\t*\r\n",
      "Steps: 39000\tLoss: 45008.28906\tPPL: 5.16198\tbleu: 19.13025\tLR: 0.00030000\t*\r\n",
      "Steps: 40000\tLoss: 44927.48828\tPPL: 5.14679\tbleu: 19.26860\tLR: 0.00030000\t*\r\n",
      "Steps: 41000\tLoss: 44799.68750\tPPL: 5.12286\tbleu: 19.82229\tLR: 0.00030000\t*\r\n",
      "Steps: 42000\tLoss: 44595.99219\tPPL: 5.08495\tbleu: 20.07113\tLR: 0.00030000\t*\r\n",
      "Steps: 43000\tLoss: 44518.01562\tPPL: 5.07051\tbleu: 20.15304\tLR: 0.00030000\t*\r\n",
      "Steps: 44000\tLoss: 44604.22656\tPPL: 5.08648\tbleu: 19.50764\tLR: 0.00030000\t\r\n",
      "Steps: 45000\tLoss: 44226.59766\tPPL: 5.01691\tbleu: 19.81995\tLR: 0.00030000\t*\r\n",
      "Steps: 46000\tLoss: 43740.90625\tPPL: 4.92883\tbleu: 19.78123\tLR: 0.00030000\t*\r\n",
      "Steps: 47000\tLoss: 43917.57031\tPPL: 4.96069\tbleu: 20.11571\tLR: 0.00030000\t\r\n",
      "Steps: 48000\tLoss: 43837.21484\tPPL: 4.94618\tbleu: 20.30522\tLR: 0.00030000\t\r\n",
      "Steps: 49000\tLoss: 43333.14062\tPPL: 4.85609\tbleu: 20.59117\tLR: 0.00030000\t*\r\n",
      "Steps: 50000\tLoss: 43285.43750\tPPL: 4.84765\tbleu: 20.42034\tLR: 0.00030000\t*\r\n",
      "Steps: 51000\tLoss: 43450.50391\tPPL: 4.87691\tbleu: 20.42331\tLR: 0.00030000\t\r\n",
      "Steps: 52000\tLoss: 43048.57031\tPPL: 4.80595\tbleu: 20.58515\tLR: 0.00030000\t*\r\n",
      "Steps: 53000\tLoss: 42810.62891\tPPL: 4.76443\tbleu: 21.11798\tLR: 0.00030000\t*\r\n",
      "Steps: 54000\tLoss: 42952.18359\tPPL: 4.78909\tbleu: 20.62784\tLR: 0.00030000\t\r\n",
      "Steps: 55000\tLoss: 42560.89453\tPPL: 4.72124\tbleu: 21.44503\tLR: 0.00030000\t*\r\n",
      "Steps: 56000\tLoss: 42400.73047\tPPL: 4.69374\tbleu: 21.26521\tLR: 0.00030000\t*\r\n",
      "Steps: 57000\tLoss: 42210.21875\tPPL: 4.66125\tbleu: 21.31058\tLR: 0.00030000\t*\r\n",
      "Steps: 58000\tLoss: 41979.88281\tPPL: 4.62226\tbleu: 21.67745\tLR: 0.00030000\t*\r\n",
      "Steps: 59000\tLoss: 42282.56250\tPPL: 4.67356\tbleu: 20.84804\tLR: 0.00030000\t\r\n",
      "Steps: 60000\tLoss: 42053.80859\tPPL: 4.63474\tbleu: 21.18251\tLR: 0.00030000\t\r\n",
      "Steps: 61000\tLoss: 41909.87109\tPPL: 4.61047\tbleu: 21.62524\tLR: 0.00030000\t*\r\n",
      "Steps: 62000\tLoss: 41541.15234\tPPL: 4.54889\tbleu: 21.41595\tLR: 0.00030000\t*\r\n",
      "Steps: 63000\tLoss: 41558.26562\tPPL: 4.55173\tbleu: 21.00293\tLR: 0.00030000\t\r\n",
      "Steps: 64000\tLoss: 41908.83594\tPPL: 4.61030\tbleu: 20.85890\tLR: 0.00030000\t\r\n",
      "Steps: 65000\tLoss: 41520.91406\tPPL: 4.54554\tbleu: 21.73595\tLR: 0.00030000\t*\r\n",
      "Steps: 66000\tLoss: 41346.65234\tPPL: 4.51674\tbleu: 22.21506\tLR: 0.00030000\t*\r\n",
      "Steps: 67000\tLoss: 41329.91406\tPPL: 4.51399\tbleu: 21.81378\tLR: 0.00030000\t*\r\n",
      "Steps: 68000\tLoss: 41089.80469\tPPL: 4.47464\tbleu: 21.86928\tLR: 0.00030000\t*\r\n",
      "Steps: 69000\tLoss: 40997.83594\tPPL: 4.45965\tbleu: 21.69884\tLR: 0.00030000\t*\r\n",
      "Steps: 70000\tLoss: 40979.94922\tPPL: 4.45675\tbleu: 22.22275\tLR: 0.00030000\t*\r\n",
      "Steps: 71000\tLoss: 40638.07422\tPPL: 4.40153\tbleu: 22.29289\tLR: 0.00030000\t*\r\n",
      "Steps: 72000\tLoss: 40649.71094\tPPL: 4.40339\tbleu: 22.32089\tLR: 0.00030000\t\r\n",
      "Steps: 73000\tLoss: 40593.98438\tPPL: 4.39446\tbleu: 22.30915\tLR: 0.00030000\t*\r\n",
      "Steps: 74000\tLoss: 40460.03906\tPPL: 4.37304\tbleu: 21.92129\tLR: 0.00030000\t*\r\n",
      "Steps: 75000\tLoss: 40404.48047\tPPL: 4.36419\tbleu: 21.79802\tLR: 0.00030000\t*\r\n",
      "Steps: 76000\tLoss: 40453.14844\tPPL: 4.37194\tbleu: 22.09940\tLR: 0.00030000\t\r\n",
      "Steps: 77000\tLoss: 40387.22656\tPPL: 4.36145\tbleu: 21.99121\tLR: 0.00030000\t*\r\n",
      "Steps: 78000\tLoss: 40202.97266\tPPL: 4.33224\tbleu: 22.53978\tLR: 0.00030000\t*\r\n",
      "Steps: 79000\tLoss: 40027.13281\tPPL: 4.30455\tbleu: 22.49454\tLR: 0.00030000\t*\r\n",
      "Steps: 80000\tLoss: 40222.30859\tPPL: 4.33530\tbleu: 22.94402\tLR: 0.00030000\t\r\n",
      "Steps: 81000\tLoss: 40111.26562\tPPL: 4.31778\tbleu: 22.34369\tLR: 0.00030000\t\r\n",
      "Steps: 82000\tLoss: 39987.89062\tPPL: 4.29839\tbleu: 22.51628\tLR: 0.00030000\t*\r\n",
      "Steps: 83000\tLoss: 39887.67188\tPPL: 4.28271\tbleu: 22.14228\tLR: 0.00030000\t*\r\n",
      "Steps: 84000\tLoss: 39712.21484\tPPL: 4.25540\tbleu: 22.63713\tLR: 0.00030000\t*\r\n",
      "Steps: 85000\tLoss: 39699.52734\tPPL: 4.25343\tbleu: 22.54750\tLR: 0.00030000\t*\r\n",
      "Steps: 86000\tLoss: 39822.35547\tPPL: 4.27252\tbleu: 22.83256\tLR: 0.00030000\t\r\n",
      "Steps: 87000\tLoss: 40062.01562\tPPL: 4.31003\tbleu: 22.34735\tLR: 0.00030000\t\r\n",
      "Steps: 88000\tLoss: 39588.43359\tPPL: 4.23623\tbleu: 22.43288\tLR: 0.00030000\t*\r\n",
      "Steps: 89000\tLoss: 39368.09375\tPPL: 4.20233\tbleu: 22.83976\tLR: 0.00030000\t*\r\n",
      "Steps: 90000\tLoss: 39452.43750\tPPL: 4.21528\tbleu: 21.94867\tLR: 0.00030000\t\r\n",
      "Steps: 91000\tLoss: 39252.95312\tPPL: 4.18472\tbleu: 22.79975\tLR: 0.00030000\t*\r\n",
      "Steps: 92000\tLoss: 39196.74609\tPPL: 4.17615\tbleu: 22.47131\tLR: 0.00030000\t*\r\n",
      "Steps: 93000\tLoss: 39379.62109\tPPL: 4.20410\tbleu: 23.16096\tLR: 0.00030000\t\r\n",
      "Steps: 94000\tLoss: 39322.86328\tPPL: 4.19540\tbleu: 23.20399\tLR: 0.00030000\t\r\n",
      "Steps: 95000\tLoss: 39088.82422\tPPL: 4.15975\tbleu: 22.83810\tLR: 0.00030000\t*\r\n",
      "Steps: 96000\tLoss: 39075.79297\tPPL: 4.15777\tbleu: 22.98239\tLR: 0.00030000\t*\r\n",
      "Steps: 97000\tLoss: 39133.95703\tPPL: 4.16660\tbleu: 22.73311\tLR: 0.00030000\t\r\n",
      "Steps: 98000\tLoss: 38947.16016\tPPL: 4.13832\tbleu: 23.27105\tLR: 0.00030000\t*\r\n",
      "Steps: 99000\tLoss: 38822.62109\tPPL: 4.11956\tbleu: 23.34243\tLR: 0.00030000\t*\r\n",
      "Steps: 100000\tLoss: 38824.01172\tPPL: 4.11977\tbleu: 23.30457\tLR: 0.00030000\t\r\n",
      "Steps: 101000\tLoss: 38837.08594\tPPL: 4.12174\tbleu: 23.17570\tLR: 0.00030000\t\r\n",
      "Steps: 102000\tLoss: 38938.53906\tPPL: 4.13701\tbleu: 22.97706\tLR: 0.00030000\t\r\n",
      "Steps: 103000\tLoss: 38957.83594\tPPL: 4.13993\tbleu: 23.61245\tLR: 0.00030000\t\r\n",
      "Steps: 104000\tLoss: 38770.32031\tPPL: 4.11171\tbleu: 23.16266\tLR: 0.00030000\t*\r\n",
      "Steps: 105000\tLoss: 38532.46094\tPPL: 4.07620\tbleu: 23.47833\tLR: 0.00030000\t*\r\n",
      "Steps: 106000\tLoss: 38456.90234\tPPL: 4.06499\tbleu: 23.20398\tLR: 0.00030000\t*\r\n",
      "Steps: 107000\tLoss: 38603.00391\tPPL: 4.08670\tbleu: 23.47297\tLR: 0.00030000\t\r\n",
      "Steps: 108000\tLoss: 38609.84766\tPPL: 4.08772\tbleu: 23.35263\tLR: 0.00030000\t\r\n",
      "Steps: 109000\tLoss: 38276.45703\tPPL: 4.03833\tbleu: 23.29704\tLR: 0.00030000\t*\r\n",
      "Steps: 110000\tLoss: 38438.14844\tPPL: 4.06221\tbleu: 23.33509\tLR: 0.00030000\t\r\n",
      "Steps: 111000\tLoss: 38361.06641\tPPL: 4.05081\tbleu: 23.13661\tLR: 0.00030000\t\r\n",
      "Steps: 112000\tLoss: 38283.51953\tPPL: 4.03937\tbleu: 23.53948\tLR: 0.00030000\t\r\n",
      "Steps: 113000\tLoss: 38362.33203\tPPL: 4.05099\tbleu: 23.79189\tLR: 0.00030000\t\r\n",
      "Steps: 114000\tLoss: 38306.84375\tPPL: 4.04280\tbleu: 23.46887\tLR: 0.00030000\t\r\n",
      "Steps: 115000\tLoss: 38309.89062\tPPL: 4.04325\tbleu: 23.60850\tLR: 0.00021000\t\r\n",
      "Steps: 116000\tLoss: 37960.96094\tPPL: 3.99213\tbleu: 23.91608\tLR: 0.00021000\t*\r\n",
      "Steps: 117000\tLoss: 37655.19141\tPPL: 3.94786\tbleu: 24.11143\tLR: 0.00021000\t*\r\n",
      "Steps: 118000\tLoss: 37697.53516\tPPL: 3.95396\tbleu: 24.29110\tLR: 0.00021000\t\r\n",
      "Steps: 119000\tLoss: 37641.87109\tPPL: 3.94595\tbleu: 24.17020\tLR: 0.00021000\t*\r\n",
      "Steps: 120000\tLoss: 37560.46484\tPPL: 3.93425\tbleu: 24.39688\tLR: 0.00021000\t*\r\n",
      "Steps: 121000\tLoss: 37624.74219\tPPL: 3.94348\tbleu: 24.20204\tLR: 0.00021000\t\r\n",
      "Steps: 122000\tLoss: 37552.70312\tPPL: 3.93314\tbleu: 24.42698\tLR: 0.00021000\t*\r\n",
      "Steps: 123000\tLoss: 37381.46094\tPPL: 3.90865\tbleu: 24.14535\tLR: 0.00021000\t*\r\n",
      "Steps: 124000\tLoss: 37220.84375\tPPL: 3.88582\tbleu: 24.34294\tLR: 0.00021000\t*\r\n",
      "Steps: 125000\tLoss: 37309.18359\tPPL: 3.89836\tbleu: 24.19451\tLR: 0.00021000\t\r\n",
      "Steps: 126000\tLoss: 37437.84375\tPPL: 3.91670\tbleu: 24.28214\tLR: 0.00021000\t\r\n",
      "Steps: 127000\tLoss: 37437.23828\tPPL: 3.91661\tbleu: 24.48442\tLR: 0.00021000\t\r\n",
      "Steps: 128000\tLoss: 37348.47656\tPPL: 3.90395\tbleu: 24.28723\tLR: 0.00021000\t\r\n",
      "Steps: 129000\tLoss: 37299.30859\tPPL: 3.89696\tbleu: 24.35202\tLR: 0.00021000\t\r\n",
      "Steps: 130000\tLoss: 37234.72656\tPPL: 3.88779\tbleu: 24.38600\tLR: 0.00014700\t\r\n",
      "Steps: 131000\tLoss: 37061.83594\tPPL: 3.86336\tbleu: 24.46883\tLR: 0.00014700\t*\r\n",
      "Steps: 132000\tLoss: 37086.16797\tPPL: 3.86679\tbleu: 24.52761\tLR: 0.00014700\t\r\n",
      "Steps: 133000\tLoss: 36999.10938\tPPL: 3.85453\tbleu: 24.58185\tLR: 0.00014700\t*\r\n",
      "Steps: 134000\tLoss: 36962.12109\tPPL: 3.84933\tbleu: 24.46629\tLR: 0.00014700\t*\r\n",
      "Steps: 135000\tLoss: 36922.65625\tPPL: 3.84380\tbleu: 24.36102\tLR: 0.00014700\t*\r\n",
      "Steps: 136000\tLoss: 36878.20703\tPPL: 3.83757\tbleu: 24.37975\tLR: 0.00014700\t*\r\n",
      "Steps: 137000\tLoss: 36916.43359\tPPL: 3.84293\tbleu: 24.41530\tLR: 0.00014700\t\r\n",
      "Steps: 138000\tLoss: 36898.53516\tPPL: 3.84042\tbleu: 24.52885\tLR: 0.00014700\t\r\n",
      "Steps: 139000\tLoss: 36940.47656\tPPL: 3.84630\tbleu: 24.83388\tLR: 0.00014700\t\r\n",
      "Steps: 140000\tLoss: 36760.76953\tPPL: 3.82117\tbleu: 24.47324\tLR: 0.00014700\t*\r\n",
      "Steps: 141000\tLoss: 36679.84375\tPPL: 3.80991\tbleu: 24.48144\tLR: 0.00014700\t*\r\n",
      "Steps: 142000\tLoss: 36667.66797\tPPL: 3.80822\tbleu: 24.54077\tLR: 0.00014700\t*\r\n",
      "Steps: 143000\tLoss: 36604.52344\tPPL: 3.79946\tbleu: 24.85794\tLR: 0.00014700\t*\r\n",
      "Steps: 144000\tLoss: 36542.67188\tPPL: 3.79090\tbleu: 24.98253\tLR: 0.00014700\t*\r\n",
      "Steps: 145000\tLoss: 36595.04297\tPPL: 3.79815\tbleu: 24.29744\tLR: 0.00014700\t\r\n",
      "Steps: 146000\tLoss: 36662.24219\tPPL: 3.80747\tbleu: 24.54266\tLR: 0.00014700\t\r\n",
      "Steps: 147000\tLoss: 36561.60156\tPPL: 3.79352\tbleu: 24.46781\tLR: 0.00014700\t\r\n",
      "Steps: 148000\tLoss: 36614.99609\tPPL: 3.80091\tbleu: 24.48913\tLR: 0.00014700\t\r\n",
      "Steps: 149000\tLoss: 36561.30469\tPPL: 3.79348\tbleu: 24.94186\tLR: 0.00014700\t\r\n",
      "Steps: 150000\tLoss: 36491.22656\tPPL: 3.78380\tbleu: 24.99520\tLR: 0.00014700\t*\r\n",
      "Steps: 151000\tLoss: 36484.06250\tPPL: 3.78281\tbleu: 24.98294\tLR: 0.00014700\t*\r\n",
      "Steps: 152000\tLoss: 36548.77734\tPPL: 3.79175\tbleu: 24.97116\tLR: 0.00014700\t\r\n",
      "Steps: 153000\tLoss: 36639.71875\tPPL: 3.80434\tbleu: 24.74850\tLR: 0.00014700\t\r\n",
      "Steps: 154000\tLoss: 36502.16406\tPPL: 3.78531\tbleu: 24.82869\tLR: 0.00014700\t\r\n",
      "Steps: 155000\tLoss: 36509.69141\tPPL: 3.78635\tbleu: 24.85599\tLR: 0.00014700\t\r\n",
      "Steps: 156000\tLoss: 36460.94141\tPPL: 3.77962\tbleu: 25.16878\tLR: 0.00014700\t*\r\n",
      "Steps: 157000\tLoss: 36453.02734\tPPL: 3.77853\tbleu: 24.95815\tLR: 0.00014700\t*\r\n",
      "Steps: 158000\tLoss: 36384.28516\tPPL: 3.76907\tbleu: 24.53031\tLR: 0.00014700\t*\r\n",
      "Steps: 159000\tLoss: 36332.47656\tPPL: 3.76196\tbleu: 24.91550\tLR: 0.00014700\t*\r\n",
      "Steps: 160000\tLoss: 36493.85547\tPPL: 3.78416\tbleu: 24.87724\tLR: 0.00014700\t\r\n",
      "Steps: 161000\tLoss: 36376.48438\tPPL: 3.76800\tbleu: 24.68776\tLR: 0.00014700\t\r\n",
      "Steps: 162000\tLoss: 36424.84375\tPPL: 3.77465\tbleu: 24.74693\tLR: 0.00014700\t\r\n",
      "Steps: 163000\tLoss: 36359.21094\tPPL: 3.76563\tbleu: 24.69640\tLR: 0.00014700\t\r\n",
      "Steps: 164000\tLoss: 36271.67578\tPPL: 3.75362\tbleu: 24.77857\tLR: 0.00014700\t*\r\n",
      "Steps: 165000\tLoss: 36375.67188\tPPL: 3.76789\tbleu: 24.67269\tLR: 0.00014700\t\r\n",
      "Steps: 166000\tLoss: 36461.55469\tPPL: 3.77971\tbleu: 24.71124\tLR: 0.00014700\t\r\n",
      "Steps: 167000\tLoss: 36372.73438\tPPL: 3.76748\tbleu: 24.73632\tLR: 0.00014700\t\r\n",
      "Steps: 168000\tLoss: 36355.90625\tPPL: 3.76517\tbleu: 25.01684\tLR: 0.00014700\t\r\n",
      "Steps: 169000\tLoss: 36412.28125\tPPL: 3.77292\tbleu: 24.76650\tLR: 0.00014700\t\r\n",
      "Steps: 170000\tLoss: 36317.55469\tPPL: 3.75991\tbleu: 24.92523\tLR: 0.00010290\t\r\n",
      "Steps: 171000\tLoss: 36043.25000\tPPL: 3.72249\tbleu: 24.96350\tLR: 0.00010290\t*\r\n",
      "Steps: 172000\tLoss: 36103.58203\tPPL: 3.73068\tbleu: 24.84813\tLR: 0.00010290\t\r\n",
      "Steps: 173000\tLoss: 36121.60156\tPPL: 3.73314\tbleu: 25.06479\tLR: 0.00010290\t\r\n",
      "Steps: 174000\tLoss: 36071.19531\tPPL: 3.72628\tbleu: 25.06656\tLR: 0.00010290\t\r\n",
      "Steps: 175000\tLoss: 36063.15234\tPPL: 3.72519\tbleu: 25.28761\tLR: 0.00010290\t\r\n",
      "Steps: 176000\tLoss: 36012.24219\tPPL: 3.71828\tbleu: 25.34094\tLR: 0.00010290\t*\r\n",
      "Steps: 177000\tLoss: 35980.06641\tPPL: 3.71392\tbleu: 25.20632\tLR: 0.00010290\t*\r\n",
      "Steps: 178000\tLoss: 36025.92188\tPPL: 3.72013\tbleu: 25.18255\tLR: 0.00010290\t\r\n",
      "Steps: 179000\tLoss: 36144.85938\tPPL: 3.73630\tbleu: 25.06112\tLR: 0.00010290\t\r\n",
      "Steps: 180000\tLoss: 36131.77344\tPPL: 3.73452\tbleu: 25.38790\tLR: 0.00010290\t\r\n",
      "Steps: 181000\tLoss: 35972.75000\tPPL: 3.71293\tbleu: 25.33007\tLR: 0.00010290\t*\r\n",
      "Steps: 182000\tLoss: 35942.91016\tPPL: 3.70889\tbleu: 25.16205\tLR: 0.00010290\t*\r\n",
      "Steps: 183000\tLoss: 35833.89062\tPPL: 3.69417\tbleu: 25.27872\tLR: 0.00010290\t*\r\n",
      "Steps: 184000\tLoss: 35872.78516\tPPL: 3.69942\tbleu: 25.20647\tLR: 0.00010290\t\r\n",
      "Steps: 185000\tLoss: 35922.33984\tPPL: 3.70611\tbleu: 25.43258\tLR: 0.00010290\t\r\n",
      "Steps: 186000\tLoss: 35907.48047\tPPL: 3.70410\tbleu: 25.34363\tLR: 0.00010290\t\r\n",
      "Steps: 187000\tLoss: 35939.73438\tPPL: 3.70846\tbleu: 25.34075\tLR: 0.00010290\t\r\n",
      "Steps: 188000\tLoss: 35964.09766\tPPL: 3.71176\tbleu: 25.07352\tLR: 0.00010290\t\r\n",
      "Steps: 189000\tLoss: 35909.78906\tPPL: 3.70441\tbleu: 25.04680\tLR: 0.00007203\t\r\n",
      "Steps: 190000\tLoss: 35842.04297\tPPL: 3.69527\tbleu: 25.47323\tLR: 0.00007203\t\r\n",
      "Steps: 191000\tLoss: 35747.55078\tPPL: 3.68256\tbleu: 25.37858\tLR: 0.00007203\t*\r\n",
      "Steps: 192000\tLoss: 35875.65234\tPPL: 3.69980\tbleu: 25.47796\tLR: 0.00007203\t\r\n",
      "Steps: 193000\tLoss: 35835.31250\tPPL: 3.69437\tbleu: 25.38831\tLR: 0.00007203\t\r\n",
      "Steps: 194000\tLoss: 35702.88281\tPPL: 3.67657\tbleu: 25.56399\tLR: 0.00007203\t*\r\n",
      "Steps: 195000\tLoss: 35749.02344\tPPL: 3.68276\tbleu: 25.74306\tLR: 0.00007203\t\r\n",
      "Steps: 196000\tLoss: 35704.27344\tPPL: 3.67675\tbleu: 25.38217\tLR: 0.00007203\t\r\n",
      "Steps: 197000\tLoss: 35716.71484\tPPL: 3.67842\tbleu: 25.18610\tLR: 0.00007203\t\r\n",
      "Steps: 198000\tLoss: 35761.97656\tPPL: 3.68450\tbleu: 25.33143\tLR: 0.00007203\t\r\n",
      "Steps: 199000\tLoss: 35756.05469\tPPL: 3.68370\tbleu: 25.74991\tLR: 0.00007203\t\r\n",
      "Steps: 200000\tLoss: 35688.59766\tPPL: 3.67465\tbleu: 25.75004\tLR: 0.00007203\t*\r\n",
      "Steps: 201000\tLoss: 35739.53516\tPPL: 3.68148\tbleu: 25.44715\tLR: 0.00007203\t\r\n",
      "Steps: 202000\tLoss: 35684.76172\tPPL: 3.67414\tbleu: 25.32426\tLR: 0.00007203\t*\r\n",
      "Steps: 203000\tLoss: 35700.25000\tPPL: 3.67621\tbleu: 25.16456\tLR: 0.00007203\t\r\n",
      "Steps: 204000\tLoss: 35610.13672\tPPL: 3.66415\tbleu: 25.55833\tLR: 0.00007203\t*\r\n",
      "Steps: 205000\tLoss: 35649.66016\tPPL: 3.66944\tbleu: 25.31584\tLR: 0.00007203\t\r\n",
      "Steps: 206000\tLoss: 35577.99609\tPPL: 3.65986\tbleu: 25.19733\tLR: 0.00007203\t*\r\n",
      "Steps: 207000\tLoss: 35679.30078\tPPL: 3.67341\tbleu: 25.37284\tLR: 0.00007203\t\r\n",
      "Steps: 208000\tLoss: 35739.56250\tPPL: 3.68149\tbleu: 25.44612\tLR: 0.00007203\t\r\n",
      "Steps: 209000\tLoss: 35686.53906\tPPL: 3.67438\tbleu: 25.63467\tLR: 0.00007203\t\r\n",
      "Steps: 210000\tLoss: 35634.19531\tPPL: 3.66737\tbleu: 25.72294\tLR: 0.00007203\t\r\n",
      "Steps: 211000\tLoss: 35625.22266\tPPL: 3.66617\tbleu: 25.87567\tLR: 0.00007203\t\r\n",
      "Steps: 212000\tLoss: 35664.34766\tPPL: 3.67140\tbleu: 25.52055\tLR: 0.00005042\t\r\n",
      "Steps: 213000\tLoss: 35665.66406\tPPL: 3.67158\tbleu: 25.55837\tLR: 0.00005042\t\r\n",
      "Steps: 214000\tLoss: 35597.09375\tPPL: 3.66241\tbleu: 25.66390\tLR: 0.00005042\t\r\n",
      "Steps: 215000\tLoss: 35623.94531\tPPL: 3.66600\tbleu: 25.36372\tLR: 0.00005042\t\r\n",
      "Steps: 216000\tLoss: 35603.84375\tPPL: 3.66331\tbleu: 25.95028\tLR: 0.00005042\t\r\n",
      "Steps: 217000\tLoss: 35519.12891\tPPL: 3.65201\tbleu: 25.81568\tLR: 0.00005042\t*\r\n",
      "Steps: 218000\tLoss: 35638.73828\tPPL: 3.66798\tbleu: 25.58719\tLR: 0.00005042\t\r\n",
      "Steps: 219000\tLoss: 35581.85547\tPPL: 3.66038\tbleu: 25.82997\tLR: 0.00005042\t\r\n",
      "Steps: 220000\tLoss: 35643.37109\tPPL: 3.66860\tbleu: 25.62460\tLR: 0.00005042\t\r\n",
      "Steps: 221000\tLoss: 35576.58984\tPPL: 3.65967\tbleu: 25.86560\tLR: 0.00005042\t\r\n",
      "Steps: 222000\tLoss: 35553.89062\tPPL: 3.65665\tbleu: 25.67420\tLR: 0.00005042\t\r\n",
      "Steps: 223000\tLoss: 35619.74219\tPPL: 3.66544\tbleu: 25.80206\tLR: 0.00003529\t\r\n",
      "Steps: 224000\tLoss: 35567.75391\tPPL: 3.65849\tbleu: 25.51542\tLR: 0.00003529\t\r\n",
      "Steps: 225000\tLoss: 35567.85938\tPPL: 3.65851\tbleu: 25.53790\tLR: 0.00003529\t\r\n",
      "Steps: 226000\tLoss: 35533.30078\tPPL: 3.65390\tbleu: 25.72201\tLR: 0.00003529\t\r\n",
      "Steps: 227000\tLoss: 35559.82812\tPPL: 3.65744\tbleu: 25.67272\tLR: 0.00003529\t\r\n",
      "Steps: 228000\tLoss: 35508.69922\tPPL: 3.65062\tbleu: 25.86938\tLR: 0.00003529\t*\r\n",
      "Steps: 229000\tLoss: 35484.63281\tPPL: 3.64742\tbleu: 25.64464\tLR: 0.00003529\t*\r\n",
      "Steps: 230000\tLoss: 35476.83203\tPPL: 3.64638\tbleu: 25.83657\tLR: 0.00003529\t*\r\n",
      "Steps: 231000\tLoss: 35483.20703\tPPL: 3.64723\tbleu: 25.76319\tLR: 0.00003529\t\r\n",
      "Steps: 232000\tLoss: 35485.14062\tPPL: 3.64749\tbleu: 25.60255\tLR: 0.00003529\t\r\n",
      "Steps: 233000\tLoss: 35461.79688\tPPL: 3.64439\tbleu: 25.84748\tLR: 0.00003529\t*\r\n",
      "Steps: 234000\tLoss: 35489.69922\tPPL: 3.64810\tbleu: 25.79881\tLR: 0.00003529\t\r\n",
      "Steps: 235000\tLoss: 35455.02734\tPPL: 3.64349\tbleu: 25.69962\tLR: 0.00003529\t*\r\n",
      "Steps: 236000\tLoss: 35423.60156\tPPL: 3.63931\tbleu: 25.81471\tLR: 0.00003529\t*\r\n",
      "Steps: 237000\tLoss: 35407.42578\tPPL: 3.63717\tbleu: 25.82644\tLR: 0.00003529\t*\r\n",
      "Steps: 238000\tLoss: 35434.66016\tPPL: 3.64078\tbleu: 25.87317\tLR: 0.00003529\t\r\n",
      "Steps: 239000\tLoss: 35453.43359\tPPL: 3.64327\tbleu: 25.70927\tLR: 0.00003529\t\r\n",
      "Steps: 240000\tLoss: 35449.62891\tPPL: 3.64277\tbleu: 25.96945\tLR: 0.00003529\t\r\n",
      "Steps: 241000\tLoss: 35425.10547\tPPL: 3.63951\tbleu: 25.91304\tLR: 0.00003529\t\r\n",
      "Steps: 242000\tLoss: 35392.91406\tPPL: 3.63524\tbleu: 25.73983\tLR: 0.00003529\t*\r\n",
      "Steps: 243000\tLoss: 35437.17969\tPPL: 3.64112\tbleu: 25.91283\tLR: 0.00003529\t\r\n",
      "Steps: 244000\tLoss: 35421.46484\tPPL: 3.63903\tbleu: 25.91689\tLR: 0.00003529\t\r\n",
      "Steps: 245000\tLoss: 35428.33594\tPPL: 3.63994\tbleu: 25.90972\tLR: 0.00003529\t\r\n",
      "Steps: 246000\tLoss: 35377.39844\tPPL: 3.63319\tbleu: 25.93869\tLR: 0.00003529\t*\r\n",
      "Steps: 247000\tLoss: 35405.59375\tPPL: 3.63692\tbleu: 26.04868\tLR: 0.00003529\t\r\n",
      "Steps: 248000\tLoss: 35375.14062\tPPL: 3.63289\tbleu: 25.80345\tLR: 0.00003529\t*\r\n",
      "Steps: 249000\tLoss: 35376.50781\tPPL: 3.63307\tbleu: 25.79546\tLR: 0.00003529\t\r\n",
      "Steps: 250000\tLoss: 35368.60547\tPPL: 3.63202\tbleu: 25.83619\tLR: 0.00003529\t*\r\n",
      "Steps: 251000\tLoss: 35431.45312\tPPL: 3.64036\tbleu: 25.98045\tLR: 0.00003529\t\r\n",
      "Steps: 252000\tLoss: 35397.84766\tPPL: 3.63590\tbleu: 25.62282\tLR: 0.00003529\t\r\n",
      "Steps: 253000\tLoss: 35396.73828\tPPL: 3.63575\tbleu: 25.91349\tLR: 0.00003529\t\r\n",
      "Steps: 254000\tLoss: 35400.69531\tPPL: 3.63627\tbleu: 25.79404\tLR: 0.00003529\t\r\n",
      "Steps: 255000\tLoss: 35373.83984\tPPL: 3.63271\tbleu: 26.00488\tLR: 0.00003529\t\r\n",
      "Steps: 256000\tLoss: 35343.31641\tPPL: 3.62867\tbleu: 25.74574\tLR: 0.00003529\t*\r\n",
      "Steps: 257000\tLoss: 35338.04688\tPPL: 3.62798\tbleu: 25.90427\tLR: 0.00003529\t*\r\n",
      "Steps: 258000\tLoss: 35363.99219\tPPL: 3.63141\tbleu: 25.88810\tLR: 0.00003529\t\r\n",
      "Steps: 259000\tLoss: 35352.41797\tPPL: 3.62988\tbleu: 25.90165\tLR: 0.00003529\t\r\n",
      "Steps: 260000\tLoss: 35439.44141\tPPL: 3.64142\tbleu: 25.66819\tLR: 0.00003529\t\r\n",
      "Steps: 261000\tLoss: 35396.10938\tPPL: 3.63567\tbleu: 25.83850\tLR: 0.00003529\t\r\n",
      "Steps: 262000\tLoss: 35398.69531\tPPL: 3.63601\tbleu: 25.84636\tLR: 0.00003529\t\r\n",
      "Steps: 263000\tLoss: 35370.63672\tPPL: 3.63229\tbleu: 25.90844\tLR: 0.00002471\t\r\n",
      "Steps: 264000\tLoss: 35358.55078\tPPL: 3.63069\tbleu: 25.96816\tLR: 0.00002471\t\r\n",
      "Steps: 265000\tLoss: 35425.92969\tPPL: 3.63962\tbleu: 25.88694\tLR: 0.00002471\t\r\n",
      "Steps: 266000\tLoss: 35360.87109\tPPL: 3.63100\tbleu: 25.80898\tLR: 0.00002471\t\r\n",
      "Steps: 267000\tLoss: 35337.44141\tPPL: 3.62790\tbleu: 25.80653\tLR: 0.00002471\t*\r\n",
      "Steps: 268000\tLoss: 35329.75000\tPPL: 3.62688\tbleu: 25.90971\tLR: 0.00002471\t*\r\n",
      "Steps: 269000\tLoss: 35331.60547\tPPL: 3.62712\tbleu: 25.79087\tLR: 0.00002471\t\r\n",
      "Steps: 270000\tLoss: 35331.38281\tPPL: 3.62709\tbleu: 25.65772\tLR: 0.00002471\t\r\n",
      "Steps: 271000\tLoss: 35319.31641\tPPL: 3.62550\tbleu: 25.89009\tLR: 0.00002471\t*\r\n",
      "Steps: 272000\tLoss: 35348.19531\tPPL: 3.62932\tbleu: 25.68393\tLR: 0.00002471\t\r\n",
      "Steps: 273000\tLoss: 35379.81250\tPPL: 3.63351\tbleu: 25.81655\tLR: 0.00002471\t\r\n",
      "Steps: 274000\tLoss: 35338.38281\tPPL: 3.62802\tbleu: 26.01635\tLR: 0.00002471\t\r\n",
      "Steps: 275000\tLoss: 35330.26172\tPPL: 3.62695\tbleu: 25.76188\tLR: 0.00002471\t\r\n",
      "Steps: 276000\tLoss: 35310.36328\tPPL: 3.62432\tbleu: 25.75515\tLR: 0.00002471\t*\r\n",
      "Steps: 277000\tLoss: 35343.89844\tPPL: 3.62875\tbleu: 25.74876\tLR: 0.00002471\t\r\n",
      "Steps: 278000\tLoss: 35353.27344\tPPL: 3.62999\tbleu: 25.71972\tLR: 0.00002471\t\r\n",
      "Steps: 279000\tLoss: 35332.88281\tPPL: 3.62729\tbleu: 25.90659\tLR: 0.00002471\t\r\n",
      "Steps: 280000\tLoss: 35312.08594\tPPL: 3.62454\tbleu: 25.60473\tLR: 0.00002471\t\r\n",
      "Steps: 281000\tLoss: 35333.80859\tPPL: 3.62742\tbleu: 25.91967\tLR: 0.00002471\t\r\n",
      "Steps: 282000\tLoss: 35300.64453\tPPL: 3.62303\tbleu: 25.89667\tLR: 0.00002471\t*\r\n",
      "Steps: 283000\tLoss: 35319.14453\tPPL: 3.62548\tbleu: 26.05449\tLR: 0.00002471\t\r\n",
      "Steps: 284000\tLoss: 35365.10547\tPPL: 3.63156\tbleu: 25.90026\tLR: 0.00002471\t\r\n",
      "Steps: 285000\tLoss: 35332.06641\tPPL: 3.62719\tbleu: 25.64572\tLR: 0.00002471\t\r\n",
      "Steps: 286000\tLoss: 35323.89844\tPPL: 3.62611\tbleu: 26.15120\tLR: 0.00002471\t\r\n",
      "Steps: 287000\tLoss: 35337.85156\tPPL: 3.62795\tbleu: 26.19771\tLR: 0.00002471\t\r\n",
      "Steps: 288000\tLoss: 35300.11719\tPPL: 3.62296\tbleu: 25.95003\tLR: 0.00001729\t*\r\n",
      "Steps: 289000\tLoss: 35303.08594\tPPL: 3.62335\tbleu: 26.02795\tLR: 0.00001729\t\r\n",
      "Steps: 290000\tLoss: 35299.01562\tPPL: 3.62282\tbleu: 25.72550\tLR: 0.00001729\t*\r\n",
      "Steps: 291000\tLoss: 35309.21094\tPPL: 3.62416\tbleu: 26.04127\tLR: 0.00001729\t\r\n",
      "Steps: 292000\tLoss: 35323.38281\tPPL: 3.62604\tbleu: 25.81217\tLR: 0.00001729\t\r\n",
      "Steps: 293000\tLoss: 35329.21875\tPPL: 3.62681\tbleu: 25.92504\tLR: 0.00001729\t\r\n",
      "Steps: 294000\tLoss: 35307.49219\tPPL: 3.62394\tbleu: 26.25329\tLR: 0.00001729\t\r\n",
      "Steps: 295000\tLoss: 35322.02734\tPPL: 3.62586\tbleu: 25.98322\tLR: 0.00001729\t\r\n",
      "Steps: 296000\tLoss: 35290.26172\tPPL: 3.62166\tbleu: 26.09704\tLR: 0.00001729\t*\r\n",
      "Steps: 297000\tLoss: 35309.61719\tPPL: 3.62422\tbleu: 25.81555\tLR: 0.00001729\t\r\n",
      "Steps: 298000\tLoss: 35329.49219\tPPL: 3.62684\tbleu: 25.85845\tLR: 0.00001729\t\r\n",
      "Steps: 299000\tLoss: 35333.42578\tPPL: 3.62736\tbleu: 25.99118\tLR: 0.00001729\t\r\n",
      "Steps: 300000\tLoss: 35281.97656\tPPL: 3.62057\tbleu: 25.96553\tLR: 0.00001729\t*\r\n",
      "Steps: 301000\tLoss: 35289.00391\tPPL: 3.62149\tbleu: 25.89258\tLR: 0.00001729\t\r\n",
      "Steps: 302000\tLoss: 35280.84375\tPPL: 3.62042\tbleu: 25.94691\tLR: 0.00001729\t*\r\n",
      "Steps: 303000\tLoss: 35303.23438\tPPL: 3.62337\tbleu: 25.82628\tLR: 0.00001729\t\r\n",
      "Steps: 304000\tLoss: 35337.96875\tPPL: 3.62797\tbleu: 25.89150\tLR: 0.00001729\t\r\n",
      "Steps: 305000\tLoss: 35304.55859\tPPL: 3.62355\tbleu: 25.72649\tLR: 0.00001729\t\r\n",
      "Steps: 306000\tLoss: 35301.97656\tPPL: 3.62321\tbleu: 25.93105\tLR: 0.00001729\t\r\n",
      "Steps: 307000\tLoss: 35305.79688\tPPL: 3.62371\tbleu: 25.91836\tLR: 0.00001729\t\r\n",
      "Steps: 308000\tLoss: 35314.07031\tPPL: 3.62481\tbleu: 25.67593\tLR: 0.00001211\t\r\n",
      "Steps: 309000\tLoss: 35292.78125\tPPL: 3.62199\tbleu: 25.94088\tLR: 0.00001211\t\r\n",
      "Steps: 310000\tLoss: 35293.79688\tPPL: 3.62213\tbleu: 26.05686\tLR: 0.00001211\t\r\n",
      "Steps: 311000\tLoss: 35312.19922\tPPL: 3.62456\tbleu: 25.89675\tLR: 0.00001211\t\r\n",
      "Steps: 312000\tLoss: 35272.92188\tPPL: 3.61937\tbleu: 25.88367\tLR: 0.00001211\t*\r\n",
      "Steps: 313000\tLoss: 35299.66797\tPPL: 3.62290\tbleu: 25.95173\tLR: 0.00001211\t\r\n",
      "Steps: 314000\tLoss: 35297.89844\tPPL: 3.62267\tbleu: 25.83882\tLR: 0.00001211\t\r\n",
      "Steps: 315000\tLoss: 35298.33594\tPPL: 3.62273\tbleu: 25.91109\tLR: 0.00001211\t\r\n",
      "Steps: 316000\tLoss: 35268.37500\tPPL: 3.61877\tbleu: 25.95875\tLR: 0.00001211\t*\r\n",
      "Steps: 317000\tLoss: 35264.03125\tPPL: 3.61820\tbleu: 25.84213\tLR: 0.00001211\t*\r\n",
      "Steps: 318000\tLoss: 35263.61328\tPPL: 3.61814\tbleu: 25.87330\tLR: 0.00001211\t*\r\n",
      "Steps: 319000\tLoss: 35267.41406\tPPL: 3.61864\tbleu: 25.73969\tLR: 0.00001211\t\r\n",
      "Steps: 320000\tLoss: 35256.56250\tPPL: 3.61721\tbleu: 25.91235\tLR: 0.00001211\t*\r\n",
      "Steps: 321000\tLoss: 35270.18359\tPPL: 3.61901\tbleu: 25.97472\tLR: 0.00001211\t\r\n",
      "Steps: 322000\tLoss: 35244.44141\tPPL: 3.61561\tbleu: 25.78369\tLR: 0.00001211\t*\r\n",
      "Steps: 323000\tLoss: 35246.20312\tPPL: 3.61585\tbleu: 26.01238\tLR: 0.00001211\t\r\n",
      "Steps: 324000\tLoss: 35301.81641\tPPL: 3.62319\tbleu: 25.82574\tLR: 0.00001211\t\r\n",
      "Steps: 325000\tLoss: 35280.89062\tPPL: 3.62042\tbleu: 26.00126\tLR: 0.00001211\t\r\n",
      "Steps: 326000\tLoss: 35269.94922\tPPL: 3.61898\tbleu: 25.78010\tLR: 0.00001211\t\r\n",
      "Steps: 327000\tLoss: 35264.97656\tPPL: 3.61832\tbleu: 25.81127\tLR: 0.00001211\t\r\n",
      "Steps: 328000\tLoss: 35253.80078\tPPL: 3.61685\tbleu: 25.75368\tLR: 0.00000847\t\r\n",
      "Steps: 329000\tLoss: 35239.36328\tPPL: 3.61494\tbleu: 25.80209\tLR: 0.00000847\t*\r\n"
     ]
    }
   ],
   "source": [
    "! cat models/${src}${tgt}_transformer/validations.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/espoir_mur_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/espoir_mur_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/espoir_mur_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/espoir_mur_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/espoir_mur_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/espoir_mur_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "2020-02-17 06:41:57,032 -  dev bleu:  26.73 [Beam search decoding with beam size = 5 and alpha = 1.0]\n",
      "2020-02-17 06:45:36,029 - test bleu:  33.73 [Beam search decoding with beam size = 5 and alpha = 1.0]\n"
     ]
    }
   ],
   "source": [
    "!python3 -m joeynmt test config/transformer_$src$tgt.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the results to the folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/329000.ckpt': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/329000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/33000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/34000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/35000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/36000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/37000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/38000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/39000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/4000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/40000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/41000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/42000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/43000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/44000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/45000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/46000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/47000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/48000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/49000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/5000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/50000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/51000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/52000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/53000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/54000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/55000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/56000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/57000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/58000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/59000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/6000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/60000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/61000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/62000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/63000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/64000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/65000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/66000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/67000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/68000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/69000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/7000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/70000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/71000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/72000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/73000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/74000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/75000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/76000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/77000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/78000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/79000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/8000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/80000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/81000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/82000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/83000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/84000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/85000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/86000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/87000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/88000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/89000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/9000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/90000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/91000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/92000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/93000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/94000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/95000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/96000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/97000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/98000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/99000.hyps': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/config.yaml': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/src_vocab.txt': No space left on device\r\n",
      "cp: cannot create directory 'fr-swc-baseline_v2/models/frswc_transformer/tensorboard': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/train.log': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/trg_vocab.txt': No space left on device\r\n",
      "cp: error writing 'fr-swc-baseline_v2/models/frswc_transformer/validations.txt': No space left on device\r\n"
     ]
    }
   ],
   "source": [
    "!mkdir $gdrive_path/models\n",
    "!mkdir $gdrive_path/models/${src}${tgt}_transformer/\n",
    "# Copy the created models from the notebook storage to google drive for persistant storage \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00329000.hyps.dev   159000.hyps  22000.hyps   281000.hyps  46000.hyps\r\n",
      "00329000.hyps.test  16000.hyps\t 220000.hyps  282000.hyps  47000.hyps\r\n",
      "1000.hyps\t    160000.hyps  221000.hyps  283000.hyps  48000.hyps\r\n",
      "10000.hyps\t    161000.hyps  222000.hyps  284000.hyps  49000.hyps\r\n",
      "100000.hyps\t    162000.hyps  223000.hyps  285000.hyps  5000.hyps\r\n",
      "101000.hyps\t    163000.hyps  224000.hyps  286000.hyps  50000.hyps\r\n",
      "102000.hyps\t    164000.hyps  225000.hyps  287000.hyps  51000.hyps\r\n",
      "103000.hyps\t    165000.hyps  226000.hyps  288000.hyps  52000.hyps\r\n",
      "104000.hyps\t    166000.hyps  227000.hyps  289000.hyps  53000.hyps\r\n",
      "105000.hyps\t    167000.hyps  228000.hyps  29000.hyps   54000.hyps\r\n",
      "106000.hyps\t    168000.hyps  229000.hyps  290000.hyps  55000.hyps\r\n",
      "107000.hyps\t    169000.hyps  23000.hyps   291000.hyps  56000.hyps\r\n",
      "108000.hyps\t    17000.hyps\t 230000.hyps  292000.hyps  57000.hyps\r\n",
      "109000.hyps\t    170000.hyps  231000.hyps  293000.hyps  58000.hyps\r\n",
      "11000.hyps\t    171000.hyps  232000.hyps  294000.hyps  59000.hyps\r\n",
      "110000.hyps\t    172000.hyps  233000.hyps  295000.hyps  6000.hyps\r\n",
      "111000.hyps\t    173000.hyps  234000.hyps  296000.hyps  60000.hyps\r\n",
      "112000.hyps\t    174000.hyps  235000.hyps  297000.hyps  61000.hyps\r\n",
      "113000.hyps\t    175000.hyps  236000.hyps  298000.hyps  62000.hyps\r\n",
      "114000.hyps\t    176000.hyps  237000.hyps  299000.hyps  63000.hyps\r\n",
      "115000.hyps\t    177000.hyps  238000.hyps  3000.hyps    64000.hyps\r\n",
      "116000.hyps\t    178000.hyps  239000.hyps  30000.hyps   65000.hyps\r\n",
      "117000.hyps\t    179000.hyps  24000.hyps   300000.hyps  66000.hyps\r\n",
      "118000.hyps\t    18000.hyps\t 240000.hyps  301000.hyps  67000.hyps\r\n",
      "119000.hyps\t    180000.hyps  241000.hyps  302000.hyps  68000.hyps\r\n",
      "12000.hyps\t    181000.hyps  242000.hyps  303000.hyps  69000.hyps\r\n",
      "120000.hyps\t    182000.hyps  243000.hyps  304000.hyps  7000.hyps\r\n",
      "121000.hyps\t    183000.hyps  244000.hyps  305000.hyps  70000.hyps\r\n",
      "122000.hyps\t    184000.hyps  245000.hyps  306000.hyps  71000.hyps\r\n",
      "123000.hyps\t    185000.hyps  246000.hyps  307000.hyps  72000.hyps\r\n",
      "124000.hyps\t    186000.hyps  247000.hyps  308000.hyps  73000.hyps\r\n",
      "125000.hyps\t    187000.hyps  248000.hyps  309000.hyps  74000.hyps\r\n",
      "126000.hyps\t    188000.hyps  249000.hyps  31000.hyps   75000.hyps\r\n",
      "127000.hyps\t    189000.hyps  25000.hyps   310000.hyps  76000.hyps\r\n",
      "128000.hyps\t    19000.hyps\t 250000.hyps  311000.hyps  77000.hyps\r\n",
      "129000.hyps\t    190000.hyps  251000.hyps  312000.hyps  78000.hyps\r\n",
      "13000.hyps\t    191000.hyps  252000.hyps  313000.hyps  79000.hyps\r\n",
      "130000.hyps\t    192000.hyps  253000.hyps  314000.hyps  8000.hyps\r\n",
      "131000.hyps\t    193000.hyps  254000.hyps  315000.hyps  80000.hyps\r\n",
      "132000.hyps\t    194000.hyps  255000.hyps  316000.hyps  81000.hyps\r\n",
      "133000.hyps\t    195000.hyps  256000.hyps  317000.hyps  82000.hyps\r\n",
      "134000.hyps\t    196000.hyps  257000.hyps  318000.hyps  83000.hyps\r\n",
      "135000.hyps\t    197000.hyps  258000.hyps  319000.hyps  84000.hyps\r\n",
      "136000.hyps\t    198000.hyps  259000.hyps  32000.hyps   85000.hyps\r\n",
      "137000.hyps\t    199000.hyps  26000.hyps   320000.hyps  86000.hyps\r\n",
      "138000.hyps\t    2000.hyps\t 260000.hyps  321000.hyps  87000.hyps\r\n",
      "139000.hyps\t    20000.hyps\t 261000.hyps  322000.hyps  88000.hyps\r\n",
      "14000.hyps\t    200000.hyps  262000.hyps  323000.hyps  89000.hyps\r\n",
      "140000.hyps\t    201000.hyps  263000.hyps  324000.hyps  9000.hyps\r\n",
      "141000.hyps\t    202000.hyps  264000.hyps  325000.hyps  90000.hyps\r\n",
      "142000.hyps\t    203000.hyps  265000.hyps  326000.hyps  91000.hyps\r\n",
      "143000.hyps\t    204000.hyps  266000.hyps  327000.hyps  92000.hyps\r\n",
      "144000.hyps\t    205000.hyps  267000.hyps  328000.hyps  93000.hyps\r\n",
      "145000.hyps\t    206000.hyps  268000.hyps  329000.hyps  94000.hyps\r\n",
      "146000.hyps\t    207000.hyps  269000.hyps  33000.hyps   95000.hyps\r\n",
      "147000.hyps\t    208000.hyps  27000.hyps   34000.hyps   96000.hyps\r\n",
      "148000.hyps\t    209000.hyps  270000.hyps  35000.hyps   97000.hyps\r\n",
      "149000.hyps\t    21000.hyps\t 271000.hyps  36000.hyps   98000.hyps\r\n",
      "15000.hyps\t    210000.hyps  272000.hyps  37000.hyps   99000.hyps\r\n",
      "150000.hyps\t    211000.hyps  273000.hyps  38000.hyps   best.ckpt\r\n",
      "151000.hyps\t    212000.hyps  274000.hyps  39000.hyps   config.yaml\r\n",
      "152000.hyps\t    213000.hyps  275000.hyps  4000.hyps    src_vocab.txt\r\n",
      "153000.hyps\t    214000.hyps  276000.hyps  40000.hyps   tensorboard\r\n",
      "154000.hyps\t    215000.hyps  277000.hyps  41000.hyps   train.log\r\n",
      "155000.hyps\t    216000.hyps  278000.hyps  42000.hyps   trg_vocab.txt\r\n",
      "156000.hyps\t    217000.hyps  279000.hyps  43000.hyps   validations.txt\r\n",
      "157000.hyps\t    218000.hyps  28000.hyps   44000.hyps\r\n",
      "158000.hyps\t    219000.hyps  280000.hyps  45000.hyps\r\n"
     ]
    }
   ],
   "source": [
    "!ls models/${src}${tgt}_transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp -r models/${src}${tgt}_transformer/best.ckpt \"$gdrive_path/models/${src}${tgt}_transformer/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp -r models/${src}${tgt}_transformer/config.yaml \"$gdrive_path/models/${src}${tgt}_transformer/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp -r models/${src}${tgt}_transformer/src_vocab.txt \"$gdrive_path/models/${src}${tgt}_transformer/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp -r models/${src}${tgt}_transformer/trg_vocab.txt \"$gdrive_path/models/${src}${tgt}_transformer/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp -r models/${src}${tgt}_transformer/validations.txt \"$gdrive_path/models/${src}${tgt}_transformer/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bpe.codes.4000\ttest.bpe.fr\t  test.en-any.en.2  test.fr-swc.fr   train.fr\r\n",
      "dev.bpe.fr\ttest.bpe.swc\t  test.en-swc.en    test.fr-swc.swc  train.swc\r\n",
      "dev.bpe.swc\ttest.en\t\t  test.en-swc.swc   test.swc\t     vocab.txt\r\n",
      "dev.fr\t\ttest.en-any.en\t  test.fr\t    train.bpe.fr\r\n",
      "dev.swc\t\ttest.en-any.en.1  test.fr-any.fr    train.bpe.swc\r\n"
     ]
    }
   ],
   "source": [
    "!ls $data_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp $data_path/test.fr \"$gdrive_path/models/${src}${tgt}_transformer/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp $data_path/test.swc \"$gdrive_path/models/${src}${tgt}_transformer/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp $data_path/vocab.txt \"$gdrive_path/models/${src}${tgt}_transformer/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp $data_path/train.bpe.swc \"$gdrive_path/models/${src}${tgt}_transformer/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp $data_path/train.bpe.fr \"$gdrive_path/models/${src}${tgt}_transformer/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00329000.hyps.dev   16000.hyps\t 221000.hyps  284000.hyps  49000.hyps\r\n",
      "00329000.hyps.test  160000.hyps  222000.hyps  285000.hyps  5000.hyps\r\n",
      "1000.hyps\t    161000.hyps  223000.hyps  286000.hyps  50000.hyps\r\n",
      "10000.hyps\t    162000.hyps  224000.hyps  287000.hyps  51000.hyps\r\n",
      "100000.hyps\t    163000.hyps  225000.hyps  288000.hyps  52000.hyps\r\n",
      "101000.hyps\t    164000.hyps  226000.hyps  289000.hyps  53000.hyps\r\n",
      "102000.hyps\t    165000.hyps  227000.hyps  29000.hyps   54000.hyps\r\n",
      "103000.hyps\t    166000.hyps  228000.hyps  290000.hyps  55000.hyps\r\n",
      "104000.hyps\t    167000.hyps  229000.hyps  291000.hyps  56000.hyps\r\n",
      "105000.hyps\t    168000.hyps  23000.hyps   292000.hyps  57000.hyps\r\n",
      "106000.hyps\t    169000.hyps  230000.hyps  293000.hyps  58000.hyps\r\n",
      "107000.hyps\t    17000.hyps\t 231000.hyps  294000.hyps  59000.hyps\r\n",
      "108000.hyps\t    170000.hyps  232000.hyps  295000.hyps  6000.hyps\r\n",
      "109000.hyps\t    171000.hyps  233000.hyps  296000.hyps  60000.hyps\r\n",
      "11000.hyps\t    172000.hyps  234000.hyps  297000.hyps  61000.hyps\r\n",
      "110000.hyps\t    173000.hyps  235000.hyps  298000.hyps  62000.hyps\r\n",
      "111000.hyps\t    174000.hyps  236000.hyps  299000.hyps  63000.hyps\r\n",
      "112000.hyps\t    175000.hyps  237000.hyps  3000.hyps    64000.hyps\r\n",
      "113000.hyps\t    176000.hyps  238000.hyps  30000.hyps   65000.hyps\r\n",
      "114000.hyps\t    177000.hyps  239000.hyps  300000.hyps  66000.hyps\r\n",
      "115000.hyps\t    178000.hyps  24000.hyps   301000.hyps  67000.hyps\r\n",
      "116000.hyps\t    179000.hyps  240000.hyps  302000.hyps  68000.hyps\r\n",
      "117000.hyps\t    18000.hyps\t 241000.hyps  303000.hyps  69000.hyps\r\n",
      "118000.hyps\t    180000.hyps  242000.hyps  304000.hyps  7000.hyps\r\n",
      "119000.hyps\t    181000.hyps  243000.hyps  305000.hyps  70000.hyps\r\n",
      "12000.hyps\t    182000.hyps  244000.hyps  306000.hyps  71000.hyps\r\n",
      "120000.hyps\t    183000.hyps  245000.hyps  307000.hyps  72000.hyps\r\n",
      "121000.hyps\t    184000.hyps  246000.hyps  308000.hyps  73000.hyps\r\n",
      "122000.hyps\t    185000.hyps  247000.hyps  309000.hyps  74000.hyps\r\n",
      "123000.hyps\t    186000.hyps  248000.hyps  31000.hyps   75000.hyps\r\n",
      "124000.hyps\t    187000.hyps  249000.hyps  310000.hyps  76000.hyps\r\n",
      "125000.hyps\t    188000.hyps  25000.hyps   311000.hyps  77000.hyps\r\n",
      "126000.hyps\t    189000.hyps  250000.hyps  312000.hyps  78000.hyps\r\n",
      "127000.hyps\t    19000.hyps\t 251000.hyps  313000.hyps  79000.hyps\r\n",
      "128000.hyps\t    190000.hyps  252000.hyps  314000.hyps  8000.hyps\r\n",
      "129000.hyps\t    191000.hyps  253000.hyps  315000.hyps  80000.hyps\r\n",
      "13000.hyps\t    192000.hyps  254000.hyps  316000.hyps  81000.hyps\r\n",
      "130000.hyps\t    193000.hyps  255000.hyps  317000.hyps  82000.hyps\r\n",
      "131000.hyps\t    194000.hyps  256000.hyps  318000.hyps  83000.hyps\r\n",
      "132000.hyps\t    195000.hyps  257000.hyps  319000.hyps  84000.hyps\r\n",
      "133000.hyps\t    196000.hyps  258000.hyps  32000.hyps   85000.hyps\r\n",
      "134000.hyps\t    197000.hyps  259000.hyps  320000.hyps  86000.hyps\r\n",
      "135000.hyps\t    198000.hyps  26000.hyps   321000.hyps  87000.hyps\r\n",
      "136000.hyps\t    199000.hyps  260000.hyps  322000.hyps  88000.hyps\r\n",
      "137000.hyps\t    2000.hyps\t 261000.hyps  323000.hyps  89000.hyps\r\n",
      "138000.hyps\t    20000.hyps\t 262000.hyps  324000.hyps  9000.hyps\r\n",
      "139000.hyps\t    200000.hyps  263000.hyps  325000.hyps  90000.hyps\r\n",
      "14000.hyps\t    201000.hyps  264000.hyps  326000.hyps  91000.hyps\r\n",
      "140000.hyps\t    202000.hyps  265000.hyps  327000.hyps  92000.hyps\r\n",
      "141000.hyps\t    203000.hyps  266000.hyps  328000.hyps  93000.hyps\r\n",
      "142000.hyps\t    204000.hyps  267000.hyps  329000.ckpt  94000.hyps\r\n",
      "143000.hyps\t    205000.hyps  268000.hyps  329000.hyps  95000.hyps\r\n",
      "144000.hyps\t    206000.hyps  269000.hyps  33000.hyps   96000.hyps\r\n",
      "145000.hyps\t    207000.hyps  27000.hyps   34000.hyps   97000.hyps\r\n",
      "146000.hyps\t    208000.hyps  270000.hyps  35000.hyps   98000.hyps\r\n",
      "147000.hyps\t    209000.hyps  271000.hyps  36000.hyps   99000.hyps\r\n",
      "148000.hyps\t    21000.hyps\t 272000.hyps  37000.hyps   best.ckpt\r\n",
      "149000.hyps\t    210000.hyps  273000.hyps  38000.hyps   config.yaml\r\n",
      "15000.hyps\t    211000.hyps  274000.hyps  39000.hyps   src_vocab.txt\r\n",
      "150000.hyps\t    212000.hyps  275000.hyps  4000.hyps    test.bpe.fr\r\n",
      "151000.hyps\t    213000.hyps  276000.hyps  40000.hyps   test.bpe.swc\r\n",
      "152000.hyps\t    214000.hyps  277000.hyps  41000.hyps   test.fr\r\n",
      "153000.hyps\t    215000.hyps  278000.hyps  42000.hyps   test.swc\r\n",
      "154000.hyps\t    216000.hyps  279000.hyps  43000.hyps   train.bpe.fr\r\n",
      "155000.hyps\t    217000.hyps  28000.hyps   44000.hyps   train.bpe.swc\r\n",
      "156000.hyps\t    218000.hyps  280000.hyps  45000.hyps   train.log\r\n",
      "157000.hyps\t    219000.hyps  281000.hyps  46000.hyps   trg_vocab.txt\r\n",
      "158000.hyps\t    22000.hyps\t 282000.hyps  47000.hyps   validations.txt\r\n",
      "159000.hyps\t    220000.hyps  283000.hyps  48000.hyps   vocab.txt\r\n"
     ]
    }
   ],
   "source": [
    "!ls fr-swc-baseline_v2/models/${src}${tgt}_transformer/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-18 13:41:36,642 Example #5\r\n",
      "2020-02-18 13:41:36,642 \tRaw source:     ['Peut', '-', 'être', 'comp@@', 'ren@@', 'ez', '-', '\\u200b', 'vous', 'ses', 'pro@@', 'ches', '.']\r\n",
      "2020-02-18 13:41:36,642 \tRaw hypothesis: ['Mbala', 'mosusu', 'oyebi', 'bandeko', 'na', 'ye', '.']\r\n",
      "2020-02-18 13:41:36,642 \tSource:     Peut - être comprenez - ​ vous ses proches .\r\n",
      "2020-02-18 13:41:36,642 \tReference:  Ntango mosusu okoki kozala na likanisi ndenge moko na bato ya libota na ye .\r\n",
      "2020-02-18 13:41:36,642 \tHypothesis: Mbala mosusu oyebi bandeko na ye .\r\n",
      "2020-02-18 13:41:36,642 Example #10\r\n",
      "2020-02-18 13:41:36,642 \tRaw source:     ['L’@@', 'endur@@', 'ance', 'face', 'à', 'de', 't@@', 'elles', 'épreuves', 'est', 'particulièrement', 'préci@@', 'euse', 'pour', 'Jéhovah', '.']\r\n",
      "2020-02-18 13:41:36,642 \tRaw hypothesis: ['Koy@@', 'ika', 'mpiko', 'na', 'komekama', 'ya', 'ndenge', 'wana', 'ezali', 'na', 'ntina', 'mingi', 'mpo', 'na', 'Yehova', '.']\r\n",
      "2020-02-18 13:41:36,642 \tSource:     L’endurance face à de telles épreuves est particulièrement précieuse pour Jéhovah .\r\n",
      "2020-02-18 13:41:36,642 \tReference:  Koyika mpiko liboso ya komekama wana ezali na motuya mingi na miso ya Yehova .\r\n",
      "2020-02-18 13:41:36,642 \tHypothesis: Koyika mpiko na komekama ya ndenge wana ezali na ntina mingi mpo na Yehova .\r\n",
      "2020-02-18 13:41:36,642 Validation result at epoch  39, step   265000: bleu:  25.84, loss: 37162.1758, ppl:   4.1573, duration: 91.9423s\r\n",
      "2020-02-18 13:42:05,706 Epoch  39 Step:   265100 Batch Loss:     1.331918 Tokens per Sec:     6900, Lr: 0.000147\r\n",
      "2020-02-18 13:42:34,490 Epoch  39 Step:   265200 Batch Loss:     1.508219 Tokens per Sec:     6953, Lr: 0.000147\r\n",
      "2020-02-18 13:42:50,547 Epoch  39: total training loss 10138.34\r\n",
      "2020-02-18 13:42:50,547 EPOCH 40\r\n",
      "2020-02-18 13:43:04,342 Epoch  40 Step:   265300 Batch Loss:     1.645450 Tokens per Sec:     6560, Lr: 0.000147\r\n",
      "2020-02-18 13:43:33,186 Epoch  40 Step:   265400 Batch Loss:     1.293967 Tokens per Sec:     7000, Lr: 0.000147\r\n",
      "2020-02-18 13:44:02,442 Epoch  40 Step:   265500 Batch Loss:     1.592970 Tokens per Sec:     7044, Lr: 0.000147\r\n"
     ]
    }
   ],
   "source": [
    "! tail -20 ../french_lingala_espoir/models/frln_transformer/train.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp $data_path/test.bpe.swc \"$gdrive_path/models/${src}${tgt}_transformer/\"\n",
    "!cp $data_path/test.bpe.fr \"$gdrive_path/models/${src}${tgt}_transformer/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-1cbef41d58e3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;34m[\u001b[0m\u001b[0;34m'espoir'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "['espoir'][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}