Commit
•
bbe961d
1
Parent(s):
128676c
Update eval.py
Browse files
eval.py
CHANGED
|
@@ -53,6 +53,11 @@ def normalize_text(text: str) -> str:
|
|
| 53 |
chars_to_ignore_regex = '[\,\?\.\!\-\;\:\"\“\%\‘\”\�\'\«\»\–\—]' # noqa: W605 IMPORTANT: this should correspond to the chars that were ignored during training
|
| 54 |
|
| 55 |
text = re.sub(chars_to_ignore_regex, "", text.lower())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
|
| 57 |
# In addition, we can normalize the target text, e.g. removing new lines characters etc...
|
| 58 |
# note that order is important here!
|
|
|
|
| 53 |
chars_to_ignore_regex = '[\,\?\.\!\-\;\:\"\“\%\‘\”\�\'\«\»\–\—]' # noqa: W605 IMPORTANT: this should correspond to the chars that were ignored during training
|
| 54 |
|
| 55 |
text = re.sub(chars_to_ignore_regex, "", text.lower())
|
| 56 |
+
text = re.sub("'", "", text)
|
| 57 |
+
text = re.sub("«", "", text)
|
| 58 |
+
text = re.sub("»", "", text)
|
| 59 |
+
text = re.sub("–", "", text)
|
| 60 |
+
text = re.sub("—", "", text)
|
| 61 |
|
| 62 |
# In addition, we can normalize the target text, e.g. removing new lines characters etc...
|
| 63 |
# note that order is important here!
|