Fix handling of long text. (#3)

- Fix exception handling in JumanppTokenizer and (514804743e0d9464e02cb292293a7e98990293b8)


Co-authored-by: KENGO SHIMIZU <tealgreen0503@users.noreply.huggingface.co>

tokenization_deberta_v2_jumanpp.py

@@ -28,9 +28,9 @@ class JumanppTokenizer:
         self.jumanpp = rhoknp.Jumanpp()
 
     def tokenize(self, text: str) -> str:
-        morphemes = self.jumanpp.apply_to_sentence(text).morphemes
-        if not morphemes:
+        try:
+            morphemes = self.jumanpp.apply_to_sentence(text).morphemes
+        except RuntimeError:
             doc = self.rhoknp.Document.from_raw_text(text)
             morphemes = self.jumanpp.apply_to_document(doc).morphemes
         return " ".join([morpheme.surf for morpheme in morphemes])
-        

tokenization_deberta_v2_jumanpp_fast.py

@@ -62,8 +62,9 @@ class JumanppPreTokenizer:
         pretok.split(self.jumanpp_split)
 
     def jumanpp_split(self, i: int, normalized_string: NormalizedString) -> List[NormalizedString]:
-        offsets = [morpheme.span for morpheme in self.jumanpp.apply_to_sentence(str(normalized_string)).morphemes]
-        if not offsets:
+        try:
+            offsets = [morpheme.span for morpheme in self.jumanpp.apply_to_sentence(str(normalized_string)).morphemes]
+        except RuntimeError:
             doc = self.rhoknp.Document.from_raw_text(str(normalized_string))
             offsets = [morpheme.span for morpheme in self.jumanpp.apply_to_document(doc).morphemes]
         return [normalized_string[offset[0]:offset[1]] for offset in offsets]