Fix jumanpp.apply_to_sentence return empty list for sentence larger than ~1700 characters (#2)

- Fix jumanpp.apply_to_sentence return empty list for sentence larger than ~1700 characters (71e8994479e42c5e0afab380bc4b41941ce1dac0)
- Fix jumanpp.apply_to_sentence return empty list (072cc83b373db1749b80d92a4d4fd5c7fbb0ba63)
- Adding rhoknp package reference (3d4a29abd2f3ba3abd94da87127d5115655fbe00)
- Adding rhoknp package reference (911013a3b7b60de980d55fa69287d9da7300c61f)


Co-authored-by: Loc Nguyen <nesv042@users.noreply.huggingface.co>

tokenization_deberta_v2_jumanpp.py

@@ -24,7 +24,13 @@ class JumanppTokenizer:
                 "You need to install rhoknp to use JumanppPreTokenizer. "
                 "See https://github.com/ku-nlp/rhoknp for installation."
             )
+        self.rhoknp = rhoknp
         self.jumanpp = rhoknp.Jumanpp()
 
     def tokenize(self, text: str) -> str:
-        return " ".join([morpheme.surf for morpheme in self.jumanpp.apply_to_sentence(text).morphemes])
+        morphemes = self.jumanpp.apply_to_sentence(text).morphemes
+        if not morphemes:
+            doc = self.rhoknp.Document.from_raw_text(text)
+            morphemes = self.jumanpp.apply_to_document(doc).morphemes
+        return " ".join([morpheme.surf for morpheme in morphemes])
+        

tokenization_deberta_v2_jumanpp_fast.py

@@ -55,6 +55,7 @@ class JumanppPreTokenizer:
                 "You need to install rhoknp to use JumanppPreTokenizer. "
                 "See https://github.com/ku-nlp/rhoknp for installation."
             )
+        self.rhoknp = rhoknp
         self.jumanpp = rhoknp.Jumanpp()
 
     def pre_tokenize(self, pretok: PreTokenizedString):
@@ -62,4 +63,7 @@ class JumanppPreTokenizer:
 
     def jumanpp_split(self, i: int, normalized_string: NormalizedString) -> List[NormalizedString]:
         offsets = [morpheme.span for morpheme in self.jumanpp.apply_to_sentence(str(normalized_string)).morphemes]
+        if not offsets:
+            doc = self.rhoknp.Document.from_raw_text(str(normalized_string))
+            offsets = [morpheme.span for morpheme in self.jumanpp.apply_to_document(doc).morphemes]
         return [normalized_string[offset[0]:offset[1]] for offset in offsets]