Update model.py

model.py

@@ -148,13 +148,13 @@ class SimilarFilter(Filter):
         return decoded_inputs
 
 class ChineseFilter:
-    def __init__(self, pinyin_lib_file='./pinyin.txt'):
+    def __init__(self, pinyin_lib_file='pinyin.txt'):
         self.name = 'chinese filter'
         self.code = []
         self.pinyin_lib = self.load_pinyin_lib(pinyin_lib_file)
 
     def load_pinyin_lib(self, file_path):
-        with open(file_path, 'r', encoding='utf-8') as f:
+        with open(os.path.join(script_dir,file_path), 'r', encoding='utf-8') as f:
             return set(line.strip().lower() for line in f)
 
     def is_valid_chinese(self, word):
@@ -407,27 +407,38 @@ class Model():
                 print(batch)
                 batch = filter_pipeline.batch_encoder(batch)
                 print(batch)
-                input_ids = self.tokenizer(batch, return_tensors="pt", padding=True).to(self.device_name)
                 temp = []
-                for target_language in target_languages:
-                    target_lang_code = self.tokenizer.lang_code_to_id[language_mapping(target_language)]
-                    generated_tokens = self.model.generate(
-                        **input_ids,
-                        forced_bos_token_id=target_lang_code,
-                    )
-                    generated_translation = self.tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
-                    
-                    print(generated_translation)
-                    generated_translation = filter_pipeline.batch_decoder(generated_translation)
-                    print(generated_translation)
-                    print(len(generated_translation))
-                    # Append result to output
-                    temp.append({
-                        "target_language": target_language,
-                        "generated_translation": generated_translation,
-                    })
-                input_ids.to('cpu')
-                del input_ids
+                if len(batch) > 0:
+                    input_ids = self.tokenizer(batch, return_tensors="pt", padding=True).to(self.device_name)
+                    for target_language in target_languages:
+                        target_lang_code = self.tokenizer.lang_code_to_id[language_mapping(target_language)]
+                        generated_tokens = self.model.generate(
+                            **input_ids,
+                            forced_bos_token_id=target_lang_code,
+                        )
+                        generated_translation = self.tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
+                        
+                        print(generated_translation)
+                        generated_translation = filter_pipeline.batch_decoder(generated_translation)
+                        print(generated_translation)
+                        print(len(generated_translation))
+                        # Append result to output
+                        temp.append({
+                            "target_language": target_language,
+                            "generated_translation": generated_translation,
+                        })
+                    input_ids.to('cpu')
+                    del input_ids
+                else:
+                    for target_language in target_languages:
+                        generated_translation = filter_pipeline.batch_decoder(batch)
+                        print(generated_translation)
+                        print(len(generated_translation))
+                        # Append result to output
+                        temp.append({
+                            "target_language": target_language,
+                            "generated_translation": generated_translation,
+                        })
                 temp_outputs.append(temp)
                 processed_num += len(batch)
                 if (index + 1) * max_batch_size // 1000 - index  * max_batch_size // 1000 == 1: