zxdu20 peakji commited on
Commit
fdb7a60
1 Parent(s): f82b180

Support single integer or empty list as input to decode (#7)

Browse files

- Update tokenization_chatglm.py (dffe870a7ef1558ebbc6f3dfdf46491cdb2b3e31)


Co-authored-by: Yichao 'Peak' Ji <peakji@users.noreply.huggingface.co>

Files changed (1) hide show
  1. tokenization_chatglm.py +4 -0
tokenization_chatglm.py CHANGED
@@ -261,6 +261,10 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
261
  spaces_between_special_tokens: bool = True,
262
  **kwargs
263
  ) -> str:
 
 
 
 
264
  if isinstance(token_ids[0], list):
265
  tokens = []
266
  for single_token_ids in token_ids:
 
261
  spaces_between_special_tokens: bool = True,
262
  **kwargs
263
  ) -> str:
264
+ if not isinstance(token_ids, list):
265
+ token_ids = [token_ids]
266
+ if len(token_ids) == 0:
267
+ return ""
268
  if isinstance(token_ids[0], list):
269
  tokens = []
270
  for single_token_ids in token_ids: