Rocketknight1 HF staff commited on
Commit
b4b114c
1 Parent(s): f92de46

Update tokenization_hyena.py

Browse files
Files changed (1) hide show
  1. tokenization_hyena.py +1 -1
tokenization_hyena.py CHANGED
@@ -94,7 +94,7 @@ class HyenaDNATokenizer(PreTrainedTokenizer):
94
  already_has_special_tokens=True,
95
  )
96
 
97
- result = [1] + ([0] * len(token_ids_0)) + [1]
98
  if token_ids_1 is not None:
99
  result += ([0] * len(token_ids_1)) + [1]
100
  return result
 
94
  already_has_special_tokens=True,
95
  )
96
 
97
+ result = ([0] * len(token_ids_0)) + [1]
98
  if token_ids_1 is not None:
99
  result += ([0] * len(token_ids_1)) + [1]
100
  return result