dirkgr commited on
Commit
46fba0d
1 Parent(s): 24ce1be

Update README.md

Browse files

Trailing space breaks tokenization

Files changed (1) hide show
  1. README.md +1 -1
README.md CHANGED
@@ -100,7 +100,7 @@ from hf_olmo import OLMoForCausalLM, OLMoTokenizerFast
100
 
101
  olmo = OLMoForCausalLM.from_pretrained("allenai/OLMo-7B")
102
  tokenizer = OLMoTokenizerFast.from_pretrained("allenai/OLMo-7B")
103
- message = ["Language modeling is "]
104
  inputs = tokenizer(message, return_tensors='pt', return_token_type_ids=False)
105
  # optional verifying cuda
106
  # inputs = {k: v.to('cuda') for k,v in inputs.items()}
 
100
 
101
  olmo = OLMoForCausalLM.from_pretrained("allenai/OLMo-7B")
102
  tokenizer = OLMoTokenizerFast.from_pretrained("allenai/OLMo-7B")
103
+ message = ["Language modeling is"]
104
  inputs = tokenizer(message, return_tensors='pt', return_token_type_ids=False)
105
  # optional verifying cuda
106
  # inputs = {k: v.to('cuda') for k,v in inputs.items()}