Update README.md
Browse files
README.md
CHANGED
@@ -78,7 +78,7 @@ input_texts = queries + documents
|
|
78 |
max_length = 512
|
79 |
|
80 |
# Tokenize the input texts
|
81 |
-
batch_dict = tokenizer(input_texts, max_length=max_length, padding=True, truncation=True, return_tensors='pt')
|
82 |
|
83 |
# Important! Adding EOS token at the end
|
84 |
batch_dict['input_ids'] = [input_ids + [tokenizer.eos_token_id] for input_ids in batch_dict['input_ids']]
|
|
|
78 |
max_length = 512
|
79 |
|
80 |
# Tokenize the input texts
|
81 |
+
batch_dict = tokenizer(input_texts, max_length=max_length-1, padding=True, truncation=True, return_tensors='pt')
|
82 |
|
83 |
# Important! Adding EOS token at the end
|
84 |
batch_dict['input_ids'] = [input_ids + [tokenizer.eos_token_id] for input_ids in batch_dict['input_ids']]
|