oscar tokenizers
Browse files- tokenizer.json +0 -0
- train_tokenizer.py +1 -1
tokenizer.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
train_tokenizer.py
CHANGED
|
@@ -2,7 +2,7 @@ from datasets import load_dataset
|
|
| 2 |
from tokenizers import ByteLevelBPETokenizer
|
| 3 |
|
| 4 |
# load dataset
|
| 5 |
-
dataset = load_dataset("
|
| 6 |
|
| 7 |
# Instantiate tokenizer
|
| 8 |
tokenizer = ByteLevelBPETokenizer()
|
|
|
|
| 2 |
from tokenizers import ByteLevelBPETokenizer
|
| 3 |
|
| 4 |
# load dataset
|
| 5 |
+
dataset = load_dataset("oscar", "unshuffled_deduplicated_id", split="train")
|
| 6 |
|
| 7 |
# Instantiate tokenizer
|
| 8 |
tokenizer = ByteLevelBPETokenizer()
|