Spaces:
Sleeping
Sleeping
Update clip/simple_tokenizer.py
Browse files- clip/simple_tokenizer.py +1 -1
clip/simple_tokenizer.py
CHANGED
@@ -62,7 +62,7 @@ class SimpleTokenizer(object):
|
|
62 |
def __init__(self, bpe_path: str = default_bpe()):
|
63 |
self.byte_encoder = bytes_to_unicode()
|
64 |
self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
|
65 |
-
if not os.
|
66 |
txt_path = bpe_path.replace(".gz", "")
|
67 |
os.system(f"gzip {txt_path}")
|
68 |
merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
|
|
|
62 |
def __init__(self, bpe_path: str = default_bpe()):
|
63 |
self.byte_encoder = bytes_to_unicode()
|
64 |
self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
|
65 |
+
if not os.path.exists(bpe_path):
|
66 |
txt_path = bpe_path.replace(".gz", "")
|
67 |
os.system(f"gzip {txt_path}")
|
68 |
merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
|