heatingma commited on
Commit
ed3cabd
1 Parent(s): 90620f9

Update clip/simple_tokenizer.py

Browse files
Files changed (1) hide show
  1. clip/simple_tokenizer.py +1 -1
clip/simple_tokenizer.py CHANGED
@@ -63,7 +63,7 @@ class SimpleTokenizer(object):
63
  self.byte_encoder = bytes_to_unicode()
64
  self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
65
  if not os.system(bpe_path):
66
- txt_path = bpe_path.replace(".gz", ".txt")
67
  os.system(f"gzip {txt_path}")
68
  merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
69
  merges = merges[1:49152-256-2+1]
 
63
  self.byte_encoder = bytes_to_unicode()
64
  self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
65
  if not os.system(bpe_path):
66
+ txt_path = bpe_path.replace(".gz" "")
67
  os.system(f"gzip {txt_path}")
68
  merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
69
  merges = merges[1:49152-256-2+1]