Update model.py
Browse files
model.py
CHANGED
@@ -16,7 +16,7 @@ def load_embedder_and_tokenizer(name: str) -> Tuple[
|
|
16 |
transformers.PreTrainedTokenizer
|
17 |
]:
|
18 |
if name.startswith("nomic") or (name == "bert-base-uncased"):
|
19 |
-
config = transformers.AutoConfig.from_pretrained(
|
20 |
model = transformers.AutoModel.from_pretrained(name, config=config, trust_remote_code=True)
|
21 |
tokenizer = transformers.AutoTokenizer.from_pretrained(name)
|
22 |
elif name in ["gtr-base", "gtr_base"]:
|
|
|
16 |
transformers.PreTrainedTokenizer
|
17 |
]:
|
18 |
if name.startswith("nomic") or (name == "bert-base-uncased"):
|
19 |
+
config = transformers.AutoConfig.from_pretrained(name, trust_remote_code=True)
|
20 |
model = transformers.AutoModel.from_pretrained(name, config=config, trust_remote_code=True)
|
21 |
tokenizer = transformers.AutoTokenizer.from_pretrained(name)
|
22 |
elif name in ["gtr-base", "gtr_base"]:
|