Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
Update src/submission/check_validity.py
Browse files
src/submission/check_validity.py
CHANGED
@@ -35,9 +35,13 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
|
|
35 |
def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
|
36 |
"""Makes sure the model is on the hub, and uses a valid configuration (in the latest transformers version)"""
|
37 |
try:
|
|
|
|
|
38 |
config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
|
|
|
39 |
if test_tokenizer:
|
40 |
tokenizer_config = get_tokenizer_config(model_name)
|
|
|
41 |
if tokenizer_config is not None:
|
42 |
tokenizer_class_candidate = tokenizer_config.get("tokenizer_class", None)
|
43 |
else:
|
@@ -45,6 +49,7 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
|
|
45 |
|
46 |
|
47 |
tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
|
|
|
48 |
if tokenizer_class is None:
|
49 |
return (
|
50 |
False,
|
|
|
35 |
def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
|
36 |
"""Makes sure the model is on the hub, and uses a valid configuration (in the latest transformers version)"""
|
37 |
try:
|
38 |
+
|
39 |
+
print("this is input :", model_name, revision, token, trust_remote_code, test_tokenizer)
|
40 |
config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
|
41 |
+
print("getting config", config)
|
42 |
if test_tokenizer:
|
43 |
tokenizer_config = get_tokenizer_config(model_name)
|
44 |
+
print("tokenizer_config", tokenizer_config)
|
45 |
if tokenizer_config is not None:
|
46 |
tokenizer_class_candidate = tokenizer_config.get("tokenizer_class", None)
|
47 |
else:
|
|
|
49 |
|
50 |
|
51 |
tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
|
52 |
+
|
53 |
if tokenizer_class is None:
|
54 |
return (
|
55 |
False,
|