Spaces:
Runtime error
Runtime error
ClΓ©mentine
commited on
Commit
Β·
e884a73
1
Parent(s):
22edb72
Fixed tokenizer message
Browse files(cherry picked from commit 8137036f6e8737c7f8700871f8169ef2d04ea322)
src/submission/check_validity.py
CHANGED
|
@@ -42,13 +42,15 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
|
|
| 42 |
config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
|
| 43 |
if test_tokenizer:
|
| 44 |
try:
|
| 45 |
-
AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
|
| 46 |
except ValueError as e:
|
| 47 |
return (
|
| 48 |
False,
|
| 49 |
f"uses a tokenizer which is not in a transformers release: {e}",
|
| 50 |
None
|
| 51 |
)
|
|
|
|
|
|
|
| 52 |
return True, None, config
|
| 53 |
|
| 54 |
except ValueError:
|
|
|
|
| 42 |
config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
|
| 43 |
if test_tokenizer:
|
| 44 |
try:
|
| 45 |
+
tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
|
| 46 |
except ValueError as e:
|
| 47 |
return (
|
| 48 |
False,
|
| 49 |
f"uses a tokenizer which is not in a transformers release: {e}",
|
| 50 |
None
|
| 51 |
)
|
| 52 |
+
except Exception as e:
|
| 53 |
+
return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
|
| 54 |
return True, None, config
|
| 55 |
|
| 56 |
except ValueError:
|