.

tokenization_xgen.py

@@ -6,11 +6,15 @@
 
 from typing import List, Optional
 
-import tiktoken
-
 from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
 from transformers.utils import logging
 
+try:
+    import tiktoken
+except ModuleNotFoundError as e:
+    raise ModuleNotFoundError("XGen requires the installation of tiktoken. Please install it via `pip install tiktoken`.") from e
+
+
 logger = logging.get_logger(__name__)
 
 MAX_MODEL_INPUT_SIZES = {