Add error logging for failed tokenizer loading

Signed-off-by: Khaled Sulayman <[email protected]>

src/instructlab/sdg/utils/chunkers.py

@@ -290,7 +290,8 @@ def fuse_texts(
 
         return fused_texts
 
-    def create_tokenizer(self, model_name: str):
+    @staticmethod
+    def create_tokenizer(cls, model_name: str):
         """
         Create a tokenizer instance from a pre-trained model or a local directory.
 
@@ -301,24 +302,26 @@ def create_tokenizer(self, model_name: str):
             AutoTokenizer: The tokenizer instance.
         """
         # Third Party
-        import ipdb
-
-        ipdb.set_trace()
         model_path = Path(model_name)
+        error_info_message = "Please run ilab model download {download_args} and try again"
         try:
             if is_model_safetensors(model_path):
                 tokenizer = AutoTokenizer.from_pretrained(model_path)
+                error_info_message = error_info_message.format(download_args=f"--repository {model_path}")
             elif is_model_gguf(model_path):
-                tokenizer = AutoTokenizer.from_pretrained(model_path.parent, gguf_file=model_path.name)
+                model_dir, model_filename = model_path.parent, model_path.name
+                tokenizer = AutoTokenizer.from_pretrained(model_dir, gguf_file=model_filename)
+                error_info_message = error_info_message.format(download_args=f"--repository {model_dir} --filename {model_filename}")
+            else:
+                raise Exception(f"Received path to invalid model format {model_path}")
             logger.info(f"Successfully loaded tokenizer from: {model_path}")
             return tokenizer
-        except Exception as e:
+
+        except (OSError, ValueError) as e:
             logger.error(
-                f"Failed to load tokenizer as model was not found at {model_path}."
-                "Please run `ilab model download {model_name} and try again\n"
-                "{str(e)}"
+                f"Failed to load tokenizer as model was not found at {model_path}. {error_info_message}"
             )
-            raise
+            raise e
 
     def get_token_count(self, text, tokenizer):
         """