Deprecate DOMSnapshotTokenizer.from_pretrained()

pyproject.toml

@@ -2,7 +2,7 @@
 name = "dom-tokenizers"
 version = "0.0.10"
 authors = [{ name = "Gary Benson", email = "[email protected]" }]
-description = "DOM-aware tokenizers for 🤗 Hugging Face language models"
+description = "DOM-aware tokenization for 🤗 Hugging Face language models"
 readme = "README.md"
 license = { text = "Apache Software License" }
 requires-python = ">=3.10"  # match..case
@@ -24,10 +24,11 @@ classifiers = [
     "Topic :: Text Processing :: Markup :: HTML",
 ]
 dependencies = [
-    "python-magic",
+    "python-magic",       # XXX review
     "tokenizers",
-    "transformers",
-    "unidecode",
+    "transformers",       # move to dev, train
+    "typing-extensions",
+    "unidecode",          # XXX review
 ]
 
 [project.urls]

src/dom_tokenizers/tokenizers.py

@@ -1,9 +1,13 @@
+from typing_extensions import deprecated
+
 from .internal.transformers import AutoTokenizer
 from .pre_tokenizers import DOMSnapshotPreTokenizer
 
 
+@deprecated("use `DOMSnapshotPreTokenizer` instead")
 class DOMSnapshotTokenizer:
     @classmethod
+    @deprecated("use `DOMSnapshotPreTokenizer.adapt()` instead")
     def from_pretrained(cls, *args, **kwargs):
         tokenizer = AutoTokenizer.from_pretrained(*args, **kwargs)
         DOMSnapshotPreTokenizer.hook_into(tokenizer)

src/dom_tokenizers/train.py

@@ -8,7 +8,8 @@
 from datasets import load_dataset
 from tokenizers.pre_tokenizers import WhitespaceSplit
 
-from .tokenizers import DOMSnapshotTokenizer
+from .internal.transformers import AutoTokenizer
+from .pre_tokenizers import DOMSnapshotPreTokenizer
 
 DEFAULT_BASE_TOKENIZER = "bert-base-cased"
 DEFAULT_SPLIT = "train"
@@ -24,8 +25,8 @@ def train_tokenizer(
 
     # Create the base tokenizer we'll train our new tokenizer from.
     if isinstance(base_tokenizer, str):
-        base_tokenizer = DOMSnapshotTokenizer.from_pretrained(
-            base_tokenizer)
+        base_tokenizer = AutoTokenizer.from_pretrained(base_tokenizer)
+        DOMSnapshotPreTokenizer.hook_into(base_tokenizer)
 
     # It's not possible to train using a custom pre-tokenizer, the Rust
     # code raises "Exception: Custom PreTokenizer cannot be serialized"

tests/conftest.py

@@ -1,6 +1,7 @@
 import pytest
 
-from dom_tokenizers import DOMSnapshotTokenizer
+from dom_tokenizers import DOMSnapshotPreTokenizer
+from dom_tokenizers.internal.transformers import AutoTokenizer
 from dom_tokenizers.train import DEFAULT_BASE_TOKENIZER
 
 
@@ -9,7 +10,9 @@ def dom_snapshot_tokenizer():
     """An instance of a tokenizer that consumes JSON-serialized
     DOM snapshots.
     """
-    return DOMSnapshotTokenizer.from_pretrained(DEFAULT_BASE_TOKENIZER)
+    tokenizer = AutoTokenizer.from_pretrained(DEFAULT_BASE_TOKENIZER)
+    DOMSnapshotPreTokenizer.hook_into(tokenizer)
+    return tokenizer
 
 
 @pytest.fixture