Lowercase tag names before splitting

gbenson · May 30, 2024 · 9da6b9a · 9da6b9a
1 parent 700a4af
commit 9da6b9a
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/src/dom_tokenizers/pre_tokenizers/dom_snapshot.py b/src/dom_tokenizers/pre_tokenizers/dom_snapshot.py
@@ -132,13 +132,13 @@ def get(
         tokens = cache.get(string_index)
         if tokens is not None:
             return tokens
+        text = self._strings[string_index]
+        if lowercase:
+            text = text.lower()
         tokens = [
             NormalizedString(token)
-            for token in self._splitter.split(self._strings[string_index])
+            for token in self._splitter.split(text)
         ]
-        if lowercase:
-            for token in tokens:
-                token.lowercase()
         cache[string_index] = tokens
         return tokens