*** what's next?

Author: gbenson

src/dom_tokenizers/pre_tokenizers/splitter.py

@@ -62,12 +62,14 @@ def special_tokens(self) -> Iterable[str]:
     FIRST_SPLIT_RE = re.compile(rf"([^\w'’{BASE64_NONWORD}]+)")
     BASE64_NONWORD_RE = re.compile("[+/=]+")
 
-    _TWOHEX = "[0-9a-fA-F]{2}"
+    _HEX = "[0-9a-fA-F]"
+    ALL_HEX_RE = re.compile(f"^{_HEX}+$")
+    _TWOHEX = f"{_HEX}{{2}}"
     TWOHEX_RE = re.compile(_TWOHEX)
     JS_CHAR_ESCAPE_RE = re.compile(f"(?:x|u{_TWOHEX}){_TWOHEX}")
     ENTITY_STARTS = {"&", "&#"}
     ESCAPE_START_RE = re.compile(r".([&%\\])")
-    PREFIXED_HEX_RE = re.compile(r"^(0x)([0-9a-f]+)([+/=]*)$", re.I)
+    PREFIXED_HEX_RE = re.compile(rf"^(0[xX])({_HEX}+)([+/=]*)$", re.I)
 
     # XXX older bits
     MAXWORDLEN = 32
@@ -461,20 +463,32 @@ def _split_base64_utf8(self, text, encoded):
             return [self.base64_token, "json"]
         except json.JSONDecodeError:
             pass
-        if self.oracle.first_is_better(encoded, text):
-            return None  # encoded is better
+        #with open("base64.matches", "a") as fp:
+        #    print("text", encoded, file=fp)
+        #if self.oracle.first_is_better(encoded, text):
+        #    return None  # encoded is better
         return [self.base64_token, "text"]
 
     def _split_base64_binary(self, data, encoded):
+        if len(encoded) < 9:  # XXX review
+            return None
         filetype = sniff_bytes(data)
-        if not filetype:
-            if self.oracle.is_texty(encoded):
-                return None
-            return [self.base64_token, "data"]
-        return [self.base64_token, filetype.name.lower()]
+        if filetype:
+            return [self.base64_token, filetype.name.lower()]
+        #with open("unsniffed.matches", "a") as fp:
+        #    print(len(encoded), encoded, file=fp)
+        #if self.oracle.is_texty(encoded):
+        #    return None
+        return [self.base64_token, "data"]
+        raise NotImplementedError(encoded)
 
     # XXX junk?
 
+    #all uppercase
+    #all lowercase
+    #all hex digits
+    #CamelCase
+
     def _sub_base64(self, splits, cursor):
         curr = splits[cursor]
         try:

tests/test_splitter.py

@@ -1,3 +1,5 @@
+from base64 import b64decode
+
 import pytest
 
 from dom_tokenizers.pre_tokenizers.splitter import TextSplitter
@@ -201,6 +203,41 @@ def test_prefixed_hex(text, expect_tokens):
     assert list(TextSplitter().split(text)) == expect_tokens
 
 
+def test_split_base64_binary():
+    ts = TextSplitter()
+    for filename in ("6-short-matches", "6-unsniffed-matches"):
+        with open(filename) as fp:
+            for line in fp.readlines():
+                check, encoded = line.split(maxsplit=1)
+                encoded = encoded.rstrip()
+                assert len(encoded) == int(check)
+                data = b64decode(encoded)
+                x = ts._split_base64_binary(data, encoded)
+                print(encoded, x)
+                assert x is not None
+
+#@pytest.mark.parametrize(
+#    "encoded,expect_tokens",
+#    [(encoded, None) for encoded in (
+#        "ajax=",
+#        "00000002",
+#        textures/spgm/contrib/overlib410/overlib
+#        ShouldAllowProductItemOutlineBorderStyle
+#        ShowAutomaticDiscountDataOnProductWidget
+#        UseQueryBySessionStartForContactBookings
+#
+#        # too short
+#        "evid=", "MA6Y=", "uapv=", "1gsQ==", "null==", "data===",
+#        "00000005", "000000de", "000001px", "16H162a2", "1/8/3/20",
+#        "1PAPISID", "20137566", "20150315", "//action", "activate",
+#        "Activate", "activeEl", "+apple/i", "appName=", "+g+k+l+h",
+#12 000000000000
+#12 000cf00eb582
+#
+#8 00000002
+#def test_split_base64_binary_encoded_(self, data, encoded):
+
+
 @pytest.mark.parametrize(
     "text,expect_tokens",
     (("That\u2019s all we know.",