*** debug code

gbenson · Jun 4, 2024 · 61b1fd0 · 61b1fd0
1 parent a4b9072
commit 61b1fd0
Showing 1 changed file with 8 additions and 4 deletions.
diff --git a/src/dom_tokenizers/pre_tokenizers/splitter.py b/src/dom_tokenizers/pre_tokenizers/splitter.py
@@ -461,15 +461,19 @@ def _split_base64_utf8(self, text, encoded):
             return [self.base64_token, "json"]
         except json.JSONDecodeError:
             pass
-        if self.oracle.first_is_better(encoded, text):
-            return None  # encoded is better
+        with open("base64.matches", "a") as fp:
+            print("text", encoded, file=fp)
+        #if self.oracle.first_is_better(encoded, text):
+        #    return None  # encoded is better
         return [self.base64_token, "text"]
 
     def _split_base64_binary(self, data, encoded):
         filetype = sniff_bytes(data)
         if not filetype:
-            if self.oracle.is_texty(encoded):
-                return None
+            with open("base64.matches", "a") as fp:
+                print("data", encoded, file=fp)
+            #if self.oracle.is_texty(encoded):
+            #    return None
             return [self.base64_token, "data"]
         return [self.base64_token, filetype.name.lower()]