Log here too

gbenson · Jun 20, 2024 · 196d354 · 196d354
1 parent e243845
commit 196d354
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/src/dom_tokenizers/train.py b/src/dom_tokenizers/train.py
@@ -1,3 +1,4 @@
+import logging
 import os
 import warnings
 
@@ -19,6 +20,8 @@
     SEND_BUGS_TO,
 )
 
+logger = logging.getLogger(__name__)
+
 
 def train_tokenizer(
         training_dataset,
@@ -55,6 +58,11 @@ def futz_input(real_input):
 
     def get_training_corpus():
         for row in training_dataset:
+            logger.info(
+                "source %d: %s",
+                row.get("source_index", -1),
+                row.get("displayed_url"),
+            )
             yield futz_input(json.dumps(row["dom_snapshot"]))
 
     # Try and get a dataset length, for the progress tracker.