*** update runner.py

gbenson · Jun 4, 2024 · 92ac899 · 92ac899
1 parent 4d178cc
commit 92ac899
Showing 1 changed file with 8 additions and 4 deletions.
diff --git a/runner.py b/runner.py
@@ -3,7 +3,6 @@
 
 from itertools import chain
 
-from dom_tokenizers.internal import json
 from dom_tokenizers.pre_tokenizers.shared_oracle import SharedOracle
 
 DEFAULT_TESTCASES = [
@@ -38,9 +37,14 @@ def main():
         lines = DEFAULT_TESTCASES
     else:
         lines = chain.from_iterable(
-            (json.loads(line)["text"]
-             for line in open(filename).readlines())
-            for filename in sys.argv[1:])
+            (line.rstrip()
+             for code, line in (
+                 line.split(maxsplit=1)
+                 for line in open(filename).readlines()
+             )
+             if code == "data")
+            for filename in sys.argv[1:]
+        )
 
     for line in lines:
         print("input:", line)