⏪ Remove change in warc reader

huggingface · Jul 11, 2023 · 7d18f66 · 7d18f66
1 parent e8279a0
commit 7d18f66
Showing 1 changed file with 2 additions and 6 deletions.
diff --git a/src/datatrove/pipeline/readers/warc.py b/src/datatrove/pipeline/readers/warc.py
@@ -17,12 +17,8 @@ def __init__(self, *args, gzip_mode: bool = False, **kwargs):
 
  def read_file(self, datafile: InputDataFile):
  with datafile.open(gzip=self.gzip_mode, binary=True) as f:
- for i, record in enumerate(ArchiveIterator(f)):
- with self.stats.time_manager:
- document = process_record(record)
-
- if i > 1000:
- return
+ for record in ArchiveIterator(f):
+ document = process_record(record)
  if document:
  document.metadata["file_path"] = datafile.path
  yield document