From 90a7760af90f56e81636f93432f8047506daa600 Mon Sep 17 00:00:00 2001
From: Junjie_ZHANG <junj.jay.zhang@gmail.com>
Date: Wed, 25 Dec 2024 10:19:15 +0800
Subject: [PATCH] fix bugs of data preprocessing with multiple json keys

---
 tools/preprocess_data.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/preprocess_data.py b/tools/preprocess_data.py
index 13e5b64a47..dfb1014adf 100644
--- a/tools/preprocess_data.py
+++ b/tools/preprocess_data.py
@@ -184,7 +184,8 @@ def process_json_file(self, file_name):
             self.print_processing_stats(i, proc_start, total_bytes_processed)
 
         fin.close()
-        builders[key].finalize(output_idx_files[key])
+        for key in builders.keys():
+            builders[key].finalize(output_idx_files[key])
 
 
 def get_args():