fix chunk_num_per_shard #1

thu-coai · Nov 7, 2024 · 6d914e8 · 6d914e8
1 parent e680467
commit 6d914e8
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/data_utils/indexed_dataset.py b/data_utils/indexed_dataset.py
@@ -141,6 +141,7 @@ def __init__(self,
         self.output_path = output_path
         self.bin_file = os.path.join(self.output_path, f"{self.split}_{self.ofid}.bin")
         self.idx_file = os.path.join(self.output_path, f"{self.split}_{self.ofid}.idx")
+        self.chunk_num_per_shard = chunk_num_per_shard
         self.tmp_output_path = tmp_output_path
         if self.tmp_output_path is not None:
             self.tmp_bin_file = os.path.join(self.tmp_output_path, f"{self.split}_{self.ofid}.bin")