Skip to content

Commit 7401646

Browse files
committed
fix export error when export_stats columns is null
1 parent 030e786 commit 7401646

File tree

1 file changed

+7
-6
lines changed

1 file changed

+7
-6
lines changed

data_juicer/core/exporter.py

+7-6
Original file line numberDiff line numberDiff line change
@@ -114,12 +114,13 @@ def _export_impl(self, dataset, export_path, suffix, export_stats=True):
114114
export_columns.append(Fields.stats)
115115
if Fields.meta in dataset.features:
116116
export_columns.append(Fields.meta)
117-
ds_stats = dataset.select_columns(export_columns)
118-
stats_file = export_path.replace('.' + suffix, '_stats.jsonl')
119-
Exporter.to_jsonl(
120-
ds_stats,
121-
stats_file,
122-
num_proc=self.num_proc if self.export_in_parallel else 1)
117+
if len(export_columns):
118+
ds_stats = dataset.select_columns(export_columns)
119+
stats_file = export_path.replace('.' + suffix, '_stats.jsonl')
120+
Exporter.to_jsonl(
121+
ds_stats,
122+
stats_file,
123+
num_proc=self.num_proc if self.export_in_parallel else 1)
123124

124125
if self.export_ds:
125126
# fetch the corresponding export method according to the suffix

0 commit comments

Comments
 (0)