Skip to content

Commit 07a056e

Browse files
dfa1claude
andcommitted
feat(csv): report progress every 10K rows instead of per-chunk
Progress now fires every 10,000 rows regardless of chunk boundaries (chunks are 65,536 rows). Counter tracks rows read; last update fires on EOF if any rows remain unreported. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 6286361 commit 07a056e

1 file changed

Lines changed: 11 additions & 3 deletions

File tree

csv/src/main/java/io/github/dfa1/vortex/csv/CsvImporter.java

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
/// O(chunkSize) regardless of file size.
2929
public final class CsvImporter {
3030

31+
private static final long PROGRESS_BATCH = 10_000;
32+
3133
private CsvImporter() {
3234
}
3335

@@ -93,27 +95,33 @@ public static void importCsv(Path csvPath, Path vortexPath, ImportOptions option
9395
VortexWriter writer = VortexWriter.create(channel, schema, options.writeOptions())) {
9496

9597
long totalRows = 0;
98+
long lastReported = 0;
9699

97100
// Write the buffered first chunk.
98101
writer.writeChunk(buildChunk(schema, firstChunk));
99102
totalRows += firstChunk.size();
103+
lastReported = totalRows;
100104
reportProgress(options, totalRows);
101105
firstChunk.clear();
102106

103107
// Stream the rest of the file through the still-open reader.
104108
List<String[]> chunk = new ArrayList<>(chunkSize);
105109
for (CsvRecord record : reader) {
106110
chunk.add(record.getFields().toArray(String[]::new));
111+
totalRows++;
107112
if (chunk.size() == chunkSize) {
108113
writer.writeChunk(buildChunk(schema, chunk));
109-
totalRows += chunk.size();
110-
reportProgress(options, totalRows);
111114
chunk.clear();
112115
}
116+
if (totalRows - lastReported >= PROGRESS_BATCH) {
117+
reportProgress(options, totalRows);
118+
lastReported = totalRows;
119+
}
113120
}
114121
if (!chunk.isEmpty()) {
115122
writer.writeChunk(buildChunk(schema, chunk));
116-
totalRows += chunk.size();
123+
}
124+
if (totalRows > lastReported) {
117125
reportProgress(options, totalRows);
118126
}
119127
}

0 commit comments

Comments
 (0)