Skip to content

Commit cb844f2

Browse files
dfa1claude
andcommitted
feat(reader): aggregate null_count across chunks in columnStats
Per-chunk null_count now flows into columnStats(): aggregateStats sums counts across flats, reporting the total only when every chunk carries one (a single missing count makes the column total unknown -> null). Also fixes the early-return guard, which dropped null_count for all-null columns that have no min/max. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 2799bdd commit cb844f2

2 files changed

Lines changed: 28 additions & 2 deletions

File tree

reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,10 @@ public Map<String, ArrayStats> columnStats() {
181181
private ArrayStats aggregateStats(List<Layout> flats) {
182182
Object globalMin = null;
183183
Object globalMax = null;
184+
// Sum is meaningful only when every chunk carries a null count; one missing makes the
185+
// column total unknown (null), so don't report a partial sum.
186+
long totalNullCount = 0L;
187+
boolean allHaveNullCount = !flats.isEmpty();
184188
for (Layout flat : flats) {
185189
ArrayStats s = readFlatStats(flat);
186190
if (s.min() != null) {
@@ -189,11 +193,17 @@ private ArrayStats aggregateStats(List<Layout> flats) {
189193
if (s.max() != null) {
190194
globalMax = globalMax == null ? s.max() : maxOf(globalMax, s.max());
191195
}
196+
if (s.nullCount() != null) {
197+
totalNullCount += s.nullCount();
198+
} else {
199+
allHaveNullCount = false;
200+
}
192201
}
193-
if (globalMin == null && globalMax == null) {
202+
Long nullCount = allHaveNullCount ? totalNullCount : null;
203+
if (globalMin == null && globalMax == null && nullCount == null) {
194204
return ArrayStats.empty();
195205
}
196-
return new ArrayStats(globalMin, globalMax, null, null, null, null);
206+
return new ArrayStats(globalMin, globalMax, null, nullCount, null, null);
197207
}
198208

199209
private ArrayStats readFlatStats(Layout flat) {

writer/src/test/java/io/github/dfa1/vortex/writer/NullCountPruningTest.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,4 +78,20 @@ void noFilter_keepsAllChunks() throws IOException {
7878
// Given / When / Then all three chunks survive
7979
assertThat(scanRowCounts(write(), null)).containsExactly(3L, 2L, 4L);
8080
}
81+
82+
@Test
83+
void columnStats_sumsNullCountAcrossChunks() throws IOException {
84+
// Given chunks carry 0, 1 and 4 nulls; every chunk records a null count so the
85+
// column total is known (0 + 1 + 4 = 5) rather than dropped to unknown
86+
Path file = write();
87+
88+
// When
89+
Long result;
90+
try (VortexReader reader = VortexReader.open(file)) {
91+
result = reader.columnStats().get("v").nullCount();
92+
}
93+
94+
// Then
95+
assertThat(result).isEqualTo(5L);
96+
}
8197
}

0 commit comments

Comments
 (0)