Skip to content

Commit 38ab5c5

Browse files
dfa1claude
andcommitted
feat(writer): zone-map MIN/MAX for extension columns via storage primitive
flushZoneMaps gated min/max on `DType.Primitive`, so extension columns (e.g. vortex.timestamp over I64) emitted NULL_COUNT-only zone-maps even though ExtEncoding already propagates the storage array's min/max scalars to each chunk. Generalise the gate via zoneStatPType, which unwraps an Extension to its storage primitive; the per-zone stat column is stored as that primitive, matching Rust (stats computed on the storage array). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 8d58d2b commit 38ab5c5

2 files changed

Lines changed: 54 additions & 4 deletions

File tree

writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -706,17 +706,19 @@ private void flushZoneMaps() throws IOException {
706706
java.util.Arrays.fill(allValid, true);
707707

708708
// NULL_COUNT is computable for every column type; MIN/MAX only for fixed-width
709-
// primitives whose chunks all carry stats. Field/bit order follows
710-
// ZonedStatsSchema: MAX(3), MIN(4), NULL_COUNT(6); each stat field is nullable.
709+
// primitives whose chunks all carry stats. Extension columns unwrap to their storage
710+
// primitive — ExtEncoding propagates the storage min/max, stored here as that primitive.
711+
// Field/bit order follows ZonedStatsSchema: MAX(3), MIN(4), NULL_COUNT(6); each nullable.
711712
DType colDtype = schema.fieldTypes().get(schema.fieldNames().indexOf(colName));
712-
boolean hasMinMax = colDtype instanceof DType.Primitive
713+
PType statPtype = zoneStatPType(colDtype);
714+
boolean hasMinMax = statPtype != null
713715
&& chunks.stream().allMatch(ChunkRef::hasStats);
714716

715717
List<String> names = new java.util.ArrayList<>();
716718
List<DType> types = new java.util.ArrayList<>();
717719
List<Object> fields = new java.util.ArrayList<>();
718720
if (hasMinMax) {
719-
PType ptype = ((DType.Primitive) colDtype).ptype();
721+
PType ptype = statPtype;
720722
DType nullablePrim = new DType.Primitive(ptype, true);
721723
boolean[] notTruncated = new boolean[nZones];
722724
names.add("max");
@@ -804,6 +806,17 @@ private static long countNulls(boolean[] validity) {
804806
return nulls;
805807
}
806808

809+
/// The primitive type whose min/max a zone-map stores for `dtype`, or `null` when the column
810+
/// has no fixed-width min/max. Extension columns resolve to their storage primitive, since
811+
/// `ExtEncoding` propagates the storage array's min/max scalars unchanged.
812+
private static PType zoneStatPType(DType dtype) {
813+
return switch (dtype) {
814+
case DType.Primitive p -> p.ptype();
815+
case DType.Extension ext when ext.storageDType() instanceof DType.Primitive p -> p.ptype();
816+
default -> null;
817+
};
818+
}
819+
807820
/// Builds the per-zone min (or max) values array in the storage shape the primitive encoder
808821
/// expects, decoding each chunk's serialised [ScalarValue] stat.
809822
private static Object statColumn(PType ptype, List<ChunkRef> chunks, boolean max) throws IOException {

writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,43 @@ void nonPrimitiveColumn_emitsNullCountOnlyZoneMap(@TempDir Path tmp) throws IOEx
255255
}
256256
}
257257

258+
@Test
259+
void extensionColumn_emitsStorageMinMaxZoneMap(@TempDir Path tmp) throws IOException {
260+
// Given an extension column over I64 storage written as a raw long[] (no spec auto-route),
261+
// two zones of two rows: zone 0 = [10, 11], zone 1 = [20, 21]. ExtEncoding propagates the
262+
// storage primitive's min/max, so the zone-map carries MAX+MIN+NULL_COUNT — same as I64.
263+
DType ext = new DType.Extension(
264+
"test.ext", new DType.Primitive(PType.I64, false), null, false);
265+
DType.Struct schema = new DType.Struct(List.of("t"), List.of(ext), false);
266+
WriteOptions opts = new WriteOptions(2, true, 0.90, 0, false, false);
267+
Path file = tmp.resolve("ext.vtx");
268+
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
269+
var sut = VortexWriter.create(ch, schema, opts)) {
270+
sut.writeChunk(Map.of("t", new long[]{10, 11}));
271+
sut.writeChunk(Map.of("t", new long[]{20, 21}));
272+
}
273+
274+
// When / Then the column is zoned with MAX+MIN+NULL_COUNT and the stats decode per zone
275+
try (VortexReader reader = VortexReader.open(file)) {
276+
Layout column = reader.layout().children().get(0);
277+
assertThat(column.isZoned()).isTrue();
278+
ByteBuffer meta = column.metadata().duplicate().order(ByteOrder.LITTLE_ENDIAN);
279+
assertThat(meta.get(meta.position() + 4)).isEqualTo((byte) 0x58); // MAX+MIN+NULL_COUNT
280+
281+
Layout zonesFlat = column.children().get(1);
282+
SegmentSpec spec = reader.footer().segmentSpecs().get(zonesFlat.segments().getFirst());
283+
try (Arena arena = Arena.ofConfined()) {
284+
StructArray stats = (StructArray) reader.decodeFlatSegment(spec, statsTableDtype(), 2, arena);
285+
LongArray max = (LongArray) ((MaskedArray) stats.field("max")).inner();
286+
LongArray min = (LongArray) ((MaskedArray) stats.field("min")).inner();
287+
assertThat(min.getLong(0)).isEqualTo(10);
288+
assertThat(max.getLong(0)).isEqualTo(11);
289+
assertThat(min.getLong(1)).isEqualTo(20);
290+
assertThat(max.getLong(1)).isEqualTo(21);
291+
}
292+
}
293+
}
294+
258295
@Test
259296
void dictColumn_emitsNullCountZoneMapWrappingDict(@TempDir Path tmp) throws IOException {
260297
// Given a low-cardinality Utf8 column across 2 chunks of 6 with heavy repeats (so the

0 commit comments

Comments
 (0)