Skip to content

Commit 7df3a0d

Browse files
dfa1claude
dfa1
and
claude
committed
feat(core): typed LayoutId; reader accepts vortex.zoned
LayoutId mirrors the sealed EncodingId shape — WellKnown constants (FLAT, CHUNKED, STRUCT, ZONED, STATS, DICT) plus Custom — because layouts are runtime-pluggable in the Rust reference (two separate footer spec namespaces sharing the string wire form; vortex.flat is layout-only). Layout's misnamed String encodingId component becomes LayoutId layoutId; unknown layouts still fail loudly (Rust default, no allowUnknown for layouts), now with a typed id in the error. Compat fix uncovered by the reference check: Rust renamed the zone-map layout id to vortex.zoned, keeping vortex.stats as legacy alias — the reader now routes BOTH through the zoned path, so files from current Rust writers scan and prune correctly. The writer keeps emitting vortex.stats, which old and new Rust readers accept; integration oracle confirms byte-identical output. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 194259d commit 7df3a0d

15 files changed

Lines changed: 426 additions & 89 deletions

File tree

‎cli/src/main/java/io/github/dfa1/vortex/cli/tui/VortexInspectorTui.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,7 @@ private String renderItem(Item item) {
442442
}
443443
String label = item.depth() == 0 && node.layout().isStruct()
444444
? "struct"
445-
: node.fieldName().map(n -> n + ": ").orElse("") + node.layout().encodingId();
445+
: node.fieldName().map(n -> n + ": ").orElse("") + node.layout().layoutId();
446446
String tag = statsChildren.contains(node) ? ", stats" : "";
447447
return " ".repeat(item.depth() * 2) + marker + label
448448
+ " (" + node.layout().rowCount() + " rows" + tag + ")";
@@ -467,7 +467,7 @@ private List<String> detailLines(InspectorTree.Node node) {
467467
List<String> lines = new ArrayList<>();
468468
Layout layout = node.layout();
469469
InspectorTree.Peek p = peek(node);
470-
lines.add("Encoding: " + (p.encoding() != null ? p.encoding() : layout.encodingId()));
470+
lines.add("Encoding: " + (p.encoding() != null ? p.encoding() : layout.layoutId().id()));
471471
node.fieldName().ifPresent(name -> lines.add("Field: " + name));
472472
String col = columnOf.get(node);
473473
if (col != null && !node.fieldName().isPresent()) {
@@ -705,14 +705,14 @@ private List<String> decodeStatsLayout(
705705
Layout child = statsLayout.children().get(i);
706706
if (!child.isFlat()) {
707707
throw new IllegalStateException(
708-
"non-flat stats chunk: " + child.encodingId());
708+
"non-flat stats chunk: " + child.layoutId());
709709
}
710710
all.addAll(decodeStatsFlat(child, statsDtype, arena));
711711
}
712712
return all;
713713
}
714714
throw new IllegalStateException(
715-
"unsupported stats layout: " + statsLayout.encodingId());
715+
"unsupported stats layout: " + statsLayout.layoutId());
716716
}
717717

718718
private List<String> decodeStatsFlat(

‎core/src/main/java/io/github/dfa1/vortex/core/model/LayoutId.java‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
package io.github.dfa1.vortex.core.model;
2+
3+
import java.util.Map;
4+
import java.util.Objects;
5+
import java.util.function.Function;
6+
import java.util.stream.Collectors;
7+
import java.util.stream.Stream;
8+
9+
/// Identity of a layout node — either a spec [WellKnown] constant or a third-party [Custom] id.
10+
///
11+
/// Layout ids and array-encoding ids ([EncodingId]) are separate namespaces — the Vortex footer
12+
/// carries two dictionary tables (`layout_specs` and `array_specs`) — even though some strings
13+
/// (`vortex.chunked`, `vortex.struct`, `vortex.dict`) appear in both. Layouts are runtime-pluggable
14+
/// upstream, so this type is open: [#parse(String)] maps any wire string to a typed value, and
15+
/// [#id()] recovers the wire string from a typed value.
16+
public sealed interface LayoutId permits LayoutId.WellKnown, LayoutId.Custom {
17+
18+
/// Returns the wire string of this layout id (e.g. `"vortex.flat"`).
19+
///
20+
/// @return the wire string of this layout id
21+
String id();
22+
23+
/// Parses a wire string into its typed representation: the matching [WellKnown] constant,
24+
/// else a [Custom] wrapping the raw string. Total over every non-blank string; blank input
25+
/// is not a valid layout id and is rejected by the [Custom] constructor — callers parsing
26+
/// untrusted input must guard blank ids and raise their own domain error.
27+
///
28+
/// @param raw the raw layout id string (e.g. `"vortex.flat"`)
29+
/// @return the matching [WellKnown] constant, or a [Custom] wrapping `raw` if none matches
30+
/// @throws NullPointerException if `raw` is `null`
31+
/// @throws IllegalArgumentException if `raw` is blank
32+
static LayoutId parse(String raw) {
33+
WellKnown known = WellKnown.byId(raw);
34+
return known != null ? known : new Custom(raw);
35+
}
36+
37+
/// Layout ids defined by the Vortex specification and understood by this build.
38+
enum WellKnown implements LayoutId {
39+
/// Flat (leaf) layout — a single encoded segment (`vortex.flat`).
40+
FLAT("vortex.flat"),
41+
/// Chunked layout — a sequence of flat layouts (`vortex.chunked`).
42+
CHUNKED("vortex.chunked"),
43+
/// Struct layout — one child per column (`vortex.struct`).
44+
STRUCT("vortex.struct"),
45+
/// Zone-map layout — wraps a child with per-chunk stats for pruning; the current canonical
46+
/// wire id upstream (`vortex.zoned`).
47+
ZONED("vortex.zoned"),
48+
/// Legacy wire alias of the zoned layout (`vortex.stats`). Rust keeps a legacy vtable for
49+
/// it, and vortex-java currently writes it; readers must route it to the zoned handling.
50+
STATS("vortex.stats"),
51+
/// Dictionary layout for low-cardinality columns (`vortex.dict`).
52+
DICT("vortex.dict"),
53+
;
54+
55+
// O(1) access to a WellKnown constant by its string representation
56+
private static final Map<String, WellKnown> LOOKUP = Stream.of(values())
57+
.collect(Collectors.toUnmodifiableMap(WellKnown::id, Function.identity()));
58+
private final String id;
59+
60+
WellKnown(String id) {
61+
this.id = id;
62+
}
63+
64+
/// Returns the well-known constant whose wire string is `id`, or `null` if none matches.
65+
///
66+
/// @param id the wire string to look up (may be `null`)
67+
/// @return the matching constant, or `null` if unrecognized
68+
static WellKnown byId(String id) {
69+
return LOOKUP.get(id);
70+
}
71+
72+
@Override
73+
public String id() {
74+
return id;
75+
}
76+
77+
@Override
78+
public String toString() {
79+
return id;
80+
}
81+
}
82+
83+
/// A third-party layout id whose wire string is not part of the [WellKnown] set.
84+
///
85+
/// @param id the wire string of this layout id; must be non-blank and must not collide
86+
/// with a [WellKnown] wire string
87+
record Custom(String id) implements LayoutId {
88+
89+
/// Validates that `id` is a usable custom layout id.
90+
///
91+
/// @param id the wire string of this layout id
92+
/// @throws NullPointerException if `id` is `null`
93+
/// @throws IllegalArgumentException if `id` is blank or matches a [WellKnown] wire string
94+
public Custom {
95+
Objects.requireNonNull(id, "id");
96+
if (id.isBlank()) {
97+
throw new IllegalArgumentException("layout id must not be blank");
98+
}
99+
WellKnown wellKnown = WellKnown.byId(id);
100+
if (wellKnown != null) {
101+
throw new IllegalArgumentException(
102+
"\"" + id + "\" is a well-known layout id; use LayoutId." + wellKnown.name() + " instead");
103+
}
104+
}
105+
106+
@Override
107+
public String toString() {
108+
return id;
109+
}
110+
}
111+
112+
// Re-export every WellKnown constant, typed as WellKnown, so `LayoutId.FLAT` call sites stay
113+
// usable wherever a WellKnown is required.
114+
115+
/// Well-known `vortex.flat` id.
116+
WellKnown FLAT = WellKnown.FLAT;
117+
/// Well-known `vortex.chunked` id.
118+
WellKnown CHUNKED = WellKnown.CHUNKED;
119+
/// Well-known `vortex.struct` id.
120+
WellKnown STRUCT = WellKnown.STRUCT;
121+
/// Well-known `vortex.zoned` id.
122+
WellKnown ZONED = WellKnown.ZONED;
123+
/// Well-known `vortex.stats` id.
124+
WellKnown STATS = WellKnown.STATS;
125+
/// Well-known `vortex.dict` id.
126+
WellKnown DICT = WellKnown.DICT;
127+
}

‎core/src/test/java/io/github/dfa1/vortex/core/model/LayoutIdTest.java‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
package io.github.dfa1.vortex.core.model;
2+
3+
import org.junit.jupiter.api.Nested;
4+
import org.junit.jupiter.api.Test;
5+
import org.junit.jupiter.params.ParameterizedTest;
6+
import org.junit.jupiter.params.provider.EnumSource;
7+
import org.junit.jupiter.params.provider.ValueSource;
8+
9+
import static org.assertj.core.api.Assertions.assertThat;
10+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
11+
12+
class LayoutIdTest {
13+
14+
@Nested
15+
class Parse {
16+
17+
@ParameterizedTest
18+
@EnumSource(LayoutId.WellKnown.class)
19+
void parse_knownId_returnsMatchingConstant(LayoutId.WellKnown id) {
20+
// Given the wire string of a well-known constant
21+
// When
22+
LayoutId result = LayoutId.parse(id.id());
23+
// Then the same constant comes back
24+
assertThat(result).isSameAs(id);
25+
}
26+
27+
@Test
28+
void parse_unknownId_returnsCustomWrappingRawId() {
29+
// Given a wire string no build knows about
30+
String raw = "supermario";
31+
// When — parse is total, so a miss is a typed Custom rather than an empty Optional
32+
LayoutId result = LayoutId.parse(raw);
33+
// Then
34+
assertThat(result).isEqualTo(new LayoutId.Custom(raw));
35+
assertThat(result.id()).isEqualTo(raw);
36+
}
37+
38+
@ParameterizedTest
39+
@ValueSource(strings = {"", " ", " "})
40+
void parse_blankId_throwsIllegalArgumentException(String blank) {
41+
// Given / When / Then — blank is not a valid id; parse must not silently wrap it,
42+
// so untrusted-input callers are forced to guard it into their own domain error
43+
assertThatThrownBy(() -> LayoutId.parse(blank))
44+
.isInstanceOf(IllegalArgumentException.class);
45+
}
46+
}
47+
48+
@Nested
49+
class CustomInvariants {
50+
51+
@Test
52+
void construct_nullId_throwsNullPointerException() {
53+
// Given / When / Then — a Custom must always carry a wire string
54+
assertThatThrownBy(() -> new LayoutId.Custom(null))
55+
.isInstanceOf(NullPointerException.class);
56+
}
57+
58+
@Test
59+
void construct_blankId_throwsIllegalArgumentException() {
60+
// Given / When / Then — blank ids have no wire representation
61+
assertThatThrownBy(() -> new LayoutId.Custom(" "))
62+
.isInstanceOf(IllegalArgumentException.class);
63+
}
64+
65+
@Test
66+
void construct_wellKnownId_throwsIllegalArgumentException() {
67+
// Given a wire string that already names a well-known constant
68+
// When / Then — Custom refuses to shadow it and points at the constant to use instead
69+
assertThatThrownBy(() -> new LayoutId.Custom("vortex.flat"))
70+
.isInstanceOf(IllegalArgumentException.class)
71+
.hasMessageContaining("FLAT");
72+
}
73+
}
74+
75+
@Nested
76+
class Properties {
77+
78+
@ParameterizedTest
79+
@EnumSource(LayoutId.WellKnown.class)
80+
void id_isNonBlankString(LayoutId.WellKnown id) {
81+
// Given / When / Then
82+
assertThat(id.id()).isNotBlank();
83+
}
84+
85+
@ParameterizedTest
86+
@EnumSource(LayoutId.WellKnown.class)
87+
void toString_equalsId(LayoutId.WellKnown id) {
88+
// Given / When / Then
89+
assertThat(id).hasToString(id.id());
90+
}
91+
}
92+
}

‎inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ private static String format(Object v) {
145145
}
146146

147147
private static void appendLayoutInline(StringBuilder sb, Layout layout) {
148-
sb.append(layout.encodingId()).append('(').append(layout.rowCount()).append(" rows)");
148+
sb.append(layout.layoutId()).append('(').append(layout.rowCount()).append(" rows)");
149149
if (layout.children().isEmpty()) {
150150
return;
151151
}

‎inspector/src/test/java/io/github/dfa1/vortex/inspect/InspectorTreeTest.java‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import io.github.dfa1.vortex.reader.CompressionScheme;
44
import io.github.dfa1.vortex.core.model.DType;
5+
import io.github.dfa1.vortex.core.model.LayoutId;
56
import io.github.dfa1.vortex.reader.Footer;
67
import io.github.dfa1.vortex.reader.Layout;
78
import io.github.dfa1.vortex.reader.SegmentSpec;
@@ -136,9 +137,9 @@ void build_reportsProgressOncePerPeekedSegment() {
136137
// Given — struct of two compressed (skipped) + two uncompressed Flat columns.
137138
// Only uncompressed leaves trigger peekFlatRoot, so progress should fire twice
138139
// with total=2.
139-
Layout c1 = new Layout("vortex.flat", 0, null, List.of(), List.of(0));
140-
Layout c2 = new Layout("vortex.flat", 0, null, List.of(), List.of(1));
141-
Layout c3 = new Layout("vortex.flat", 0, null, List.of(), List.of(2));
140+
Layout c1 = new Layout(LayoutId.parse("vortex.flat"), 0, null, List.of(), List.of(0));
141+
Layout c2 = new Layout(LayoutId.parse("vortex.flat"), 0, null, List.of(), List.of(1));
142+
Layout c3 = new Layout(LayoutId.parse("vortex.flat"), 0, null, List.of(), List.of(2));
142143
Layout root = struct(0, List.of(c1, c2, c3));
143144
DType dtype = new DType.Struct(List.of("a", "b", "c"),
144145
List.of(DType.I32,
@@ -178,8 +179,8 @@ void buildShallow_skipsAllSlicesAndStillNamesColumns() {
178179
// Given — shallow build is the path the TUI uses; it must touch zero segment
179180
// bytes (so opening a remote file is instant) yet still populate fieldName on
180181
// top-level struct children.
181-
Layout col0 = new Layout("vortex.flat", 10, null, List.of(), List.of(0));
182-
Layout col1 = new Layout("vortex.flat", 10, null, List.of(), List.of(1));
182+
Layout col0 = new Layout(LayoutId.parse("vortex.flat"), 10, null, List.of(), List.of(0));
183+
Layout col1 = new Layout(LayoutId.parse("vortex.flat"), 10, null, List.of(), List.of(1));
183184
Layout root = struct(10, List.of(col0, col1));
184185
DType dtype = new DType.Struct(List.of("id", "value"),
185186
List.of(DType.I64,
@@ -226,7 +227,7 @@ void peek_nonFlatNode_returnsEmptyWithoutSlicing() {
226227
void peek_compressedFlatSegment_returnsEmptyWithoutSlicing() {
227228
// Given — compressed segments would need the encoding to decompress before
228229
// their FlatBuffer can be parsed; peek skips them rather than slicing garbage.
229-
Layout flat = new Layout("vortex.flat", 10, null, List.of(), List.of(0));
230+
Layout flat = new Layout(LayoutId.parse("vortex.flat"), 10, null, List.of(), List.of(0));
230231
InspectorTree.Node node = new InspectorTree.Node(flat, java.util.Optional.empty(),
231232
Set.of(), io.github.dfa1.vortex.reader.ArrayStats.empty(), List.of());
232233
given(handle.footer()).willReturn(new io.github.dfa1.vortex.reader.Footer(
@@ -248,7 +249,7 @@ void build_flatChildWithCompressedSegment_skipsRootEncodingPeek() {
248249
// Given — peekRootEncoding() reads the segment as a FlatBuffer; compressed segments
249250
// are intentionally skipped so a malformed or compressed payload can't crash the
250251
// inspector. With code != NONE we should still build a tree, with no encodings used.
251-
Layout root = new Layout("vortex.flat", 0, null, List.of(), List.of(0));
252+
Layout root = new Layout(LayoutId.parse("vortex.flat"), 0, null, List.of(), List.of(0));
252253
DType dtype = DType.I32;
253254
SegmentSpec compressed = new SegmentSpec(0, 1024, (byte) 0, CompressionScheme.ZSTD);
254255
givenHandle(dtype, root, List.of("vortex.flat"), List.of(compressed));
@@ -268,10 +269,10 @@ private void givenHandle(DType dtype, Layout layout, List<String> arraySpecs, Li
268269
}
269270

270271
private static Layout struct(long rows, List<Layout> children) {
271-
return new Layout("vortex.struct", rows, null, children, List.of());
272+
return new Layout(LayoutId.parse("vortex.struct"), rows, null, children, List.of());
272273
}
273274

274275
private static Layout leaf(String encodingId, long rows) {
275-
return new Layout(encodingId, rows, null, List.of(), List.of());
276+
return new Layout(LayoutId.parse(encodingId), rows, null, List.of(), List.of());
276277
}
277278
}

‎inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import io.github.dfa1.vortex.reader.ArrayStats;
44
import io.github.dfa1.vortex.reader.CompressionScheme;
55
import io.github.dfa1.vortex.core.model.DType;
6+
import io.github.dfa1.vortex.core.model.LayoutId;
67
import io.github.dfa1.vortex.reader.Layout;
78
import io.github.dfa1.vortex.reader.SegmentSpec;
89
import org.junit.jupiter.api.Test;
@@ -67,7 +68,7 @@ void render_segmentTable_listsEverySegment() {
6768
@Test
6869
void render_nonStruct_inlinesSingleColumnLayout() {
6970
// Given
70-
Layout leaf = new Layout("vortex.flat", 100, null, List.of(), List.of());
71+
Layout leaf = new Layout(LayoutId.parse("vortex.flat"), 100, null, List.of(), List.of());
7172
InspectorTree.Node root = new InspectorTree.Node(leaf, Optional.empty(), Set.of(), ArrayStats.empty(), List.of());
7273
InspectorTree sut = new InspectorTree(
7374
1, 256L,
@@ -101,10 +102,10 @@ void render_formatsBytesAcrossUnits() {
101102
@Test
102103
void render_chainsChildrenWithArrow() {
103104
// Given — nested zoned → chunked → flat chain
104-
Layout flat = new Layout("vortex.flat", 1000, null, List.of(), List.of());
105-
Layout chunked = new Layout("vortex.chunked", 1000, null, List.of(flat), List.of());
106-
Layout zoned = new Layout("vortex.stats", 1000, null, List.of(chunked), List.of());
107-
Layout structLayout = new Layout("vortex.struct", 1000, null, List.of(zoned), List.of());
105+
Layout flat = new Layout(LayoutId.parse("vortex.flat"), 1000, null, List.of(), List.of());
106+
Layout chunked = new Layout(LayoutId.parse("vortex.chunked"), 1000, null, List.of(flat), List.of());
107+
Layout zoned = new Layout(LayoutId.parse("vortex.stats"), 1000, null, List.of(chunked), List.of());
108+
Layout structLayout = new Layout(LayoutId.parse("vortex.struct"), 1000, null, List.of(zoned), List.of());
108109

109110
InspectorTree.Node flatN = new InspectorTree.Node(flat, Optional.empty(), Set.of(), ArrayStats.empty(), List.of());
110111
InspectorTree.Node chunkedN = new InspectorTree.Node(chunked, Optional.empty(), Set.of(), ArrayStats.empty(), List.of(flatN));
@@ -127,10 +128,10 @@ void render_chainsChildrenWithArrow() {
127128
@Test
128129
void render_aggregatesMinMaxAcrossChunks() {
129130
// Given — column with two chunked Flat leaves; aggregate should fold each leaf's stats
130-
Layout chunk1 = new Layout("vortex.flat", 500, null, List.of(), List.of());
131-
Layout chunk2 = new Layout("vortex.flat", 500, null, List.of(), List.of());
132-
Layout chunked = new Layout("vortex.chunked", 1000, null, List.of(chunk1, chunk2), List.of());
133-
Layout structLayout = new Layout("vortex.struct", 1000, null, List.of(chunked), List.of());
131+
Layout chunk1 = new Layout(LayoutId.parse("vortex.flat"), 500, null, List.of(), List.of());
132+
Layout chunk2 = new Layout(LayoutId.parse("vortex.flat"), 500, null, List.of(), List.of());
133+
Layout chunked = new Layout(LayoutId.parse("vortex.chunked"), 1000, null, List.of(chunk1, chunk2), List.of());
134+
Layout structLayout = new Layout(LayoutId.parse("vortex.struct"), 1000, null, List.of(chunked), List.of());
134135

135136
InspectorTree.Node c1 = new InspectorTree.Node(chunk1, Optional.empty(), Set.of(),
136137
new ArrayStats(10L, 50L, null, null, null, null, null), List.of());
@@ -178,9 +179,9 @@ void render_emptyUsedEncodings_omitsBracketSuffix() {
178179
}
179180

180181
private static InspectorTree struct2col(int version, long fileSize, List<SegmentSpec> specs, Set<String> usedById) {
181-
Layout idLeaf = new Layout("fastlanes.bitpacked", 1000, null, List.of(), List.of());
182-
Layout valLeaf = new Layout("vortex.constant", 1000, null, List.of(), List.of());
183-
Layout root = new Layout("vortex.struct", 1000, null, List.of(idLeaf, valLeaf), List.of());
182+
Layout idLeaf = new Layout(LayoutId.parse("fastlanes.bitpacked"), 1000, null, List.of(), List.of());
183+
Layout valLeaf = new Layout(LayoutId.parse("vortex.constant"), 1000, null, List.of(), List.of());
184+
Layout root = new Layout(LayoutId.parse("vortex.struct"), 1000, null, List.of(idLeaf, valLeaf), List.of());
184185

185186
InspectorTree.Node idNode = new InspectorTree.Node(idLeaf,
186187
Optional.of("id"), Set.of("fastlanes.bitpacked"), ArrayStats.empty(), List.of());

0 commit comments

Comments
 (0)