Skip to content

Commit 1370276

Browse files
dfa1claude
dfa1
and
claude
committed
test(integration): cover patched, fastlanes.delta, masked round-trips
Close three integration-coverage gaps found in the state review: - fastlanes.delta — Java→Rust round-trip (forced DeltaEncodingEncoder over a monotonic I64 ramp; Rust trims the padded 1024-chunk back to n rows). - vortex.masked — direct Masked-over-primitive Java→Rust round-trip (nullable I64 with interleaved nulls via the boxed Long[] builder path). Previously only covered indirectly through nullable_date (Masked → Ext → Primitive). - vortex.patched — the bundled vortex-jni build rejects a standalone patched array ("Unknown encoding: vortex.patched"), so this is a Java write → Java read round-trip in a new JavaRoundTripIntegrationTest. Still exercises the writer encode + file format + reader decode end to end; four outliers (< n/20) force the patch path. Full integration suite: 271 green, 2 @disabled (f16/uuid, JNI-blocked). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent e85b0d1 commit 1370276

2 files changed

Lines changed: 138 additions & 0 deletions

File tree

‎integration/src/test/java/io/github/dfa1/vortex/integration/JavaRoundTripIntegrationTest.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package io.github.dfa1.vortex.integration;
2+
3+
import io.github.dfa1.vortex.core.DType;
4+
import io.github.dfa1.vortex.core.PType;
5+
import io.github.dfa1.vortex.reader.ReadRegistry;
6+
import io.github.dfa1.vortex.reader.ScanOptions;
7+
import io.github.dfa1.vortex.reader.VortexReader;
8+
import io.github.dfa1.vortex.reader.array.IntArray;
9+
import io.github.dfa1.vortex.writer.VortexWriter;
10+
import io.github.dfa1.vortex.writer.WriteOptions;
11+
import io.github.dfa1.vortex.writer.encode.PatchedEncodingEncoder;
12+
import org.junit.jupiter.api.Test;
13+
import org.junit.jupiter.api.io.TempDir;
14+
15+
import java.io.IOException;
16+
import java.nio.channels.FileChannel;
17+
import java.nio.file.Path;
18+
import java.nio.file.StandardOpenOption;
19+
import java.util.ArrayList;
20+
import java.util.List;
21+
import java.util.Map;
22+
23+
import static org.assertj.core.api.Assertions.assertThat;
24+
25+
/// Java writer → Java reader round-trips for encodings the bundled `vortex-jni` build cannot read
26+
/// back, so they have no Java→Rust coverage. This is still a real cross-module integration test: it
27+
/// drives the writer's encode, the on-disk file format, and the reader's decode end to end.
28+
///
29+
/// `vortex.patched` is the case here — the JNI reader rejects a standalone patched array with
30+
/// "Unknown encoding: vortex.patched", so the round-trip is asserted on the Java side instead.
31+
class JavaRoundTripIntegrationTest {
32+
33+
private static final DType.Struct I32_SCHEMA = new DType.Struct(
34+
List.of("v"),
35+
List.of(new DType.Primitive(PType.I32, false)),
36+
false);
37+
38+
@Test
39+
void patched_i32_javaWriteJavaRead(@TempDir Path tmp) throws IOException {
40+
// Given — most values fit ~6 bits; four large outliers (< n/20) force the patch path
41+
// (base inner array + patch index / patch value children). Non-negative, so the bit width
42+
// is computed from the value itself rather than sign-extension.
43+
Path file = tmp.resolve("java_patched_i32.vtx");
44+
int[] data = new int[120];
45+
for (int i = 0; i < data.length; i++) {
46+
data[i] = i % 50;
47+
}
48+
data[7] = 5_000_000;
49+
data[23] = 6_000_000;
50+
data[61] = 7_000_000;
51+
data[88] = 8_000_000;
52+
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
53+
var sut = VortexWriter.create(ch, I32_SCHEMA, WriteOptions.defaults(),
54+
List.of(new PatchedEncodingEncoder()))) {
55+
// When
56+
sut.writeChunk(Map.of("v", data));
57+
}
58+
59+
// Then — the Java reader reconstructs base values + patched outliers exactly
60+
int[] decoded = readIntColumn(file, "v");
61+
assertThat(decoded).containsExactly(data);
62+
}
63+
64+
private static int[] readIntColumn(Path file, String column) throws IOException {
65+
try (var vf = VortexReader.open(file, ReadRegistry.loadAll());
66+
var iter = vf.scan(ScanOptions.columns(column))) {
67+
var ints = new ArrayList<Integer>();
68+
iter.forEachRemaining(c -> {
69+
IntArray arr = (IntArray) c.columns().get(column);
70+
for (long i = 0; i < arr.length(); i++) {
71+
ints.add(arr.getInt(i));
72+
}
73+
});
74+
return ints.stream().mapToInt(Integer::intValue).toArray();
75+
}
76+
}
77+
}

‎integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import io.github.dfa1.vortex.writer.encode.ByteBoolEncodingEncoder;
1616
import io.github.dfa1.vortex.writer.encode.ConstantEncodingEncoder;
1717
import io.github.dfa1.vortex.writer.encode.DateExtensionEncoder;
18+
import io.github.dfa1.vortex.writer.encode.DeltaEncodingEncoder;
1819
import io.github.dfa1.vortex.writer.encode.FsstEncodingEncoder;
1920
import io.github.dfa1.vortex.writer.encode.ListData;
2021
import io.github.dfa1.vortex.writer.encode.ListEncodingEncoder;
@@ -994,6 +995,66 @@ void javaWriter_rustReader_sparse_i32(@TempDir Path tmp) throws IOException {
994995
assertThat(decoded).containsExactly(data);
995996
}
996997

998+
@Test
999+
void javaWriter_rustReader_delta_i64(@TempDir Path tmp) throws IOException {
1000+
// Given — FastLanes delta: monotonic data with varied positive steps. The encoder pads to a
1001+
// 1024-element chunk and stores per-lane bases + deltas; Rust must trim back to n rows.
1002+
Path file = tmp.resolve("java_delta_i64.vtx");
1003+
long[] data = new long[50];
1004+
long acc = 1_000L;
1005+
for (int i = 0; i < data.length; i++) {
1006+
acc += 1 + (i % 7);
1007+
data[i] = acc;
1008+
}
1009+
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
1010+
var sut = VortexWriter.create(ch, TS_SCHEMA, WriteOptions.defaults(),
1011+
List.of(new DeltaEncodingEncoder()))) {
1012+
// When
1013+
sut.writeChunk(Map.of("ts", data));
1014+
}
1015+
1016+
// Then
1017+
long[] decoded = readLongColumn(file, "ts");
1018+
assertThat(decoded).containsExactly(data);
1019+
}
1020+
1021+
@Test
1022+
void javaWriter_rustReader_masked_nullableI64(@TempDir Path tmp) throws IOException {
1023+
// Given — nullable primitive I64 with interleaved nulls. The default write path wraps a
1024+
// nullable column as MaskedEncoding → PrimitiveEncoding (values + validity bool child).
1025+
// This is the direct Masked-over-primitive layout; nullable_date only covers Masked → Ext.
1026+
Path file = tmp.resolve("java_masked_i64.vtx");
1027+
DType.Struct schema = new DType.Struct(
1028+
List.of("v"),
1029+
List.of(new DType.Primitive(PType.I64, true)),
1030+
false);
1031+
Long[] data = {10L, null, 30L, null, 50L, 60L};
1032+
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
1033+
var sut = VortexWriter.create(ch, schema, WriteOptions.defaults())) {
1034+
// When — boxed Long[] via the builder routes through the nullable → masked path
1035+
sut.writeChunk(c -> c.put("v", data));
1036+
}
1037+
1038+
// Then — Rust reads a nullable BigInt vector; null positions survive, values round-trip
1039+
String uri = file.toAbsolutePath().toUri().toString();
1040+
DataSource ds = DataSource.open(SESSION, uri);
1041+
Scan scan = ds.scan(ScanOptions.of());
1042+
var values = new ArrayList<Long>();
1043+
while (scan.hasNext()) {
1044+
Partition partition = scan.next();
1045+
try (ArrowReader reader = partition.scanArrow(ALLOCATOR)) {
1046+
while (reader.loadNextBatch()) {
1047+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
1048+
BigIntVector vec = (BigIntVector) root.getVector("v");
1049+
for (int i = 0; i < root.getRowCount(); i++) {
1050+
values.add(vec.isNull(i) ? null : vec.get(i));
1051+
}
1052+
}
1053+
}
1054+
}
1055+
assertThat(values).containsExactly(10L, null, 30L, null, 50L, 60L);
1056+
}
1057+
9971058
@Test
9981059
void javaWriter_rustReader_bool_boolEncoding(@TempDir Path tmp) throws IOException {
9991060
// Given — BoolEncoding: bit-packed boolean column

0 commit comments

Comments
 (0)