Skip to content

Commit d7953e1

Browse files
dfa1claude
dfa1
and
claude
committed
refactor(reader): drop FoR Materialized fallbacks (ADR 0013)
Add LazyForByteArray and LazyForShortArray for I8/U8 and I16/U16 columns. Drop materialiseEager and applyReference — all integer PTypes now return a Lazy array. F64 was dead code (encoder rejects floats); replaced with a throw in the switch default. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent cd59fef commit d7953e1

6 files changed

Lines changed: 253 additions & 38 deletions

File tree

‎docs/compatibility.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ decoder falls into one of three shapes:
121121
| `vortex.pco` | Materialized | Materialized | range-encoded decompression |
122122
| `fastlanes.bitpacked` | Materialized | Materialized | window unpacks bits |
123123
| `fastlanes.delta` | Materialized | Materialized | cumulative sum requires sequential decode |
124-
| `fastlanes.for` | Lazy | Lazy | `LazyForLongArray`/`LazyForIntArray`, ADR 0010 |
124+
| `fastlanes.for` | Lazy | Lazy | `LazyForXxxArray` (I8/U8/I16/U16/I32/U32/I64/U64), ADR 0010 + 0013 |
125125
| `fastlanes.rle` | Materialized | Lazy | run-locating accessor possible |
126126
| `vortex.patched` | Materialized | Materialized | inner is full base + chunked patches (1024-elem blocks, lane-window-sorted); per-row access requires 2 laneOffsets reads + binary search inside the chunk window, so eager scatter wins for full scans |
127127
| `vortex.variant` | Materialized | TBD | shredded child reassembly |

‎reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyForByteArray.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package io.github.dfa1.vortex.reader.array;
2+
3+
import io.github.dfa1.vortex.core.DType;
4+
import io.github.dfa1.vortex.core.PType;
5+
6+
import java.lang.foreign.MemorySegment;
7+
import java.lang.foreign.ValueLayout;
8+
import java.util.function.LongBinaryOperator;
9+
10+
/// Lazy [ByteArray] backed by the `fastlanes.for` encoded `i8`/`u8` child segment.
11+
///
12+
/// Decode is deferred to element access: `getByte(i) = encoded[i] + ref` (byte-wrapping addition).
13+
/// Returned by {@link io.github.dfa1.vortex.reader.decode.FrameOfReferenceEncodingDecoder} when
14+
/// `ref != 0` for I8 or U8 columns.
15+
///
16+
/// @param dtype logical I8/U8 type
17+
/// @param length number of logical elements
18+
/// @param encoded backing `i8` segment (one byte per row, frame-of-reference residual)
19+
/// @param ref reference byte added to each encoded element (frame value)
20+
public record LazyForByteArray(DType dtype, long length, MemorySegment encoded, byte ref)
21+
implements ByteArray {
22+
23+
@Override
24+
public byte getByte(long i) {
25+
return (byte) (encoded.get(ValueLayout.JAVA_BYTE, i) + ref);
26+
}
27+
28+
@Override
29+
public int getInt(long i) {
30+
byte raw = getByte(i);
31+
boolean unsigned = dtype instanceof DType.Primitive p && p.ptype() == PType.U8;
32+
return unsigned ? Byte.toUnsignedInt(raw) : (int) raw;
33+
}
34+
35+
@Override
36+
public long fold(long identity, LongBinaryOperator op) {
37+
MemorySegment seg = encoded;
38+
long n = length;
39+
long result = identity;
40+
byte r = ref;
41+
boolean unsigned = dtype instanceof DType.Primitive p && p.ptype() == PType.U8;
42+
for (long i = 0; i < n; i++) {
43+
byte v = (byte) (seg.get(ValueLayout.JAVA_BYTE, i) + r);
44+
result = op.applyAsLong(result, unsigned ? Byte.toUnsignedLong(v) : (long) v);
45+
}
46+
return result;
47+
}
48+
}

‎reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyForShortArray.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package io.github.dfa1.vortex.reader.array;
2+
3+
import io.github.dfa1.vortex.core.DType;
4+
import io.github.dfa1.vortex.core.PType;
5+
import io.github.dfa1.vortex.encoding.PTypeIO;
6+
7+
import java.lang.foreign.MemorySegment;
8+
import java.util.function.LongBinaryOperator;
9+
10+
/// Lazy [ShortArray] backed by the `fastlanes.for` encoded `i16`/`u16` child segment.
11+
///
12+
/// Decode is deferred to element access: `getShort(i) = encoded[i] + ref` (short-wrapping addition).
13+
/// Returned by {@link io.github.dfa1.vortex.reader.decode.FrameOfReferenceEncodingDecoder} when
14+
/// `ref != 0` for I16 or U16 columns.
15+
///
16+
/// @param dtype logical I16/U16 type
17+
/// @param length number of logical elements
18+
/// @param encoded backing `i16` segment (one short per row, frame-of-reference residual)
19+
/// @param ref reference short added to each encoded element (frame value)
20+
public record LazyForShortArray(DType dtype, long length, MemorySegment encoded, short ref)
21+
implements ShortArray {
22+
23+
@Override
24+
public short getShort(long i) {
25+
return (short) (encoded.getAtIndex(PTypeIO.LE_SHORT, i) + ref);
26+
}
27+
28+
@Override
29+
public int getInt(long i) {
30+
short raw = getShort(i);
31+
boolean unsigned = dtype instanceof DType.Primitive p && p.ptype() == PType.U16;
32+
return unsigned ? Short.toUnsignedInt(raw) : (int) raw;
33+
}
34+
35+
@Override
36+
public long fold(long identity, LongBinaryOperator op) {
37+
MemorySegment seg = encoded;
38+
long n = length;
39+
long result = identity;
40+
short r = ref;
41+
boolean unsigned = dtype instanceof DType.Primitive p && p.ptype() == PType.U16;
42+
for (long i = 0; i < n; i++) {
43+
short v = (short) (seg.getAtIndex(PTypeIO.LE_SHORT, i) + r);
44+
result = op.applyAsLong(result, unsigned ? Short.toUnsignedLong(v) : (long) v);
45+
}
46+
return result;
47+
}
48+
}

‎reader/src/main/java/io/github/dfa1/vortex/reader/decode/FrameOfReferenceEncodingDecoder.java‎

Lines changed: 5 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,20 @@
11
package io.github.dfa1.vortex.reader.decode;
22

33
import io.github.dfa1.vortex.core.DType;
4-
import io.github.dfa1.vortex.core.PType;
54
import io.github.dfa1.vortex.core.VortexException;
65
import io.github.dfa1.vortex.encoding.EncodingId;
7-
import io.github.dfa1.vortex.encoding.PTypeIO;
86
import io.github.dfa1.vortex.proto.ScalarValue;
97
import io.github.dfa1.vortex.reader.array.Array;
108
import io.github.dfa1.vortex.reader.array.ArraySegments;
119
import io.github.dfa1.vortex.reader.array.BoolArray;
10+
import io.github.dfa1.vortex.reader.array.LazyForByteArray;
1211
import io.github.dfa1.vortex.reader.array.LazyForIntArray;
1312
import io.github.dfa1.vortex.reader.array.LazyForLongArray;
13+
import io.github.dfa1.vortex.reader.array.LazyForShortArray;
1414
import io.github.dfa1.vortex.reader.array.MaskedArray;
15-
import io.github.dfa1.vortex.reader.array.MaterializedByteArray;
16-
import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray;
17-
import io.github.dfa1.vortex.reader.array.MaterializedShortArray;
1815

1916
import java.io.IOException;
2017
import java.lang.foreign.MemorySegment;
21-
import java.lang.foreign.SegmentAllocator;
22-
import java.lang.foreign.ValueLayout;
2318
import java.nio.ByteBuffer;
2419

2520
/// Read-only decoder for `fastlanes.for` (Frame of Reference).
@@ -76,21 +71,13 @@ public Array decode(DecodeContext ctx) {
7671
Array result = switch (p.ptype()) {
7772
case I64, U64 -> new LazyForLongArray(ctx.dtype(), n, src, ref);
7873
case I32, U32 -> new LazyForIntArray(ctx.dtype(), n, src, (int) ref);
79-
default -> materialiseEager(ctx, src, n, p.ptype(), ref);
74+
case I16, U16 -> new LazyForShortArray(ctx.dtype(), n, src, (short) ref);
75+
case I8, U8 -> new LazyForByteArray(ctx.dtype(), n, src, (byte) ref);
76+
default -> throw new VortexException(EncodingId.FASTLANES_FOR, "unsupported ptype " + p.ptype());
8077
};
8178
return validity != null ? new MaskedArray(result, validity) : result;
8279
}
8380

84-
private static Array materialiseEager(DecodeContext ctx, MemorySegment src, long n, PType ptype, long ref) {
85-
MemorySegment dst = applyReference(src, n, ptype, ref, ctx.arena());
86-
return switch (ptype) {
87-
case F64 -> new MaterializedDoubleArray(ctx.dtype(), n, dst);
88-
case I16, U16 -> new MaterializedShortArray(ctx.dtype(), n, dst);
89-
case I8, U8 -> new MaterializedByteArray(ctx.dtype(), n, dst);
90-
default -> throw new VortexException(EncodingId.FASTLANES_FOR, "unsupported ptype " + ptype);
91-
};
92-
}
93-
9481
private static long referenceValue(ScalarValue scalar) {
9582
if (scalar.int64_value() != null) {
9683
return scalar.int64_value();
@@ -100,23 +87,4 @@ private static long referenceValue(ScalarValue scalar) {
10087
}
10188
return 0L;
10289
}
103-
104-
private static MemorySegment applyReference(MemorySegment src, long n, PType ptype, long ref, SegmentAllocator arena) {
105-
int wordBytes = ptype.byteSize();
106-
MemorySegment dst = arena.allocate(n * wordBytes);
107-
switch (ptype) {
108-
case I8, U8 -> {
109-
for (long off = 0, end = n; off < end; off++) {
110-
dst.set(ValueLayout.JAVA_BYTE, off, (byte) (src.get(ValueLayout.JAVA_BYTE, off) + (byte) ref));
111-
}
112-
}
113-
case I16, U16 -> {
114-
for (long off = 0, end = n * 2; off < end; off += 2) {
115-
dst.set(PTypeIO.LE_SHORT, off, (short) (src.get(PTypeIO.LE_SHORT, off) + (short) ref));
116-
}
117-
}
118-
default -> throw new VortexException(EncodingId.FASTLANES_FOR, "unsupported ptype " + ptype);
119-
}
120-
return dst;
121-
}
12290
}

‎reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyForByteArrayTest.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package io.github.dfa1.vortex.reader.array;
2+
3+
import io.github.dfa1.vortex.core.DType;
4+
import io.github.dfa1.vortex.core.PType;
5+
import org.junit.jupiter.api.Test;
6+
7+
import java.lang.foreign.Arena;
8+
import java.lang.foreign.MemorySegment;
9+
import java.lang.foreign.ValueLayout;
10+
import java.util.ArrayList;
11+
import java.util.List;
12+
13+
import static org.assertj.core.api.Assertions.assertThat;
14+
15+
class LazyForByteArrayTest {
16+
17+
private static final DType I8 = new DType.Primitive(PType.I8, false);
18+
19+
private static LazyForByteArray of(byte ref, byte... encoded) {
20+
MemorySegment seg = Arena.ofAuto().allocate(encoded.length);
21+
for (int i = 0; i < encoded.length; i++) {
22+
seg.set(ValueLayout.JAVA_BYTE, i, encoded[i]);
23+
}
24+
return new LazyForByteArray(I8, encoded.length, seg, ref);
25+
}
26+
27+
@Test
28+
void getByteAddsRef() {
29+
// Given — residuals [1, 2, 3] with ref=10 → decoded [11, 12, 13]
30+
LazyForByteArray sut = of((byte) 10, (byte) 1, (byte) 2, (byte) 3);
31+
32+
// When + Then
33+
assertThat(sut.getByte(0)).isEqualTo((byte) 11);
34+
assertThat(sut.getByte(1)).isEqualTo((byte) 12);
35+
assertThat(sut.getByte(2)).isEqualTo((byte) 13);
36+
}
37+
38+
@Test
39+
void getByteWrapsOnOverflow() {
40+
// Given — wrapping addition: 120 + 20 = 140 overflows to -116 for signed byte
41+
LazyForByteArray sut = of((byte) 20, (byte) 120);
42+
43+
// When + Then
44+
assertThat(sut.getByte(0)).isEqualTo((byte) 140);
45+
}
46+
47+
@Test
48+
void getIntWidensSignedByte() {
49+
// Given — ref=0, residual=-1 → decoded=-1, getInt returns -1 for I8
50+
LazyForByteArray sut = of((byte) 0, (byte) -1);
51+
52+
// When + Then
53+
assertThat(sut.getInt(0)).isEqualTo(-1);
54+
}
55+
56+
@Test
57+
void forEachByteVisitsAll() {
58+
// Given
59+
LazyForByteArray sut = of((byte) 5, (byte) 1, (byte) 2);
60+
List<Byte> got = new ArrayList<>();
61+
62+
// When
63+
sut.forEachByte(got::add);
64+
65+
// Then
66+
assertThat(got).containsExactly((byte) 6, (byte) 7);
67+
}
68+
69+
@Test
70+
void foldSumApplies() {
71+
// Given — residuals [1,2,3] ref=10 → decoded [11,12,13] → sum 36
72+
LazyForByteArray sut = of((byte) 10, (byte) 1, (byte) 2, (byte) 3);
73+
74+
// When
75+
long sum = sut.fold(0L, Long::sum);
76+
77+
// Then
78+
assertThat(sum).isEqualTo(36L);
79+
}
80+
}

‎reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyForShortArrayTest.java‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
package io.github.dfa1.vortex.reader.array;
2+
3+
import io.github.dfa1.vortex.core.DType;
4+
import io.github.dfa1.vortex.core.PType;
5+
import io.github.dfa1.vortex.encoding.PTypeIO;
6+
import org.junit.jupiter.api.Test;
7+
8+
import java.lang.foreign.Arena;
9+
import java.lang.foreign.MemorySegment;
10+
import java.util.ArrayList;
11+
import java.util.List;
12+
13+
import static org.assertj.core.api.Assertions.assertThat;
14+
15+
class LazyForShortArrayTest {
16+
17+
private static final DType I16 = new DType.Primitive(PType.I16, false);
18+
19+
private static LazyForShortArray of(short ref, short... encoded) {
20+
MemorySegment seg = Arena.ofAuto().allocate((long) encoded.length * 2, 2);
21+
for (int i = 0; i < encoded.length; i++) {
22+
seg.setAtIndex(PTypeIO.LE_SHORT, i, encoded[i]);
23+
}
24+
return new LazyForShortArray(I16, encoded.length, seg, ref);
25+
}
26+
27+
@Test
28+
void getShortAddsRef() {
29+
// Given — residuals [1, 2, 3] with ref=100 → decoded [101, 102, 103]
30+
LazyForShortArray sut = of((short) 100, (short) 1, (short) 2, (short) 3);
31+
32+
// When + Then
33+
assertThat(sut.getShort(0)).isEqualTo((short) 101);
34+
assertThat(sut.getShort(1)).isEqualTo((short) 102);
35+
assertThat(sut.getShort(2)).isEqualTo((short) 103);
36+
}
37+
38+
@Test
39+
void getIntWidensSignedShort() {
40+
// Given — ref=0, residual=-1 → decoded=-1, getInt returns -1 for I16
41+
LazyForShortArray sut = of((short) 0, (short) -1);
42+
43+
// When + Then
44+
assertThat(sut.getInt(0)).isEqualTo(-1);
45+
}
46+
47+
@Test
48+
void forEachShortVisitsAll() {
49+
// Given
50+
LazyForShortArray sut = of((short) 10, (short) 1, (short) 2);
51+
List<Short> got = new ArrayList<>();
52+
53+
// When
54+
sut.forEachShort(got::add);
55+
56+
// Then
57+
assertThat(got).containsExactly((short) 11, (short) 12);
58+
}
59+
60+
@Test
61+
void foldSumApplies() {
62+
// Given — residuals [1,2,3] ref=100 → decoded [101,102,103] → sum 306
63+
LazyForShortArray sut = of((short) 100, (short) 1, (short) 2, (short) 3);
64+
65+
// When
66+
long sum = sut.fold(0L, Long::sum);
67+
68+
// Then
69+
assertThat(sum).isEqualTo(306L);
70+
}
71+
}

0 commit comments

Comments
 (0)