Skip to content

Commit 0bbcb81

Browse files
dfa1claude
dfa1
and
claude
committed
refactor(reader): drop RunEnd Bool Materialized fallback (ADR 0013)
Add LazyRunEndBoolArray: binary search through runs on getBoolean(i), walkRuns loop on forEachBoolean. Drop expandBool and its bit-scatter O(n) allocation. Utf8/Binary RunEnd stays Materialized — offset rebasing into VarBin offsets does not trivially express as per-element lookup. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent a6a9611 commit 0bbcb81

4 files changed

Lines changed: 55 additions & 34 deletions

File tree

‎docs/compatibility.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ decoder falls into one of three shapes:
9999
| `vortex.zigzag` | Lazy | Lazy | `LazyZigZagXxxArray` (I8/I16/I32/I64); broadcast → `LazyConstantXxxArray`, ADR 0010 + 0013 |
100100
| `vortex.constant` | Lazy | Lazy | `LazyConstantXxxArray` (primitive + bool + decimal); per-row broadcast, no buffer, ADR 0013 |
101101
| `vortex.ext` | Zero-copy | Zero-copy | wraps storage |
102-
| `vortex.runend` | Materialized | Lazy | could expose run-locating accessor (similar to Dict) |
102+
| `vortex.runend` | Lazy | Lazy | `LazyRunEndXxxArray` (primitive + bool); Utf8/Binary stays Materialized (offset rebasing), ADR 0013 |
103103
| `vortex.varbin` | Zero-copy | Zero-copy | bytes + offsets slices |
104104
| `vortex.varbinview` | Lazy | Lazy | `VarBinArray.ViewMode` — keeps views + data buffers as mmap slices |
105105
| `vortex.alp` | Lazy | Lazy | `LazyAlpXxxArray`; broadcast → `LazyConstantXxxArray`; patched stays Materialized, ADR 0010 + 0013 |

‎reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import io.github.dfa1.vortex.reader.array.LazyConstantDecimalArray;
3535
import io.github.dfa1.vortex.reader.array.LazyDecimalArray;
3636
import io.github.dfa1.vortex.reader.array.LazyDecimalBytePartsArray;
37+
import io.github.dfa1.vortex.reader.array.LazyRunEndBoolArray;
3738
import io.github.dfa1.vortex.reader.array.LongArray;
3839
import io.github.dfa1.vortex.reader.array.MaskedArray;
3940
import io.github.dfa1.vortex.reader.array.MaterializedBoolArray;
@@ -332,6 +333,8 @@ private static Array truncateArray(Array arr, long rows, SegmentAllocator arena)
332333
case ShortArray a ->
333334
new MaterializedShortArray(a.dtype(), rows, ArraySegments.of(a, arena).asSlice(0, rows * Short.BYTES));
334335
case ByteArray a -> new MaterializedByteArray(a.dtype(), rows, ArraySegments.of(a, arena).asSlice(0, rows));
336+
case LazyRunEndBoolArray a ->
337+
new LazyRunEndBoolArray(a.dtype(), rows, a.values(), a.runEnds(), a.offset());
335338
case BoolArray a ->
336339
new MaterializedBoolArray(a.dtype(), rows, ArraySegments.of(a, arena).asSlice(0, (rows + 7) / 8));
337340
case NullArray a -> new NullArray(a.dtype(), rows);

‎reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyRunEndBoolArray.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package io.github.dfa1.vortex.reader.array;
2+
3+
import io.github.dfa1.vortex.core.DType;
4+
5+
/// Lazy RunEnd-encoded [BoolArray]. `getBoolean(i) = values.getBoolean(findRun(i + offset))`.
6+
/// `forEachBoolean` walks runs (one binary search at start, then per-run loops)
7+
/// so sequential reads are O(numRuns) work plus length emissions.
8+
///
9+
/// @param dtype logical Bool type
10+
/// @param length total logical row count
11+
/// @param values boolean values per run; length = `numRuns`
12+
/// @param runEnds cumulative run-end positions (absolute, before `offset`);
13+
/// length = `numRuns`; typed as [Array] because ends ptype varies
14+
/// @param offset starting absolute position; logical row `i` maps to absolute `i + offset`
15+
public record LazyRunEndBoolArray(DType dtype, long length, BoolArray values, Array runEnds, long offset)
16+
implements BoolArray {
17+
18+
@Override
19+
public boolean getBoolean(long i) {
20+
int k = RunEndArrays.findRun(runEnds, values.length(), i + offset);
21+
return values.getBoolean(k);
22+
}
23+
24+
@Override
25+
public void forEachBoolean(BooleanConsumer c) {
26+
long numRuns = values.length();
27+
long startAbs = offset;
28+
long endAbs = offset + length;
29+
int run = RunEndArrays.findRun(runEnds, numRuns, startAbs);
30+
long emittedFrom = startAbs;
31+
while (emittedFrom < endAbs && run < numRuns) {
32+
long runEnd = RunEndArrays.readRunEnd(runEnds, run);
33+
long emitTo = Math.min(runEnd, endAbs);
34+
long count = emitTo - emittedFrom;
35+
if (count > 0) {
36+
boolean v = values.getBoolean(run);
37+
for (long r = 0; r < count; r++) {
38+
c.accept(v);
39+
}
40+
}
41+
emittedFrom = emitTo;
42+
run++;
43+
}
44+
}
45+
}

‎reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java‎

Lines changed: 6 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,13 @@
1111
import io.github.dfa1.vortex.reader.array.BoolArray;
1212
import io.github.dfa1.vortex.reader.array.ByteArray;
1313
import io.github.dfa1.vortex.reader.array.IntArray;
14+
import io.github.dfa1.vortex.reader.array.LazyRunEndBoolArray;
1415
import io.github.dfa1.vortex.reader.array.LazyRunEndByteArray;
1516
import io.github.dfa1.vortex.reader.array.LazyRunEndIntArray;
1617
import io.github.dfa1.vortex.reader.array.LazyRunEndLongArray;
1718
import io.github.dfa1.vortex.reader.array.LazyRunEndShortArray;
1819
import io.github.dfa1.vortex.reader.array.LongArray;
1920
import io.github.dfa1.vortex.reader.array.MaskedArray;
20-
import io.github.dfa1.vortex.reader.array.MaterializedBoolArray;
2121
import io.github.dfa1.vortex.reader.array.ShortArray;
2222
import io.github.dfa1.vortex.reader.array.VarBinArray;
2323

@@ -74,7 +74,9 @@ public Array decode(DecodeContext ctx) {
7474

7575
if (ctx.dtype() instanceof DType.Bool) {
7676
Array valuesArr = ctx.decodeChild(1, ctx.dtype(), numRuns);
77-
return expandBool(endsArr, (BoolArray) valuesArr, endsPtype, numRuns, offset, n, ctx.dtype(), ctx.arena());
77+
Array valuesData = valuesArr instanceof MaskedArray m ? m.inner() : valuesArr;
78+
Array endsData = endsArr instanceof MaskedArray m ? m.inner() : endsArr;
79+
return new LazyRunEndBoolArray(ctx.dtype(), n, (BoolArray) valuesData, endsData, offset);
7880
}
7981

8082
if (!(ctx.dtype() instanceof DType.Primitive p)) {
@@ -83,8 +85,8 @@ public Array decode(DecodeContext ctx) {
8385
PType valuePtype = p.ptype();
8486

8587
// Lazy path: wrap values + ends without expanding into an n-sized buffer.
86-
// Bool and VarBin keep the eager path above (different shapes — bit-packing
87-
// and offset rebasing don't trivially express as a binary-search-on-read).
88+
// VarBin keeps the eager path above — offset rebasing doesn't trivially
89+
// express as binary-search-on-read.
8890
Array valuesArr = ctx.decodeChild(1, ctx.dtype(), numRuns);
8991
Array valuesData = valuesArr instanceof MaskedArray m ? m.inner() : valuesArr;
9092
Array endsData = endsArr instanceof MaskedArray m ? m.inner() : endsArr;
@@ -97,35 +99,6 @@ public Array decode(DecodeContext ctx) {
9799
};
98100
}
99101

100-
private static Array expandBool(
101-
Array endsArr, BoolArray valuesArr,
102-
PType endsPtype, long numRuns, long offset, long n,
103-
DType dtype, SegmentAllocator arena
104-
) {
105-
MemorySegment endsSeg = ArraySegments.of(endsArr);
106-
long endsCap = SegmentBroadcast.capacity(endsSeg, endsPtype.byteSize());
107-
long numBytes = (n + 7) >>> 3;
108-
MemorySegment out = arena.allocate(numBytes);
109-
110-
long outIdx = 0;
111-
long logicalPos = 0;
112-
for (long run = 0; run < numRuns && outIdx < n; run++) {
113-
long runEnd = readUnsigned(endsSeg, run % endsCap, endsPtype);
114-
boolean val = valuesArr.getBoolean(run);
115-
long lo = Math.max(logicalPos, offset);
116-
long hi = Math.min(runEnd, offset + n);
117-
for (long lp = lo; lp < hi; lp++, outIdx++) {
118-
if (val) {
119-
long byteIdx = outIdx >>> 3;
120-
byte cur = out.get(ValueLayout.JAVA_BYTE, byteIdx);
121-
out.set(ValueLayout.JAVA_BYTE, byteIdx, (byte) ((cur & 0xff) | (1 << (outIdx & 7))));
122-
}
123-
}
124-
logicalPos = runEnd;
125-
}
126-
return new MaterializedBoolArray(dtype, n, out.asReadOnly());
127-
}
128-
129102
private static Array expandStrings(
130103
Array endsArr, VarBinArray.OffsetMode valuesArr,
131104
PType endsPtype, long numRuns, long offset, long n,

0 commit comments

Comments
 (0)