Skip to content

Commit a74263c

Browse files
dfa1claude
dfa1
and
claude
committed
refactor(encoding): extract shared PrimitiveArrays.toLongs/fromLongs to core
toLongs (integer array -> long[] widen) was copy-pasted in 4 writer encoders (Delta, Bitpacked, FrameOfReference, Patched), and fromLongs (long[] -> off-heap segment) in the Delta encoder and decoder — identical but for the EncodingId in the error throw. Since fromLongs is needed by both reader and writer, the shared home is core (io.github.dfa1.vortex.encoding), the only module both depend on. New PrimitiveArrays holds the inverse pair: toLongs(data, ptype, encoding) and fromLongs(longs, ptype, arena). The EncodingId is now a parameter so each caller keeps its own error attribution. Deliberately NOT folded in: RleEncodingEncoder.toLongs (a superset that also raw-bit-packs F32/F64/F16), Rle.fromLongs (takes a count prefix), and Patched.fromLongs (returns a boxed java array, opposite direction). Those are genuinely different methods, left alone. Ground truth green: JavaWritesRustReads (213) + JavaRoundTrip round-trips pass, so the extracted widen/narrow produces byte-identical output. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 8362a35 commit a74263c

6 files changed

Lines changed: 123 additions & 236 deletions

File tree

‎core/src/main/java/io/github/dfa1/vortex/encoding/PrimitiveArrays.java‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
package io.github.dfa1.vortex.encoding;
2+
3+
import io.github.dfa1.vortex.core.PType;
4+
import io.github.dfa1.vortex.core.VortexException;
5+
6+
import java.lang.foreign.MemorySegment;
7+
import java.lang.foreign.SegmentAllocator;
8+
import java.lang.foreign.ValueLayout;
9+
10+
/// Conversions between a boxed Java primitive value array and its wide / off-heap forms,
11+
/// shared by the integer encodings on both the read and write sides.
12+
///
13+
/// [#toLongs(Object, PType, EncodingId)] and [#fromLongs(long[], PType, SegmentAllocator)] are
14+
/// inverses: the first widens any 8–64 bit integer array to a `long[]`, the second writes a
15+
/// `long[]` back to a little-endian off-heap segment of the target width. Floating-point ptypes
16+
/// are not handled here — they reinterpret to raw bits or take type-specific encode paths instead.
17+
public final class PrimitiveArrays {
18+
19+
private PrimitiveArrays() {
20+
}
21+
22+
/// Widens a boxed primitive integer array to `long[]`, zero-extending the unsigned ptypes and
23+
/// sign-extending the signed ones. The I64/U64 case returns the input array directly (no copy).
24+
///
25+
/// @param data the value array; its runtime type must match `ptype`
26+
/// (`byte[]` for I8/U8, `short[]` for I16/U16, `int[]` for I32/U32, `long[]` for I64/U64)
27+
/// @param ptype the logical primitive type of `data`
28+
/// @param encoding the encoding requesting the widening, used for error attribution
29+
/// @return a `long[]` holding every element of `data` widened to 64 bits
30+
/// @throws VortexException if `ptype` is not an integer ptype
31+
public static long[] toLongs(Object data, PType ptype, EncodingId encoding) {
32+
return switch (ptype) {
33+
case I8 -> {
34+
byte[] arr = (byte[]) data;
35+
long[] r = new long[arr.length];
36+
for (int i = 0; i < arr.length; i++) {
37+
r[i] = arr[i];
38+
}
39+
yield r;
40+
}
41+
case U8 -> {
42+
byte[] arr = (byte[]) data;
43+
long[] r = new long[arr.length];
44+
for (int i = 0; i < arr.length; i++) {
45+
r[i] = Byte.toUnsignedLong(arr[i]);
46+
}
47+
yield r;
48+
}
49+
case I16 -> {
50+
short[] arr = (short[]) data;
51+
long[] r = new long[arr.length];
52+
for (int i = 0; i < arr.length; i++) {
53+
r[i] = arr[i];
54+
}
55+
yield r;
56+
}
57+
case U16 -> {
58+
short[] arr = (short[]) data;
59+
long[] r = new long[arr.length];
60+
for (int i = 0; i < arr.length; i++) {
61+
r[i] = Short.toUnsignedLong(arr[i]);
62+
}
63+
yield r;
64+
}
65+
case I32 -> {
66+
int[] arr = (int[]) data;
67+
long[] r = new long[arr.length];
68+
for (int i = 0; i < arr.length; i++) {
69+
r[i] = arr[i];
70+
}
71+
yield r;
72+
}
73+
case U32 -> {
74+
int[] arr = (int[]) data;
75+
long[] r = new long[arr.length];
76+
for (int i = 0; i < arr.length; i++) {
77+
r[i] = Integer.toUnsignedLong(arr[i]);
78+
}
79+
yield r;
80+
}
81+
case I64, U64 -> (long[]) data;
82+
default -> throw new VortexException(encoding, "unsupported ptype: " + ptype);
83+
};
84+
}
85+
86+
/// Writes a `long[]` to a freshly allocated little-endian off-heap segment whose element width
87+
/// is that of `ptype`, narrowing each element to the low bytes. Inverse of
88+
/// [#toLongs(Object, PType, EncodingId)]. The I64/U64 case bulk-copies; narrower widths write
89+
/// element by element through [PTypeIO#set(MemorySegment, long, PType, long)].
90+
///
91+
/// @param longs the wide values to write
92+
/// @param ptype the target primitive width
93+
/// @param arena allocator for the output segment
94+
/// @return a little-endian segment of `longs.length` elements at `ptype`'s width
95+
public static MemorySegment fromLongs(long[] longs, PType ptype, SegmentAllocator arena) {
96+
if (ptype == PType.I64 || ptype == PType.U64) {
97+
MemorySegment dst = arena.allocate((long) longs.length * 8);
98+
MemorySegment.copy(MemorySegment.ofArray(longs), ValueLayout.JAVA_LONG, 0L, dst, PTypeIO.LE_LONG, 0L, longs.length);
99+
return dst;
100+
}
101+
int n = longs.length;
102+
long elemSize = ptype.byteSize();
103+
MemorySegment seg = arena.allocate(n * elemSize);
104+
for (int i = 0; i < n; i++) {
105+
PTypeIO.set(seg, i * elemSize, ptype, longs[i]);
106+
}
107+
return seg;
108+
}
109+
}

‎reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import io.github.dfa1.vortex.core.PType;
55
import io.github.dfa1.vortex.core.VortexException;
66
import io.github.dfa1.vortex.encoding.EncodingId;
7+
import io.github.dfa1.vortex.encoding.PrimitiveArrays;
78
import io.github.dfa1.vortex.encoding.PTypeIO;
89
import io.github.dfa1.vortex.proto.DeltaMetadata;
910
import io.github.dfa1.vortex.reader.array.Array;
@@ -14,7 +15,6 @@
1415

1516
import java.io.IOException;
1617
import java.lang.foreign.MemorySegment;
17-
import java.lang.foreign.SegmentAllocator;
1818
import java.lang.foreign.ValueLayout;
1919
import java.nio.ByteBuffer;
2020

@@ -107,7 +107,7 @@ public Array decode(DecodeContext ctx) {
107107
long[] result = new long[(int) rowCount];
108108
System.arraycopy(decoded, offset, result, 0, (int) rowCount);
109109

110-
MemorySegment seg = fromLongs(result, ptype, ctx.arena());
110+
MemorySegment seg = PrimitiveArrays.fromLongs(result, ptype, ctx.arena());
111111
return switch (ptype) {
112112
case I64, U64 -> new MaterializedLongArray(ctx.dtype(), rowCount, seg);
113113
case I32, U32 -> new MaterializedIntArray(ctx.dtype(), rowCount, seg);
@@ -179,19 +179,4 @@ private static long typeMask(PType ptype) {
179179
return bits == 64 ? -1L : (1L << bits) - 1;
180180
}
181181

182-
private static MemorySegment fromLongs(long[] longs, PType ptype, SegmentAllocator arena) {
183-
if (ptype == PType.I64 || ptype == PType.U64) {
184-
MemorySegment dst = arena.allocate((long) longs.length * 8);
185-
MemorySegment.copy(MemorySegment.ofArray(longs), ValueLayout.JAVA_LONG, 0L, dst, PTypeIO.LE_LONG, 0L, longs.length);
186-
return dst;
187-
}
188-
int n = longs.length;
189-
long elemSize = ptype.byteSize();
190-
MemorySegment seg = arena.allocate(n * elemSize);
191-
for (int i = 0; i < n; i++) {
192-
PTypeIO.set(seg, i * elemSize, ptype, longs[i]);
193-
}
194-
return seg;
195-
}
196-
197182
}

‎writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java‎

Lines changed: 2 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import io.github.dfa1.vortex.core.PType;
55
import io.github.dfa1.vortex.core.VortexException;
66
import io.github.dfa1.vortex.encoding.EncodingId;
7+
import io.github.dfa1.vortex.encoding.PrimitiveArrays;
78
import io.github.dfa1.vortex.encoding.PTypeIO;
89
import io.github.dfa1.vortex.proto.BitPackedMetadata;
910
import io.github.dfa1.vortex.proto.PatchesMetadata;
@@ -43,7 +44,7 @@ public boolean accepts(DType dtype) {
4344
@Override
4445
public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
4546
PType ptype = ((DType.Primitive) dtype).ptype();
46-
long[] longs = toLongs(data, ptype);
47+
long[] longs = PrimitiveArrays.toLongs(data, ptype, EncodingId.FASTLANES_BITPACKED);
4748
int n = longs.length;
4849
int typeBits = ptype.byteSize() * 8;
4950
long typeMask = typeMask(typeBits);
@@ -237,60 +238,6 @@ private static MemorySegment packFastLanes(long[] values, int n, int bitWidth, i
237238
return seg;
238239
}
239240

240-
private static long[] toLongs(Object data, PType ptype) {
241-
return switch (ptype) {
242-
case I8 -> {
243-
byte[] arr = (byte[]) data;
244-
long[] r = new long[arr.length];
245-
for (int i = 0; i < arr.length; i++) {
246-
r[i] = arr[i];
247-
}
248-
yield r;
249-
}
250-
case U8 -> {
251-
byte[] arr = (byte[]) data;
252-
long[] r = new long[arr.length];
253-
for (int i = 0; i < arr.length; i++) {
254-
r[i] = Byte.toUnsignedLong(arr[i]);
255-
}
256-
yield r;
257-
}
258-
case I16 -> {
259-
short[] arr = (short[]) data;
260-
long[] r = new long[arr.length];
261-
for (int i = 0; i < arr.length; i++) {
262-
r[i] = arr[i];
263-
}
264-
yield r;
265-
}
266-
case U16 -> {
267-
short[] arr = (short[]) data;
268-
long[] r = new long[arr.length];
269-
for (int i = 0; i < arr.length; i++) {
270-
r[i] = Short.toUnsignedLong(arr[i]);
271-
}
272-
yield r;
273-
}
274-
case I32 -> {
275-
int[] arr = (int[]) data;
276-
long[] r = new long[arr.length];
277-
for (int i = 0; i < arr.length; i++) {
278-
r[i] = arr[i];
279-
}
280-
yield r;
281-
}
282-
case U32 -> {
283-
int[] arr = (int[]) data;
284-
long[] r = new long[arr.length];
285-
for (int i = 0; i < arr.length; i++) {
286-
r[i] = Integer.toUnsignedLong(arr[i]);
287-
}
288-
yield r;
289-
}
290-
case I64, U64 -> (long[]) data;
291-
default -> throw new VortexException(EncodingId.FASTLANES_BITPACKED, "unsupported ptype: " + ptype);
292-
};
293-
}
294241

295242
private static long typeMask(int typeBits) {
296243
return typeBits == 64 ? -1L : (1L << typeBits) - 1L;

‎writer/src/main/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoder.java‎

Lines changed: 4 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,12 @@
22

33
import io.github.dfa1.vortex.core.DType;
44
import io.github.dfa1.vortex.core.PType;
5-
import io.github.dfa1.vortex.core.VortexException;
65
import io.github.dfa1.vortex.encoding.EncodingId;
7-
import io.github.dfa1.vortex.encoding.PTypeIO;
6+
import io.github.dfa1.vortex.encoding.PrimitiveArrays;
87
import io.github.dfa1.vortex.proto.DeltaMetadata;
98
import io.github.dfa1.vortex.proto.ScalarValue;
109

1110
import java.lang.foreign.MemorySegment;
12-
import java.lang.foreign.SegmentAllocator;
13-
import java.lang.foreign.ValueLayout;
1411
import java.nio.ByteBuffer;
1512
import java.util.List;
1613

@@ -40,7 +37,7 @@ public boolean accepts(DType dtype) {
4037
@Override
4138
public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
4239
PType ptype = ((DType.Primitive) dtype).ptype();
43-
long[] longs = toLongs(data, ptype);
40+
long[] longs = PrimitiveArrays.toLongs(data, ptype, EncodingId.FASTLANES_DELTA);
4441
int n = longs.length;
4542
int typeBits = typeBits(ptype);
4643
int lanes = lanes(ptype);
@@ -93,8 +90,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
9390
System.arraycopy(chunkDelta, 0, deltasAll, chunk * FL_CHUNK_SIZE, FL_CHUNK_SIZE);
9491
}
9592

96-
MemorySegment basesSeg = fromLongs(basesAll, ptype, ctx.arena());
97-
MemorySegment deltasSeg = fromLongs(deltasAll, ptype, ctx.arena());
93+
MemorySegment basesSeg = PrimitiveArrays.fromLongs(basesAll, ptype, ctx.arena());
94+
MemorySegment deltasSeg = PrimitiveArrays.fromLongs(deltasAll, ptype, ctx.arena());
9895

9996
byte[] metaBytes = new DeltaMetadata(paddedLen, 0).encode();
10097

@@ -120,61 +117,6 @@ private static void deltaChunk(long[] transposed, long[] bases, int lanes, int t
120117
}
121118
}
122119

123-
private static long[] toLongs(Object data, PType ptype) {
124-
return switch (ptype) {
125-
case I8 -> {
126-
byte[] arr = (byte[]) data;
127-
long[] r = new long[arr.length];
128-
for (int i = 0; i < arr.length; i++) {
129-
r[i] = arr[i];
130-
}
131-
yield r;
132-
}
133-
case U8 -> {
134-
byte[] arr = (byte[]) data;
135-
long[] r = new long[arr.length];
136-
for (int i = 0; i < arr.length; i++) {
137-
r[i] = Byte.toUnsignedLong(arr[i]);
138-
}
139-
yield r;
140-
}
141-
case I16 -> {
142-
short[] arr = (short[]) data;
143-
long[] r = new long[arr.length];
144-
for (int i = 0; i < arr.length; i++) {
145-
r[i] = arr[i];
146-
}
147-
yield r;
148-
}
149-
case U16 -> {
150-
short[] arr = (short[]) data;
151-
long[] r = new long[arr.length];
152-
for (int i = 0; i < arr.length; i++) {
153-
r[i] = Short.toUnsignedLong(arr[i]);
154-
}
155-
yield r;
156-
}
157-
case I32 -> {
158-
int[] arr = (int[]) data;
159-
long[] r = new long[arr.length];
160-
for (int i = 0; i < arr.length; i++) {
161-
r[i] = arr[i];
162-
}
163-
yield r;
164-
}
165-
case U32 -> {
166-
int[] arr = (int[]) data;
167-
long[] r = new long[arr.length];
168-
for (int i = 0; i < arr.length; i++) {
169-
r[i] = Integer.toUnsignedLong(arr[i]);
170-
}
171-
yield r;
172-
}
173-
case I64, U64 -> (long[]) data;
174-
default -> throw new VortexException(EncodingId.FASTLANES_DELTA, "unsupported ptype: " + ptype);
175-
};
176-
}
177-
178120
private static boolean isUnsigned(PType ptype) {
179121
return switch (ptype) {
180122
case U8, U16, U32, U64 -> true;
@@ -219,19 +161,5 @@ private static long typeMask(PType ptype) {
219161
return bits == 64 ? -1L : (1L << bits) - 1;
220162
}
221163

222-
private static MemorySegment fromLongs(long[] longs, PType ptype, SegmentAllocator arena) {
223-
if (ptype == PType.I64 || ptype == PType.U64) {
224-
MemorySegment dst = arena.allocate((long) longs.length * 8);
225-
MemorySegment.copy(MemorySegment.ofArray(longs), ValueLayout.JAVA_LONG, 0L, dst, PTypeIO.LE_LONG, 0L, longs.length);
226-
return dst;
227-
}
228-
int n = longs.length;
229-
long elemSize = ptype.byteSize();
230-
MemorySegment seg = arena.allocate(n * elemSize);
231-
for (int i = 0; i < n; i++) {
232-
PTypeIO.set(seg, i * elemSize, ptype, longs[i]);
233-
}
234-
return seg;
235-
}
236164

237165
}

0 commit comments

Comments
 (0)