diff --git a/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java b/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java index 29640bb6..0fd9a060 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java @@ -9,6 +9,7 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; import java.lang.foreign.ValueLayout; +import java.lang.reflect.Array; /// Conversions between a boxed Java primitive value array and its wide / off-heap forms, /// shared by the integer encodings on both the read and write sides. @@ -219,6 +220,31 @@ public static MemorySegment fromLongs(long[] longs, PType ptype, SegmentAllocato return seg; } + /// Copies a heap carrier array of any primitive type into a freshly allocated little-endian + /// off-heap segment, aligned to `ptype`'s element width (the alignment an `EncodedBuffer` + /// declares). One bulk copy per call; [MemorySegment#copy(Object, int, MemorySegment, + /// ValueLayout, long, int)] does any byte swap. + /// + /// @param data the heap carrier array (`byte[]` for I8/U8, `short[]` for I16/U16/F16, + /// `int[]` for I32/U32, `long[]` for I64/U64, `float[]` for F32, `double[]` for F64) + /// @param ptype the primitive type of `data` + /// @param arena allocator for the output segment + /// @return a native little-endian segment holding every element of `data` + public static MemorySegment toSegment(Object data, PType ptype, SegmentAllocator arena) { + ValueLayout layout = switch (ptype) { + case I8, U8 -> ValueLayout.JAVA_BYTE; + case I16, U16, F16 -> VortexFormat.LE_SHORT; + case I32, U32 -> VortexFormat.LE_INT; + case I64, U64 -> VortexFormat.LE_LONG; + case F32 -> VortexFormat.LE_FLOAT; + case F64 -> VortexFormat.LE_DOUBLE; + }; + int n = Array.getLength(data); + MemorySegment seg = arena.allocate((long) n * ptype.byteSize(), ptype.byteSize()); + MemorySegment.copy(data, 0, seg, layout, 0, n); + return seg; + } + /// Narrows `longs` back to `ptype`'s heap carrier array (`byte[]` for I8/U8, `short[]` for /// I16/U16, `int[]` for I32/U32, `long[]` for I64/U64) — the inverse of /// [#toLongs(Object, PType, EncodingId)]. Truncation keeps the low bits, which round-trips diff --git a/core/src/test/java/io/github/dfa1/vortex/core/compute/PrimitiveArraysTest.java b/core/src/test/java/io/github/dfa1/vortex/core/compute/PrimitiveArraysTest.java index 522a7cc2..8ab397de 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/compute/PrimitiveArraysTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/compute/PrimitiveArraysTest.java @@ -500,4 +500,34 @@ void fromBitsArray_integers_delegateToFromLongsArray() { // Then assertThat((short[]) result).containsExactly((short) -1, (short) 0, (short) 7); } + @ParameterizedTest + @EnumSource(PType.class) + void toSegment_writesLittleEndianElementAlignedBytes(PType ptype) { + // Given — -2 sets every byte of its width, so a missing byte swap or a short copy shows up; + // fromBitsArray builds the right carrier type for every ptype, floats included. + long[] bits = {1, -2, 0x7f}; + Object carrier = PrimitiveArrays.fromBitsArray(bits, ptype, EncodingId.VORTEX_PRIMITIVE); + int width = ptype.byteSize(); + + try (Arena arena = Arena.ofConfined()) { + // When + MemorySegment result = PrimitiveArrays.toSegment(carrier, ptype, arena); + + // Then — Rust rejects a buffer below its element alignment, so the address is asserted too + assertThat(result.byteSize()).isEqualTo(3L * width); + assertThat(result.address() % width).isZero(); + long mask = width == 8 ? -1L : (1L << (width * 8)) - 1; + for (int i = 0; i < bits.length; i++) { + assertThat(leBits(result, (long) i * width, width)).isEqualTo(bits[i] & mask); + } + } + } + + private static long leBits(MemorySegment seg, long offset, int width) { + long bits = 0; + for (int k = 0; k < width; k++) { + bits |= (seg.get(ValueLayout.JAVA_BYTE, offset + k) & 0xFFL) << (8 * k); + } + return bits; + } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombiner.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombiner.java index 32246568..6524afa8 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombiner.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombiner.java @@ -56,7 +56,16 @@ public static Array combine(DType dtype, long totalRows, List chunks, return validity != null ? new MaskedArray(data, validity) : data; } - private static Array combinePrimitive(PType ptype, DType dtype, long totalRows, + /// Folds per-chunk primitive arrays into the zero-copy `ChunkedXxxArray` view for `ptype`. + /// Validity is not handled here; [#combine(DType, long, List, SegmentAllocator)] adds it. + /// + /// @param ptype the chunks' primitive type + /// @param dtype the dtype the combined view reports + /// @param totalRows the total logical row count across all chunks + /// @param chunks the decoded per-chunk arrays, in row order + /// @return the combined primitive view + /// @throws VortexException for a ptype with no chunked view (F16) + public static Array combinePrimitive(PType ptype, DType dtype, long totalRows, List chunks) { return switch (ptype) { case I64, U64 -> ChunkedLongArray.of(dtype, totalRows, chunks); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoder.java index a95e9930..dfe7043b 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoder.java @@ -1,17 +1,11 @@ package io.github.dfa1.vortex.reader.decode; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.ChunkedArrayCombiner; import io.github.dfa1.vortex.reader.array.ChunkedBoolArray; -import io.github.dfa1.vortex.reader.array.ChunkedByteArray; -import io.github.dfa1.vortex.reader.array.ChunkedDoubleArray; -import io.github.dfa1.vortex.reader.array.ChunkedFloatArray; -import io.github.dfa1.vortex.reader.array.ChunkedIntArray; -import io.github.dfa1.vortex.reader.array.ChunkedLongArray; -import io.github.dfa1.vortex.reader.array.ChunkedShortArray; import io.github.dfa1.vortex.reader.array.StructArray; import java.lang.foreign.MemorySegment; @@ -100,7 +94,7 @@ private static long[] readOffsets(DecodeContext ctx, int nchunks, long rowCount) /// [DType.Struct]. No concat / no per-row materialize. private static Array wrap(List chunks, DType dtype, long totalRows) { if (dtype instanceof DType.Primitive pt) { - return wrapPrimitive(chunks, pt, dtype, totalRows); + return ChunkedArrayCombiner.combinePrimitive(pt.ptype(), dtype, totalRows, chunks); } if (dtype instanceof DType.Bool) { return ChunkedBoolArray.of(dtype, totalRows, chunks); @@ -117,7 +111,7 @@ private static Array wrap(List chunks, DType dtype, long totalRows) { } DType innerDtype = chunks.get(0).dtype(); if (innerDtype instanceof DType.Primitive innerPt) { - return wrapPrimitive(chunks, innerPt, innerDtype, totalRows); + return ChunkedArrayCombiner.combinePrimitive(innerPt.ptype(), innerDtype, totalRows, chunks); } if (innerDtype instanceof DType.Bool) { return ChunkedBoolArray.of(innerDtype, totalRows, chunks); @@ -129,22 +123,6 @@ private static Array wrap(List chunks, DType dtype, long totalRows) { "chunked not supported for dtype: " + dtype); } - private static Array wrapPrimitive( - List chunks, DType.Primitive pt, DType dtype, long totalRows - ) { - PType ptype = pt.ptype(); - return switch (ptype) { - case I64, U64 -> ChunkedLongArray.of(dtype, totalRows, chunks); - case I32, U32 -> ChunkedIntArray.of(dtype, totalRows, chunks); - case F64 -> ChunkedDoubleArray.of(dtype, totalRows, chunks); - case F32 -> ChunkedFloatArray.of(dtype, totalRows, chunks); - case I16, U16 -> ChunkedShortArray.of(dtype, totalRows, chunks); - case I8, U8 -> ChunkedByteArray.of(dtype, totalRows, chunks); - default -> throw new VortexException(EncodingId.VORTEX_CHUNKED, - "unsupported ptype for chunked: " + ptype); - }; - } - private static StructArray wrapStruct(List chunks, DType.Struct struct, long totalRows) { int nfields = struct.fieldTypes().size(); List wrappedFields = new ArrayList<>(nfields); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ArrayStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ArrayStats.java index 98d66aad..53a30621 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ArrayStats.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ArrayStats.java @@ -4,6 +4,8 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; +import java.lang.reflect.Array; + /// Read-only stats over a primitive array, computed in a single scan and shared across /// all encoders that requested a given stat via [StatsOptions]. Replaces the per-encoder /// sample-encoding probe that biases on leading rows. @@ -39,7 +41,7 @@ public record ArrayStats( /// @param options which stats to compute; merged options from all eligible encoders /// @return immutable [ArrayStats] public static ArrayStats compute(PType ptype, Object data, StatsOptions options) { - int n = arrayLength(ptype, data); + int n = Array.getLength(data); if (n == 0) { return EMPTY; } @@ -200,17 +202,6 @@ private static boolean scanF64(double[] a, int n, LongIntMap counts, long cap) { return false; } - private static int arrayLength(PType ptype, Object data) { - return switch (ptype) { - case I8, U8 -> ((byte[]) data).length; - case I16, U16, F16 -> ((short[]) data).length; - case I32, U32 -> ((int[]) data).length; - case I64, U64 -> ((long[]) data).length; - case F32 -> ((float[]) data).length; - case F64 -> ((double[]) data).length; - }; - } - /// @return whether [#distinctCount()] was computed during this scan public boolean hasDistinctCount() { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java index 1ca886e1..4a2b3a36 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java @@ -7,6 +7,7 @@ import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import java.lang.foreign.MemorySegment; +import java.lang.reflect.Array; /// Write-only encoder for `vortex.constant`. public final class ConstantEncodingEncoder implements EncodingEncoder { @@ -57,7 +58,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { byte[] scalarBytes = scalar.encode(); // A constant array's min and max are both the one repeated value, by construction -- no // scan needed. Empty arrays report no stats, matching every other encoder's convention. - byte[] stats = arrayLength(data, ptype) > 0 ? scalarBytes : null; + byte[] stats = Array.getLength(data) > 0 ? scalarBytes : null; return EncodeResult.simple(EncodingId.VORTEX_CONSTANT, EncodedBuffer.bytes(MemorySegment.ofArray(scalarBytes)), stats, stats); } @@ -109,21 +110,9 @@ private static long readFirstRaw(Object data, PType ptype) { }; } - private static int arrayLength(Object data, PType ptype) { - return switch (ptype) { - case I8, U8 -> ((byte[]) data).length; - case I16, U16 -> ((short[]) data).length; - case I32, U32 -> ((int[]) data).length; - case I64, U64 -> ((long[]) data).length; - case F32 -> ((float[]) data).length; - case F64 -> ((double[]) data).length; - default -> throw new VortexException(EncodingId.VORTEX_CONSTANT, "unsupported ptype: " + ptype); - }; - } - private static boolean isConstant(Object data, PType ptype) { long firstRaw = readFirstRaw(data, ptype); - int len = arrayLength(data, ptype); + int len = Array.getLength(data); for (int i = 1; i < len; i++) { long raw = switch (ptype) { case I8, U8 -> ((byte[]) data)[i]; diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java index b72a104d..0e1601dc 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java @@ -14,6 +14,7 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; +import java.lang.reflect.Array; import java.nio.charset.StandardCharsets; import java.util.Set; import java.util.LinkedHashMap; @@ -319,7 +320,7 @@ private static EncodeResult encodeUtf8(String[] strings, EncodeContext ctx) { private static DictData buildDictData(DType dtype, Object data) { PType ptype = ((DType.Primitive) dtype).ptype(); - int len = arrayLength(data, ptype); + int len = Array.getLength(data); // Dedup on raw value bits rather than boxed keys. This probed a // LinkedHashMap, boxing a value per ROW just to look it up - the same @@ -369,17 +370,6 @@ private static PType codePType(int dictSize) { return PType.U32; } - private static int arrayLength(Object data, PType ptype) { - return switch (ptype) { - case I8, U8 -> ((byte[]) data).length; - case I16, U16, F16 -> ((short[]) data).length; - case I32, U32 -> ((int[]) data).length; - case I64, U64 -> ((long[]) data).length; - case F32 -> ((float[]) data).length; - case F64 -> ((double[]) data).length; - }; - } - /// Builds the dictionary pool from the first-seen raw bit patterns, in order. /// /// @param ptype the column's primitive type diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java index 96e7c7fa..224a06fa 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java @@ -3,10 +3,9 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.compute.PrimitiveArrays; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; -import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; /// Write-only encoder for `vortex.primitive` — raw little-endian primitive arrays. @@ -25,7 +24,10 @@ public boolean accepts(DType dtype) { @Override public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { PType ptype = ((DType.Primitive) dtype).ptype(); - MemorySegment seg = encodePrimitive(ptype, data, ctx.arena()); + // byte[] needs no byte swap, so it is wrapped in place rather than copied off-heap. + MemorySegment seg = data instanceof byte[] bytes + ? MemorySegment.ofArray(bytes) + : PrimitiveArrays.toSegment(data, ptype, ctx.arena()); byte[] min = null; byte[] max = null; byte[][] stats = minMaxStats(ptype, data); @@ -36,52 +38,6 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { return EncodeResult.simple(EncodingId.VORTEX_PRIMITIVE, EncodedBuffer.of(seg, ptype), min, max); } - private static MemorySegment encodePrimitive(PType ptype, Object data, Arena arena) { - return switch (ptype) { - case I8, U8 -> MemorySegment.ofArray((byte[]) data); - case I16, U16, F16 -> { - short[] arr = (short[]) data; - MemorySegment seg = arena.allocate((long) arr.length * 2, 2); - for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(VortexFormat.LE_SHORT, i, arr[i]); - } - yield seg; - } - case I32, U32 -> { - int[] arr = (int[]) data; - MemorySegment seg = arena.allocate((long) arr.length * 4, 4); - for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(VortexFormat.LE_INT, i, arr[i]); - } - yield seg; - } - case I64, U64 -> { - long[] arr = (long[]) data; - MemorySegment seg = arena.allocate((long) arr.length * 8, 8); - for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(VortexFormat.LE_LONG, i, arr[i]); - } - yield seg; - } - case F32 -> { - float[] arr = (float[]) data; - MemorySegment seg = arena.allocate((long) arr.length * 4, 4); - for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(VortexFormat.LE_FLOAT, i, arr[i]); - } - yield seg; - } - case F64 -> { - double[] arr = (double[]) data; - MemorySegment seg = arena.allocate((long) arr.length * 8, 8); - for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(VortexFormat.LE_DOUBLE, i, arr[i]); - } - yield seg; - } - }; - } - /// Computes the serialized min/max [io.github.dfa1.vortex.core.proto.ProtoScalarValue] pair for a raw /// primitive array, in the same signed/unsigned/float shape the per-segment stats use. Returns /// `null` for an empty array. Shared so the dictionary zone-map path computes per-chunk min/max diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java index 28e5ce25..2a812cb1 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java @@ -11,6 +11,7 @@ import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import java.lang.foreign.MemorySegment; +import java.lang.reflect.Array; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -117,7 +118,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { throw new VortexException(EncodingId.VORTEX_RUNEND, "encode only supports Primitive dtype, got " + dtype); } PType ptype = p.ptype(); - int n = arrayLength(data, ptype); + int n = Array.getLength(data); boolean unsign = ptype.isUnsigned(); int elemBytes = ptype.byteSize(); @@ -228,7 +229,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { return CascadeStep.notApplicable(); } PType ptype = p.ptype(); - int n = arrayLength(data, ptype); + int n = Array.getLength(data); if (n == 0) { return CascadeStep.notApplicable(); } @@ -269,15 +270,4 @@ private static byte[] statsBytes(PType ptype, long value) { } return ProtoScalarValue.ofInt64Value(value).encode(); } - - private static int arrayLength(Object data, PType ptype) { - return switch (ptype) { - case I8, U8 -> ((byte[]) data).length; - case I16, U16 -> ((short[]) data).length; - case I32, U32 -> ((int[]) data).length; - case I64, U64 -> ((long[]) data).length; - default -> throw new VortexException(EncodingId.VORTEX_RUNEND, "unsupported ptype: " + ptype); - }; - } - } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java index 626ad6b5..c5be3c4e 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java @@ -5,6 +5,7 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; import java.lang.foreign.MemorySegment; +import java.lang.reflect.Array; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.proto.ProtoSequenceMetadata; @@ -51,7 +52,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { } private static EncodeResult encodeInteger(PType pt, Object data) { - int n = intArrayLength(pt, data); + int n = Array.getLength(data); long base = 0; long multiplier = 0; if (n > 0) { @@ -156,16 +157,6 @@ private static ProtoScalarValue buildIntScalar(PType pt, long value) { }; } - private static int intArrayLength(PType pt, Object data) { - return switch (pt) { - case I8, U8 -> ((byte[]) data).length; - case I16, U16 -> ((short[]) data).length; - case I32, U32 -> ((int[]) data).length; - case I64, U64 -> ((long[]) data).length; - default -> throw new VortexException(EncodingId.VORTEX_SEQUENCE, "unsupported ptype: " + pt); - }; - } - private static long readLong(PType pt, Object data, int i) { return switch (pt) { case I8, U8 -> ((byte[]) data)[i]; diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java index cc4b71b9..7a3ed457 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java @@ -2,16 +2,16 @@ import io.github.dfa1.zstd.ZstdCompressContext; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.compute.PrimitiveArrays; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoZstdFrameMetadata; import io.github.dfa1.vortex.core.proto.ProtoZstdMetadata; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; +import java.lang.reflect.Array; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -125,8 +125,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { private EncodeResult encodePrimitive(DType.Primitive dt, Object data, Arena arena) { int byteWidth = dt.ptype().byteSize(); - MemorySegment raw = primitiveToLeBytes(dt.ptype(), data, arena); - long n = primitiveLength(dt.ptype(), data); + MemorySegment raw = PrimitiveArrays.toSegment(data, dt.ptype(), arena); + long n = Array.getLength(data); return buildResult(raw, uniformLayout(n, byteWidth), arena); } @@ -150,7 +150,7 @@ private EncodeResult encodeNullablePrimitive(DType.Primitive dt, NullableData nd boolean[] validity = nd.validity(); // Strip null positions: only valid values reach the compressed payload (mirrors the Rust // reference). The decoder scatters them back over the validity mask carried by child[0]. - MemorySegment full = primitiveToLeBytes(dt.ptype(), nd.values(), arena); + MemorySegment full = PrimitiveArrays.toSegment(nd.values(), dt.ptype(), arena); MemorySegment packed = packValidBytes(full, validity, byteWidth, arena); return buildNullableResult(packed, uniformLayout(countValid(validity), byteWidth), validity, ctx); } @@ -344,68 +344,6 @@ private static int countNonNull(byte[][] rows) { return count; } - private static MemorySegment primitiveToLeBytes(PType ptype, Object data, Arena arena) { - return switch (ptype) { - case I8, U8 -> { - byte[] arr = (byte[]) data; - MemorySegment seg = arena.allocate(arr.length); - MemorySegment.copy(arr, 0, seg, ValueLayout.JAVA_BYTE, 0, arr.length); - yield seg; - } - case I16, U16, F16 -> { - short[] arr = (short[]) data; - MemorySegment seg = arena.allocate((long) arr.length * 2, 2); - for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(VortexFormat.LE_SHORT, i, arr[i]); - } - yield seg; - } - case I32, U32 -> { - int[] arr = (int[]) data; - MemorySegment seg = arena.allocate((long) arr.length * 4, 4); - for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(VortexFormat.LE_INT, i, arr[i]); - } - yield seg; - } - case I64, U64 -> { - long[] arr = (long[]) data; - MemorySegment seg = arena.allocate((long) arr.length * 8, 8); - for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(VortexFormat.LE_LONG, i, arr[i]); - } - yield seg; - } - case F32 -> { - float[] arr = (float[]) data; - MemorySegment seg = arena.allocate((long) arr.length * 4, 4); - for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(VortexFormat.LE_FLOAT, i, arr[i]); - } - yield seg; - } - case F64 -> { - double[] arr = (double[]) data; - MemorySegment seg = arena.allocate((long) arr.length * 8, 8); - for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(VortexFormat.LE_DOUBLE, i, arr[i]); - } - yield seg; - } - }; - } - - private static long primitiveLength(PType ptype, Object data) { - return switch (ptype) { - case I8, U8 -> ((byte[]) data).length; - case I16, U16, F16 -> ((short[]) data).length; - case I32, U32 -> ((int[]) data).length; - case F32 -> ((float[]) data).length; - case I64, U64 -> ((long[]) data).length; - case F64 -> ((double[]) data).length; - }; - } - private static MemorySegment buildLengthPrefixed(byte[][] encoded, Arena arena) { int total = 0; for (byte[] bytes : encoded) {