refactor: share primitive length, LE-segment and chunked-combine helpers

core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java

@@ -9,6 +9,7 @@
 import java.lang.foreign.MemorySegment;
 import java.lang.foreign.SegmentAllocator;
 import java.lang.foreign.ValueLayout;
+import java.lang.reflect.Array;
 
 /// Conversions between a boxed Java primitive value array and its wide / off-heap forms,
 /// shared by the integer encodings on both the read and write sides.
@@ -219,6 +220,31 @@ public static MemorySegment fromLongs(long[] longs, PType ptype, SegmentAllocato
         return seg;
     }
 
+    /// Copies a heap carrier array of any primitive type into a freshly allocated little-endian
+    /// off-heap segment, aligned to `ptype`'s element width (the alignment an `EncodedBuffer`
+    /// declares). One bulk copy per call; [MemorySegment#copy(Object, int, MemorySegment,
+    /// ValueLayout, long, int)] does any byte swap.
+    ///
+    /// @param data  the heap carrier array (`byte[]` for I8/U8, `short[]` for I16/U16/F16,
+    ///              `int[]` for I32/U32, `long[]` for I64/U64, `float[]` for F32, `double[]` for F64)
+    /// @param ptype the primitive type of `data`
+    /// @param arena allocator for the output segment
+    /// @return a native little-endian segment holding every element of `data`
+    public static MemorySegment toSegment(Object data, PType ptype, SegmentAllocator arena) {
+        ValueLayout layout = switch (ptype) {
+            case I8, U8 -> ValueLayout.JAVA_BYTE;
+            case I16, U16, F16 -> VortexFormat.LE_SHORT;
+            case I32, U32 -> VortexFormat.LE_INT;
+            case I64, U64 -> VortexFormat.LE_LONG;
+            case F32 -> VortexFormat.LE_FLOAT;
+            case F64 -> VortexFormat.LE_DOUBLE;
+        };
+        int n = Array.getLength(data);
+        MemorySegment seg = arena.allocate((long) n * ptype.byteSize(), ptype.byteSize());
+        MemorySegment.copy(data, 0, seg, layout, 0, n);
+        return seg;
+    }
+
     /// Narrows `longs` back to `ptype`'s heap carrier array (`byte[]` for I8/U8, `short[]` for
     /// I16/U16, `int[]` for I32/U32, `long[]` for I64/U64) — the inverse of
     /// [#toLongs(Object, PType, EncodingId)]. Truncation keeps the low bits, which round-trips

core/src/test/java/io/github/dfa1/vortex/core/compute/PrimitiveArraysTest.java

@@ -500,4 +500,34 @@ void fromBitsArray_integers_delegateToFromLongsArray() {
         // Then
         assertThat((short[]) result).containsExactly((short) -1, (short) 0, (short) 7);
     }
+    @ParameterizedTest
+    @EnumSource(PType.class)
+    void toSegment_writesLittleEndianElementAlignedBytes(PType ptype) {
+        // Given — -2 sets every byte of its width, so a missing byte swap or a short copy shows up;
+        // fromBitsArray builds the right carrier type for every ptype, floats included.
+        long[] bits = {1, -2, 0x7f};
+        Object carrier = PrimitiveArrays.fromBitsArray(bits, ptype, EncodingId.VORTEX_PRIMITIVE);
+        int width = ptype.byteSize();
+
+        try (Arena arena = Arena.ofConfined()) {
+            // When
+            MemorySegment result = PrimitiveArrays.toSegment(carrier, ptype, arena);
+
+            // Then — Rust rejects a buffer below its element alignment, so the address is asserted too
+            assertThat(result.byteSize()).isEqualTo(3L * width);
+            assertThat(result.address() % width).isZero();
+            long mask = width == 8 ? -1L : (1L << (width * 8)) - 1;
+            for (int i = 0; i < bits.length; i++) {
+                assertThat(leBits(result, (long) i * width, width)).isEqualTo(bits[i] & mask);
+            }
+        }
+    }
+
+    private static long leBits(MemorySegment seg, long offset, int width) {
+        long bits = 0;
+        for (int k = 0; k < width; k++) {
+            bits |= (seg.get(ValueLayout.JAVA_BYTE, offset + k) & 0xFFL) << (8 * k);
+        }
+        return bits;
+    }
 }

reader/src/main/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombiner.java

@@ -56,7 +56,16 @@ public static Array combine(DType dtype, long totalRows, List<Array> chunks,
         return validity != null ? new MaskedArray(data, validity) : data;
     }
 
-    private static Array combinePrimitive(PType ptype, DType dtype, long totalRows,
+    /// Folds per-chunk primitive arrays into the zero-copy `ChunkedXxxArray` view for `ptype`.
+    /// Validity is not handled here; [#combine(DType, long, List, SegmentAllocator)] adds it.
+    ///
+    /// @param ptype     the chunks' primitive type
+    /// @param dtype     the dtype the combined view reports
+    /// @param totalRows the total logical row count across all chunks
+    /// @param chunks    the decoded per-chunk arrays, in row order
+    /// @return the combined primitive view
+    /// @throws VortexException for a ptype with no chunked view (F16)
+    public static Array combinePrimitive(PType ptype, DType dtype, long totalRows,
             List<Array> chunks) {
         return switch (ptype) {
             case I64, U64 -> ChunkedLongArray.of(dtype, totalRows, chunks);

reader/src/main/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoder.java

@@ -1,17 +1,11 @@
 package io.github.dfa1.vortex.reader.decode;
 
 import io.github.dfa1.vortex.core.model.DType;
-import io.github.dfa1.vortex.core.model.PType;
 import io.github.dfa1.vortex.core.error.VortexException;
 import io.github.dfa1.vortex.core.model.EncodingId;
 import io.github.dfa1.vortex.reader.array.Array;
+import io.github.dfa1.vortex.reader.array.ChunkedArrayCombiner;
 import io.github.dfa1.vortex.reader.array.ChunkedBoolArray;
-import io.github.dfa1.vortex.reader.array.ChunkedByteArray;
-import io.github.dfa1.vortex.reader.array.ChunkedDoubleArray;
-import io.github.dfa1.vortex.reader.array.ChunkedFloatArray;
-import io.github.dfa1.vortex.reader.array.ChunkedIntArray;
-import io.github.dfa1.vortex.reader.array.ChunkedLongArray;
-import io.github.dfa1.vortex.reader.array.ChunkedShortArray;
 import io.github.dfa1.vortex.reader.array.StructArray;
 
 import java.lang.foreign.MemorySegment;
@@ -100,7 +94,7 @@ private static long[] readOffsets(DecodeContext ctx, int nchunks, long rowCount)
     /// [DType.Struct]. No concat / no per-row materialize.
     private static Array wrap(List<Array> chunks, DType dtype, long totalRows) {
         if (dtype instanceof DType.Primitive pt) {
-            return wrapPrimitive(chunks, pt, dtype, totalRows);
+            return ChunkedArrayCombiner.combinePrimitive(pt.ptype(), dtype, totalRows, chunks);
         }
         if (dtype instanceof DType.Bool) {
             return ChunkedBoolArray.of(dtype, totalRows, chunks);
@@ -117,7 +111,7 @@ private static Array wrap(List<Array> chunks, DType dtype, long totalRows) {
             }
             DType innerDtype = chunks.get(0).dtype();
             if (innerDtype instanceof DType.Primitive innerPt) {
-                return wrapPrimitive(chunks, innerPt, innerDtype, totalRows);
+                return ChunkedArrayCombiner.combinePrimitive(innerPt.ptype(), innerDtype, totalRows, chunks);
             }
             if (innerDtype instanceof DType.Bool) {
                 return ChunkedBoolArray.of(innerDtype, totalRows, chunks);
@@ -129,22 +123,6 @@ private static Array wrap(List<Array> chunks, DType dtype, long totalRows) {
                 "chunked not supported for dtype: " + dtype);
     }
 
-    private static Array wrapPrimitive(
-            List<Array> chunks, DType.Primitive pt, DType dtype, long totalRows
-    ) {
-        PType ptype = pt.ptype();
-        return switch (ptype) {
-            case I64, U64 -> ChunkedLongArray.of(dtype, totalRows, chunks);
-            case I32, U32 -> ChunkedIntArray.of(dtype, totalRows, chunks);
-            case F64 -> ChunkedDoubleArray.of(dtype, totalRows, chunks);
-            case F32 -> ChunkedFloatArray.of(dtype, totalRows, chunks);
-            case I16, U16 -> ChunkedShortArray.of(dtype, totalRows, chunks);
-            case I8, U8 -> ChunkedByteArray.of(dtype, totalRows, chunks);
-            default -> throw new VortexException(EncodingId.VORTEX_CHUNKED,
-                    "unsupported ptype for chunked: " + ptype);
-        };
-    }
-
     private static StructArray wrapStruct(List<Array> chunks, DType.Struct struct, long totalRows) {
         int nfields = struct.fieldTypes().size();
         List<Array> wrappedFields = new ArrayList<>(nfields);

writer/src/main/java/io/github/dfa1/vortex/writer/encode/ArrayStats.java

@@ -4,6 +4,8 @@
 import io.github.dfa1.vortex.core.error.VortexException;
 import io.github.dfa1.vortex.core.model.EncodingId;
 
+import java.lang.reflect.Array;
+
 /// Read-only stats over a primitive array, computed in a single scan and shared across
 /// all encoders that requested a given stat via [StatsOptions]. Replaces the per-encoder
 /// sample-encoding probe that biases on leading rows.
@@ -39,7 +41,7 @@ public record ArrayStats(
     /// @param options which stats to compute; merged options from all eligible encoders
     /// @return immutable [ArrayStats]
     public static ArrayStats compute(PType ptype, Object data, StatsOptions options) {
-        int n = arrayLength(ptype, data);
+        int n = Array.getLength(data);
         if (n == 0) {
             return EMPTY;
         }
@@ -200,17 +202,6 @@ private static boolean scanF64(double[] a, int n, LongIntMap counts, long cap) {
         return false;
     }
 
-    private static int arrayLength(PType ptype, Object data) {
-        return switch (ptype) {
-            case I8, U8 -> ((byte[]) data).length;
-            case I16, U16, F16 -> ((short[]) data).length;
-            case I32, U32 -> ((int[]) data).length;
-            case I64, U64 -> ((long[]) data).length;
-            case F32 -> ((float[]) data).length;
-            case F64 -> ((double[]) data).length;
-        };
-    }
-
 
     /// @return whether [#distinctCount()] was computed during this scan
     public boolean hasDistinctCount() {

writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java

@@ -7,6 +7,7 @@
 import io.github.dfa1.vortex.core.proto.ProtoScalarValue;
 
 import java.lang.foreign.MemorySegment;
+import java.lang.reflect.Array;
 
 /// Write-only encoder for `vortex.constant`.
 public final class ConstantEncodingEncoder implements EncodingEncoder {
@@ -57,7 +58,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
         byte[] scalarBytes = scalar.encode();
         // A constant array's min and max are both the one repeated value, by construction -- no
         // scan needed. Empty arrays report no stats, matching every other encoder's convention.
-        byte[] stats = arrayLength(data, ptype) > 0 ? scalarBytes : null;
+        byte[] stats = Array.getLength(data) > 0 ? scalarBytes : null;
         return EncodeResult.simple(EncodingId.VORTEX_CONSTANT, EncodedBuffer.bytes(MemorySegment.ofArray(scalarBytes)), stats, stats);
     }
 
@@ -109,21 +110,9 @@ private static long readFirstRaw(Object data, PType ptype) {
         };
     }
 
-    private static int arrayLength(Object data, PType ptype) {
-        return switch (ptype) {
-            case I8, U8 -> ((byte[]) data).length;
-            case I16, U16 -> ((short[]) data).length;
-            case I32, U32 -> ((int[]) data).length;
-            case I64, U64 -> ((long[]) data).length;
-            case F32 -> ((float[]) data).length;
-            case F64 -> ((double[]) data).length;
-            default -> throw new VortexException(EncodingId.VORTEX_CONSTANT, "unsupported ptype: " + ptype);
-        };
-    }
-
     private static boolean isConstant(Object data, PType ptype) {
         long firstRaw = readFirstRaw(data, ptype);
-        int len = arrayLength(data, ptype);
+        int len = Array.getLength(data);
         for (int i = 1; i < len; i++) {
             long raw = switch (ptype) {
                 case I8, U8 -> ((byte[]) data)[i];

writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java

@@ -14,6 +14,7 @@
 import java.lang.foreign.Arena;
 import java.lang.foreign.MemorySegment;
 import java.lang.foreign.ValueLayout;
+import java.lang.reflect.Array;
 import java.nio.charset.StandardCharsets;
 import java.util.Set;
 import java.util.LinkedHashMap;
@@ -319,7 +320,7 @@ private static EncodeResult encodeUtf8(String[] strings, EncodeContext ctx) {
 
     private static DictData buildDictData(DType dtype, Object data) {
         PType ptype = ((DType.Primitive) dtype).ptype();
-        int len = arrayLength(data, ptype);
+        int len = Array.getLength(data);
 
         // Dedup on raw value bits rather than boxed keys. This probed a
         // LinkedHashMap<Object, Integer>, boxing a value per ROW just to look it up - the same
@@ -369,17 +370,6 @@ private static PType codePType(int dictSize) {
         return PType.U32;
     }
 
-    private static int arrayLength(Object data, PType ptype) {
-        return switch (ptype) {
-            case I8, U8 -> ((byte[]) data).length;
-            case I16, U16, F16 -> ((short[]) data).length;
-            case I32, U32 -> ((int[]) data).length;
-            case I64, U64 -> ((long[]) data).length;
-            case F32 -> ((float[]) data).length;
-            case F64 -> ((double[]) data).length;
-        };
-    }
-
     /// Builds the dictionary pool from the first-seen raw bit patterns, in order.
     ///
     /// @param ptype    the column's primitive type

writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java

@@ -3,10 +3,9 @@
 import io.github.dfa1.vortex.core.model.DType;
 import io.github.dfa1.vortex.core.model.PType;
 import io.github.dfa1.vortex.core.model.EncodingId;
-import io.github.dfa1.vortex.core.io.VortexFormat;
+import io.github.dfa1.vortex.core.compute.PrimitiveArrays;
 import io.github.dfa1.vortex.core.proto.ProtoScalarValue;
 
-import java.lang.foreign.Arena;
 import java.lang.foreign.MemorySegment;
 
 /// Write-only encoder for `vortex.primitive` — raw little-endian primitive arrays.
@@ -25,7 +24,10 @@ public boolean accepts(DType dtype) {
     @Override
     public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
         PType ptype = ((DType.Primitive) dtype).ptype();
-        MemorySegment seg = encodePrimitive(ptype, data, ctx.arena());
+        // byte[] needs no byte swap, so it is wrapped in place rather than copied off-heap.
+        MemorySegment seg = data instanceof byte[] bytes
+                ? MemorySegment.ofArray(bytes)
+                : PrimitiveArrays.toSegment(data, ptype, ctx.arena());
         byte[] min = null;
         byte[] max = null;
         byte[][] stats = minMaxStats(ptype, data);
@@ -36,52 +38,6 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
         return EncodeResult.simple(EncodingId.VORTEX_PRIMITIVE, EncodedBuffer.of(seg, ptype), min, max);
     }
 
-    private static MemorySegment encodePrimitive(PType ptype, Object data, Arena arena) {
-        return switch (ptype) {
-            case I8, U8 -> MemorySegment.ofArray((byte[]) data);
-            case I16, U16, F16 -> {
-                short[] arr = (short[]) data;
-                MemorySegment seg = arena.allocate((long) arr.length * 2, 2);
-                for (int i = 0; i < arr.length; i++) {
-                    seg.setAtIndex(VortexFormat.LE_SHORT, i, arr[i]);
-                }
-                yield seg;
-            }
-            case I32, U32 -> {
-                int[] arr = (int[]) data;
-                MemorySegment seg = arena.allocate((long) arr.length * 4, 4);
-                for (int i = 0; i < arr.length; i++) {
-                    seg.setAtIndex(VortexFormat.LE_INT, i, arr[i]);
-                }
-                yield seg;
-            }
-            case I64, U64 -> {
-                long[] arr = (long[]) data;
-                MemorySegment seg = arena.allocate((long) arr.length * 8, 8);
-                for (int i = 0; i < arr.length; i++) {
-                    seg.setAtIndex(VortexFormat.LE_LONG, i, arr[i]);
-                }
-                yield seg;
-            }
-            case F32 -> {
-                float[] arr = (float[]) data;
-                MemorySegment seg = arena.allocate((long) arr.length * 4, 4);
-                for (int i = 0; i < arr.length; i++) {
-                    seg.setAtIndex(VortexFormat.LE_FLOAT, i, arr[i]);
-                }
-                yield seg;
-            }
-            case F64 -> {
-                double[] arr = (double[]) data;
-                MemorySegment seg = arena.allocate((long) arr.length * 8, 8);
-                for (int i = 0; i < arr.length; i++) {
-                    seg.setAtIndex(VortexFormat.LE_DOUBLE, i, arr[i]);
-                }
-                yield seg;
-            }
-        };
-    }
-
     /// Computes the serialized min/max [io.github.dfa1.vortex.core.proto.ProtoScalarValue] pair for a raw
     /// primitive array, in the same signed/unsigned/float shape the per-segment stats use. Returns
     /// `null` for an empty array. Shared so the dictionary zone-map path computes per-chunk min/max

writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java

@@ -11,6 +11,7 @@
 import io.github.dfa1.vortex.core.proto.ProtoScalarValue;
 
 import java.lang.foreign.MemorySegment;
+import java.lang.reflect.Array;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -117,7 +118,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
             throw new VortexException(EncodingId.VORTEX_RUNEND, "encode only supports Primitive dtype, got " + dtype);
         }
         PType ptype = p.ptype();
-        int n = arrayLength(data, ptype);
+        int n = Array.getLength(data);
         boolean unsign = ptype.isUnsigned();
 
         int elemBytes = ptype.byteSize();
@@ -228,7 +229,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) {
             return CascadeStep.notApplicable();
         }
         PType ptype = p.ptype();
-        int n = arrayLength(data, ptype);
+        int n = Array.getLength(data);
         if (n == 0) {
             return CascadeStep.notApplicable();
         }
@@ -269,15 +270,4 @@ private static byte[] statsBytes(PType ptype, long value) {
         }
         return ProtoScalarValue.ofInt64Value(value).encode();
     }
-
-    private static int arrayLength(Object data, PType ptype) {
-        return switch (ptype) {
-            case I8, U8 -> ((byte[]) data).length;
-            case I16, U16 -> ((short[]) data).length;
-            case I32, U32 -> ((int[]) data).length;
-            case I64, U64 -> ((long[]) data).length;
-            default -> throw new VortexException(EncodingId.VORTEX_RUNEND, "unsupported ptype: " + ptype);
-        };
-    }
-
 }