diff --git a/CHANGELOG.md b/CHANGELOG.md index 835ac4bf..e884dc29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Null rows of Rust-written `vortex.list` columns read as empty lists: the decoder ignored the list's validity child. - CSV export (`CsvExporter`, `vortex export`) failed on list-view columns with `unsupported array type for CSV export: ListViewArray`. - `vortex inspect --html` grouped the Chunks panel by chunk index, so files whose columns chunk differently (e.g. Rust's `tpch_orders.compact`) showed overlapping row ranges and mixed sizes; the panel now lists one entry per distinct row range. +- `vortex.patched` columns of F16 values failed to read with `unsupported ptype: F16`; Rust accepts every primitive type there. ([#449](https://github.com/dfa1/vortex-java/pull/449)) ## [0.15.2] — 2026-10-03 diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java index 8c6bad80..c4feb8c1 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java @@ -10,10 +10,6 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.MaskedArray; -import io.github.dfa1.vortex.reader.array.MaterializedByteArray; -import io.github.dfa1.vortex.reader.array.MaterializedIntArray; -import io.github.dfa1.vortex.reader.array.MaterializedLongArray; -import io.github.dfa1.vortex.reader.array.MaterializedShortArray; import java.io.IOException; import java.lang.foreign.MemorySegment; @@ -53,6 +49,9 @@ public Array decode(DecodeContext ctx) { int bitWidth = meta.bit_width(); int offset = meta.offset(); PType ptype = ((DType.Primitive) ctx.dtype()).ptype(); + if (ptype.isFloating()) { + throw new VortexException(EncodingId.FASTLANES_BITPACKED, "unsupported ptype " + ptype); + } int typeBits = ptype.bits(); long rowCount = ctx.rowCount(); @@ -89,13 +88,7 @@ public Array decode(DecodeContext ctx) { applyPatches(ctx, meta.patches(), output, ptype.byteSize()); } - Array values = switch (ptype) { - case I64, U64 -> new MaterializedLongArray(ctx.dtype(), rowCount, output); - case I32, U32 -> new MaterializedIntArray(ctx.dtype(), rowCount, output); - case I16, U16 -> new MaterializedShortArray(ctx.dtype(), rowCount, output); - case I8, U8 -> new MaterializedByteArray(ctx.dtype(), rowCount, output); - default -> throw new VortexException(EncodingId.FASTLANES_BITPACKED, "unsupported ptype " + ptype); - }; + Array values = MaterializedArrays.of(ctx.dtype(), ptype, rowCount, output); return wrapValidity(ctx, meta, values, rowCount); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java index 9ab77b62..7a572796 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java @@ -9,10 +9,6 @@ import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoDeltaMetadata; import io.github.dfa1.vortex.reader.array.Array; -import io.github.dfa1.vortex.reader.array.MaterializedByteArray; -import io.github.dfa1.vortex.reader.array.MaterializedIntArray; -import io.github.dfa1.vortex.reader.array.MaterializedLongArray; -import io.github.dfa1.vortex.reader.array.MaterializedShortArray; import java.io.IOException; import java.lang.foreign.MemorySegment; @@ -47,6 +43,9 @@ public Array decode(DecodeContext ctx) { } PType ptype = ((DType.Primitive) ctx.dtype()).ptype(); + if (ptype.isFloating()) { + throw new VortexException(EncodingId.FASTLANES_DELTA, "unsupported ptype: " + ptype); + } long rowCount = ctx.rowCount(); int typeBits = ptype.bits(); int lanes = FastLanes.lanes(ptype); @@ -56,7 +55,7 @@ public Array decode(DecodeContext ctx) { int offset = meta.offset(); if (deltasLen == 0L || rowCount == 0L) { - return array(ctx, ptype, 0L, ctx.arena().allocate(0)); + return MaterializedArrays.of(ctx.dtype(), ptype, 0L, ctx.arena().allocate(0)); } // Rows come from the window `[offset, offset + rowCount)` of the `deltasLen` elements @@ -104,7 +103,7 @@ public Array decode(DecodeContext ctx) { undeltaChunk(chunkDeltas, chunkBases, lanes, typeBits, mask, chunkUndelta); scatterChunk(out, ptype, chunkUndelta, chunk * FastLanes.CHUNK - offset, rowCount); } - return array(ctx, ptype, rowCount, out.asReadOnly()); + return MaterializedArrays.of(ctx.dtype(), ptype, rowCount, out.asReadOnly()); } /// Untransposes one chunk straight into the output window. @@ -163,24 +162,6 @@ private static void scatterChunk(MemorySegment out, PType ptype, long[] values, } } - /// Wraps a decoded segment in the `Materialized*Array` matching `ptype`. - /// - /// @param ctx decode context, source of the logical dtype - /// @param ptype physical type of the values in `seg` - /// @param length row count - /// @param seg the decoded values, little-endian at `ptype`'s width - /// @return the typed array view over `seg` - /// @throws VortexException if `ptype` is not an integer ptype - private static Array array(DecodeContext ctx, PType ptype, long length, MemorySegment seg) { - return switch (ptype) { - case I64, U64 -> new MaterializedLongArray(ctx.dtype(), length, seg); - case I32, U32 -> new MaterializedIntArray(ctx.dtype(), length, seg); - case I16, U16 -> new MaterializedShortArray(ctx.dtype(), length, seg); - case I8, U8 -> new MaterializedByteArray(ctx.dtype(), length, seg); - default -> throw new VortexException(EncodingId.FASTLANES_DELTA, "unsupported ptype: " + ptype); - }; - } - private static void undeltaChunk(long[] deltas, long[] bases, int lanes, int typeBits, long mask, long[] out) { for (int lane = 0; lane < lanes; lane++) { long prev = bases[lane] & mask; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java index cfb1bf04..84bfd18d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java @@ -21,12 +21,6 @@ import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.MaterializedBoolArray; -import io.github.dfa1.vortex.reader.array.MaterializedByteArray; -import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray; -import io.github.dfa1.vortex.reader.array.MaterializedFloatArray; -import io.github.dfa1.vortex.reader.array.MaterializedIntArray; -import io.github.dfa1.vortex.reader.array.MaterializedLongArray; -import io.github.dfa1.vortex.reader.array.MaterializedShortArray; import io.github.dfa1.vortex.reader.array.ShortArray; import io.github.dfa1.vortex.reader.array.VarBinArray; import io.github.dfa1.vortex.reader.array.VarBinOffsetArray; @@ -97,8 +91,8 @@ private static Array decodeLegacyJava(DecodeContext ctx, byte codeTypeByte) { valuesBuf.byteSize(), valPType.byteSize()); long poolLength = valuesBuf.byteSize() / valPType.byteSize(); - Array values = typedArray(ctx.dtype(), valPType, poolLength, valuesBuf); - Array codes = typedArray(codesDtype, codePType, rowCount, codesBuf); + Array values = MaterializedArrays.of(ctx.dtype(), valPType, poolLength, valuesBuf); + Array codes = MaterializedArrays.of(codesDtype, codePType, rowCount, codesBuf); validateCodesInRange(codes, poolLength); return buildLazyDict(ctx.dtype(), valPType, rowCount, values, codes); } @@ -390,16 +384,4 @@ private static Array decodeUtf8DictProto(DecodeContext ctx, MemorySegment metaBu return rowValidity == null ? dict : new MaskedArray(dict, rowValidity); } - - private static Array typedArray(DType dtype, PType ptype, long n, MemorySegment seg) { - return switch (ptype) { - case I64, U64 -> new MaterializedLongArray(dtype, n, seg); - case I32, U32 -> new MaterializedIntArray(dtype, n, seg); - case F64 -> new MaterializedDoubleArray(dtype, n, seg); - case F32 -> new MaterializedFloatArray(dtype, n, seg); - case I16, U16 -> new MaterializedShortArray(dtype, n, seg); - case I8, U8 -> new MaterializedByteArray(dtype, n, seg); - default -> throw new VortexException(EncodingId.VORTEX_DICT, "unsupported ptype " + ptype); - }; - } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/MaterializedArrays.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/MaterializedArrays.java new file mode 100644 index 00000000..cb8d6df7 --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/MaterializedArrays.java @@ -0,0 +1,39 @@ +package io.github.dfa1.vortex.reader.decode; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.PType; +import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.MaterializedByteArray; +import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray; +import io.github.dfa1.vortex.reader.array.MaterializedFloat16Array; +import io.github.dfa1.vortex.reader.array.MaterializedFloatArray; +import io.github.dfa1.vortex.reader.array.MaterializedIntArray; +import io.github.dfa1.vortex.reader.array.MaterializedLongArray; +import io.github.dfa1.vortex.reader.array.MaterializedShortArray; + +import java.lang.foreign.MemorySegment; + +/// Wraps a decoded little-endian segment in the `MaterializedXxxArray` for its ptype. Callers +/// that only accept some ptypes (e.g. integer-only encodings) reject the rest before decoding. +final class MaterializedArrays { + + private MaterializedArrays() { + } + + /// @param dtype the dtype the array reports + /// @param ptype the physical element type of `seg` + /// @param n the element count + /// @param seg the decoded elements, little-endian + /// @return a zero-copy array view over `seg` + static Array of(DType dtype, PType ptype, long n, MemorySegment seg) { + return switch (ptype) { + case I8, U8 -> new MaterializedByteArray(dtype, n, seg); + case I16, U16 -> new MaterializedShortArray(dtype, n, seg); + case I32, U32 -> new MaterializedIntArray(dtype, n, seg); + case I64, U64 -> new MaterializedLongArray(dtype, n, seg); + case F16 -> new MaterializedFloat16Array(dtype, n, seg); + case F32 -> new MaterializedFloatArray(dtype, n, seg); + case F64 -> new MaterializedDoubleArray(dtype, n, seg); + }; + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java index 3db3af62..4e2c57fb 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java @@ -7,12 +7,6 @@ import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoPatchedMetadata; import io.github.dfa1.vortex.reader.array.Array; -import io.github.dfa1.vortex.reader.array.MaterializedByteArray; -import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray; -import io.github.dfa1.vortex.reader.array.MaterializedFloatArray; -import io.github.dfa1.vortex.reader.array.MaterializedIntArray; -import io.github.dfa1.vortex.reader.array.MaterializedLongArray; -import io.github.dfa1.vortex.reader.array.MaterializedShortArray; import java.io.IOException; import java.lang.foreign.MemorySegment; @@ -69,16 +63,7 @@ public Array decode(DecodeContext ctx) { MemorySegment out = patchedOutput(ctx, innerSeg, n, elemBytes, nPatches, nChunks, nLanes, offset, laneOffsetsSeg, patchIndicesSeg, patchValuesSeg); - return switch (ptype) { - case I8, U8 -> new MaterializedByteArray(ctx.dtype(), n, out); - case I16, U16 -> new MaterializedShortArray(ctx.dtype(), n, out); - case I32, U32 -> new MaterializedIntArray(ctx.dtype(), n, out); - case I64, U64 -> new MaterializedLongArray(ctx.dtype(), n, out); - case F32 -> new MaterializedFloatArray(ctx.dtype(), n, out); - case F64 -> new MaterializedDoubleArray(ctx.dtype(), n, out); - default -> throw new VortexException(EncodingId.VORTEX_PATCHED, - "unsupported ptype: " + ptype); - }; + return MaterializedArrays.of(ctx.dtype(), ptype, n, out); } /// Produces the patched values buffer. diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PrimitiveEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PrimitiveEncodingDecoder.java index 259c0023..fc310fe6 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PrimitiveEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PrimitiveEncodingDecoder.java @@ -7,13 +7,6 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.MaskedArray; -import io.github.dfa1.vortex.reader.array.MaterializedByteArray; -import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray; -import io.github.dfa1.vortex.reader.array.MaterializedFloat16Array; -import io.github.dfa1.vortex.reader.array.MaterializedFloatArray; -import io.github.dfa1.vortex.reader.array.MaterializedIntArray; -import io.github.dfa1.vortex.reader.array.MaterializedLongArray; -import io.github.dfa1.vortex.reader.array.MaterializedShortArray; import java.lang.foreign.MemorySegment; @@ -38,15 +31,7 @@ public Array decode(DecodeContext ctx) { throw new VortexException(EncodingId.VORTEX_PRIMITIVE, "expected primitive dtype, got " + dt); } PType ptype = primitiveDt.ptype(); - Array values = switch (ptype) { - case I64, U64 -> new MaterializedLongArray(dt, n, buf); - case I32, U32 -> new MaterializedIntArray(dt, n, buf); - case F64 -> new MaterializedDoubleArray(dt, n, buf); - case F32 -> new MaterializedFloatArray(dt, n, buf); - case I16, U16 -> new MaterializedShortArray(dt, n, buf); - case I8, U8 -> new MaterializedByteArray(dt, n, buf); - case F16 -> new MaterializedFloat16Array(dt, n, buf); - }; + Array values = MaterializedArrays.of(dt, ptype, n, buf); if (ctx.node().children().length == 1) { Array va = ctx.decodeChild(0, DType.BOOL, n); BoolArray validity = MaskedArray.requireBoolArray(va, EncodingId.VORTEX_PRIMITIVE, "validity child"); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java index c9e22292..b562b6df 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java @@ -10,13 +10,6 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.MaskedArray; -import io.github.dfa1.vortex.reader.array.MaterializedByteArray; -import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray; -import io.github.dfa1.vortex.reader.array.MaterializedFloat16Array; -import io.github.dfa1.vortex.reader.array.MaterializedFloatArray; -import io.github.dfa1.vortex.reader.array.MaterializedIntArray; -import io.github.dfa1.vortex.reader.array.MaterializedLongArray; -import io.github.dfa1.vortex.reader.array.MaterializedShortArray; import io.github.dfa1.vortex.reader.array.VarBinArray; import io.github.dfa1.vortex.reader.array.VarBinOffsetArray; @@ -144,7 +137,7 @@ private static Array buildScatteredPrimitive( } } DType.Primitive nonNull = new DType.Primitive(dt.ptype(), false); - return buildPrimitive(nonNull, rowCount, out); + return MaterializedArrays.of(nonNull, nonNull.ptype(), rowCount, out); } private static VarBinArray buildScatteredVarBin( @@ -260,7 +253,7 @@ private static MemorySegment asNative(MemorySegment seg, Arena scratch) { private static Array buildArray(DType dtype, long n, MemorySegment decompressed, DecodeContext ctx) { if (dtype instanceof DType.Primitive dt) { - return buildPrimitive(dt, n, decompressed); + return MaterializedArrays.of(dt, dt.ptype(), n, decompressed); } if (dtype instanceof DType.Utf8 || dtype instanceof DType.Binary) { return buildVarBin(dtype, n, decompressed, ctx); @@ -268,19 +261,6 @@ private static Array buildArray(DType dtype, long n, MemorySegment decompressed, throw new VortexException(EncodingId.VORTEX_ZSTD, "unsupported dtype: " + dtype); } - private static Array buildPrimitive(DType.Primitive dt, long n, MemorySegment decompressed) { - PType ptype = dt.ptype(); - return switch (ptype) { - case I64, U64 -> new MaterializedLongArray(dt, n, decompressed); - case I32, U32 -> new MaterializedIntArray(dt, n, decompressed); - case F64 -> new MaterializedDoubleArray(dt, n, decompressed); - case F32 -> new MaterializedFloatArray(dt, n, decompressed); - case I16, U16 -> new MaterializedShortArray(dt, n, decompressed); - case I8, U8 -> new MaterializedByteArray(dt, n, decompressed); - case F16 -> new MaterializedFloat16Array(dt, n, decompressed); - }; - } - /// Reads a 4-byte little-endian length prefix at `pos` from a decompressed VarBin payload and /// validates that both the prefix and the `len` bytes that follow lie within `src`. Without this, /// a crafted payload with a negative or oversized length would advance the cursor out of bounds diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java index c3cf72a5..bf55e6dc 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java @@ -6,6 +6,7 @@ import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.Float16Array; import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.core.model.EncodingId; @@ -197,6 +198,25 @@ void decode_i64_singlePatch() { assertThat(longs.getLong(2)).isEqualTo(300L); } + @Test + void decode_f16_singlePatch() { + // Given — Rust's Patched accepts every native ptype (match_each_native_ptype), F16 included; + // this decoder used to reject F16. 0x3c00 = 1.0, 0x4000 = 2.0, 0x4200 = 3.0, 0x4900 = 10.0. + DType dtype = DType.F16; + + // When + Array result = decode(dtype, 3, + TestSegments.leShorts((short) 0x3c00, (short) 0x4000, (short) 0x4200), TestSegments.leInts(0, 1), + TestSegments.leShorts((short) 1), TestSegments.leShorts((short) 0x4900), 1); + + // Then + assertThat(result).isInstanceOf(Float16Array.class); + Float16Array halves = (Float16Array) result; + assertThat(halves.getFloat(0)).isEqualTo(1.0f); + assertThat(halves.getFloat(1)).isEqualTo(10.0f); + assertThat(halves.getFloat(2)).isEqualTo(3.0f); + } + @Test void decode_missingMetadata_throws() { // Given