diff --git a/CHANGELOG.md b/CHANGELOG.md index e10d33ab..481e3c3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,9 +25,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `vortex.sparse` propagates nullability: a `fill_value: null` array nulls every unpatched position (world-energy-consumption `biofuel_cons_change_pct` f64? previously decoded them as 0.0), and a null patch value nulls its own position. Row validity is a lazy sparse bool whose fill is `fill_value.is_valid()` and whose patch bits are the patch values' validity, matching the Rust `ValidityVTable`. ([#226](https://github.com/dfa1/vortex-java/issues/226)) - `vortex.sparse` over utf8/binary values now carries the same row validity as the primitive path: a `fill_value: null` string column nulls every unpatched position instead of rendering an empty string, and a nullable patch value nulls its own position instead of surfacing its raw bytes. The Rust `ValidityVTable` is generic over the values encoding, so VarBin reuses the identical sparse-bool validity — completing the #226 fix for string/binary columns. ([#232](https://github.com/dfa1/vortex-java/issues/232)) - `vortex.datetimeparts` propagates null component rows instead of throwing `DateTimeParts: null cell at index`: when any part (days/seconds/subseconds) decodes to a null — as a nullable `vortex.runend` child now does after #225 — the reassembled timestamp row is null, unblocking scans of bi-yalelanguages and bi-euro2016. ([#235](https://github.com/dfa1/vortex-java/issues/235)) +- `fastlanes.alprd` now propagates left_parts validity: null float rows no longer decode as `0.0`. ([#234](https://github.com/dfa1/vortex-java/issues/234)) ### Added +- Integration test for null-fill Sparse and null-run RunEnd round-trip, running on every gate. ([#233](https://github.com/dfa1/vortex-java/issues/233)) - Real-world conformance suite against the [Raincloud](https://github.com/spiraldb/raincloud) corpus: 247 public datasets whose Vortex files are written by the Python bindings, each validated value-for-value against its Parquet sibling. `scripts/hydrate-raincloud-corpus.sh` hydrates any subset (cache → mirror → local build), `RaincloudConformanceIntegrationTest` tests whatever is hydrated against the checked-in per-dataset status matrix, and a weekly workflow runs a size-capped sweep. First triage found four reader gaps on real data, including one silent-corruption bug (unsigned integers rendered as signed). ([#205](https://github.com/dfa1/vortex-java/issues/205)) ## [0.12.0] — 2026-07-04 diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv index ff628924..8b760e75 100644 --- a/integration/src/test/resources/raincloud/expected-status.csv +++ b/integration/src/test/resources/raincloud/expected-status.csv @@ -29,7 +29,7 @@ bi-cmsprovider,untriaged bi-commongovernment,untriaged bi-corporations,ok bi-eixo,untriaged -bi-euro2016,gap:234 +bi-euro2016,ok bi-food,ok bi-generico,untriaged bi-hashtags,untriaged diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java index cc0578a8..7658da44 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java @@ -8,6 +8,7 @@ import io.github.dfa1.vortex.core.proto.ProtoALPRDMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.LazyAlpRdDoubleArray; import io.github.dfa1.vortex.reader.array.LazyAlpRdFloatArray; @@ -45,14 +46,22 @@ public Array decode(DecodeContext ctx) { long n = ctx.rowCount(); PType ptype = p.ptype(); - // Lazy path: keep left/right as typed Arrays + patches as a small short[] + - // a lazy indices Array. No n-sized output buffer allocated. + // Validity mirrors the Rust reference: an ALP-RD array's validity IS its + // left_parts child's validity. Decode the child as an Array so a nullable + // left_parts surfaces its MaskedArray; capture the mask and re-wrap the decoded + // result rather than flattening it (which silently dropped nulls — #234). Array leftRaw = ctx.decodeChild(0, DType.U16, n); - ShortArray leftArr = (ShortArray) unwrap(leftRaw); + BoolArray validity = null; + Array leftInner = leftRaw; + if (leftRaw instanceof MaskedArray masked) { + leftInner = masked.inner(); + validity = masked.validity(); + } + ShortArray leftArr = (ShortArray) leftInner; Patches patches = decodePatches(ctx, meta.patches()); - return switch (ptype) { + Array decoded = switch (ptype) { case F64 -> { Array rightRaw = ctx.decodeChild(1, DType.U64, n); LongArray rightArr = (LongArray) unwrap(rightRaw); @@ -67,6 +76,7 @@ yield new LazyAlpRdFloatArray(ctx.dtype(), n, dict, rightBitWidth, } default -> throw new VortexException(EncodingId.VORTEX_ALPRD, "unsupported dtype " + ptype); }; + return validity != null ? new MaskedArray(decoded, validity) : decoded; } private static Array unwrap(Array arr) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoderTest.java new file mode 100644 index 00000000..ff120972 --- /dev/null +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoderTest.java @@ -0,0 +1,126 @@ +package io.github.dfa1.vortex.reader.decode; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.proto.ProtoALPRDMetadata; +import io.github.dfa1.vortex.core.proto.ProtoPType; +import io.github.dfa1.vortex.reader.ReadRegistry; +import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.DoubleArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import org.junit.jupiter.api.Test; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.within; + +class AlpRdEncodingDecoderTest { + + private static final AlpRdEncodingDecoder SUT = new AlpRdEncodingDecoder(); + private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders( + SUT, new PrimitiveEncodingDecoder(), new BoolEncodingDecoder()); + + // ALP-RD reconstruction is `longBitsToDouble((dict[left] << rightBitWidth) | right)`. + // With rightBitWidth=48, the dict codes are the top 16 bits of an f64 bit pattern: + // 1.0 = 0x3FF0_0000_0000_0000 -> 0x3FF0; 2.0 = 0x4000_0000_0000_0000 -> 0x4000. + private static final int RIGHT_BIT_WIDTH = 48; + private static final int DICT_ONE = 0x3FF0; + private static final int DICT_TWO = 0x4000; + + @Test + void decode_nullLeftParts_yieldsNullRows() { + // Given — left codes [0,1,0] with row 1 null via the left_parts validity bitmap; + // the decoder must carry that mask onto the reconstructed doubles (#234). + DecodeContext ctx = maskedContext(new short[]{0, 1, 0}, new long[]{0, 0, 0}, + boolBitmap(true, false, true)); + + // When + Array result = SUT.decode(ctx); + + // Then — the null row survives and valid rows decode to their dictionary doubles + MaskedArray masked = (MaskedArray) result; + assertThat(masked.isValid(0)).isTrue(); + assertThat(masked.isValid(1)).isFalse(); + assertThat(masked.isValid(2)).isTrue(); + DoubleArray inner = (DoubleArray) masked.inner(); + assertThat(inner.getDouble(0)).isCloseTo(1.0, within(1e-12)); + assertThat(inner.getDouble(2)).isCloseTo(1.0, within(1e-12)); + } + + @Test + void decode_allValidLeftParts_yieldsPlainArray() { + // Given — a non-nullable left_parts child (no validity): no-regression path must NOT mask + DecodeContext ctx = plainContext(new short[]{0, 1, 0}, new long[]{0, 0, 0}); + + // When + Array result = SUT.decode(ctx); + + // Then — a bare DoubleArray, and every row decodes through the dictionary + assertThat(result).isInstanceOf(DoubleArray.class).isNotInstanceOf(MaskedArray.class); + DoubleArray inner = (DoubleArray) result; + assertThat(inner.getDouble(0)).isCloseTo(1.0, within(1e-12)); + assertThat(inner.getDouble(1)).isCloseTo(2.0, within(1e-12)); + assertThat(inner.getDouble(2)).isCloseTo(1.0, within(1e-12)); + } + + private static DecodeContext maskedContext(short[] leftCodes, long[] rightBits, MemorySegment validity) { + // buffers: 0 = left u16 codes, 1 = right u64 payloads, 2 = validity bitmap + ArrayNode validityNode = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{2}); + ArrayNode leftNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, + new ArrayNode[]{validityNode}, new int[]{0}); + ArrayNode rightNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALPRD, metaSegment(), + new ArrayNode[]{leftNode, rightNode}, new int[0]); + MemorySegment[] segs = {leShorts(leftCodes), leLongs(rightBits), validity}; + return new DecodeContext(node, DType.F64, leftCodes.length, segs, REGISTRY, Arena.ofAuto()); + } + + private static DecodeContext plainContext(short[] leftCodes, long[] rightBits) { + // buffers: 0 = left u16 codes, 1 = right u64 payloads (no validity child) + ArrayNode leftNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode rightNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALPRD, metaSegment(), + new ArrayNode[]{leftNode, rightNode}, new int[0]); + MemorySegment[] segs = {leShorts(leftCodes), leLongs(rightBits)}; + return new DecodeContext(node, DType.F64, leftCodes.length, segs, REGISTRY, Arena.ofAuto()); + } + + private static MemorySegment metaSegment() { + byte[] meta = new ProtoALPRDMetadata(RIGHT_BIT_WIDTH, 2, + List.of(DICT_ONE, DICT_TWO), ProtoPType.U16, null).encode(); + return MemorySegment.ofArray(meta); + } + + private static MemorySegment leShorts(short... vs) { + byte[] b = new byte[vs.length * 2]; + ByteBuffer bb = ByteBuffer.wrap(b).order(ByteOrder.LITTLE_ENDIAN); + for (short v : vs) { + bb.putShort(v); + } + return MemorySegment.ofArray(b); + } + + private static MemorySegment leLongs(long... vs) { + byte[] b = new byte[vs.length * 8]; + ByteBuffer bb = ByteBuffer.wrap(b).order(ByteOrder.LITTLE_ENDIAN); + for (long v : vs) { + bb.putLong(v); + } + return MemorySegment.ofArray(b); + } + + private static MemorySegment boolBitmap(boolean... valid) { + byte[] bytes = new byte[(valid.length + 7) / 8]; + for (int i = 0; i < valid.length; i++) { + if (valid[i]) { + bytes[i >>> 3] |= (byte) (1 << (i & 7)); + } + } + return MemorySegment.ofArray(bytes); + } +}