fix: AlpRd propagates left_parts validity; null float rows no longer decode as 0.0 (#234)

CHANGELOG.md

@@ -25,9 +25,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `vortex.sparse` propagates nullability: a `fill_value: null` array nulls every unpatched position (world-energy-consumption `biofuel_cons_change_pct` f64? previously decoded them as 0.0), and a null patch value nulls its own position. Row validity is a lazy sparse bool whose fill is `fill_value.is_valid()` and whose patch bits are the patch values' validity, matching the Rust `ValidityVTable<Sparse>`. ([#226](https://github.com/dfa1/vortex-java/issues/226))
 - `vortex.sparse` over utf8/binary values now carries the same row validity as the primitive path: a `fill_value: null` string column nulls every unpatched position instead of rendering an empty string, and a nullable patch value nulls its own position instead of surfacing its raw bytes. The Rust `ValidityVTable<Sparse>` is generic over the values encoding, so VarBin reuses the identical sparse-bool validity — completing the #226 fix for string/binary columns. ([#232](https://github.com/dfa1/vortex-java/issues/232))
 - `vortex.datetimeparts` propagates null component rows instead of throwing `DateTimeParts: null cell at index`: when any part (days/seconds/subseconds) decodes to a null — as a nullable `vortex.runend` child now does after #225 — the reassembled timestamp row is null, unblocking scans of bi-yalelanguages and bi-euro2016. ([#235](https://github.com/dfa1/vortex-java/issues/235))
+- `fastlanes.alprd` now propagates left_parts validity: null float rows no longer decode as `0.0`. ([#234](https://github.com/dfa1/vortex-java/issues/234))
 
 ### Added
 
+- Integration test for null-fill Sparse and null-run RunEnd round-trip, running on every gate. ([#233](https://github.com/dfa1/vortex-java/issues/233))
 - Real-world conformance suite against the [Raincloud](https://github.com/spiraldb/raincloud) corpus: 247 public datasets whose Vortex files are written by the Python bindings, each validated value-for-value against its Parquet sibling. `scripts/hydrate-raincloud-corpus.sh` hydrates any subset (cache → mirror → local build), `RaincloudConformanceIntegrationTest` tests whatever is hydrated against the checked-in per-dataset status matrix, and a weekly workflow runs a size-capped sweep. First triage found four reader gaps on real data, including one silent-corruption bug (unsigned integers rendered as signed). ([#205](https://github.com/dfa1/vortex-java/issues/205))
 
 ## [0.12.0] — 2026-07-04

integration/src/test/resources/raincloud/expected-status.csv

@@ -29,7 +29,7 @@ bi-cmsprovider,untriaged
 bi-commongovernment,untriaged
 bi-corporations,ok
 bi-eixo,untriaged
-bi-euro2016,gap:234
+bi-euro2016,ok
 bi-food,ok
 bi-generico,untriaged
 bi-hashtags,untriaged

reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java

@@ -8,6 +8,7 @@
 import io.github.dfa1.vortex.core.proto.ProtoALPRDMetadata;
 import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata;
 import io.github.dfa1.vortex.reader.array.Array;
+import io.github.dfa1.vortex.reader.array.BoolArray;
 import io.github.dfa1.vortex.reader.array.IntArray;
 import io.github.dfa1.vortex.reader.array.LazyAlpRdDoubleArray;
 import io.github.dfa1.vortex.reader.array.LazyAlpRdFloatArray;
@@ -45,14 +46,22 @@ public Array decode(DecodeContext ctx) {
         long n = ctx.rowCount();
         PType ptype = p.ptype();
 
-        // Lazy path: keep left/right as typed Arrays + patches as a small short[] +
-        // a lazy indices Array. No n-sized output buffer allocated.
+        // Validity mirrors the Rust reference: an ALP-RD array's validity IS its
+        // left_parts child's validity. Decode the child as an Array so a nullable
+        // left_parts surfaces its MaskedArray; capture the mask and re-wrap the decoded
+        // result rather than flattening it (which silently dropped nulls — #234).
         Array leftRaw = ctx.decodeChild(0, DType.U16, n);
-        ShortArray leftArr = (ShortArray) unwrap(leftRaw);
+        BoolArray validity = null;
+        Array leftInner = leftRaw;
+        if (leftRaw instanceof MaskedArray masked) {
+            leftInner = masked.inner();
+            validity = masked.validity();
+        }
+        ShortArray leftArr = (ShortArray) leftInner;
 
         Patches patches = decodePatches(ctx, meta.patches());
 
-        return switch (ptype) {
+        Array decoded = switch (ptype) {
             case F64 -> {
                 Array rightRaw = ctx.decodeChild(1, DType.U64, n);
                 LongArray rightArr = (LongArray) unwrap(rightRaw);
@@ -67,6 +76,7 @@ yield new LazyAlpRdFloatArray(ctx.dtype(), n, dict, rightBitWidth,
             }
             default -> throw new VortexException(EncodingId.VORTEX_ALPRD, "unsupported dtype " + ptype);
         };
+        return validity != null ? new MaskedArray(decoded, validity) : decoded;
     }
 
     private static Array unwrap(Array arr) {

reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoderTest.java

@@ -0,0 +1,126 @@
+package io.github.dfa1.vortex.reader.decode;
+
+import io.github.dfa1.vortex.core.model.DType;
+import io.github.dfa1.vortex.core.model.EncodingId;
+import io.github.dfa1.vortex.core.proto.ProtoALPRDMetadata;
+import io.github.dfa1.vortex.core.proto.ProtoPType;
+import io.github.dfa1.vortex.reader.ReadRegistry;
+import io.github.dfa1.vortex.reader.array.Array;
+import io.github.dfa1.vortex.reader.array.DoubleArray;
+import io.github.dfa1.vortex.reader.array.MaskedArray;
+import org.junit.jupiter.api.Test;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.within;
+
+class AlpRdEncodingDecoderTest {
+
+    private static final AlpRdEncodingDecoder SUT = new AlpRdEncodingDecoder();
+    private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders(
+            SUT, new PrimitiveEncodingDecoder(), new BoolEncodingDecoder());
+
+    // ALP-RD reconstruction is `longBitsToDouble((dict[left] << rightBitWidth) | right)`.
+    // With rightBitWidth=48, the dict codes are the top 16 bits of an f64 bit pattern:
+    // 1.0 = 0x3FF0_0000_0000_0000 -> 0x3FF0; 2.0 = 0x4000_0000_0000_0000 -> 0x4000.
+    private static final int RIGHT_BIT_WIDTH = 48;
+    private static final int DICT_ONE = 0x3FF0;
+    private static final int DICT_TWO = 0x4000;
+
+    @Test
+    void decode_nullLeftParts_yieldsNullRows() {
+        // Given — left codes [0,1,0] with row 1 null via the left_parts validity bitmap;
+        // the decoder must carry that mask onto the reconstructed doubles (#234).
+        DecodeContext ctx = maskedContext(new short[]{0, 1, 0}, new long[]{0, 0, 0},
+                boolBitmap(true, false, true));
+
+        // When
+        Array result = SUT.decode(ctx);
+
+        // Then — the null row survives and valid rows decode to their dictionary doubles
+        MaskedArray masked = (MaskedArray) result;
+        assertThat(masked.isValid(0)).isTrue();
+        assertThat(masked.isValid(1)).isFalse();
+        assertThat(masked.isValid(2)).isTrue();
+        DoubleArray inner = (DoubleArray) masked.inner();
+        assertThat(inner.getDouble(0)).isCloseTo(1.0, within(1e-12));
+        assertThat(inner.getDouble(2)).isCloseTo(1.0, within(1e-12));
+    }
+
+    @Test
+    void decode_allValidLeftParts_yieldsPlainArray() {
+        // Given — a non-nullable left_parts child (no validity): no-regression path must NOT mask
+        DecodeContext ctx = plainContext(new short[]{0, 1, 0}, new long[]{0, 0, 0});
+
+        // When
+        Array result = SUT.decode(ctx);
+
+        // Then — a bare DoubleArray, and every row decodes through the dictionary
+        assertThat(result).isInstanceOf(DoubleArray.class).isNotInstanceOf(MaskedArray.class);
+        DoubleArray inner = (DoubleArray) result;
+        assertThat(inner.getDouble(0)).isCloseTo(1.0, within(1e-12));
+        assertThat(inner.getDouble(1)).isCloseTo(2.0, within(1e-12));
+        assertThat(inner.getDouble(2)).isCloseTo(1.0, within(1e-12));
+    }
+
+    private static DecodeContext maskedContext(short[] leftCodes, long[] rightBits, MemorySegment validity) {
+        // buffers: 0 = left u16 codes, 1 = right u64 payloads, 2 = validity bitmap
+        ArrayNode validityNode = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{2});
+        ArrayNode leftNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null,
+                new ArrayNode[]{validityNode}, new int[]{0});
+        ArrayNode rightNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1});
+        ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALPRD, metaSegment(),
+                new ArrayNode[]{leftNode, rightNode}, new int[0]);
+        MemorySegment[] segs = {leShorts(leftCodes), leLongs(rightBits), validity};
+        return new DecodeContext(node, DType.F64, leftCodes.length, segs, REGISTRY, Arena.ofAuto());
+    }
+
+    private static DecodeContext plainContext(short[] leftCodes, long[] rightBits) {
+        // buffers: 0 = left u16 codes, 1 = right u64 payloads (no validity child)
+        ArrayNode leftNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0});
+        ArrayNode rightNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1});
+        ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALPRD, metaSegment(),
+                new ArrayNode[]{leftNode, rightNode}, new int[0]);
+        MemorySegment[] segs = {leShorts(leftCodes), leLongs(rightBits)};
+        return new DecodeContext(node, DType.F64, leftCodes.length, segs, REGISTRY, Arena.ofAuto());
+    }
+
+    private static MemorySegment metaSegment() {
+        byte[] meta = new ProtoALPRDMetadata(RIGHT_BIT_WIDTH, 2,
+                List.of(DICT_ONE, DICT_TWO), ProtoPType.U16, null).encode();
+        return MemorySegment.ofArray(meta);
+    }
+
+    private static MemorySegment leShorts(short... vs) {
+        byte[] b = new byte[vs.length * 2];
+        ByteBuffer bb = ByteBuffer.wrap(b).order(ByteOrder.LITTLE_ENDIAN);
+        for (short v : vs) {
+            bb.putShort(v);
+        }
+        return MemorySegment.ofArray(b);
+    }
+
+    private static MemorySegment leLongs(long... vs) {
+        byte[] b = new byte[vs.length * 8];
+        ByteBuffer bb = ByteBuffer.wrap(b).order(ByteOrder.LITTLE_ENDIAN);
+        for (long v : vs) {
+            bb.putLong(v);
+        }
+        return MemorySegment.ofArray(b);
+    }
+
+    private static MemorySegment boolBitmap(boolean... valid) {
+        byte[] bytes = new byte[(valid.length + 7) / 8];
+        for (int i = 0; i < valid.length; i++) {
+            if (valid[i]) {
+                bytes[i >>> 3] |= (byte) (1 << (i & 7));
+            }
+        }
+        return MemorySegment.ofArray(bytes);
+    }
+}