fix: propagate validity in RunEnd and Sparse lazy decoders (#225, #226)

CHANGELOG.md

@@ -21,6 +21,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - CSV export handles all-null (`DType.Null`) columns as empty fields instead of throwing `unsupported array type: NullArray`. ([#211](https://github.com/dfa1/vortex-java/issues/211))
 - String-dict columns whose values are FSST-compressed no longer fail to scan with `IndexOutOfBoundsException` — the dictionary value offsets are now read at their true ptype width instead of a hardcoded 8-byte stride (uci-magic-gamma-telescope's `class` column decompressed to 4-byte offsets). ([#215](https://github.com/dfa1/vortex-java/issues/215))
 - Null rows no longer silently decode as values: wrapper decoders now propagate the row validity that files from the Python bindings carry deep in the encoding tree. Three representations were dropped — a trailing validity child on `fastlanes.bitpacked` (reached through `vortex.alp`/`vortex.zigzag`/`fastlanes.for`, which delegate validity to their encoded child per the Rust `ValidityChild` contract), dict pools with invalid slots, and dict codes with their own validity — across both the eager `vortex.dict` decoder and the lazy dict layout path. Found on real data: penguins and kepler exported invented values (`32.1`, `0.0`) for thousands of null cells. ([#210](https://github.com/dfa1/vortex-java/issues/210))
+- `vortex.runend` propagates nullable run-values' validity: a null run now nulls every row it covers instead of expanding to a filler value (uci-online-retail `customerid` u16? nulls previously decoded as the FoR base). Row validity is a lazy run-end bool over the same run-ends, matching the Rust `ValidityVTable<RunEnd>`. ([#225](https://github.com/dfa1/vortex-java/issues/225))
+- `vortex.sparse` propagates nullability: a `fill_value: null` array nulls every unpatched position (world-energy-consumption `biofuel_cons_change_pct` f64? previously decoded them as 0.0), and a null patch value nulls its own position. Row validity is a lazy sparse bool whose fill is `fill_value.is_valid()` and whose patch bits are the patch values' validity, matching the Rust `ValidityVTable<Sparse>`. ([#226](https://github.com/dfa1/vortex-java/issues/226))
 
 ### Added
 

docs/compatibility.md

@@ -62,13 +62,12 @@ Per-slug status lives in `integration/src/test/resources/raincloud/expected-stat
 (`ok` must pass; `gap:<issue>` must still fail, so a fix flips the entry in the same change;
 `untriaged` runs and reports without failing the build). A scheduled workflow
 (`raincloud-conformance.yml`) hydrates a size-capped subset weekly. Current triage —
-38 `ok`, 2 known gaps (both silent null loss in lazy decoders:
-[#225](https://github.com/dfa1/vortex-java/issues/225) RunEnd run-values validity,
-[#226](https://github.com/dfa1/vortex-java/issues/226) Sparse null fill / nullable patches);
-207 slugs untriaged. Every gap found by earlier rounds is fixed
+40 `ok`, 0 known gaps; 207 slugs untriaged. Every gap found so far is fixed
 ([#206](https://github.com/dfa1/vortex-java/issues/206)–[#211](https://github.com/dfa1/vortex-java/issues/211),
 [#215](https://github.com/dfa1/vortex-java/issues/215)–[#217](https://github.com/dfa1/vortex-java/issues/217),
-[#221](https://github.com/dfa1/vortex-java/issues/221)).
+[#221](https://github.com/dfa1/vortex-java/issues/221),
+[#225](https://github.com/dfa1/vortex-java/issues/225) RunEnd run-values validity,
+[#226](https://github.com/dfa1/vortex-java/issues/226) Sparse null fill / nullable patches).
 
 ## Encodings
 

integration/src/test/resources/raincloud/expected-status.csv

@@ -219,7 +219,7 @@ uci-individual-household-electric-power-consumption,untriaged
 uci-iris,ok
 uci-magic-gamma-telescope,ok
 uci-mushroom,ok
-uci-online-retail,gap:225
+uci-online-retail,ok
 uci-online-retail-ii,untriaged
 uci-online-shoppers-purchasing-intention,untriaged
 uci-optical-recognition-of-handwritten-digits,ok
@@ -250,7 +250,7 @@ waxal-dagbani-asr-test,untriaged
 wdi,untriaged
 websight-v01,untriaged
 wikipedia-en,untriaged
-world-energy-consumption,gap:226
+world-energy-consumption,ok
 yellow_tripdata_2025,untriaged
 youtube-commons-sample,untriaged
 zoo-animal-classification,untriaged

reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java

@@ -55,18 +55,33 @@ public Array decode(DecodeContext ctx) {
         long n = ctx.rowCount();
         DType endsDtype = new DType.Primitive(endsPtype, false);
         Array endsArr = ctx.decodeChild(0, endsDtype, numRuns);
+        Array endsData = endsArr instanceof MaskedArray m ? m.inner() : endsArr;
+
+        // Values-side validity mirrors the Rust reference `ValidityVTable<RunEnd>`: a
+        // RunEnd array's validity IS a RunEnd over the same ends whose per-run value is
+        // the run-value's validity bit. Row i's validity is thus the validity of the run
+        // it falls in — expressed lazily as `LazyRunEndBoolArray`, O(numRuns) not O(n)
+        // (n can be huge, e.g. uci-online-retail 499712 rows). A nullable values child
+        // surfaces as a MaskedArray; dropping its mask expanded null runs to a filler
+        // value (e.g. u16? null rows emitting the FoR base) — #225.
+        Array valuesArr = ctx.decodeChild(1, ctx.dtype(), numRuns);
+        BoolArray valuesValidity = null;
+        Array valuesData = valuesArr;
+        if (valuesArr instanceof MaskedArray m) {
+            valuesData = m.inner();
+            valuesValidity = m.validity();
+        }
 
         if (ctx.dtype() instanceof DType.Utf8 || ctx.dtype() instanceof DType.Binary) {
-            VarBinArray valuesArr = (VarBinArray) ctx.decodeChild(1, ctx.dtype(), numRuns);
-            MemorySegment endsSeg = ctx.materialize(endsArr);
-            return expandStrings(endsSeg, VarBinArray.toOffsetMode(valuesArr, ctx.arena()), endsPtype, numRuns, offset, n, ctx.dtype(), ctx.arena());
+            MemorySegment endsSeg = ctx.materialize(endsData);
+            Array result = expandStrings(endsSeg, VarBinArray.toOffsetMode((VarBinArray) valuesData, ctx.arena()),
+                    endsPtype, numRuns, offset, n, ctx.dtype(), ctx.arena());
+            return withRunValidity(result, valuesValidity, endsData, n, offset);
         }
 
         if (ctx.dtype() instanceof DType.Bool) {
-            Array valuesArr = ctx.decodeChild(1, ctx.dtype(), numRuns);
-            Array valuesData = valuesArr instanceof MaskedArray m ? m.inner() : valuesArr;
-            Array endsData = endsArr instanceof MaskedArray m ? m.inner() : endsArr;
-            return new LazyRunEndBoolArray(ctx.dtype(), n, (BoolArray) valuesData, endsData, offset);
+            Array result = new LazyRunEndBoolArray(ctx.dtype(), n, (BoolArray) valuesData, endsData, offset);
+            return withRunValidity(result, valuesValidity, endsData, n, offset);
         }
 
         if (!(ctx.dtype() instanceof DType.Primitive p)) {
@@ -77,16 +92,32 @@ public Array decode(DecodeContext ctx) {
         // Lazy path: wrap values + ends without expanding into an n-sized buffer.
         // VarBin keeps the eager path above — offset rebasing doesn't trivially
         // express as binary-search-on-read.
-        Array valuesArr = ctx.decodeChild(1, ctx.dtype(), numRuns);
-        Array valuesData = valuesArr instanceof MaskedArray m ? m.inner() : valuesArr;
-        Array endsData = endsArr instanceof MaskedArray m ? m.inner() : endsArr;
-        return switch (valuePtype) {
+        Array result = switch (valuePtype) {
             case I64, U64 -> new LazyRunEndLongArray(ctx.dtype(), n, (LongArray) valuesData, endsData, offset);
             case I32, U32 -> new LazyRunEndIntArray(ctx.dtype(), n, (IntArray) valuesData, endsData, offset);
             case I16, U16 -> new LazyRunEndShortArray(ctx.dtype(), n, (ShortArray) valuesData, endsData, offset);
             case I8, U8 -> new LazyRunEndByteArray(ctx.dtype(), n, (ByteArray) valuesData, endsData, offset);
             default -> throw new VortexException(EncodingId.VORTEX_RUNEND, "unsupported ptype " + valuePtype);
         };
+        return withRunValidity(result, valuesValidity, endsData, n, offset);
+    }
+
+    /// Wraps `result` in a [MaskedArray] whose per-row validity is a run-end bool array
+    /// over the same run-ends: row `i` is valid iff the run it falls in has a valid value.
+    /// Returns `result` unchanged when the values child carried no validity (all valid).
+    ///
+    /// @param result         the decoded run-end value array
+    /// @param valuesValidity per-run validity bits from the values child, or `null`
+    /// @param endsData       the run-ends array (raw, mask-unwrapped)
+    /// @param n              logical row count
+    /// @param offset         starting absolute position
+    /// @return `result`, or a [MaskedArray] carrying the lazy per-row validity
+    private static Array withRunValidity(Array result, BoolArray valuesValidity, Array endsData, long n, long offset) {
+        if (valuesValidity == null) {
+            return result;
+        }
+        BoolArray rowValidity = new LazyRunEndBoolArray(DType.BOOL, n, valuesValidity, endsData, offset);
+        return new MaskedArray(result, rowValidity);
     }
 
     private static Array expandStrings(

reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java

@@ -23,6 +23,7 @@
 import io.github.dfa1.vortex.reader.array.LazySparseShortArray;
 import io.github.dfa1.vortex.reader.array.LongArray;
 import io.github.dfa1.vortex.reader.array.MaskedArray;
+import io.github.dfa1.vortex.reader.array.MaterializedBoolArray;
 import io.github.dfa1.vortex.reader.array.ShortArray;
 import io.github.dfa1.vortex.reader.array.VarBinArray;
 
@@ -63,27 +64,37 @@ public Array decode(DecodeContext ctx) {
             return decodeVarBin(ctx, n, numPatches, offset, indicesPtype);
         }
 
+        // Row validity mirrors the Rust reference `ValidityVTable<Sparse>`: it is a sparse
+        // bool array whose fill is `fill_value.is_valid()` and whose per-patch value is the
+        // patch value's validity bit. So a position is valid iff (it is a patch AND that
+        // patch is valid) OR (it is unpatched AND the fill is non-null). Dropping either
+        // facet lost nulls: a `fill_value: null` array (world-energy `biofuel_cons_change_pct`
+        // f64?) decoded unpatched rows as 0.0, and a null patch (nuclear_share_energy)
+        // decoded to raw 0 — #226.
+        MemorySegment fillBuf = ctx.buffer(0);
+        ProtoScalarValue fillScalar = decodeFill(fillBuf);
+        boolean fillValid = !isNullScalar(fillScalar);
+
         if (ctx.dtype() instanceof DType.Bool) {
             DType indicesDtype = new DType.Primitive(indicesPtype, false);
             Array patchIndices = ctx.decodeChild(0, indicesDtype, numPatches);
             Array patchValues = ctx.decodeChild(1, ctx.dtype(), numPatches);
             Array idxData = patchIndices instanceof MaskedArray m ? m.inner() : patchIndices;
-            Array valData = patchValues instanceof MaskedArray m ? m.inner() : patchValues;
-            return new LazySparseBoolArray(ctx.dtype(), n, false, (BoolArray) valData, idxData, offset);
+            BoolArray patchValidity = null;
+            Array valData = patchValues;
+            if (patchValues instanceof MaskedArray m) {
+                valData = m.inner();
+                patchValidity = m.validity();
+            }
+            boolean fillValue = Boolean.TRUE.equals(fillScalar.bool_value());
+            Array result = new LazySparseBoolArray(ctx.dtype(), n, fillValue, (BoolArray) valData, idxData, offset);
+            return withSparseValidity(ctx, result, fillValid, patchValidity, idxData, numPatches, n, offset);
         }
 
         if (!(ctx.dtype() instanceof DType.Primitive)) {
             throw new VortexException(EncodingId.VORTEX_SPARSE, "expected primitive dtype, got " + ctx.dtype());
         }
         PType valuePtype = ((DType.Primitive) ctx.dtype()).ptype();
-
-        MemorySegment fillBuf = ctx.buffer(0);
-        ProtoScalarValue fillScalar;
-        try {
-            fillScalar = ProtoScalarValue.decode(fillBuf, 0, fillBuf.byteSize());
-        } catch (IOException e) {
-            throw new VortexException(EncodingId.VORTEX_SPARSE, "invalid fill value", e);
-        }
         long fillBits = scalarToLong(fillScalar);
 
         // Lazy path: keep fill bits + decoded patches; no n-sized buffer allocated.
@@ -93,9 +104,14 @@ public Array decode(DecodeContext ctx) {
         Array patchIndices = ctx.decodeChild(0, indicesDtype, numPatches);
         Array patchValues = ctx.decodeChild(1, ctx.dtype(), numPatches);
         Array idxData = patchIndices instanceof MaskedArray m ? m.inner() : patchIndices;
-        Array valData = patchValues instanceof MaskedArray m ? m.inner() : patchValues;
+        BoolArray patchValidity = null;
+        Array valData = patchValues;
+        if (patchValues instanceof MaskedArray m) {
+            valData = m.inner();
+            patchValidity = m.validity();
+        }
 
-        return switch (valuePtype) {
+        Array result = switch (valuePtype) {
             case I64, U64 -> new LazySparseLongArray(ctx.dtype(), n, fillBits,
                     (LongArray) valData, idxData, offset);
             case I32, U32 -> new LazySparseIntArray(ctx.dtype(), n, (int) fillBits,
@@ -114,6 +130,59 @@ public Array decode(DecodeContext ctx) {
                     (ByteArray) valData, idxData, offset);
             default -> throw new VortexException(EncodingId.VORTEX_SPARSE, "unsupported ptype " + valuePtype);
         };
+        return withSparseValidity(ctx, result, fillValid, patchValidity, idxData, numPatches, n, offset);
+    }
+
+    /// Wraps `result` in a [MaskedArray] whose per-row validity is a sparse bool array:
+    /// fill = `fillValid` (the fill scalar is non-null), each patch bit = that patch's
+    /// validity. Returns `result` unchanged only when the fill is non-null and no patch
+    /// carried a null (the all-valid no-regression path).
+    ///
+    /// @param ctx           decode context (allocation arena)
+    /// @param result        the decoded sparse value array
+    /// @param fillValid     `true` when the fill scalar is non-null
+    /// @param patchValidity per-patch validity bits, or `null` when all patches are valid
+    /// @param idxData       sorted absolute patch positions (raw, mask-unwrapped)
+    /// @param numPatches    number of patches
+    /// @param n             logical row count
+    /// @param offset        starting absolute position
+    /// @return `result`, or a [MaskedArray] carrying the lazy per-row validity
+    private static Array withSparseValidity(DecodeContext ctx, Array result, boolean fillValid,
+            BoolArray patchValidity, Array idxData, long numPatches, long n, long offset) {
+        if (fillValid && patchValidity == null) {
+            return result;
+        }
+        BoolArray patchBits = patchValidity != null ? patchValidity : allValid(ctx, numPatches);
+        BoolArray rowValidity = new LazySparseBoolArray(DType.BOOL, n, fillValid, patchBits, idxData, offset);
+        return new MaskedArray(result, rowValidity);
+    }
+
+    /// Builds an all-valid bit-packed [BoolArray] of `len` bits — the patch-validity stand-in
+    /// when the patch values are non-nullable but the fill is null (every patch punches in a
+    /// valid position over an all-invalid base).
+    private static BoolArray allValid(DecodeContext ctx, long len) {
+        MemorySegment bits = ctx.arena().allocate(Math.max(1, (len + 7) >>> 3));
+        bits.fill((byte) 0xFF);
+        return new MaterializedBoolArray(DType.BOOL, len, bits.asReadOnly());
+    }
+
+    private static ProtoScalarValue decodeFill(MemorySegment fillBuf) {
+        try {
+            return ProtoScalarValue.decode(fillBuf, 0, fillBuf.byteSize());
+        } catch (IOException e) {
+            throw new VortexException(EncodingId.VORTEX_SPARSE, "invalid fill value", e);
+        }
+    }
+
+    /// Detects a null fill scalar: either an explicit `null_value`, or a scalar with no
+    /// value-bearing field set (Rust encodes a null fill as `ScalarValue::Null`, and a
+    /// non-null fill always sets exactly one typed field — even integer/float `0`).
+    private static boolean isNullScalar(ProtoScalarValue s) {
+        return s.null_value() != null
+                || (s.bool_value() == null && s.int64_value() == null && s.uint64_value() == null
+                        && s.f32_value() == null && s.f64_value() == null && s.string_value() == null
+                        && s.bytes_value() == null && s.f16_value() == null && s.list_value() == null
+                        && s.variant_value() == null);
     }
 
     private static Array decodeVarBin(