From 3367c8b5dff9e132363f15c52d4bf808a3c1e498 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 4 Jul 2026 20:32:43 +0000 Subject: [PATCH 1/2] build(deps): Bump dev.vortex:vortex-jni from 0.75.0 to 0.76.0 Bumps [dev.vortex:vortex-jni](https://github.com/spiraldb/vortex) from 0.75.0 to 0.76.0. - [Release notes](https://github.com/spiraldb/vortex/releases) - [Commits](https://github.com/spiraldb/vortex/compare/0.75.0...0.76.0) --- updated-dependencies: - dependency-name: dev.vortex:vortex-jni dependency-version: 0.76.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bfc95643..fb3f81af 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,7 @@ 6.1.1 3.27.7 5.23.0 - 0.75.0 + 0.76.0 19.0.0 2.0.18 From 94dec382a7d9f73e63a20d5c39140d03fd658f60 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Tue, 7 Jul 2026 07:28:55 +0200 Subject: [PATCH 2/2] fix(reader): decode vortex.zoned aggregate-spec zone maps (vortex-jni 0.76.0) Bumping vortex-jni to 0.76.0 broke RustWritesJavaReadsIntegrationTest #jniWriter_perZoneSum_readFromZoneMapTable with a ClassCastException (DType$Bool -> DType$Primitive) while decoding the per-zone stats table. 0.76.0's zoned layout (spiraldb/vortex #7938) splits the format in two: the canonical `vortex.zoned` id now stores an ordered aggregate-function spec list in its metadata (proto: version byte + ZonedMetadataProto) and its stats table drops the legacy per-stat `_is_truncated` Bool flags, while the legacy `vortex.stats` alias keeps the Stat-bitset metadata. Our reader treated the two ids as pure aliases and always reconstructed the legacy schema, so the extra Bool fields no longer matched the encoded table and the positional struct decode mismatched. The reader now dispatches on the layout id: `vortex.zoned` goes through a new ZonedStatsSchema.aggregateStatsTableDtype that parses the aggregate specs and maps each known aggregate (max/min/sum/null_count/nan_count/...) to its stored stat dtype, with no truncation flags; `vortex.stats` keeps the bitset path. Unknown/nested-state aggregates (e.g. vortex.bounded_max) bail to the per-chunk fallback rather than decode a misaligned table. The CLI stats inspector uses the same dispatch. This is a genuine compatibility fix on top of the dependency bump, not a rebase-conflict resolution (the rebase onto main was clean). Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 1 + .../vortex/cli/tui/VortexInspectorTui.java | 9 +- .../dfa1/vortex/reader/ScanIterator.java | 20 +- .../reader/layout/ZonedStatsSchema.java | 212 ++++++++++++++++++ .../reader/layout/ZonedStatsSchemaTest.java | 120 ++++++++++ 5 files changed, 355 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c41ee8e7..f36569f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Per-zone stats from current Rust writers (`vortex.zoned`, vortex-jni 0.76.0) decode again. The 0.76 zoned layout replaced the legacy `vortex.stats` bit-set metadata with an aggregate-function spec list and dropped the per-stat truncation flags; the reader now reconstructs the stats table from that spec list (min/max/sum/null_count), so `columnZoneStats` and aggregate push-down work against those files instead of throwing `ClassCastException`. ([#197](https://github.com/dfa1/vortex-java/pull/197)) - CSV export renders nested struct columns as JSON object cells (`{"field":value,...}`, strings JSON-escaped, null fields as JSON `null`, nested structs recursed) instead of throwing `unsupported array type: StructArray`. ([#217](https://github.com/dfa1/vortex-java/issues/217)) - Scanning a struct whose columns include a nested struct no longer fails chunk planning. A nested `vortex.struct` layout column (e.g. Raincloud `countries-of-the-world`'s `data` column) is now treated as a single full-range chunk source and decoded through the layout registry's new struct decoder into a `StructArray`, matching the Rust reference (a nested struct spans its parent's row range, not an independent chunking). `schema`/`count`/`inspect` already worked; plain scans and `select` of sibling columns now work too. CSV export of the struct column itself remains unsupported (a separate rendering limitation). ([#207](https://github.com/dfa1/vortex-java/issues/207)) - CSV export renders unsigned integer columns (U8–U64) with their unsigned values — high-half values previously printed as two's-complement negatives (uci-wine `magnesium` U8 132 exported as -124), silent corruption found by the Raincloud conformance suite. ([#208](https://github.com/dfa1/vortex-java/issues/208)) diff --git a/cli/src/main/java/io/github/dfa1/vortex/cli/tui/VortexInspectorTui.java b/cli/src/main/java/io/github/dfa1/vortex/cli/tui/VortexInspectorTui.java index da2fc566..2d180a6c 100644 --- a/cli/src/main/java/io/github/dfa1/vortex/cli/tui/VortexInspectorTui.java +++ b/cli/src/main/java/io/github/dfa1/vortex/cli/tui/VortexInspectorTui.java @@ -4,6 +4,7 @@ import io.github.dfa1.vortex.reader.SegmentSpec; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.LayoutId; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.cli.tui.term.Ansi; import io.github.dfa1.vortex.cli.tui.term.Key; @@ -673,8 +674,12 @@ private void runStatsLoad(InspectorTree.Node anchor) { statsCache.put(anchor, new DataState.Failed("no column dtype")); return; } - DType.Struct statsDtype = ZonedStatsSchema.statsTableDtype(columnDtype, anchorLayout.metadata()); - if (statsDtype.fieldNames().isEmpty()) { + // `vortex.zoned` (Rust >= 0.76) carries aggregate-spec metadata; the legacy + // `vortex.stats` alias carries a Stat bitset — reconstruct the schema accordingly. + DType.Struct statsDtype = anchorLayout.layoutId() == LayoutId.ZONED + ? ZonedStatsSchema.aggregateStatsTableDtype(columnDtype, anchorLayout.metadata()) + : ZonedStatsSchema.statsTableDtype(columnDtype, anchorLayout.metadata()); + if (statsDtype == null || statsDtype.fieldNames().isEmpty()) { statsCache.put(anchor, new DataState.Failed("no stats present in metadata")); return; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index a13e48a7..e3c95d87 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -3,6 +3,7 @@ import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.LayoutId; import io.github.dfa1.vortex.core.io.IoBounds; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.reader.array.Array; @@ -395,10 +396,11 @@ public List columnZoneStats(String column) { return out; } - /// Decodes the column's `vortex.stats` zone-map table into one [ArrayStats] per zone, or - /// returns `null` when the column has no zone map (so the caller falls back to per-chunk - /// node stats). The table is a single flat segment encoding a struct with a subset of the - /// `min`/`max`/`sum`/`null_count` fields (see [ZonedStatsSchema]); it is decoded into a + /// Decodes the column's zone-map table into one [ArrayStats] per zone, or returns `null` when + /// the column has no zone map (so the caller falls back to per-chunk node stats). The table is + /// a single flat segment encoding a struct with a subset of the `min`/`max`/`sum`/`null_count` + /// fields; the schema is reconstructed from the layout metadata (legacy `vortex.stats` bitset + /// or newer `vortex.zoned` aggregate specs — see [ZonedStatsSchema]). It is decoded into a /// short-lived confined arena and the scalar values are boxed out before the arena closes. private List decodeZoneTable(ColumnName column) { Layout zoned = findZonedLayout(file.layout(), column); @@ -417,7 +419,15 @@ private List decodeZoneTable(ColumnName column) { if (segIdx < 0 || segIdx >= file.footer().segmentSpecs().size()) { return null; } - DType.Struct statsDtype = ZonedStatsSchema.statsTableDtype(columnDtype, zoned.metadata()); + // The canonical `vortex.zoned` id (Rust >= 0.76) stores an aggregate-spec metadata blob; + // the legacy `vortex.stats` alias (what vortex-java writes) stores a Stat bitset. They + // reconstruct the table schema differently — dispatch on the layout id. + DType.Struct statsDtype = zoned.layoutId() == LayoutId.ZONED + ? ZonedStatsSchema.aggregateStatsTableDtype(columnDtype, zoned.metadata()) + : ZonedStatsSchema.statsTableDtype(columnDtype, zoned.metadata()); + if (statsDtype == null || statsDtype.fieldNames().isEmpty()) { + return null; + } long nZones = statsFlat.rowCount(); SegmentSpec spec = file.footer().segmentSpecs().get(segIdx); try (Arena tableArena = Arena.ofConfined()) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchema.java b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchema.java index cb8d18a9..fd31c07d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchema.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchema.java @@ -135,6 +135,52 @@ public static DType.Struct statsTableDtype(DType columnDtype, MemorySegment meta return statsTableDtype(columnDtype, presentStats(metadata)); } + /// Reconstructs the per-zone stats-table dtype for a newer `vortex.zoned` layout, whose + /// metadata carries an ordered list of aggregate-function specs instead of the legacy + /// [Stat] bitset. + /// + /// The shape mirrors Rust's `aggregate_stats_table_dtype` + /// ([vortex-layout/src/layouts/zoned/schema.rs](https://github.com/spiraldb/vortex/blob/develop/vortex-layout/src/layouts/zoned/schema.rs)): + /// for every aggregate whose state dtype is defined for the column, append one + /// `(name, nullable state-dtype)` field, in spec order — no `_is_truncated` flags. Field + /// names are the aggregate's canonical [Stat#fieldName()] so the reader can look them up by + /// stat; the struct decode itself is positional, so alignment with the encoded table is what + /// matters. + /// + /// Returns `null` when the metadata is not a decodable aggregate-spec blob or names an + /// aggregate this reader cannot map to a [Stat] (e.g. `vortex.bounded_max`, whose state is a + /// nested struct). Bailing keeps the positional schema faithful: the caller falls back to + /// per-chunk stats rather than decoding a misaligned table. + /// + /// @param columnDtype the column's logical dtype (the `data` child's dtype) + /// @param metadata raw `vortex.zoned` layout metadata + /// @return reconstructed non-nullable struct dtype, or `null` when it cannot be reconstructed + public static DType.Struct aggregateStatsTableDtype(DType columnDtype, MemorySegment metadata) { + List aggregateIds = aggregateIds(metadata); + if (aggregateIds == null) { + return null; + } + List names = new ArrayList<>(aggregateIds.size()); + List types = new ArrayList<>(aggregateIds.size()); + for (String aggregateId : aggregateIds) { + Stat stat = statForAggregate(aggregateId); + if (stat == null) { + // Unknown aggregate — the encoded table has a field here we cannot describe, so + // any reconstruction would be positionally misaligned. Bail to the fallback path. + return null; + } + DType stype = statDtype(stat, columnDtype); + if (stype == null) { + // Rust's schema drops aggregates with no state dtype for this column (e.g. + // nan_count over a non-float); skip to stay aligned with the encoded table. + continue; + } + names.add(ColumnName.of(stat.fieldName())); + types.add(stype.withNullable(true)); + } + return new DType.Struct(List.copyOf(names), List.copyOf(types), false); + } + /// Reconstructs the per-zone stats-table dtype for the given column dtype /// and explicit stat list (already decoded from metadata). /// @@ -231,4 +277,170 @@ private static DType sumDtype(DType columnDtype) { // so the stat is dropped from the schema rather than causing a decode mismatch. return null; } + + /// First byte of `vortex.zoned` metadata: the protobuf envelope version Rust writes. + private static final int AGGREGATE_METADATA_VERSION = 1; + + /// Maps a well-known aggregate-function id (Rust `AggregateFnId`) to the [Stat] whose stored + /// dtype and canonical field name it uses in the zone-map table, or `null` when the reader + /// has no faithful mapping (nested-state aggregates like `vortex.bounded_max`, or functions + /// not stored as a scalar stat). + private static Stat statForAggregate(String aggregateId) { + return switch (aggregateId) { + case "vortex.max" -> Stat.MAX; + case "vortex.min" -> Stat.MIN; + case "vortex.sum" -> Stat.SUM; + case "vortex.null_count" -> Stat.NULL_COUNT; + case "vortex.nan_count" -> Stat.NAN_COUNT; + case "vortex.is_constant" -> Stat.IS_CONSTANT; + case "vortex.is_sorted" -> Stat.IS_SORTED; + case "vortex.is_strict_sorted" -> Stat.IS_STRICT_SORTED; + case "vortex.uncompressed_size_in_bytes" -> Stat.UNCOMPRESSED_SIZE_IN_BYTES; + default -> null; + }; + } + + /// Extracts the ordered aggregate-function ids from `vortex.zoned` metadata. + /// + /// The metadata is a version byte (value [#AGGREGATE_METADATA_VERSION]) followed by a + /// protobuf `ZonedMetadataProto { uint32 zone_len = 1; repeated AggregateSpecProto + /// aggregate_specs = 2; }`, where `AggregateSpecProto { string id = 1; bytes options = 2; }`. + /// Only the `id` of each spec is needed to reconstruct the table schema. + /// + /// Returns `null` when the blob is empty, carries an unsupported version, or is not + /// well-formed protobuf — signalling the caller to fall back rather than trust a partial parse. + private static List aggregateIds(MemorySegment metadata) { + if (metadata == null || metadata.byteSize() < 1) { + return null; + } + if ((metadata.get(ValueLayout.JAVA_BYTE, 0) & 0xff) != AGGREGATE_METADATA_VERSION) { + return null; + } + ProtoCursor cursor = new ProtoCursor(metadata, 1, metadata.byteSize()); + List ids = new ArrayList<>(); + while (cursor.hasRemaining()) { + long tag = cursor.readVarint(); + if (tag < 0) { + return null; + } + int fieldNumber = (int) (tag >>> 3); + int wireType = (int) (tag & 0x7); + if (fieldNumber == 2 && wireType == ProtoCursor.WIRE_LEN) { + String id = cursor.readAggregateSpecId(); + if (id == null) { + return null; + } + ids.add(id); + } else if (!cursor.skipField(wireType)) { + return null; + } + } + return List.copyOf(ids); + } + + /// Minimal forward cursor over a protobuf byte range within a [MemorySegment]. Reads only the + /// varints and length-delimited fields the zoned metadata uses; any malformed or unexpected + /// wire shape is reported as a failure so the caller can bail rather than misread. + private static final class ProtoCursor { + static final int WIRE_VARINT = 0; + static final int WIRE_I64 = 1; + static final int WIRE_LEN = 2; + static final int WIRE_I32 = 5; + + private final MemorySegment segment; + private long pos; + private final long end; + + ProtoCursor(MemorySegment segment, long start, long end) { + this.segment = segment; + this.pos = start; + this.end = end; + } + + boolean hasRemaining() { + return pos < end; + } + + /// Reads a base-128 varint, or `-1` when it runs past the end or exceeds 64 bits. + long readVarint() { + long result = 0; + int shift = 0; + while (pos < end) { + int b = segment.get(ValueLayout.JAVA_BYTE, pos) & 0xff; + pos++; + if (shift < 64) { + result |= (long) (b & 0x7f) << shift; + } + if ((b & 0x80) == 0) { + return result; + } + shift += 7; + if (shift > 63) { + return -1; + } + } + return -1; + } + + /// Skips a field of the given wire type, returning `false` on an unknown type or overrun. + boolean skipField(int wireType) { + return switch (wireType) { + case WIRE_VARINT -> readVarint() >= 0; + case WIRE_I64 -> advance(8); + case WIRE_I32 -> advance(4); + case WIRE_LEN -> { + long len = readVarint(); + yield len >= 0 && advance(len); + } + default -> false; + }; + } + + /// Reads an `AggregateSpecProto` length-delimited message and returns its `id` (field 1), + /// or `null` if the message is malformed or carries no id. + String readAggregateSpecId() { + long len = readVarint(); + if (len < 0 || pos + len > end) { + return null; + } + long messageEnd = pos + len; + String id = null; + while (pos < messageEnd) { + long tag = readVarint(); + if (tag < 0) { + return null; + } + int fieldNumber = (int) (tag >>> 3); + int wireType = (int) (tag & 0x7); + if (fieldNumber == 1 && wireType == WIRE_LEN) { + id = readString(messageEnd); + if (id == null) { + return null; + } + } else if (!skipField(wireType)) { + return null; + } + } + return id; + } + + private String readString(long limit) { + long len = readVarint(); + if (len < 0 || pos + len > limit) { + return null; + } + byte[] bytes = new byte[(int) len]; + MemorySegment.copy(segment, ValueLayout.JAVA_BYTE, pos, bytes, 0, (int) len); + pos += len; + return new String(bytes, java.nio.charset.StandardCharsets.UTF_8); + } + + private boolean advance(long count) { + if (count < 0 || pos + count > end) { + return false; + } + pos += count; + return true; + } + } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java index 0be91e98..09f85c71 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java @@ -246,10 +246,130 @@ void allStatsTogetherForI32() { } } + @Nested + class AggregateStatsTableDtype { + @Test + void buildsNumericSchemaWithoutTruncationFlags() { + // Given — a Rust >= 0.76 `vortex.zoned` column: default numeric aggregates in spec + // order. nan_count has no state dtype for i64, so Rust drops it from the table. + MemorySegment meta = aggregateMeta(8192, + "vortex.max", "vortex.min", "vortex.sum", "vortex.nan_count", "vortex.null_count"); + + // When + DType.Struct schema = ZonedStatsSchema.aggregateStatsTableDtype(DType.I64, meta); + + // Then — no `_is_truncated` fields (the new format has none) and nan_count is dropped + assertThat(schema.fieldNames().stream().map(ColumnName::value).toList()) + .containsExactly("max", "min", "sum", "null_count"); + assertThat(schema.fieldTypes()).containsExactly( + new DType.Primitive(PType.I64, true), + new DType.Primitive(PType.I64, true), + new DType.Primitive(PType.I64, true), + new DType.Primitive(PType.U64, true)); + assertThat(schema.nullable()).isFalse(); + } + + @Test + void keepsNanCountForFloatColumn() { + // Given — nan_count resolves to u64 for a float column, so it stays in the table + MemorySegment meta = aggregateMeta(1024, "vortex.max", "vortex.nan_count"); + + // When + DType.Struct schema = ZonedStatsSchema.aggregateStatsTableDtype(DType.F64, meta); + + // Then + assertThat(schema.fieldNames().stream().map(ColumnName::value).toList()) + .containsExactly("max", "nan_count"); + assertThat(schema.fieldTypes()).element(1).isEqualTo(new DType.Primitive(PType.U64, true)); + } + + @Test + void bailsOnUnknownAggregate() { + // Given — bounded_max stores a nested struct we cannot faithfully describe. Returning a + // partial schema would misalign the positional decode, so reconstruction must bail. + MemorySegment meta = aggregateMeta(4096, "vortex.bounded_max", "vortex.null_count"); + + // When + DType.Struct schema = ZonedStatsSchema.aggregateStatsTableDtype(DType.UTF8, meta); + + // Then + assertThat(schema).isNull(); + } + + @Test + void bailsOnUnsupportedVersion() { + // Given — a version byte other than the one Rust writes for `vortex.zoned` metadata + MemorySegment meta = aggregateMeta(4096, "vortex.max"); + meta.set(java.lang.foreign.ValueLayout.JAVA_BYTE, 0, (byte) 2); + + // When + DType.Struct schema = ZonedStatsSchema.aggregateStatsTableDtype(DType.I64, meta); + + // Then + assertThat(schema).isNull(); + } + + @Test + void bailsOnNullMetadata() { + // Given / When — a zoned layout with no metadata cannot be reconstructed + DType.Struct schema = ZonedStatsSchema.aggregateStatsTableDtype(DType.I64, null); + + // Then + assertThat(schema).isNull(); + } + + @Test + void returnsEmptyStructWhenNoAggregatesPresent() { + // Given — a well-formed envelope carrying only zone_len, no aggregate specs + MemorySegment meta = aggregateMeta(8192); + + // When + DType.Struct schema = ZonedStatsSchema.aggregateStatsTableDtype(DType.I64, meta); + + // Then — an empty (non-null) struct; the caller falls back to per-chunk stats + assertThat(schema).isNotNull(); + assertThat(schema.fieldNames()).isEmpty(); + } + } + private static MemorySegment metaWithBitset(int zoneLen, int firstByte) { MemorySegment meta = MemorySegment.ofArray(new byte[4 + 1]); meta.set(io.github.dfa1.vortex.core.io.VortexFormat.LE_INT, 0, zoneLen); meta.set(java.lang.foreign.ValueLayout.JAVA_BYTE, 4, (byte) firstByte); return meta; } + + /// Builds `vortex.zoned` metadata: a version byte (1) followed by a protobuf + /// `ZonedMetadataProto { uint32 zone_len = 1; repeated AggregateSpecProto specs = 2; }`, + /// where each spec is `AggregateSpecProto { string id = 1; }` (options omitted). + private static MemorySegment aggregateMeta(int zoneLen, String... aggregateIds) { + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + out.write(1); // envelope version + writeTag(out, 1, 0); // zone_len, wire VARINT + writeVarint(out, zoneLen); + for (String id : aggregateIds) { + byte[] idBytes = id.getBytes(java.nio.charset.StandardCharsets.UTF_8); + java.io.ByteArrayOutputStream spec = new java.io.ByteArrayOutputStream(); + writeTag(spec, 1, 2); // id, wire LEN + writeVarint(spec, idBytes.length); + spec.writeBytes(idBytes); + writeTag(out, 2, 2); // aggregate_specs, wire LEN + writeVarint(out, spec.size()); + out.writeBytes(spec.toByteArray()); + } + return MemorySegment.ofArray(out.toByteArray()); + } + + private static void writeTag(java.io.ByteArrayOutputStream out, int fieldNumber, int wireType) { + writeVarint(out, (fieldNumber << 3) | wireType); + } + + private static void writeVarint(java.io.ByteArrayOutputStream out, int value) { + int v = value; + while ((v & ~0x7f) != 0) { + out.write((v & 0x7f) | 0x80); + v >>>= 7; + } + out.write(v); + } }