Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ claim measured against the Rust (JNI) oracle; behavior divergences documented in

### Changed

- Column names are typed as `ColumnName` throughout vortex's own read and write APIs, not raw `String`. `DType.Struct.fieldNames()` returns `List<ColumnName>` (the builder already validated then discarded the type — now it keeps it), `VortexReader.columnStats()` is keyed by `ColumnName`, `RowFilter.Column.column` is a `ColumnName`, and the writer's internal column maps and typed `Chunk.put(ColumnName, …)` builder follow. User-facing sugar is unchanged — `structBuilder().field(String)`, `RowFilter.eq(String, …)`, `ScanOptions.columns(String...)`, and `writeChunk(Map<String,Object>)` still take strings and validate at the edge. A footgun field name (blank/control) is now rejected structurally at `ColumnName` construction — a footgun-named schema can no longer be built. Framework/IO boundaries (Calcite, CSV, JDBC, Parquet, display) keep `String`. Wire output unchanged (integration oracle green). ([84769863](https://github.com/dfa1/vortex-java/commit/84769863))

- `ScanOptions.columns()` returns `List<ColumnName>` instead of `List<String>`. The `columns(String...)` / `withColumns(String...)` factories still take strings at the API boundary but validate them into `ColumnName`, so a blank or control-character projection name fails fast rather than silently matching nothing. ([e478a3a7](https://github.com/dfa1/vortex-java/commit/e478a3a7))

- `Footer.arraySpecs()` / `Footer.layoutSpecs()` return typed `List<EncodingId>` / `List<LayoutId>` instead of `List<String>`. The wire strings are parsed to their typed ids once at the footer boundary (with the blank-id guard), so array and layout nodes index directly into typed dictionaries — completing "strings at the boundary, types inside": the two scattered per-node `parse` calls in `SerializedArrayDecoder` and the layout converter are gone. ([0dd677ef](https://github.com/dfa1/vortex-java/commit/0dd677ef))
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.github.dfa1.vortex.calcite;

import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.reader.Chunk;
import io.github.dfa1.vortex.reader.ScanIterator;
Expand Down Expand Up @@ -58,7 +59,7 @@ private VortexAggregates() {
/// @param column the numeric column name
/// @return the column's aggregate summary
public static Summary of(VortexReader reader, String column) {
ArrayStats stats = reader.columnStats().getOrDefault(column, ArrayStats.empty());
ArrayStats stats = reader.columnStats().getOrDefault(ColumnName.of(column), ArrayStats.empty());
long totalRows = totalRows(reader);
long nullCount = stats.nullCount() == null ? 0L : stats.nullCount();
long count = totalRows - nullCount;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.github.dfa1.vortex.calcite;

import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.reader.Chunk;
Expand Down Expand Up @@ -103,7 +104,7 @@ public record ColumnStats(ArrayStats stats, long totalRows) {
/// @return the column's aggregated statistics paired with the file's total row count
public ColumnStats statsAndRows(String column) {
try (VortexReader reader = VortexReader.open(file)) {
ArrayStats stats = reader.columnStats().getOrDefault(column, ArrayStats.empty());
ArrayStats stats = reader.columnStats().getOrDefault(ColumnName.of(column), ArrayStats.empty());
return new ColumnStats(stats, countRows(reader));
} catch (IOException e) {
throw new UncheckedIOException("cannot read stats of " + file, e);
Expand Down Expand Up @@ -137,7 +138,7 @@ public ZoneSum zoneSum(String column) {
try (VortexReader reader = VortexReader.open(file)) {
Number sum = new ZoneReducer(reader).sum(column);
Long nullCount = reader.columnStats()
.getOrDefault(column, ArrayStats.empty()).nullCount();
.getOrDefault(ColumnName.of(column), ArrayStats.empty()).nullCount();
return new ZoneSum(sum, nullCount, countRows(reader));
} catch (IOException e) {
throw new UncheckedIOException("cannot read zone sum of " + file, e);
Expand Down Expand Up @@ -420,7 +421,7 @@ private FilteredFold result() {
/// @return the conjoined filter, or empty when any predicate is not fully translatable
public Optional<RowFilter> translatePushedFilters(List<RexNode> filters) {
DType.Struct struct = struct();
List<String> names = struct.fieldNames();
List<String> names = struct.fieldNames().stream().map(ColumnName::value).toList();
List<DType> types = struct.fieldTypes();
List<RowFilter> translated = new ArrayList<>();
for (RexNode node : filters) {
Expand Down Expand Up @@ -467,7 +468,7 @@ private static Match classify(RowFilter filter, int zone,
yield allIn ? Match.IN : Match.BOUNDARY;
}
case RowFilter.Column(var col, var predicate) ->
classifyColumn(predicate, zoneStats.get(col).get(zone), rowCount);
classifyColumn(predicate, zoneStats.get(col.value()).get(zone), rowCount);
};
}

Expand Down Expand Up @@ -567,15 +568,15 @@ private static Match compare(ArrayStats s, Object value,
/// Whether `column` is an unsigned primitive, whose signed stat ordering [#compareStat] cannot
/// safely classify (the fold abandons such columns to the scan).
private static boolean isUnsigned(DType.Struct struct, String column) {
int idx = struct.fieldNames().indexOf(column);
int idx = struct.fieldNames().indexOf(ColumnName.of(column));
return idx >= 0 && struct.fieldTypes().get(idx) instanceof DType.Primitive p && p.ptype().isUnsigned();
}

/// Whether `column` is a floating-point primitive, whose compute-kernel compare
/// ([Compare#values(Object, Object, DType)]) sorts NaN as the maximum and so cannot soundly fold a
/// boundary partition (the fold abandons such a filter column to the NaN-correct scan).
private static boolean isFloating(DType.Struct struct, String column) {
int idx = struct.fieldNames().indexOf(column);
int idx = struct.fieldNames().indexOf(ColumnName.of(column));
return idx >= 0 && struct.fieldTypes().get(idx) instanceof DType.Primitive p && p.ptype().isFloating();
}

Expand Down Expand Up @@ -640,15 +641,15 @@ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
DType.Struct struct = struct();
RelDataTypeFactory.Builder builder = typeFactory.builder();
for (int i = 0; i < struct.fieldNames().size(); i++) {
builder.add(struct.fieldNames().get(i), toSqlType(typeFactory, struct.fieldTypes().get(i)));
builder.add(struct.fieldNames().get(i).value(), toSqlType(typeFactory, struct.fieldTypes().get(i)));
}
return builder.build();
}

@Override
public Enumerable<Object[]> scan(DataContext root, List<RexNode> filters, int[] projects) {
DType.Struct struct = struct();
List<String> allNames = struct.fieldNames();
List<String> allNames = struct.fieldNames().stream().map(ColumnName::value).toList();

// Projection: the columns to decode and emit, in the order Calcite asked for. A null
// projects array means "all columns".
Expand Down Expand Up @@ -847,7 +848,7 @@ private static Optional<RowFilter> toRowFilter(List<RexNode> filters, List<Strin
private static void collectColumns(RowFilter filter, java.util.Set<String> out) {
switch (filter) {
case RowFilter.And(var parts) -> parts.forEach(f -> collectColumns(f, out));
case RowFilter.Column(var col, var ignored) -> out.add(col);
case RowFilter.Column(var col, var ignored) -> out.add(col.value());
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.github.dfa1.vortex.calcite;

import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.PType;
import io.github.dfa1.vortex.writer.VortexWriter;
Expand Down Expand Up @@ -45,7 +46,7 @@ class AggregateSumNullTest {
/// one per-zone SUM row. `values` holds the raw longs; `valid[i] == false` marks row `i` null.
private SchemaPlus tableOf(long[] values, boolean[] valid) throws IOException {
DType.Struct schema = new DType.Struct(
List.of("v"), List.of(new DType.Primitive(PType.I64, true)), false);
List.of(ColumnName.of("v")), List.of(new DType.Primitive(PType.I64, true)), false);
Path file = tmp.resolve("sum-nulls.vortex");
// Large chunk so the whole column is one chunk; zone maps on so the SUM stat is emitted.
WriteOptions opts = new WriteOptions(1024, true, 0.90, 0, false, false);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.github.dfa1.vortex.calcite;

import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.PType;
import io.github.dfa1.vortex.writer.VortexWriter;
Expand Down Expand Up @@ -201,7 +202,7 @@ void nullableAggColumnAcrossBoundaryChunk() throws Exception {
// predicate `id > 0 AND id < 100` cuts chunk 0 (drops id 0, keeps ids 1,2,3) — a boundary
// zone — and selects chunk 1 whole (IN).
DType.Struct schema = new DType.Struct(
List.of("id", "val"),
List.of(ColumnName.of("id"), ColumnName.of("val")),
List.of(new DType.Primitive(PType.I64, false), new DType.Primitive(PType.I64, true)),
false);
Path f = tmp.resolve("nullable-boundary.vortex");
Expand Down Expand Up @@ -238,7 +239,7 @@ void nonNumericMinAcrossBoundaryAbandonsButScanIsCorrect() throws Exception {
// rewrite must abandon so the scan returns the value. chunk 0 id 0..3 / name {"a".."d"},
// chunk 1 id 10..13 / name {"e".."h"}; `id > 0 AND id < 100` cuts chunk 0 (keeps b,c,d).
DType.Struct schema = new DType.Struct(
List.of("id", "name"),
List.of(ColumnName.of("id"), ColumnName.of("name")),
List.of(new DType.Primitive(PType.I64, false), new DType.Utf8(false)),
false);
Path f = tmp.resolve("strmin-boundary.vortex");
Expand Down Expand Up @@ -267,7 +268,7 @@ void unsignedKeyAcrossBoundaryAbandonsButScanIsCorrect() throws Exception {
// past the high bit, so the fold abandons any unsigned column outright — even on the boundary
// path. chunk 0 id 0..3, chunk 1 id 10..13; `id > 1 AND id < 13` cuts both chunks.
DType.Struct schema = new DType.Struct(
List.of("id", "val"),
List.of(ColumnName.of("id"), ColumnName.of("val")),
List.of(new DType.Primitive(PType.U64, false), new DType.Primitive(PType.I64, false)),
false);
Path f = tmp.resolve("u64-boundary.vortex");
Expand Down Expand Up @@ -334,7 +335,7 @@ void neqWithNullInFilterColumnAcrossBoundaryExcludesNullAndValue() throws Except
// null row AND the 30 row must both drop. This confirms the boundary mask's three-valued
// logic for `<>` — a case the equality/range tests do not reach.
DType.Struct schema = new DType.Struct(
List.of("id", "amt"),
List.of(ColumnName.of("id"), ColumnName.of("amt")),
List.of(new DType.Primitive(PType.I64, false), new DType.Primitive(PType.I64, true)),
false);
Path f = tmp.resolve("neq-null-boundary.vortex");
Expand Down Expand Up @@ -385,7 +386,7 @@ void floatFilterColumnAcrossBoundaryAbandonsButScanIsNaNCorrect() throws Excepti
// NaN-correct. chunk 0 f {1.0,NaN,2.0,3.0}, chunk 1 f {4.0,5.0,6.0,7.0}; `f >= 1.5` cuts
// chunk 0 (a boundary) and keeps chunk 1 whole.
DType.Struct schema = new DType.Struct(
List.of("f", "val"),
List.of(ColumnName.of("f"), ColumnName.of("val")),
List.of(new DType.Primitive(PType.F64, false), new DType.Primitive(PType.I64, false)),
false);
Path f = tmp.resolve("float-nan-boundary.vortex");
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.github.dfa1.vortex.calcite;

import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.PType;
import io.github.dfa1.vortex.writer.VortexWriter;
Expand Down Expand Up @@ -155,7 +156,7 @@ void narrowIntegerKeyFoldsFromStats() throws Exception {
// a same-type comparison would never fire — this guards that the widening compare folds
// narrow columns. Two chunks: ids 0..3, then 10..13.
DType.Struct schema = new DType.Struct(
List.of("id", "val"),
List.of(ColumnName.of("id"), ColumnName.of("val")),
List.of(new DType.Primitive(PType.I32, false), new DType.Primitive(PType.I32, false)),
false);
Path f = tmp.resolve("i32.vortex");
Expand Down Expand Up @@ -185,7 +186,7 @@ void narrowIntegerKeyFoldsFromStats() throws Exception {
void nullAwareSumAndCountOverKeptZones() throws Exception {
// Given a non-null clustered key and a nullable value with two nulls in the kept chunk
DType.Struct schema = new DType.Struct(
List.of("id", "val"),
List.of(ColumnName.of("id"), ColumnName.of("val")),
List.of(new DType.Primitive(PType.I64, false), new DType.Primitive(PType.I64, true)),
false);
Path f = tmp.resolve("nullable.vortex");
Expand Down Expand Up @@ -217,7 +218,7 @@ void unsignedKeyColumnAbandonsToScan() throws Exception {
// Given a clustered U64 key: a signed stat compare could fold the wrong zones past the high
// bit, so the fold must abandon unsigned columns to the (unsigned-aware) scan
DType.Struct schema = new DType.Struct(
List.of("id", "val"),
List.of(ColumnName.of("id"), ColumnName.of("val")),
List.of(new DType.Primitive(PType.U64, false), new DType.Primitive(PType.I64, false)),
false);
Path f = tmp.resolve("u64.vortex");
Expand Down Expand Up @@ -301,7 +302,7 @@ void isNotNullBoundaryChunkFoldsViaDecode() throws Exception {
// builds an IS NOT NULL mask, and reduces only its selected rows, so the fold answers
// without a full scan.
DType.Struct schema = new DType.Struct(
List.of("id", "val"),
List.of(ColumnName.of("id"), ColumnName.of("val")),
List.of(new DType.Primitive(PType.I64, false), new DType.Primitive(PType.I64, true)),
false);
Path f = tmp.resolve("isnotnull-boundary.vortex");
Expand Down Expand Up @@ -413,7 +414,7 @@ void isNullOverExpressionAbandonsToScan() throws Exception {
/// `IS NULL` / `IS NOT NULL` clean-partition tests.
private static Path nullPartitionedFile(String name) throws Exception {
DType.Struct schema = new DType.Struct(
List.of("id", "val"),
List.of(ColumnName.of("id"), ColumnName.of("val")),
List.of(new DType.Primitive(PType.I64, false), new DType.Primitive(PType.I64, true)),
false);
Path f = tmp.resolve(name);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.github.dfa1.vortex.calcite;

import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.reader.ScanIterator;
import io.github.dfa1.vortex.reader.ScanOptions;
import io.github.dfa1.vortex.reader.VortexReader;
Expand Down Expand Up @@ -224,7 +225,8 @@ private static Pushdown runPushdown(Path file) throws Exception {
total += c;
}
}
return new Pushdown(stats.get("low").min(), stats.get("high").max(), total);
return new Pushdown(stats.get(ColumnName.of("low")).min(),
stats.get(ColumnName.of("high")).max(), total);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ private static Comparable<?> parseValue(String raw) {

private static RowPredicate toRowPredicate(RowFilter filter) {
return switch (filter) {
case RowFilter.Column(var col, var predicate) -> columnPredicate(col, predicate);
case RowFilter.Column(var col, var predicate) -> columnPredicate(col.value(), predicate);
case RowFilter.And(var filters) -> {
RowPredicate[] preds = filters.stream().map(FilterCommand::toRowPredicate).toArray(RowPredicate[]::new);
yield (chunk, rowIdx) -> {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.github.dfa1.vortex.cli;

import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.reader.VortexReader;

Expand Down Expand Up @@ -41,8 +42,8 @@ private static void printSchema(PrintStream out, Path path, DType dtype, long ro
out.printf("%s (%s rows, %d columns)%n%n",
path.getFileName(), formatRows(rowCount), n);
int idxWidth = Integer.toString(n).length();
int nameWidth = s.fieldNames().stream().mapToInt(String::length).max().orElse(0);
List<String> names = s.fieldNames();
List<String> names = s.fieldNames().stream().map(ColumnName::value).toList();
int nameWidth = names.stream().mapToInt(String::length).max().orElse(0);
List<DType> types = s.fieldTypes();
for (int i = 0; i < n; i++) {
out.printf(" %" + idxWidth + "d %-" + nameWidth + "s %s%n",
Expand All @@ -67,7 +68,7 @@ static String formatDType(DType dtype) {
if (i > 0) {
sb.append(", ");
}
sb.append(s.fieldNames().get(i)).append(": ").append(formatDType(s.fieldTypes().get(i)));
sb.append(s.fieldNames().get(i).value()).append(": ").append(formatDType(s.fieldTypes().get(i)));
}
sb.append('>');
yield sb.toString();
Expand Down
9 changes: 5 additions & 4 deletions cli/src/main/java/io/github/dfa1/vortex/cli/StatsCommand.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.github.dfa1.vortex.cli;

import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.reader.VortexReader;

Expand Down Expand Up @@ -32,21 +33,21 @@ static int run(String[] args) {
return ExitStatus.ERROR;
}
long totalRows = reader.layout().rowCount();
Map<String, ArrayStats> stats = reader.columnStats();
Map<ColumnName, ArrayStats> stats = reader.columnStats();

System.out.printf("rows: %,d%n%n", totalRows);
System.out.printf("%-20s %-12s %15s %15s%n", "column", "type", "min", "max");
System.out.println("-".repeat(67));

List<String> names = schema.fieldNames();
List<ColumnName> names = schema.fieldNames();
List<DType> types = schema.fieldTypes();
for (int i = 0; i < names.size(); i++) {
String name = names.get(i);
ColumnName name = names.get(i);
String type = formatDType(types.get(i));
ArrayStats s = stats.getOrDefault(name, ArrayStats.empty());
String min = s.min() != null ? s.min().toString() : "n/a";
String max = s.max() != null ? s.max().toString() : "n/a";
System.out.printf("%-20s %-12s %15s %15s%n", name, type, min, max);
System.out.printf("%-20s %-12s %15s %15s%n", name.value(), type, min, max);
}
return ExitStatus.OK;
} catch (IOException e) {
Expand Down
Loading
Loading