Skip to content

[core] Add option for catalog options system table #8303

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
JingsongLi wants to merge 2 commits into
base: master
Choose a base branch
from
Open

[core] Add option for catalog options system table #8303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
776d886
[core] Add option for catalog options system table
JingsongLi Jun 20, 2026
36ac72f
[hotfix] Fix catalog table test formatting
JingsongLi Jun 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/docs/concepts/system-tables.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -609,7 +609,8 @@ SELECT * FROM sys.all_table_options;

### Catalog Options Table

You can query the catalog's option information through catalog options table. The options not shown will be the default value. You can take reference to [Configuration](../maintenance/configurations#coreoptions).
You can query the catalog's option information through catalog options table when `catalog-options-table.enabled` is set to `true`.
The options not shown will be the default value. You can take reference to [Configuration](../maintenance/configurations#coreoptions).

```sql
SELECT * FROM sys.catalog_options;
Expand Down
6 changes: 6 additions & 0 deletions docs/generated/catalog_configuration.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,12 @@
<td>Boolean</td>
<td>Indicates whether this catalog is case-sensitive.</td>
</tr>
<tr>
<td><h5>catalog-options-table.enabled</h5></td>
<td style="word-wrap: break-word;">false</td>
<td>Boolean</td>
<td>Whether to support the sys.catalog_options table.</td>
</tr>
<tr>
<td><h5>client-pool-size</h5></td>
<td style="word-wrap: break-word;">2</td>
Expand Down
6 changes: 6 additions & 0 deletions paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,12 @@ public class CatalogOptions {
+ "However, during these processes, it does not connect to the metastore; hence, newly added partitions will not be reflected in"
+ " the metastore and need to be manually added as separate partition operations.");

public static final ConfigOption<Boolean> CATALOG_OPTIONS_TABLE_ENABLED =
ConfigOptions.key("catalog-options-table.enabled")
.booleanType()
.defaultValue(false)
.withDescription("Whether to support the sys.catalog_options table.");

public static final ConfigOption<Boolean> RESOLVING_FILE_IO_ENABLED =
ConfigOptions.key("resolving-file-io.enabled")
.booleanType()
Expand Down
4 changes: 2 additions & 2 deletions paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ protected abstract void alterDatabaseImpl(String name, List<PropertyChange> chan
@Override
public List<String> listTables(String databaseName) throws DatabaseNotExistException {
if (isSystemDatabase(databaseName)) {
return SystemTableLoader.loadGlobalTableNames();
return SystemTableLoader.loadGlobalTableNames(context.options());
}

// check db exists
Expand Down Expand Up @@ -289,7 +289,7 @@ public PagedList<Table> listTableDetailsPaged(
CatalogUtils.validateTableType(this, tableType);
if (isSystemDatabase(databaseName)) {
List<Table> systemTables =
SystemTableLoader.loadGlobalTableNames().stream()
SystemTableLoader.loadGlobalTableNames(context.options()).stream()
.map(
tableName -> {
try {
Expand Down
8 changes: 7 additions & 1 deletion paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
import org.apache.paimon.manifest.PartitionEntry;
import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.options.Options;
import org.apache.paimon.partition.Partition;
import org.apache.paimon.rest.exceptions.NotImplementedException;
Expand Down Expand Up @@ -328,7 +329,12 @@ private static Table createGlobalSystemTable(String tableName, Catalog catalog)
return AllPartitionsTable.fromPartitions(
toAllPartitions(catalog, listAllTables(catalog)));
case CATALOG_OPTIONS:
return new CatalogOptionsTable(Options.fromMap(catalog.options()));
Options catalogOptions = Options.fromMap(catalog.options());
if (!catalogOptions.get(CatalogOptions.CATALOG_OPTIONS_TABLE_ENABLED)) {
throw new Catalog.TableNotExistException(
Identifier.create(SYSTEM_DATABASE_NAME, tableName));
}
return new CatalogOptionsTable(catalogOptions);
default:
throw new Catalog.TableNotExistException(
Identifier.create(SYSTEM_DATABASE_NAME, tableName));
Expand Down
36 changes: 35 additions & 1 deletion paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ public void alterDatabase(String name, List<PropertyChange> changes, boolean ign
public List<String> listTables(String databaseName) throws DatabaseNotExistException {
try {
if (isSystemDatabase(databaseName)) {
return SystemTableLoader.loadGlobalTableNames();
return SystemTableLoader.loadGlobalTableNames(context.options());
}
return api.listTables(databaseName);
} catch (NoSuchResourceException e) {
Expand All @@ -241,6 +241,12 @@ public PagedList<String> listTablesPaged(
@Nullable String tableType)
throws DatabaseNotExistException {
try {
if (isSystemDatabase(databaseName)) {
CatalogUtils.validateNamePattern(this, tableNamePattern);
CatalogUtils.validateTableType(this, tableType);
return new PagedList<>(

@JingsongLi JingsongLi Jun 23, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The new system-database path accepts the paged-list parameters but then returns the full global system table list with a null next token. Since RESTCatalog advertises support for paging, name patterns, and table-type filtering, calls such as listTablesPaged("sys", 1, null, null, null) or listTablesPaged("sys", null, null, "catalog_%", null) now return unpaged/unfiltered results instead of following the REST catalog contract. Could we either apply the same filtering/paging to loadGlobalTableNames(...) here and in listTableDetailsPaged, or reject these parameters for the system database?

SystemTableLoader.loadGlobalTableNames(context.options()), null);
}
return api.listTablesPaged(
databaseName, maxResults, pageToken, tableNamePattern, tableType);
} catch (NoSuchResourceException e) {
Expand All @@ -257,6 +263,23 @@ public PagedList<Table> listTableDetailsPaged(
@Nullable String tableType)
throws DatabaseNotExistException {
try {
if (isSystemDatabase(db)) {
CatalogUtils.validateNamePattern(this, tableNamePattern);
CatalogUtils.validateTableType(this, tableType);
List<Table> systemTables =
SystemTableLoader.loadGlobalTableNames(context.options()).stream()
.map(
tableName -> {
try {
return getTable(Identifier.create(db, tableName));
} catch (TableNotExistException ignored) {
return null;
}
})
.filter(Objects::nonNull)
.collect(Collectors.toList());
return new PagedList<>(systemTables, null);
}
PagedList<GetTableResponse> tables =
api.listTableDetailsPaged(
db, maxResults, pageToken, tableNamePattern, tableType);
Expand All @@ -273,6 +296,17 @@ public PagedList<Table> listTableDetailsPaged(
@Override
public List<Table> listTableDetails(String databaseName) throws DatabaseNotExistException {
try {
if (isSystemDatabase(databaseName)) {
List<Table> result = new ArrayList<>();
for (String tableName : SystemTableLoader.loadGlobalTableNames(context.options())) {
try {
result.add(getTable(Identifier.create(databaseName, tableName)));
} catch (TableNotExistException ignored) {
// ignore
}
}
return result;
}
List<GetTableResponse> tables = api.listTableDetails(databaseName);
return tables.stream().map(t -> toTable(databaseName, t)).collect(Collectors.toList());
} catch (NoSuchResourceException e) {
Expand Down
12 changes: 11 additions & 1 deletion paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

package org.apache.paimon.table.system;

import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.options.Options;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.Table;

Expand Down Expand Up @@ -92,7 +94,15 @@ public static Table load(String type, FileStoreTable dataTable) {
.orElse(null);
}

public static List<String> loadGlobalTableNames(Options catalogOptions) {
List<String> tableNames = new ArrayList<>(GLOBAL_SYSTEM_TABLES);
if (!catalogOptions.get(CatalogOptions.CATALOG_OPTIONS_TABLE_ENABLED)) {
tableNames.remove(CATALOG_OPTIONS);
}
return tableNames;
}

public static List<String> loadGlobalTableNames() {
return GLOBAL_SYSTEM_TABLES;
return loadGlobalTableNames(new Options());
}
}
14 changes: 9 additions & 5 deletions paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.apache.paimon.table.sink.CommitMessage;
import org.apache.paimon.table.source.ReadBuilder;
import org.apache.paimon.table.source.TableScan;
import org.apache.paimon.table.system.SystemTableLoader;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
Expand Down Expand Up @@ -85,7 +86,6 @@
import static org.apache.paimon.catalog.Catalog.SYSTEM_DATABASE_NAME;
import static org.apache.paimon.table.system.AllTableOptionsTable.ALL_TABLE_OPTIONS;
import static org.apache.paimon.table.system.CatalogOptionsTable.CATALOG_OPTIONS;
import static org.apache.paimon.table.system.SystemTableLoader.GLOBAL_SYSTEM_TABLES;
import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
Expand Down Expand Up @@ -1037,15 +1037,19 @@ public void testGetTable() throws Exception {
Table allTableOptionsTable =
catalog.getTable(Identifier.create(SYSTEM_DATABASE_NAME, ALL_TABLE_OPTIONS));
assertThat(allTableOptionsTable).isNotNull();
Table catalogOptionsTable =
catalog.getTable(Identifier.create(SYSTEM_DATABASE_NAME, CATALOG_OPTIONS));
assertThat(catalogOptionsTable).isNotNull();
assertThatExceptionOfType(Catalog.TableNotExistException.class)
.isThrownBy(
() ->
catalog.getTable(
Identifier.create(SYSTEM_DATABASE_NAME, CATALOG_OPTIONS)));
assertThatExceptionOfType(Catalog.TableNotExistException.class)
.isThrownBy(
() -> catalog.getTable(Identifier.create(SYSTEM_DATABASE_NAME, "1111")));

List<String> sysTables = catalog.listTables(SYSTEM_DATABASE_NAME);
assertThat(sysTables).containsAll(GLOBAL_SYSTEM_TABLES);
assertThat(sysTables)
.containsExactlyInAnyOrderElementsOf(
SystemTableLoader.loadGlobalTableNames(new Options()));

assertThat(catalog.listViews(SYSTEM_DATABASE_NAME)).isEmpty();
}
Expand Down
1 change: 1 addition & 0 deletions paimon-core/src/test/java/org/apache/paimon/table/system/CatalogOptionsTableTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ public class CatalogOptionsTableTest extends TableTestBase {
public void before() throws Exception {
catalogOptions = new Options();
catalogOptions.set(CatalogOptions.TABLE_TYPE, CatalogTableType.MANAGED);
catalogOptions.set(CatalogOptions.CATALOG_OPTIONS_TABLE_ENABLED, true);
catalogOptions.set("table-default.scan.infer-parallelism", "false");
catalogOptions.set(CatalogOptions.WAREHOUSE, tempDir.toUri().toString());
catalog = CatalogFactory.createCatalog(CatalogContext.create(catalogOptions));
Expand Down
11 changes: 10 additions & 1 deletion ...n-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,13 @@
/** ITCase for catalog tables. */
public class CatalogTableITCase extends CatalogITCaseBase {

@Override
protected Map<String, String> catalogOptions() {
Map<String, String> options = new HashMap<>();
options.put("catalog-options-table.enabled", "true");
return options;
}

@Override
protected boolean inferScanParallelism() {
return true;
Expand Down Expand Up @@ -183,7 +190,9 @@ public void testAllTableOptions() {
@Test
public void testCatalogOptionsTable() {
List<Row> result = sql("SELECT * FROM sys.catalog_options");
assertThat(result).containsExactly(Row.of("warehouse", path));
assertThat(result)
.containsExactlyInAnyOrder(
Row.of("catalog-options-table.enabled", "true"), Row.of("warehouse", path));
}

@Test
Expand Down
2 changes: 1 addition & 1 deletion .../paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonSystemTableTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ class PaimonSystemTableTest extends PaimonSparkTestBase {
test("system table: show all global system tables") {
checkAnswer(
sql("SHOW TABLES IN sys").select("tableName"),
SystemTableLoader.GLOBAL_SYSTEM_TABLES.asScala.map(Row(_)).toSeq
SystemTableLoader.loadGlobalTableNames().asScala.map(Row(_)).toSeq
)
}
}
Loading