diff --git a/docs/docs/concepts/system-tables.mdx b/docs/docs/concepts/system-tables.mdx index 164d68a61cca..29806d7435a2 100644 --- a/docs/docs/concepts/system-tables.mdx +++ b/docs/docs/concepts/system-tables.mdx @@ -609,7 +609,8 @@ SELECT * FROM sys.all_table_options; ### Catalog Options Table -You can query the catalog's option information through catalog options table. The options not shown will be the default value. You can take reference to [Configuration](../maintenance/configurations#coreoptions). +You can query the catalog's option information through catalog options table when `catalog-options-table.enabled` is set to `true`. +The options not shown will be the default value. You can take reference to [Configuration](../maintenance/configurations#coreoptions). ```sql SELECT * FROM sys.catalog_options; diff --git a/docs/generated/catalog_configuration.html b/docs/generated/catalog_configuration.html index 5464bccf3d38..806edac3f870 100644 --- a/docs/generated/catalog_configuration.html +++ b/docs/generated/catalog_configuration.html @@ -92,6 +92,12 @@ Boolean Indicates whether this catalog is case-sensitive. + +
catalog-options-table.enabled
+ false + Boolean + Whether to support the sys.catalog_options table. +
client-pool-size
2 diff --git a/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java b/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java index 609d7ea436b7..b1ccfe1caada 100644 --- a/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java @@ -183,6 +183,12 @@ public class CatalogOptions { + "However, during these processes, it does not connect to the metastore; hence, newly added partitions will not be reflected in" + " the metastore and need to be manually added as separate partition operations."); + public static final ConfigOption CATALOG_OPTIONS_TABLE_ENABLED = + ConfigOptions.key("catalog-options-table.enabled") + .booleanType() + .defaultValue(false) + .withDescription("Whether to support the sys.catalog_options table."); + public static final ConfigOption RESOLVING_FILE_IO_ENABLED = ConfigOptions.key("resolving-file-io.enabled") .booleanType() diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java index 4a8a8b1b91b6..87d335652d18 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java @@ -255,7 +255,7 @@ protected abstract void alterDatabaseImpl(String name, List chan @Override public List listTables(String databaseName) throws DatabaseNotExistException { if (isSystemDatabase(databaseName)) { - return SystemTableLoader.loadGlobalTableNames(); + return SystemTableLoader.loadGlobalTableNames(context.options()); } // check db exists @@ -289,7 +289,7 @@ public PagedList listTableDetailsPaged( CatalogUtils.validateTableType(this, tableType); if (isSystemDatabase(databaseName)) { List
systemTables = - SystemTableLoader.loadGlobalTableNames().stream() + SystemTableLoader.loadGlobalTableNames(context.options()).stream() .map( tableName -> { try { diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java index a33dc2c0c106..03a9cd08ad3c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java @@ -23,6 +23,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; import org.apache.paimon.rest.exceptions.NotImplementedException; @@ -328,7 +329,12 @@ private static Table createGlobalSystemTable(String tableName, Catalog catalog) return AllPartitionsTable.fromPartitions( toAllPartitions(catalog, listAllTables(catalog))); case CATALOG_OPTIONS: - return new CatalogOptionsTable(Options.fromMap(catalog.options())); + Options catalogOptions = Options.fromMap(catalog.options()); + if (!catalogOptions.get(CatalogOptions.CATALOG_OPTIONS_TABLE_ENABLED)) { + throw new Catalog.TableNotExistException( + Identifier.create(SYSTEM_DATABASE_NAME, tableName)); + } + return new CatalogOptionsTable(catalogOptions); default: throw new Catalog.TableNotExistException( Identifier.create(SYSTEM_DATABASE_NAME, tableName)); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 9d76cfdf4fef..91ae15132b41 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -222,7 +222,7 @@ public void alterDatabase(String name, List changes, boolean ign public List listTables(String databaseName) throws DatabaseNotExistException { try { if (isSystemDatabase(databaseName)) { - return SystemTableLoader.loadGlobalTableNames(); + return SystemTableLoader.loadGlobalTableNames(context.options()); } return api.listTables(databaseName); } catch (NoSuchResourceException e) { @@ -241,6 +241,12 @@ public PagedList listTablesPaged( @Nullable String tableType) throws DatabaseNotExistException { try { + if (isSystemDatabase(databaseName)) { + CatalogUtils.validateNamePattern(this, tableNamePattern); + CatalogUtils.validateTableType(this, tableType); + return new PagedList<>( + SystemTableLoader.loadGlobalTableNames(context.options()), null); + } return api.listTablesPaged( databaseName, maxResults, pageToken, tableNamePattern, tableType); } catch (NoSuchResourceException e) { @@ -257,6 +263,23 @@ public PagedList
listTableDetailsPaged( @Nullable String tableType) throws DatabaseNotExistException { try { + if (isSystemDatabase(db)) { + CatalogUtils.validateNamePattern(this, tableNamePattern); + CatalogUtils.validateTableType(this, tableType); + List
systemTables = + SystemTableLoader.loadGlobalTableNames(context.options()).stream() + .map( + tableName -> { + try { + return getTable(Identifier.create(db, tableName)); + } catch (TableNotExistException ignored) { + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + return new PagedList<>(systemTables, null); + } PagedList tables = api.listTableDetailsPaged( db, maxResults, pageToken, tableNamePattern, tableType); @@ -273,6 +296,17 @@ public PagedList
listTableDetailsPaged( @Override public List
listTableDetails(String databaseName) throws DatabaseNotExistException { try { + if (isSystemDatabase(databaseName)) { + List
result = new ArrayList<>(); + for (String tableName : SystemTableLoader.loadGlobalTableNames(context.options())) { + try { + result.add(getTable(Identifier.create(databaseName, tableName))); + } catch (TableNotExistException ignored) { + // ignore + } + } + return result; + } List tables = api.listTableDetails(databaseName); return tables.stream().map(t -> toTable(databaseName, t)).collect(Collectors.toList()); } catch (NoSuchResourceException e) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java index 93e0453ab744..bcbffe21ca21 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java @@ -18,6 +18,8 @@ package org.apache.paimon.table.system; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.Options; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; @@ -92,7 +94,15 @@ public static Table load(String type, FileStoreTable dataTable) { .orElse(null); } + public static List loadGlobalTableNames(Options catalogOptions) { + List tableNames = new ArrayList<>(GLOBAL_SYSTEM_TABLES); + if (!catalogOptions.get(CatalogOptions.CATALOG_OPTIONS_TABLE_ENABLED)) { + tableNames.remove(CATALOG_OPTIONS); + } + return tableNames; + } + public static List loadGlobalTableNames() { - return GLOBAL_SYSTEM_TABLES; + return loadGlobalTableNames(new Options()); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java index 9087eee6c7fd..ba7443554b77 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java @@ -47,6 +47,7 @@ import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.table.system.SystemTableLoader; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; @@ -85,7 +86,6 @@ import static org.apache.paimon.catalog.Catalog.SYSTEM_DATABASE_NAME; import static org.apache.paimon.table.system.AllTableOptionsTable.ALL_TABLE_OPTIONS; import static org.apache.paimon.table.system.CatalogOptionsTable.CATALOG_OPTIONS; -import static org.apache.paimon.table.system.SystemTableLoader.GLOBAL_SYSTEM_TABLES; import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; @@ -1037,15 +1037,19 @@ public void testGetTable() throws Exception { Table allTableOptionsTable = catalog.getTable(Identifier.create(SYSTEM_DATABASE_NAME, ALL_TABLE_OPTIONS)); assertThat(allTableOptionsTable).isNotNull(); - Table catalogOptionsTable = - catalog.getTable(Identifier.create(SYSTEM_DATABASE_NAME, CATALOG_OPTIONS)); - assertThat(catalogOptionsTable).isNotNull(); + assertThatExceptionOfType(Catalog.TableNotExistException.class) + .isThrownBy( + () -> + catalog.getTable( + Identifier.create(SYSTEM_DATABASE_NAME, CATALOG_OPTIONS))); assertThatExceptionOfType(Catalog.TableNotExistException.class) .isThrownBy( () -> catalog.getTable(Identifier.create(SYSTEM_DATABASE_NAME, "1111"))); List sysTables = catalog.listTables(SYSTEM_DATABASE_NAME); - assertThat(sysTables).containsAll(GLOBAL_SYSTEM_TABLES); + assertThat(sysTables) + .containsExactlyInAnyOrderElementsOf( + SystemTableLoader.loadGlobalTableNames(new Options())); assertThat(catalog.listViews(SYSTEM_DATABASE_NAME)).isEmpty(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/CatalogOptionsTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/CatalogOptionsTableTest.java index f23c5be359b4..dd12c96f6d07 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/CatalogOptionsTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/CatalogOptionsTableTest.java @@ -55,6 +55,7 @@ public class CatalogOptionsTableTest extends TableTestBase { public void before() throws Exception { catalogOptions = new Options(); catalogOptions.set(CatalogOptions.TABLE_TYPE, CatalogTableType.MANAGED); + catalogOptions.set(CatalogOptions.CATALOG_OPTIONS_TABLE_ENABLED, true); catalogOptions.set("table-default.scan.infer-parallelism", "false"); catalogOptions.set(CatalogOptions.WAREHOUSE, tempDir.toUri().toString()); catalog = CatalogFactory.createCatalog(CatalogContext.create(catalogOptions)); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java index ee14caa381ba..802bca58f561 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java @@ -59,6 +59,13 @@ /** ITCase for catalog tables. */ public class CatalogTableITCase extends CatalogITCaseBase { + @Override + protected Map catalogOptions() { + Map options = new HashMap<>(); + options.put("catalog-options-table.enabled", "true"); + return options; + } + @Override protected boolean inferScanParallelism() { return true; @@ -183,7 +190,9 @@ public void testAllTableOptions() { @Test public void testCatalogOptionsTable() { List result = sql("SELECT * FROM sys.catalog_options"); - assertThat(result).containsExactly(Row.of("warehouse", path)); + assertThat(result) + .containsExactlyInAnyOrder( + Row.of("catalog-options-table.enabled", "true"), Row.of("warehouse", path)); } @Test diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonSystemTableTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonSystemTableTest.scala index ada2ff35729f..47453b664b4e 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonSystemTableTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonSystemTableTest.scala @@ -30,7 +30,7 @@ class PaimonSystemTableTest extends PaimonSparkTestBase { test("system table: show all global system tables") { checkAnswer( sql("SHOW TABLES IN sys").select("tableName"), - SystemTableLoader.GLOBAL_SYSTEM_TABLES.asScala.map(Row(_)).toSeq + SystemTableLoader.loadGlobalTableNames().asScala.map(Row(_)).toSeq ) } }