From 582d63babab717e4dbe99086bfa809ea4bf2df9f Mon Sep 17 00:00:00 2001 From: huangxiaoping <1754789345@qq.com> Date: Sun, 21 Jun 2026 00:11:19 +0800 Subject: [PATCH 1/2] [spark] Expose external table type to Spark --- .../paimon/spark/PaimonSparkTableBase.scala | 6 ++++- .../spark/SparkCatalogWithHiveTest.java | 24 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala index bd69174be712..017d7f6e1988 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala @@ -21,12 +21,13 @@ package org.apache.paimon.spark import org.apache.paimon.CoreOptions import org.apache.paimon.CoreOptions.BucketFunctionType import org.apache.paimon.options.Options +import org.apache.paimon.options.CatalogOptions.TABLE_TYPE import org.apache.paimon.spark.catalog.functions.BucketFunction import org.apache.paimon.spark.read.PaimonSplitScanBuilder import org.apache.paimon.spark.schema.PaimonMetadataColumn import org.apache.paimon.spark.util.OptionUtils import org.apache.paimon.spark.write.{PaimonV2WriteBuilder, PaimonWriteBuilder} -import org.apache.paimon.table.{Table, _} +import org.apache.paimon.table.{CatalogTableType, Table, _} import org.apache.paimon.table.BucketMode.{BUCKET_UNAWARE, HASH_FIXED, POSTPONE_MODE} import org.apache.spark.sql.connector.catalog._ @@ -84,6 +85,9 @@ abstract class PaimonSparkTableBase(val table: Table) if (properties.containsKey(CoreOptions.PATH.key())) { properties.put(TableCatalog.PROP_LOCATION, properties.get(CoreOptions.PATH.key())) } + if (CatalogTableType.EXTERNAL.toString.equalsIgnoreCase(dataTable.options().get(TABLE_TYPE.key()))) { + properties.put(TableCatalog.PROP_EXTERNAL, "true") + } properties case _ => Collections.emptyMap() } diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java index 583522822b9e..c56705c45418 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java @@ -156,6 +156,30 @@ public void testCreateExternalTable() throws IOException { } } + @Test + public void testDescribeExternalAndManagedTableType() throws IOException { + try (SparkSession spark = createSessionBuilder().getOrCreate()) { + spark.sql("CREATE DATABASE IF NOT EXISTS test_db"); + spark.sql("USE spark_catalog.test_db"); + + spark.sql("CREATE EXTERNAL TABLE external_table (a INT, bb INT, c STRING)"); + assertThat( + spark.sql("DESC FORMATTED external_table") + .filter("col_name = 'Type'") + .head() + .getString(1)) + .isEqualTo("EXTERNAL"); + + spark.sql("CREATE TABLE managed_table (a INT)"); + assertThat( + spark.sql("DESC FORMATTED managed_table") + .filter("col_name = 'Type'") + .head() + .getString(1)) + .isEqualTo("MANAGED"); + } + } + private SparkSession.Builder createSessionBuilder() { Path warehousePath = new Path("file:" + tempDir.toString()); return SparkSession.builder() From 0bd05b5333fa229d89038463a61bf6c0112f9590 Mon Sep 17 00:00:00 2001 From: huangxiaoping <1754789345@qq.com> Date: Sun, 21 Jun 2026 08:19:23 +0800 Subject: [PATCH 2/2] Fix code style --- .../org/apache/paimon/spark/PaimonSparkTableBase.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala index 017d7f6e1988..6be314cee847 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala @@ -20,8 +20,8 @@ package org.apache.paimon.spark import org.apache.paimon.CoreOptions import org.apache.paimon.CoreOptions.BucketFunctionType -import org.apache.paimon.options.Options import org.apache.paimon.options.CatalogOptions.TABLE_TYPE +import org.apache.paimon.options.Options import org.apache.paimon.spark.catalog.functions.BucketFunction import org.apache.paimon.spark.read.PaimonSplitScanBuilder import org.apache.paimon.spark.schema.PaimonMetadataColumn @@ -85,7 +85,10 @@ abstract class PaimonSparkTableBase(val table: Table) if (properties.containsKey(CoreOptions.PATH.key())) { properties.put(TableCatalog.PROP_LOCATION, properties.get(CoreOptions.PATH.key())) } - if (CatalogTableType.EXTERNAL.toString.equalsIgnoreCase(dataTable.options().get(TABLE_TYPE.key()))) { + if ( + CatalogTableType.EXTERNAL.toString.equalsIgnoreCase( + dataTable.options().get(TABLE_TYPE.key())) + ) { properties.put(TableCatalog.PROP_EXTERNAL, "true") } properties