diff --git a/docs/generated/flink_catalog_configuration.html b/docs/generated/flink_catalog_configuration.html index 6241c224eaf8..b1ce3ac1507c 100644 --- a/docs/generated/flink_catalog_configuration.html +++ b/docs/generated/flink_catalog_configuration.html @@ -38,5 +38,11 @@ Boolean If true, creating table in default database is not allowed. Default is false. + +
lineage-use-catalog-key-as-identifier
+ false + Boolean + Whether to use the 'catalog-key' as the catalog identifier in the lineage dataset name (i.e., catalog_key.db_name.table_name) when using a JDBC metastore. If false, the internal Flink catalog name is used instead. + diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalogOptions.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalogOptions.java index 8f6214ee08d1..206c0984e481 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalogOptions.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalogOptions.java @@ -36,4 +36,13 @@ public class FlinkCatalogOptions { .defaultValue(false) .withDescription( "If true, creating table in default database is not allowed. Default is false."); + + public static final ConfigOption LINEAGE_USE_CATALOG_KEY = + ConfigOptions.key("lineage-use-catalog-key-as-identifier") + .booleanType() + .defaultValue(false) + .withDescription( + "Whether to use the 'catalog-key' as the catalog identifier in the lineage dataset name " + + "(i.e., catalog_key.db_name.table_name) when using a JDBC metastore. " + + "If false, the internal Flink catalog name is used instead."); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java index df69031fcf31..e7a13ceb9a93 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java @@ -22,6 +22,7 @@ import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.flink.FlinkCatalogFactory; +import org.apache.paimon.flink.FlinkCatalogOptions; import org.apache.paimon.fs.Path; import org.apache.paimon.iceberg.IcebergOptions; import org.apache.paimon.jdbc.JdbcCatalogFactory; @@ -138,23 +139,26 @@ public static String getNamespace(Table table, @Nullable CatalogContext catalogC @VisibleForTesting static String resolveNameByMetastore(Table table, @Nullable String defaultName) { - if (defaultName != null) { - return defaultName; - } - CatalogContext ctx = catalogContext(table); if (ctx != null) { Options catalogOptions = ctx.options(); - // If jdbc metastore is used, use catalog-key as the catalog identifier. - if (JdbcCatalogFactory.IDENTIFIER.equals( - catalogOptions.get(CatalogOptions.METASTORE))) { + // Use catalog-key as the identifier when explicitly opted-in via config, + // or when no explicit name is provided (DataStream API path). + boolean useCatalogKey = + defaultName == null + || catalogOptions.get(FlinkCatalogOptions.LINEAGE_USE_CATALOG_KEY); + if (useCatalogKey + && JdbcCatalogFactory.IDENTIFIER.equals( + catalogOptions.get(CatalogOptions.METASTORE))) { String catalogKeyValue = catalogOptions.get(JdbcCatalogOptions.CATALOG_KEY); if (!StringUtils.isNullOrWhitespaceOnly(catalogKeyValue)) { return catalogKeyValue + "." + table.fullName(); } } } - return DEFAULT_CATALOG_IDENTIFIER + "." + table.fullName(); + return defaultName != null + ? defaultName + : DEFAULT_CATALOG_IDENTIFIER + "." + table.fullName(); } /** diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java index 2686a9e404dc..b8ab16ec8116 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java @@ -188,6 +188,31 @@ void testResolveNameByMetastoreIgnoresCatalogKeyForNonJdbcMetastore() throws Exc .isEqualTo("paimon.db.src"); } + @Test + void testResolveNameByMetastoreUsesCatalogKeyWhenFlagEnabled() throws Exception { + Map catalogOptions = new HashMap<>(); + catalogOptions.put("metastore", "jdbc"); + catalogOptions.put("catalog-key", "jdbc-warehouse"); + catalogOptions.put("lineage-use-catalog-key-as-identifier", "true"); + FileStoreTable table = createTableWithCatalogOptions(catalogOptions); + + // With flag enabled, catalog-key overrides even when explicit name is provided + assertThat(LineageUtils.resolveNameByMetastore(table, "my_catalog.db.src")) + .isEqualTo("jdbc-warehouse." + table.fullName()); + } + + @Test + void testResolveNameByMetastoreKeepsCatalogNameWhenFlagDisabled() throws Exception { + Map catalogOptions = new HashMap<>(); + catalogOptions.put("metastore", "jdbc"); + catalogOptions.put("catalog-key", "jdbc-warehouse"); + FileStoreTable table = createTableWithCatalogOptions(catalogOptions); + + // With flag disabled (default), Flink catalog name is preserved + assertThat(LineageUtils.resolveNameByMetastore(table, "my_catalog.db.src")) + .isEqualTo("my_catalog.db.src"); + } + @Test void testDataStreamSourceLineageVertexUsesCatalogKey() throws Exception { Map catalogOptions = new HashMap<>();