Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/generated/flink_catalog_configuration.html
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,11 @@
<td>Boolean</td>
<td>If true, creating table in default database is not allowed. Default is false.</td>
</tr>
<tr>
<td><h5>lineage-use-catalog-key-as-identifier</h5></td>
<td style="word-wrap: break-word;">false</td>
<td>Boolean</td>
<td>Whether to use the 'catalog-key' as the catalog identifier in the lineage dataset name (i.e., catalog_key.db_name.table_name) when using a JDBC metastore. If false, the internal Flink catalog name is used instead.</td>
</tr>
</tbody>
</table>
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,13 @@ public class FlinkCatalogOptions {
.defaultValue(false)
.withDescription(
"If true, creating table in default database is not allowed. Default is false.");

public static final ConfigOption<Boolean> LINEAGE_USE_CATALOG_KEY =
ConfigOptions.key("lineage-use-catalog-key-as-identifier")
.booleanType()
.defaultValue(false)
.withDescription(
"Whether to use the 'catalog-key' as the catalog identifier in the lineage dataset name "
+ "(i.e., catalog_key.db_name.table_name) when using a JDBC metastore. "
+ "If false, the internal Flink catalog name is used instead.");
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.flink.FlinkCatalogFactory;
import org.apache.paimon.flink.FlinkCatalogOptions;
import org.apache.paimon.fs.Path;
import org.apache.paimon.iceberg.IcebergOptions;
import org.apache.paimon.jdbc.JdbcCatalogFactory;
Expand Down Expand Up @@ -138,23 +139,26 @@ public static String getNamespace(Table table, @Nullable CatalogContext catalogC

@VisibleForTesting
static String resolveNameByMetastore(Table table, @Nullable String defaultName) {
if (defaultName != null) {
return defaultName;
}

CatalogContext ctx = catalogContext(table);
if (ctx != null) {
Options catalogOptions = ctx.options();
// If jdbc metastore is used, use catalog-key as the catalog identifier.
if (JdbcCatalogFactory.IDENTIFIER.equals(
catalogOptions.get(CatalogOptions.METASTORE))) {
// Use catalog-key as the identifier when explicitly opted-in via config,
// or when no explicit name is provided (DataStream API path).
boolean useCatalogKey =
defaultName == null
|| catalogOptions.get(FlinkCatalogOptions.LINEAGE_USE_CATALOG_KEY);
if (useCatalogKey
&& JdbcCatalogFactory.IDENTIFIER.equals(
catalogOptions.get(CatalogOptions.METASTORE))) {
String catalogKeyValue = catalogOptions.get(JdbcCatalogOptions.CATALOG_KEY);
if (!StringUtils.isNullOrWhitespaceOnly(catalogKeyValue)) {
return catalogKeyValue + "." + table.fullName();
}
}
}
return DEFAULT_CATALOG_IDENTIFIER + "." + table.fullName();
return defaultName != null
? defaultName
: DEFAULT_CATALOG_IDENTIFIER + "." + table.fullName();
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,31 @@ void testResolveNameByMetastoreIgnoresCatalogKeyForNonJdbcMetastore() throws Exc
.isEqualTo("paimon.db.src");
}

@Test
void testResolveNameByMetastoreUsesCatalogKeyWhenFlagEnabled() throws Exception {
Map<String, String> catalogOptions = new HashMap<>();
catalogOptions.put("metastore", "jdbc");
catalogOptions.put("catalog-key", "jdbc-warehouse");
catalogOptions.put("lineage-use-catalog-key-as-identifier", "true");
FileStoreTable table = createTableWithCatalogOptions(catalogOptions);

// With flag enabled, catalog-key overrides even when explicit name is provided
assertThat(LineageUtils.resolveNameByMetastore(table, "my_catalog.db.src"))
.isEqualTo("jdbc-warehouse." + table.fullName());
}

@Test
void testResolveNameByMetastoreKeepsCatalogNameWhenFlagDisabled() throws Exception {
Map<String, String> catalogOptions = new HashMap<>();
catalogOptions.put("metastore", "jdbc");
catalogOptions.put("catalog-key", "jdbc-warehouse");
FileStoreTable table = createTableWithCatalogOptions(catalogOptions);

// With flag disabled (default), Flink catalog name is preserved
assertThat(LineageUtils.resolveNameByMetastore(table, "my_catalog.db.src"))
.isEqualTo("my_catalog.db.src");
}

@Test
void testDataStreamSourceLineageVertexUsesCatalogKey() throws Exception {
Map<String, String> catalogOptions = new HashMap<>();
Expand Down
Loading