From 95c52e6a7684fa16dae56c8a9d1a72b0720fcc9b Mon Sep 17 00:00:00 2001 From: jkukreja Date: Wed, 1 Apr 2026 13:52:28 -0400 Subject: [PATCH 1/5] [flink] fix namespace for lineage for table path to warehouse path --- .../apache/paimon/flink/lineage/LineageUtils.java | 12 ++++++------ .../paimon/flink/lineage/LineageUtilsTest.java | 14 +++++++++----- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java index 110365c76ee4..ab4b77993386 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java @@ -19,6 +19,7 @@ package org.apache.paimon.flink.lineage; import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.CatalogUtils; import org.apache.paimon.table.Table; import org.apache.flink.api.connector.source.Boundedness; @@ -38,8 +39,6 @@ */ public class LineageUtils { - private static final String PAIMON_DATASET_PREFIX = "paimon://"; - private static final Set PAIMON_OPTION_KEYS = CoreOptions.getOptions().stream().map(opt -> opt.key()).collect(Collectors.toSet()); @@ -49,6 +48,7 @@ public class LineageUtils { */ private static Map buildConfigMap(Table table) { Map config = new HashMap<>(); + config.put("type", "paimon"); config.put("partition-keys", String.join(",", table.partitionKeys())); config.put("primary-keys", String.join(",", table.primaryKeys())); @@ -60,12 +60,12 @@ private static Map buildConfigMap(Table table) { } /** - * Returns the lineage namespace for a Paimon table. The namespace uses the {@code paimon://} - * scheme followed by the table's physical warehouse path, e.g. {@code - * "paimon://s3://my-bucket/warehouse/mydb.db/mytable"}. + * Returns the lineage namespace for a Paimon table. The namespace is the warehouse path derived + * via {@link CatalogUtils#warehouse(String)}, e.g. {@code "s3://my-bucket/warehouse"} for + * object stores or {@code "file:/tmp/warehouse"} for local paths. */ public static String getNamespace(Table table) { - return PAIMON_DATASET_PREFIX + CoreOptions.path(table.options()); + return CatalogUtils.warehouse(CoreOptions.path(table.options()).toString()); } /** diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java index 62d601ec1b23..44c95cec8e06 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java @@ -54,11 +54,14 @@ class LineageUtilsTest { @TempDir java.nio.file.Path temp; + private Path warehouse; private Path tablePath; @BeforeEach void setUp() { - tablePath = new Path(temp.toUri().toString()); + // mirror real Paimon layout: /.db/ + warehouse = new Path(temp.toUri().toString()); + tablePath = new Path(warehouse, "test_db.db/test_table"); } private FileStoreTable createTable( @@ -85,8 +88,8 @@ void testGetNamespace() throws Exception { String namespace = LineageUtils.getNamespace(table); - assertThat(namespace).startsWith("paimon://"); - assertThat(namespace).contains(tablePath.toString()); + // namespace is the warehouse root (2 levels up from the table path) + assertThat(namespace).isEqualTo("file:" + warehouse.toUri().getPath()); } @Test @@ -102,7 +105,7 @@ void testSourceLineageVertexBounded() throws Exception { LineageDataset dataset = vertex.datasets().get(0); assertThat(dataset.name()).isEqualTo("paimon.db.src"); - assertThat(dataset.namespace()).startsWith("paimon://"); + assertThat(dataset.namespace()).isEqualTo("file:" + warehouse.toUri().getPath()); } @Test @@ -128,7 +131,7 @@ void testSinkLineageVertex() throws Exception { LineageDataset dataset = vertex.datasets().get(0); assertThat(dataset.name()).isEqualTo("paimon.db.sink"); - assertThat(dataset.namespace()).startsWith("paimon://"); + assertThat(dataset.namespace()).isEqualTo("file:" + warehouse.toUri().getPath()); } @Test @@ -144,6 +147,7 @@ void testConfigFacetContainsPartitionAndPrimaryKeys() throws Exception { DatasetConfigFacet configFacet = (DatasetConfigFacet) facets.get("config"); Map config = configFacet.config(); + assertThat(config).containsEntry("type", "paimon"); assertThat(config).containsEntry("partition-keys", "f2"); assertThat(config).containsEntry("primary-keys", "f0,f2"); } From aa90d96ea52633025d81005939f5ae5180755d54 Mon Sep 17 00:00:00 2001 From: jkukreja Date: Mon, 20 Apr 2026 00:34:28 -0400 Subject: [PATCH 2/5] retrigger CI From c2d9e064f68999a94224c1edee92943149cb4464 Mon Sep 17 00:00:00 2001 From: jkukreja Date: Tue, 28 Apr 2026 16:44:41 -0400 Subject: [PATCH 3/5] only add specific keys for catalog options --- .../paimon/flink/lineage/LineageUtils.java | 72 ++++++++++++---- .../flink/lineage/PaimonLineageDataset.java | 2 +- .../flink/lineage/LineageUtilsTest.java | 84 +++++++++++++++---- 3 files changed, 127 insertions(+), 31 deletions(-) diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java index ab4b77993386..9b77d9a0845b 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java @@ -19,7 +19,8 @@ package org.apache.paimon.flink.lineage; import org.apache.paimon.CoreOptions; -import org.apache.paimon.catalog.CatalogUtils; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.table.Table; import org.apache.flink.api.connector.source.Boundedness; @@ -27,26 +28,41 @@ import org.apache.flink.streaming.api.lineage.LineageVertex; import org.apache.flink.streaming.api.lineage.SourceLineageVertex; +import javax.annotation.Nullable; + +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; /** * Lineage utilities for building {@link SourceLineageVertex} and {@link LineageVertex} from a - * Paimon table name and its physical warehouse path (namespace). + * Paimon table name and catalog options. */ public class LineageUtils { + /** Default namespace when the catalog warehouse path is not available. */ + private static final String DEFAULT_NAMESPACE = "paimon"; + + private static final String CATALOG_PREFIX = "catalog."; + + /** Catalog option keys safe to include in lineage facets (no credentials or secrets). */ + private static final Set CATALOG_OPTION_ALLOWLIST = + new HashSet<>( + Arrays.asList(CatalogOptions.WAREHOUSE.key(), CatalogOptions.METASTORE.key())); + private static final Set PAIMON_OPTION_KEYS = CoreOptions.getOptions().stream().map(opt -> opt.key()).collect(Collectors.toSet()); /** - * Builds the config map for a dataset facet from a {@link Table}. Includes filtered Paimon - * {@link CoreOptions}, partition keys, primary keys, and the table comment (if present). + * Builds the config map for a dataset facet from a {@link Table} and {@link Catalog}. Includes + * filtered Paimon {@link CoreOptions}, partition keys, primary keys, and a safe subset of + * catalog-level options (warehouse, metastore) prefixed with {@code "catalog."}. */ - private static Map buildConfigMap(Table table) { + private static Map buildConfigMap(Table table, @Nullable Catalog catalog) { Map config = new HashMap<>(); config.put("type", "paimon"); config.put("partition-keys", String.join(",", table.partitionKeys())); @@ -56,16 +72,35 @@ private static Map buildConfigMap(Table table) { .filter(e -> PAIMON_OPTION_KEYS.contains(e.getKey())) .forEach(e -> config.put(e.getKey(), e.getValue())); + if (catalog != null) { + catalog.options() + .forEach( + (k, v) -> { + if (CATALOG_OPTION_ALLOWLIST.contains(k)) { + config.put(CATALOG_PREFIX + k, v); + } + }); + } + return config; } /** - * Returns the lineage namespace for a Paimon table. The namespace is the warehouse path derived - * via {@link CatalogUtils#warehouse(String)}, e.g. {@code "s3://my-bucket/warehouse"} for - * object stores or {@code "file:/tmp/warehouse"} for local paths. + * Returns the lineage namespace for a Paimon table. Uses the catalog {@code 'warehouse'} option + * directly (e.g. from {@code CREATE CATALOG ... 'warehouse' = '...'}). Falls back to {@code + * "paimon"} when the warehouse is not available (e.g. standalone connector usage or REST + * catalogs without a warehouse option). + * + * @param catalog the Paimon catalog, or null if not available */ - public static String getNamespace(Table table) { - return CatalogUtils.warehouse(CoreOptions.path(table.options()).toString()); + public static String getNamespace(@Nullable Catalog catalog) { + if (catalog != null) { + String warehouse = catalog.options().get(CatalogOptions.WAREHOUSE.key()); + if (warehouse != null) { + return warehouse; + } + } + return DEFAULT_NAMESPACE; } /** @@ -73,12 +108,14 @@ public static String getNamespace(Table table) { * * @param name fully qualified table name, e.g. {@code "paimon.mydb.mytable"} * @param isBounded whether the source is bounded (batch) or unbounded (streaming) - * @param table the Paimon table (namespace is derived from its {@code path} option) + * @param table the Paimon table + * @param catalog the Paimon catalog, or null if not available */ public static SourceLineageVertex sourceLineageVertex( - String name, boolean isBounded, Table table) { + String name, boolean isBounded, Table table, @Nullable Catalog catalog) { LineageDataset dataset = - new PaimonLineageDataset(name, getNamespace(table), buildConfigMap(table)); + new PaimonLineageDataset( + name, getNamespace(catalog), buildConfigMap(table, catalog)); Boundedness boundedness = isBounded ? Boundedness.BOUNDED : Boundedness.CONTINUOUS_UNBOUNDED; return new PaimonSourceLineageVertex(boundedness, Collections.singletonList(dataset)); @@ -88,11 +125,14 @@ public static SourceLineageVertex sourceLineageVertex( * Creates a {@link LineageVertex} for a Paimon sink table. * * @param name fully qualified table name, e.g. {@code "paimon.mydb.mytable"} - * @param table the Paimon table (namespace is derived from its {@code path} option) + * @param table the Paimon table + * @param catalog the Paimon catalog, or null if not available */ - public static LineageVertex sinkLineageVertex(String name, Table table) { + public static LineageVertex sinkLineageVertex( + String name, Table table, @Nullable Catalog catalog) { LineageDataset dataset = - new PaimonLineageDataset(name, getNamespace(table), buildConfigMap(table)); + new PaimonLineageDataset( + name, getNamespace(catalog), buildConfigMap(table, catalog)); return new PaimonSinkLineageVertex(Collections.singletonList(dataset)); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/PaimonLineageDataset.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/PaimonLineageDataset.java index 5e99df0b2d4d..cdc072623b1a 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/PaimonLineageDataset.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/PaimonLineageDataset.java @@ -27,7 +27,7 @@ /** * A {@link LineageDataset} representing a Paimon table, identified by its fully qualified name and - * physical warehouse path as the namespace. + * catalog warehouse option as the namespace. */ public class PaimonLineageDataset implements LineageDataset { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java index 44c95cec8e06..add3fa191028 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java @@ -19,10 +19,15 @@ package org.apache.paimon.flink.lineage; import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.flink.PaimonDataStreamScanProvider; import org.apache.paimon.flink.PaimonDataStreamSinkProvider; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.Options; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; @@ -82,14 +87,63 @@ private FileStoreTable createTable( } @Test - void testGetNamespace() throws Exception { + void testGetNamespaceWithNullCatalog() { + // null catalog falls back to default namespace + assertThat(LineageUtils.getNamespace(null)).isEqualTo("paimon"); + } + + @Test + void testGetNamespaceWithCatalog() throws Exception { + Options options = new Options(); + options.set(CatalogOptions.WAREHOUSE, warehouse.toString()); + Catalog catalog = CatalogFactory.createCatalog(CatalogContext.create(options)); + + assertThat(LineageUtils.getNamespace(catalog)).isEqualTo(warehouse.toString()); + catalog.close(); + } + + @Test + void testConfigFacetIncludesCatalogOptions() throws Exception { + Options options = new Options(); + options.set(CatalogOptions.WAREHOUSE, warehouse.toString()); + Catalog catalog = CatalogFactory.createCatalog(CatalogContext.create(options)); + FileStoreTable table = createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); - String namespace = LineageUtils.getNamespace(table); + LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.t", table, catalog); + LineageDataset dataset = vertex.datasets().get(0); + + assertThat(dataset.namespace()).isEqualTo(warehouse.toString()); - // namespace is the warehouse root (2 levels up from the table path) - assertThat(namespace).isEqualTo("file:" + warehouse.toUri().getPath()); + DatasetConfigFacet configFacet = (DatasetConfigFacet) dataset.facets().get("config"); + Map config = configFacet.config(); + assertThat(config).containsEntry("catalog.warehouse", warehouse.toString()); + + catalog.close(); + } + + @Test + void testConfigFacetExcludesNonAllowlistedCatalogOptions() throws Exception { + Options options = new Options(); + options.set(CatalogOptions.WAREHOUSE, warehouse.toString()); + options.setString("s3.access-key", "SECRET_KEY"); + options.setString("s3.secret-key", "SECRET_VALUE"); + Catalog catalog = CatalogFactory.createCatalog(CatalogContext.create(options)); + + FileStoreTable table = + createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); + + LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.t", table, catalog); + LineageDataset dataset = vertex.datasets().get(0); + + DatasetConfigFacet configFacet = (DatasetConfigFacet) dataset.facets().get("config"); + Map config = configFacet.config(); + assertThat(config).containsEntry("catalog.warehouse", warehouse.toString()); + assertThat(config).doesNotContainKey("catalog.s3.access-key"); + assertThat(config).doesNotContainKey("catalog.s3.secret-key"); + + catalog.close(); } @Test @@ -97,7 +151,8 @@ void testSourceLineageVertexBounded() throws Exception { FileStoreTable table = createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); - SourceLineageVertex vertex = LineageUtils.sourceLineageVertex("paimon.db.src", true, table); + SourceLineageVertex vertex = + LineageUtils.sourceLineageVertex("paimon.db.src", true, table, null); assertThat(vertex).isInstanceOf(PaimonSourceLineageVertex.class); assertThat(vertex.boundedness()).isEqualTo(Boundedness.BOUNDED); @@ -105,7 +160,7 @@ void testSourceLineageVertexBounded() throws Exception { LineageDataset dataset = vertex.datasets().get(0); assertThat(dataset.name()).isEqualTo("paimon.db.src"); - assertThat(dataset.namespace()).isEqualTo("file:" + warehouse.toUri().getPath()); + assertThat(dataset.namespace()).isEqualTo("paimon"); } @Test @@ -114,7 +169,7 @@ void testSourceLineageVertexUnbounded() throws Exception { createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); SourceLineageVertex vertex = - LineageUtils.sourceLineageVertex("paimon.db.src", false, table); + LineageUtils.sourceLineageVertex("paimon.db.src", false, table, null); assertThat(vertex.boundedness()).isEqualTo(Boundedness.CONTINUOUS_UNBOUNDED); } @@ -124,14 +179,14 @@ void testSinkLineageVertex() throws Exception { FileStoreTable table = createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); - LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.sink", table); + LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.sink", table, null); assertThat(vertex).isInstanceOf(PaimonSinkLineageVertex.class); assertThat(vertex.datasets()).hasSize(1); LineageDataset dataset = vertex.datasets().get(0); assertThat(dataset.name()).isEqualTo("paimon.db.sink"); - assertThat(dataset.namespace()).isEqualTo("file:" + warehouse.toUri().getPath()); + assertThat(dataset.namespace()).isEqualTo("paimon"); } @Test @@ -139,7 +194,7 @@ void testConfigFacetContainsPartitionAndPrimaryKeys() throws Exception { FileStoreTable table = createTable(new HashMap<>(), Arrays.asList("f2"), Arrays.asList("f0", "f2")); - LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.t", table); + LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.t", table, null); LineageDataset dataset = vertex.datasets().get(0); Map facets = dataset.facets(); @@ -159,7 +214,8 @@ void testConfigFacetIncludesPaimonOptions() throws Exception { FileStoreTable table = createTable(options, Collections.emptyList(), Arrays.asList("f0")); - SourceLineageVertex vertex = LineageUtils.sourceLineageVertex("paimon.db.t", true, table); + SourceLineageVertex vertex = + LineageUtils.sourceLineageVertex("paimon.db.t", true, table, null); LineageDataset dataset = vertex.datasets().get(0); DatasetConfigFacet configFacet = (DatasetConfigFacet) dataset.facets().get("config"); @@ -172,7 +228,7 @@ void testConfigFacetWithEmptyKeys() throws Exception { FileStoreTable table = createTable(new HashMap<>(), Collections.emptyList(), Collections.emptyList()); - LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.t", table); + LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.t", table, null); LineageDataset dataset = vertex.datasets().get(0); DatasetConfigFacet configFacet = (DatasetConfigFacet) dataset.facets().get("config"); @@ -187,7 +243,7 @@ void testScanProviderImplementsLineageVertexProvider() throws Exception { createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); PaimonDataStreamScanProvider provider = - new PaimonDataStreamScanProvider(true, env -> null, "paimon.db.src", table); + new PaimonDataStreamScanProvider(true, env -> null, "paimon.db.src", table, null); assertThat(provider).isInstanceOf(LineageVertexProvider.class); LineageVertex vertex = provider.getLineageVertex(); @@ -202,7 +258,7 @@ void testSinkProviderImplementsLineageVertexProvider() throws Exception { createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); PaimonDataStreamSinkProvider provider = - new PaimonDataStreamSinkProvider(dataStream -> null, "paimon.db.sink", table); + new PaimonDataStreamSinkProvider(dataStream -> null, "paimon.db.sink", table, null); assertThat(provider).isInstanceOf(LineageVertexProvider.class); LineageVertex vertex = provider.getLineageVertex(); From 5cb1a3d8b4db5032bf8fb07339e777adcc9a3042 Mon Sep 17 00:00:00 2001 From: jkukreja Date: Fri, 1 May 2026 00:21:30 -0400 Subject: [PATCH 4/5] use catalogcontext instead of passing catalog as param --- .../paimon/flink/lineage/LineageUtils.java | 60 ++++++++------- .../flink/lineage/LineageUtilsTest.java | 74 ++++--------------- 2 files changed, 50 insertions(+), 84 deletions(-) diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java index 9b77d9a0845b..d3c7f3d11081 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java @@ -19,8 +19,10 @@ package org.apache.paimon.flink.lineage; import org.apache.paimon.CoreOptions; -import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Table; import org.apache.flink.api.connector.source.Boundedness; @@ -57,12 +59,25 @@ public class LineageUtils { private static final Set PAIMON_OPTION_KEYS = CoreOptions.getOptions().stream().map(opt -> opt.key()).collect(Collectors.toSet()); + /** Extracts the {@link CatalogContext} from a table, or null if not available. */ + @Nullable + private static CatalogContext catalogContext(Table table) { + if (table instanceof FileStoreTable) { + return ((FileStoreTable) table).catalogEnvironment().catalogContext(); + } + if (table instanceof FormatTable) { + return ((FormatTable) table).catalogContext(); + } + return null; + } + /** - * Builds the config map for a dataset facet from a {@link Table} and {@link Catalog}. Includes - * filtered Paimon {@link CoreOptions}, partition keys, primary keys, and a safe subset of - * catalog-level options (warehouse, metastore) prefixed with {@code "catalog."}. + * Builds the config map for a dataset facet. Includes filtered Paimon {@link CoreOptions}, + * partition keys, primary keys, and a safe subset of catalog-level options (warehouse, + * metastore) prefixed with {@code "catalog."}. */ - private static Map buildConfigMap(Table table, @Nullable Catalog catalog) { + private static Map buildConfigMap( + Table table, @Nullable CatalogContext catalogContext) { Map config = new HashMap<>(); config.put("type", "paimon"); config.put("partition-keys", String.join(",", table.partitionKeys())); @@ -72,8 +87,10 @@ private static Map buildConfigMap(Table table, @Nullable Catalog .filter(e -> PAIMON_OPTION_KEYS.contains(e.getKey())) .forEach(e -> config.put(e.getKey(), e.getValue())); - if (catalog != null) { - catalog.options() + if (catalogContext != null) { + catalogContext + .options() + .toMap() .forEach( (k, v) -> { if (CATALOG_OPTION_ALLOWLIST.contains(k)) { @@ -86,16 +103,12 @@ private static Map buildConfigMap(Table table, @Nullable Catalog } /** - * Returns the lineage namespace for a Paimon table. Uses the catalog {@code 'warehouse'} option - * directly (e.g. from {@code CREATE CATALOG ... 'warehouse' = '...'}). Falls back to {@code - * "paimon"} when the warehouse is not available (e.g. standalone connector usage or REST - * catalogs without a warehouse option). - * - * @param catalog the Paimon catalog, or null if not available + * Returns the catalog warehouse path as the lineage namespace, or {@code "paimon"} when the + * warehouse is not available. */ - public static String getNamespace(@Nullable Catalog catalog) { - if (catalog != null) { - String warehouse = catalog.options().get(CatalogOptions.WAREHOUSE.key()); + public static String getNamespace(@Nullable CatalogContext catalogContext) { + if (catalogContext != null) { + String warehouse = catalogContext.options().get(CatalogOptions.WAREHOUSE); if (warehouse != null) { return warehouse; } @@ -109,13 +122,12 @@ public static String getNamespace(@Nullable Catalog catalog) { * @param name fully qualified table name, e.g. {@code "paimon.mydb.mytable"} * @param isBounded whether the source is bounded (batch) or unbounded (streaming) * @param table the Paimon table - * @param catalog the Paimon catalog, or null if not available */ public static SourceLineageVertex sourceLineageVertex( - String name, boolean isBounded, Table table, @Nullable Catalog catalog) { + String name, boolean isBounded, Table table) { + CatalogContext ctx = catalogContext(table); LineageDataset dataset = - new PaimonLineageDataset( - name, getNamespace(catalog), buildConfigMap(table, catalog)); + new PaimonLineageDataset(name, getNamespace(ctx), buildConfigMap(table, ctx)); Boundedness boundedness = isBounded ? Boundedness.BOUNDED : Boundedness.CONTINUOUS_UNBOUNDED; return new PaimonSourceLineageVertex(boundedness, Collections.singletonList(dataset)); @@ -126,13 +138,11 @@ public static SourceLineageVertex sourceLineageVertex( * * @param name fully qualified table name, e.g. {@code "paimon.mydb.mytable"} * @param table the Paimon table - * @param catalog the Paimon catalog, or null if not available */ - public static LineageVertex sinkLineageVertex( - String name, Table table, @Nullable Catalog catalog) { + public static LineageVertex sinkLineageVertex(String name, Table table) { + CatalogContext ctx = catalogContext(table); LineageDataset dataset = - new PaimonLineageDataset( - name, getNamespace(catalog), buildConfigMap(table, catalog)); + new PaimonLineageDataset(name, getNamespace(ctx), buildConfigMap(table, ctx)); return new PaimonSinkLineageVertex(Collections.singletonList(dataset)); } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java index add3fa191028..685b1c3466d8 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java @@ -19,9 +19,7 @@ package org.apache.paimon.flink.lineage; import org.apache.paimon.CoreOptions; -import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; -import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.flink.PaimonDataStreamScanProvider; import org.apache.paimon.flink.PaimonDataStreamSinkProvider; import org.apache.paimon.fs.Path; @@ -87,63 +85,23 @@ private FileStoreTable createTable( } @Test - void testGetNamespaceWithNullCatalog() { - // null catalog falls back to default namespace + void testGetNamespaceWithNullCatalogContext() { assertThat(LineageUtils.getNamespace(null)).isEqualTo("paimon"); } @Test - void testGetNamespaceWithCatalog() throws Exception { + void testGetNamespaceWithCatalogContext() { Options options = new Options(); options.set(CatalogOptions.WAREHOUSE, warehouse.toString()); - Catalog catalog = CatalogFactory.createCatalog(CatalogContext.create(options)); + CatalogContext ctx = CatalogContext.create(options); - assertThat(LineageUtils.getNamespace(catalog)).isEqualTo(warehouse.toString()); - catalog.close(); + assertThat(LineageUtils.getNamespace(ctx)).isEqualTo(warehouse.toString()); } @Test - void testConfigFacetIncludesCatalogOptions() throws Exception { - Options options = new Options(); - options.set(CatalogOptions.WAREHOUSE, warehouse.toString()); - Catalog catalog = CatalogFactory.createCatalog(CatalogContext.create(options)); - - FileStoreTable table = - createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); - - LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.t", table, catalog); - LineageDataset dataset = vertex.datasets().get(0); - - assertThat(dataset.namespace()).isEqualTo(warehouse.toString()); - - DatasetConfigFacet configFacet = (DatasetConfigFacet) dataset.facets().get("config"); - Map config = configFacet.config(); - assertThat(config).containsEntry("catalog.warehouse", warehouse.toString()); - - catalog.close(); - } - - @Test - void testConfigFacetExcludesNonAllowlistedCatalogOptions() throws Exception { - Options options = new Options(); - options.set(CatalogOptions.WAREHOUSE, warehouse.toString()); - options.setString("s3.access-key", "SECRET_KEY"); - options.setString("s3.secret-key", "SECRET_VALUE"); - Catalog catalog = CatalogFactory.createCatalog(CatalogContext.create(options)); - - FileStoreTable table = - createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); - - LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.t", table, catalog); - LineageDataset dataset = vertex.datasets().get(0); - - DatasetConfigFacet configFacet = (DatasetConfigFacet) dataset.facets().get("config"); - Map config = configFacet.config(); - assertThat(config).containsEntry("catalog.warehouse", warehouse.toString()); - assertThat(config).doesNotContainKey("catalog.s3.access-key"); - assertThat(config).doesNotContainKey("catalog.s3.secret-key"); - - catalog.close(); + void testGetNamespaceWithCatalogContextNoWarehouse() { + CatalogContext ctx = CatalogContext.create(new Options()); + assertThat(LineageUtils.getNamespace(ctx)).isEqualTo("paimon"); } @Test @@ -151,8 +109,7 @@ void testSourceLineageVertexBounded() throws Exception { FileStoreTable table = createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); - SourceLineageVertex vertex = - LineageUtils.sourceLineageVertex("paimon.db.src", true, table, null); + SourceLineageVertex vertex = LineageUtils.sourceLineageVertex("paimon.db.src", true, table); assertThat(vertex).isInstanceOf(PaimonSourceLineageVertex.class); assertThat(vertex.boundedness()).isEqualTo(Boundedness.BOUNDED); @@ -169,7 +126,7 @@ void testSourceLineageVertexUnbounded() throws Exception { createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); SourceLineageVertex vertex = - LineageUtils.sourceLineageVertex("paimon.db.src", false, table, null); + LineageUtils.sourceLineageVertex("paimon.db.src", false, table); assertThat(vertex.boundedness()).isEqualTo(Boundedness.CONTINUOUS_UNBOUNDED); } @@ -179,7 +136,7 @@ void testSinkLineageVertex() throws Exception { FileStoreTable table = createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); - LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.sink", table, null); + LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.sink", table); assertThat(vertex).isInstanceOf(PaimonSinkLineageVertex.class); assertThat(vertex.datasets()).hasSize(1); @@ -194,7 +151,7 @@ void testConfigFacetContainsPartitionAndPrimaryKeys() throws Exception { FileStoreTable table = createTable(new HashMap<>(), Arrays.asList("f2"), Arrays.asList("f0", "f2")); - LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.t", table, null); + LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.t", table); LineageDataset dataset = vertex.datasets().get(0); Map facets = dataset.facets(); @@ -214,8 +171,7 @@ void testConfigFacetIncludesPaimonOptions() throws Exception { FileStoreTable table = createTable(options, Collections.emptyList(), Arrays.asList("f0")); - SourceLineageVertex vertex = - LineageUtils.sourceLineageVertex("paimon.db.t", true, table, null); + SourceLineageVertex vertex = LineageUtils.sourceLineageVertex("paimon.db.t", true, table); LineageDataset dataset = vertex.datasets().get(0); DatasetConfigFacet configFacet = (DatasetConfigFacet) dataset.facets().get("config"); @@ -228,7 +184,7 @@ void testConfigFacetWithEmptyKeys() throws Exception { FileStoreTable table = createTable(new HashMap<>(), Collections.emptyList(), Collections.emptyList()); - LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.t", table, null); + LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.t", table); LineageDataset dataset = vertex.datasets().get(0); DatasetConfigFacet configFacet = (DatasetConfigFacet) dataset.facets().get("config"); @@ -243,7 +199,7 @@ void testScanProviderImplementsLineageVertexProvider() throws Exception { createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); PaimonDataStreamScanProvider provider = - new PaimonDataStreamScanProvider(true, env -> null, "paimon.db.src", table, null); + new PaimonDataStreamScanProvider(true, env -> null, "paimon.db.src", table); assertThat(provider).isInstanceOf(LineageVertexProvider.class); LineageVertex vertex = provider.getLineageVertex(); @@ -258,7 +214,7 @@ void testSinkProviderImplementsLineageVertexProvider() throws Exception { createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); PaimonDataStreamSinkProvider provider = - new PaimonDataStreamSinkProvider(dataStream -> null, "paimon.db.sink", table, null); + new PaimonDataStreamSinkProvider(dataStream -> null, "paimon.db.sink", table); assertThat(provider).isInstanceOf(LineageVertexProvider.class); LineageVertex vertex = provider.getLineageVertex(); From 07afc86a895f90f0ac96727d572d50324438224a Mon Sep 17 00:00:00 2001 From: jkukreja Date: Wed, 17 Jun 2026 05:08:51 -0400 Subject: [PATCH 5/5] [flink] Fix type key conflict and improve namespace fallback in lineage --- .../paimon/flink/lineage/LineageUtils.java | 20 +++++-- .../flink/lineage/LineageUtilsTest.java | 60 +++++++++++++++---- 2 files changed, 64 insertions(+), 16 deletions(-) diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java index d3c7f3d11081..e9c51d489a42 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lineage/LineageUtils.java @@ -20,6 +20,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.fs.Path; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FormatTable; @@ -79,7 +80,6 @@ private static CatalogContext catalogContext(Table table) { private static Map buildConfigMap( Table table, @Nullable CatalogContext catalogContext) { Map config = new HashMap<>(); - config.put("type", "paimon"); config.put("partition-keys", String.join(",", table.partitionKeys())); config.put("primary-keys", String.join(",", table.primaryKeys())); @@ -87,6 +87,8 @@ private static Map buildConfigMap( .filter(e -> PAIMON_OPTION_KEYS.contains(e.getKey())) .forEach(e -> config.put(e.getKey(), e.getValue())); + config.put("type", "paimon"); + if (catalogContext != null) { catalogContext .options() @@ -103,16 +105,20 @@ private static Map buildConfigMap( } /** - * Returns the catalog warehouse path as the lineage namespace, or {@code "paimon"} when the - * warehouse is not available. + * Returns the catalog warehouse path as the lineage namespace. Falls back to the table path + * when no warehouse is configured, or {@code "paimon"} as a last resort. */ - public static String getNamespace(@Nullable CatalogContext catalogContext) { + public static String getNamespace(Table table, @Nullable CatalogContext catalogContext) { if (catalogContext != null) { String warehouse = catalogContext.options().get(CatalogOptions.WAREHOUSE); if (warehouse != null) { return warehouse; } } + Path path = CoreOptions.path(table.options()); + if (path != null) { + return path.toString(); + } return DEFAULT_NAMESPACE; } @@ -127,7 +133,8 @@ public static SourceLineageVertex sourceLineageVertex( String name, boolean isBounded, Table table) { CatalogContext ctx = catalogContext(table); LineageDataset dataset = - new PaimonLineageDataset(name, getNamespace(ctx), buildConfigMap(table, ctx)); + new PaimonLineageDataset( + name, getNamespace(table, ctx), buildConfigMap(table, ctx)); Boundedness boundedness = isBounded ? Boundedness.BOUNDED : Boundedness.CONTINUOUS_UNBOUNDED; return new PaimonSourceLineageVertex(boundedness, Collections.singletonList(dataset)); @@ -142,7 +149,8 @@ public static SourceLineageVertex sourceLineageVertex( public static LineageVertex sinkLineageVertex(String name, Table table) { CatalogContext ctx = catalogContext(table); LineageDataset dataset = - new PaimonLineageDataset(name, getNamespace(ctx), buildConfigMap(table, ctx)); + new PaimonLineageDataset( + name, getNamespace(table, ctx), buildConfigMap(table, ctx)); return new PaimonSinkLineageVertex(Collections.singletonList(dataset)); } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java index 685b1c3466d8..ed06e45430b9 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java @@ -19,7 +19,9 @@ package org.apache.paimon.flink.lineage; import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.flink.PaimonDataStreamScanProvider; import org.apache.paimon.flink.PaimonDataStreamSinkProvider; import org.apache.paimon.fs.Path; @@ -85,29 +87,50 @@ private FileStoreTable createTable( } @Test - void testGetNamespaceWithNullCatalogContext() { - assertThat(LineageUtils.getNamespace(null)).isEqualTo("paimon"); + void testGetNamespaceWithNullCatalogContext() throws Exception { + FileStoreTable table = + createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); + assertThat(LineageUtils.getNamespace(table, null)).isEqualTo(table.options().get("path")); } @Test - void testGetNamespaceWithCatalogContext() { + void testGetNamespaceWithCatalogContext() throws Exception { + FileStoreTable table = + createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); Options options = new Options(); options.set(CatalogOptions.WAREHOUSE, warehouse.toString()); CatalogContext ctx = CatalogContext.create(options); - assertThat(LineageUtils.getNamespace(ctx)).isEqualTo(warehouse.toString()); + assertThat(LineageUtils.getNamespace(table, ctx)).isEqualTo(warehouse.toString()); } @Test - void testGetNamespaceWithCatalogContextNoWarehouse() { + void testGetNamespaceWithCatalogContextNoWarehouse() throws Exception { + FileStoreTable table = + createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); CatalogContext ctx = CatalogContext.create(new Options()); - assertThat(LineageUtils.getNamespace(ctx)).isEqualTo("paimon"); + assertThat(LineageUtils.getNamespace(table, ctx)).isEqualTo(table.options().get("path")); } @Test void testSourceLineageVertexBounded() throws Exception { + Options catalogOptions = new Options(); + catalogOptions.set(CatalogOptions.WAREHOUSE, warehouse.toString()); + Catalog catalog = CatalogFactory.createCatalog(CatalogContext.create(catalogOptions)); + catalog.createDatabase("test_db", true); + catalog.createTable( + org.apache.paimon.catalog.Identifier.create("test_db", "src"), + new Schema( + RowType.of(new IntType(), new VarCharType(100), new IntType()).getFields(), + Collections.emptyList(), + Arrays.asList("f0"), + new HashMap<>(), + ""), + false); FileStoreTable table = - createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); + (FileStoreTable) + catalog.getTable( + org.apache.paimon.catalog.Identifier.create("test_db", "src")); SourceLineageVertex vertex = LineageUtils.sourceLineageVertex("paimon.db.src", true, table); @@ -117,7 +140,8 @@ void testSourceLineageVertexBounded() throws Exception { LineageDataset dataset = vertex.datasets().get(0); assertThat(dataset.name()).isEqualTo("paimon.db.src"); - assertThat(dataset.namespace()).isEqualTo("paimon"); + assertThat(dataset.namespace()).isEqualTo(warehouse.toString()); + catalog.close(); } @Test @@ -133,8 +157,23 @@ void testSourceLineageVertexUnbounded() throws Exception { @Test void testSinkLineageVertex() throws Exception { + Options catalogOptions = new Options(); + catalogOptions.set(CatalogOptions.WAREHOUSE, warehouse.toString()); + Catalog catalog = CatalogFactory.createCatalog(CatalogContext.create(catalogOptions)); + catalog.createDatabase("test_db", true); + catalog.createTable( + org.apache.paimon.catalog.Identifier.create("test_db", "sink"), + new Schema( + RowType.of(new IntType(), new VarCharType(100), new IntType()).getFields(), + Collections.emptyList(), + Arrays.asList("f0"), + new HashMap<>(), + ""), + false); FileStoreTable table = - createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0")); + (FileStoreTable) + catalog.getTable( + org.apache.paimon.catalog.Identifier.create("test_db", "sink")); LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.sink", table); @@ -143,7 +182,8 @@ void testSinkLineageVertex() throws Exception { LineageDataset dataset = vertex.datasets().get(0); assertThat(dataset.name()).isEqualTo("paimon.db.sink"); - assertThat(dataset.namespace()).isEqualTo("paimon"); + assertThat(dataset.namespace()).isEqualTo(warehouse.toString()); + catalog.close(); } @Test