Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,35 +19,66 @@
package org.apache.paimon.flink.lineage;

import org.apache.paimon.CoreOptions;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.fs.Path;
import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.FormatTable;
import org.apache.paimon.table.Table;

import org.apache.flink.api.connector.source.Boundedness;
import org.apache.flink.streaming.api.lineage.LineageDataset;
import org.apache.flink.streaming.api.lineage.LineageVertex;
import org.apache.flink.streaming.api.lineage.SourceLineageVertex;

import javax.annotation.Nullable;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

/**
* Lineage utilities for building {@link SourceLineageVertex} and {@link LineageVertex} from a
* Paimon table name and its physical warehouse path (namespace).
* Paimon table name and catalog options.
*/
public class LineageUtils {

private static final String PAIMON_DATASET_PREFIX = "paimon://";
/** Default namespace when the catalog warehouse path is not available. */
private static final String DEFAULT_NAMESPACE = "paimon";

private static final String CATALOG_PREFIX = "catalog.";

/** Catalog option keys safe to include in lineage facets (no credentials or secrets). */
private static final Set<String> CATALOG_OPTION_ALLOWLIST =
new HashSet<>(
Arrays.asList(CatalogOptions.WAREHOUSE.key(), CatalogOptions.METASTORE.key()));

private static final Set<String> PAIMON_OPTION_KEYS =
CoreOptions.getOptions().stream().map(opt -> opt.key()).collect(Collectors.toSet());

/** Extracts the {@link CatalogContext} from a table, or null if not available. */
@Nullable
private static CatalogContext catalogContext(Table table) {
if (table instanceof FileStoreTable) {
return ((FileStoreTable) table).catalogEnvironment().catalogContext();
}
if (table instanceof FormatTable) {
return ((FormatTable) table).catalogContext();
}
return null;
}

/**
* Builds the config map for a dataset facet from a {@link Table}. Includes filtered Paimon
* {@link CoreOptions}, partition keys, primary keys, and the table comment (if present).
* Builds the config map for a dataset facet. Includes filtered Paimon {@link CoreOptions},
* partition keys, primary keys, and a safe subset of catalog-level options (warehouse,
* metastore) prefixed with {@code "catalog."}.
*/
private static Map<String, String> buildConfigMap(Table table) {
private static Map<String, String> buildConfigMap(
Table table, @Nullable CatalogContext catalogContext) {
Map<String, String> config = new HashMap<>();
config.put("partition-keys", String.join(",", table.partitionKeys()));
config.put("primary-keys", String.join(",", table.primaryKeys()));
Expand All @@ -56,29 +87,54 @@ private static Map<String, String> buildConfigMap(Table table) {
.filter(e -> PAIMON_OPTION_KEYS.contains(e.getKey()))
.forEach(e -> config.put(e.getKey(), e.getValue()));

config.put("type", "paimon");

if (catalogContext != null) {
catalogContext
.options()
.toMap()
.forEach(
(k, v) -> {
if (CATALOG_OPTION_ALLOWLIST.contains(k)) {
config.put(CATALOG_PREFIX + k, v);
}
});
}

return config;
}

/**
* Returns the lineage namespace for a Paimon table. The namespace uses the {@code paimon://}
* scheme followed by the table's physical warehouse path, e.g. {@code
* "paimon://s3://my-bucket/warehouse/mydb.db/mytable"}.
* Returns the catalog warehouse path as the lineage namespace. Falls back to the table path
* when no warehouse is configured, or {@code "paimon"} as a last resort.
*/
public static String getNamespace(Table table) {
return PAIMON_DATASET_PREFIX + CoreOptions.path(table.options());
public static String getNamespace(Table table, @Nullable CatalogContext catalogContext) {
if (catalogContext != null) {
String warehouse = catalogContext.options().get(CatalogOptions.WAREHOUSE);
if (warehouse != null) {
return warehouse;
}
}
Path path = CoreOptions.path(table.options());
if (path != null) {
return path.toString();
}
return DEFAULT_NAMESPACE;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For direct Flink connector tables, CatalogContext is built from table options and may have path but
no warehouse. This makes all such tables fall back to the same namespace "paimon"; can we fall back to table path instead?

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@XiaoHongbo-Hope Yes, I have fixed this as you suggested.

}

/**
* Creates a {@link SourceLineageVertex} for a Paimon source table.
*
* @param name fully qualified table name, e.g. {@code "paimon.mydb.mytable"}
* @param isBounded whether the source is bounded (batch) or unbounded (streaming)
* @param table the Paimon table (namespace is derived from its {@code path} option)
* @param table the Paimon table
*/
public static SourceLineageVertex sourceLineageVertex(
String name, boolean isBounded, Table table) {
CatalogContext ctx = catalogContext(table);
LineageDataset dataset =
new PaimonLineageDataset(name, getNamespace(table), buildConfigMap(table));
new PaimonLineageDataset(
name, getNamespace(table, ctx), buildConfigMap(table, ctx));
Boundedness boundedness =
isBounded ? Boundedness.BOUNDED : Boundedness.CONTINUOUS_UNBOUNDED;
return new PaimonSourceLineageVertex(boundedness, Collections.singletonList(dataset));
Expand All @@ -88,11 +144,13 @@ public static SourceLineageVertex sourceLineageVertex(
* Creates a {@link LineageVertex} for a Paimon sink table.
*
* @param name fully qualified table name, e.g. {@code "paimon.mydb.mytable"}
* @param table the Paimon table (namespace is derived from its {@code path} option)
* @param table the Paimon table
*/
public static LineageVertex sinkLineageVertex(String name, Table table) {
CatalogContext ctx = catalogContext(table);
LineageDataset dataset =
new PaimonLineageDataset(name, getNamespace(table), buildConfigMap(table));
new PaimonLineageDataset(
name, getNamespace(table, ctx), buildConfigMap(table, ctx));
return new PaimonSinkLineageVertex(Collections.singletonList(dataset));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

/**
* A {@link LineageDataset} representing a Paimon table, identified by its fully qualified name and
* physical warehouse path as the namespace.
* catalog warehouse option as the namespace.
*/
public class PaimonLineageDataset implements LineageDataset {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,15 @@
package org.apache.paimon.flink.lineage;

import org.apache.paimon.CoreOptions;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.CatalogFactory;
import org.apache.paimon.flink.PaimonDataStreamScanProvider;
import org.apache.paimon.flink.PaimonDataStreamSinkProvider;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.options.Options;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.table.FileStoreTable;
Expand Down Expand Up @@ -54,11 +59,14 @@ class LineageUtilsTest {

@TempDir java.nio.file.Path temp;

private Path warehouse;
private Path tablePath;

@BeforeEach
void setUp() {
tablePath = new Path(temp.toUri().toString());
// mirror real Paimon layout: <warehouse>/<database>.db/<table>
warehouse = new Path(temp.toUri().toString());
tablePath = new Path(warehouse, "test_db.db/test_table");
}

private FileStoreTable createTable(
Expand All @@ -79,20 +87,50 @@ private FileStoreTable createTable(
}

@Test
void testGetNamespace() throws Exception {
void testGetNamespaceWithNullCatalogContext() throws Exception {
FileStoreTable table =
createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0"));
assertThat(LineageUtils.getNamespace(table, null)).isEqualTo(table.options().get("path"));
}

String namespace = LineageUtils.getNamespace(table);
@Test
void testGetNamespaceWithCatalogContext() throws Exception {
FileStoreTable table =
createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0"));
Options options = new Options();
options.set(CatalogOptions.WAREHOUSE, warehouse.toString());
CatalogContext ctx = CatalogContext.create(options);

assertThat(namespace).startsWith("paimon://");
assertThat(namespace).contains(tablePath.toString());
assertThat(LineageUtils.getNamespace(table, ctx)).isEqualTo(warehouse.toString());
}

@Test
void testSourceLineageVertexBounded() throws Exception {
void testGetNamespaceWithCatalogContextNoWarehouse() throws Exception {
FileStoreTable table =
createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0"));
CatalogContext ctx = CatalogContext.create(new Options());
assertThat(LineageUtils.getNamespace(table, ctx)).isEqualTo(table.options().get("path"));
}

@Test
void testSourceLineageVertexBounded() throws Exception {
Options catalogOptions = new Options();
catalogOptions.set(CatalogOptions.WAREHOUSE, warehouse.toString());
Catalog catalog = CatalogFactory.createCatalog(CatalogContext.create(catalogOptions));
catalog.createDatabase("test_db", true);
catalog.createTable(
org.apache.paimon.catalog.Identifier.create("test_db", "src"),
new Schema(
RowType.of(new IntType(), new VarCharType(100), new IntType()).getFields(),
Collections.emptyList(),
Arrays.asList("f0"),
new HashMap<>(),
""),
false);
FileStoreTable table =
(FileStoreTable)
catalog.getTable(
org.apache.paimon.catalog.Identifier.create("test_db", "src"));

SourceLineageVertex vertex = LineageUtils.sourceLineageVertex("paimon.db.src", true, table);

Expand All @@ -102,7 +140,8 @@ void testSourceLineageVertexBounded() throws Exception {

LineageDataset dataset = vertex.datasets().get(0);
assertThat(dataset.name()).isEqualTo("paimon.db.src");
assertThat(dataset.namespace()).startsWith("paimon://");
assertThat(dataset.namespace()).isEqualTo(warehouse.toString());
catalog.close();
}

@Test
Expand All @@ -118,8 +157,23 @@ void testSourceLineageVertexUnbounded() throws Exception {

@Test
void testSinkLineageVertex() throws Exception {
Options catalogOptions = new Options();
catalogOptions.set(CatalogOptions.WAREHOUSE, warehouse.toString());
Catalog catalog = CatalogFactory.createCatalog(CatalogContext.create(catalogOptions));
catalog.createDatabase("test_db", true);
catalog.createTable(
org.apache.paimon.catalog.Identifier.create("test_db", "sink"),
new Schema(
RowType.of(new IntType(), new VarCharType(100), new IntType()).getFields(),
Collections.emptyList(),
Arrays.asList("f0"),
new HashMap<>(),
""),
false);
FileStoreTable table =
createTable(new HashMap<>(), Collections.emptyList(), Arrays.asList("f0"));
(FileStoreTable)
catalog.getTable(
org.apache.paimon.catalog.Identifier.create("test_db", "sink"));

LineageVertex vertex = LineageUtils.sinkLineageVertex("paimon.db.sink", table);

Expand All @@ -128,7 +182,8 @@ void testSinkLineageVertex() throws Exception {

LineageDataset dataset = vertex.datasets().get(0);
assertThat(dataset.name()).isEqualTo("paimon.db.sink");
assertThat(dataset.namespace()).startsWith("paimon://");
assertThat(dataset.namespace()).isEqualTo(warehouse.toString());
catalog.close();
}

@Test
Expand All @@ -144,6 +199,7 @@ void testConfigFacetContainsPartitionAndPrimaryKeys() throws Exception {

DatasetConfigFacet configFacet = (DatasetConfigFacet) facets.get("config");
Map<String, String> config = configFacet.config();
assertThat(config).containsEntry("type", "paimon");
assertThat(config).containsEntry("partition-keys", "f2");
assertThat(config).containsEntry("primary-keys", "f0,f2");
}
Expand Down