diff --git a/docs/content.zh/docs/sql/built-in-functions/process-table-functions.md b/docs/content.zh/docs/sql/built-in-functions/process-table-functions.md
index c89fa07b7f3872..de05defee76b8c 100644
--- a/docs/content.zh/docs/sql/built-in-functions/process-table-functions.md
+++ b/docs/content.zh/docs/sql/built-in-functions/process-table-functions.md
@@ -31,6 +31,7 @@ The built-in process table functions (PTFs) take a table as input and return a t
|---------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `FROM_CHANGELOG(input => TABLE t [, ...])` | Converts an append-only table with an explicit operation column into a dynamic table. See Changelog Conversion for the full list of arguments, semantics, and usage. |
| `TO_CHANGELOG(input => TABLE t [, ...])` | Converts a dynamic table into an append-only table with an explicit operation column. See Changelog Conversion for the full list of arguments, semantics, and usage. |
+| `DEDUPLICATE(input => TABLE t PARTITION BY k [, ...])` | Keeps the first or the last row per key of an append-only table. See [Deduplication]({{< ref "docs/sql/reference/queries/deduplication" >}}#deduplicate) for the full list of arguments, semantics, and usage. |
| `SNAPSHOT(input => TABLE t [, ...])` | Returns the current state of a dynamic table `t`. `SNAPSHOT` can only be used in a `LATERAL` context and not as a stand-alone table function. See [LATERAL SNAPSHOT join]({{< ref "docs/sql/reference/queries/joins" >}}#lateral-snapshot-join) for the full list of arguments, the join semantics, and usage. |
| `ML_PREDICT(TABLE t, MODEL m, DESCRIPTOR(...) [, ...])` | Applies the model `m` to the columns of table `t` that the `DESCRIPTOR` names, and returns the columns of `t` plus the prediction columns of the model. See [Model Inference]({{< ref "docs/sql/reference/queries/model-inference" >}}#ml_predict-函数) for the full list of arguments, semantics, and usage. |
diff --git a/docs/content.zh/docs/sql/reference/queries/deduplication.md b/docs/content.zh/docs/sql/reference/queries/deduplication.md
index d580f62e431905..0403f51f6dfa3e 100644
--- a/docs/content.zh/docs/sql/reference/queries/deduplication.md
+++ b/docs/content.zh/docs/sql/reference/queries/deduplication.md
@@ -69,4 +69,143 @@ FROM (
WHERE row_num = 1
```
+## DEDUPLICATE
+
+{{< label Streaming >}}
+
+The `DEDUPLICATE` process table function (PTF) keeps the first or the last row per key of an append-only table, as long as the state of the key is retained. Rows are ordered by arrival or by `sequence_by` columns.
+
+### Syntax
+
+```sql
+SELECT * FROM DEDUPLICATE(
+ input => TABLE source_table PARTITION BY key_col[, ...],
+ [keep => 'first' | 'last',]
+ [sequence_by => DESCRIPTOR(col[, ...]),]
+ [state_ttl => INTERVAL '1' HOUR,]
+ [reset_ttl_on_duplicate => true | false,]
+ [uid => 'my-id']
+)
+```
+
+### Parameters
+
+| Parameter | Required | Description |
+|:----------|:---------|:------------|
+| `input` | Yes | The append-only input table. `PARTITION BY` is required and its columns are the key of the output.|
+| `keep` | No | `'first'` (default) keeps the earliest arrival or the smallest `sequence_by` value, `'last'` keeps the latest arrival or the greatest. Must be a literal. Arrival order is not deterministic across reruns, use `sequence_by` for a repeatable result. |
+| `sequence_by` | No | One or more columns that order the records of a key instead of their arrival. The first column decides, later columns break ties, and `NULL` orders below every other value. On a tie, the later arrival wins for `'last'` and the earlier one for `'first'`. Columns must be numeric, string, binary, boolean or date-time, and not part of `PARTITION BY`. |
+| `state_ttl` | No | How long the state of an idle key is retained, as an `INTERVAL` literal. Defaults to `table.exec.state.ttl`, and `INTERVAL '0' SECOND` disables expiry. After the state of a key expires, its next record is emitted as `+I` again. |
+| `reset_ttl_on_duplicate` | No | Whether a record that is dropped refreshes the TTL of its key. Defaults to `true`. Has no effect without a TTL. |
+
+
+### Output
+
+The output columns are the `PARTITION BY` columns, in the order listed, followed by the remaining input columns. Time attributes are emitted as regular timestamps.
+
+The result has one row per key. When a later row replaces the kept one, the row of that key is updated, so the result must be written to a table that accepts updates, for example one with a primary key on the `PARTITION BY` columns. Currently this holds for every `keep` mode, including `keep => 'first'` without `sequence_by`, which never replaces a row.
+
+{{< hint info >}}
+The first record of a key is emitted as `+I`. A record that replaces the kept one is emitted as `+U`, preceded by `-U` if the consumer needs retractions. A record that equals the kept one, or does not replace it, emits nothing. See [Updates and Changelogs]({{< ref "docs/dev/table/functions/ptfs" >}}#updates-and-changelogs).
+{{< /hint >}}
+
+### Examples
+
+#### Keep the first or the last record
+
+Rows of `user_events` arrive in this order:
+
+| `user_events` |
+|:----------------|
+| `(u1, login)` |
+| `(u2, login)` |
+| `(u1, click)` |
+
+`keep` defaults to `'first'`, which keeps the first row of each user:
+
+```sql
+SELECT * FROM DEDUPLICATE(
+ input => TABLE user_events PARTITION BY user_id)
+```
+
+| Result |
+|:----------------|
+| `(u1, login)` |
+| `(u2, login)` |
+
+With `keep => 'last'`, a later row replaces the kept one:
+
+```sql
+SELECT * FROM DEDUPLICATE(
+ input => TABLE user_events PARTITION BY user_id,
+ keep => 'last')
+```
+
+| Result |
+|:----------------|
+| `(u1, click)` |
+| `(u2, login)` |
+
+#### Order by a column
+
+`sequence_by` ranks the rows of a key by a column instead of by arrival, so a late, older row does not replace a newer one:
+
+| `customer_changes` |
+|:----------------------------|
+| `(c1, Berlin, 100)` |
+| `(c1, Munich, 120)` |
+| `(c1, Hamburg, 110)` |
+
+```sql
+SELECT * FROM DEDUPLICATE(
+ input => TABLE customer_changes PARTITION BY customer_id,
+ keep => 'last',
+ sequence_by => DESCRIPTOR(lsn),
+ state_ttl => INTERVAL '1' DAY)
+```
+
+| Result |
+|:----------------------|
+| `(c1, Munich, 120)` |
+
+`(c1, Hamburg, 110)` is dropped because its `lsn` is smaller than that of the kept row.
+
+#### Composite key
+
+The columns of `PARTITION BY` come first in the output, in the order listed:
+
+| `sessions` (`user_id`, `page`, `device`) |
+|:-----------------------------------------|
+| `(u1, a, web)` |
+| `(u1, b, mobile)` |
+| `(u1, c, web)` |
+
+```sql
+SELECT * FROM DEDUPLICATE(
+ input => TABLE sessions PARTITION BY (device, user_id),
+ keep => 'last')
+```
+
+| Result (`device`, `user_id`, `page`) |
+|:-------------------------------------|
+| `(web, u1, c)` |
+| `(mobile, u1, b)` |
+
+#### Table API
+
+```java
+// Keep the first record of each payment
+Table result = payments
+ .partitionBy($("payment_id"))
+ .deduplicate();
+
+// Keep the record with the greatest lsn per customer
+Table result = customerChanges
+ .partitionBy($("customer_id"))
+ .deduplicate(
+ lit("last").asArgument("keep"),
+ descriptor("lsn").asArgument("sequence_by"),
+ lit(Duration.ofDays(1)).asArgument("state_ttl"));
+```
+
{{< top >}}
diff --git a/docs/content/docs/sql/built-in-functions/process-table-functions.md b/docs/content/docs/sql/built-in-functions/process-table-functions.md
index 0c2c3c35930a49..86a21808356ced 100644
--- a/docs/content/docs/sql/built-in-functions/process-table-functions.md
+++ b/docs/content/docs/sql/built-in-functions/process-table-functions.md
@@ -31,6 +31,7 @@ The built-in process table functions (PTFs) take a table as input and return a t
|---------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `FROM_CHANGELOG(input => TABLE t [, ...])` | Converts an append-only table with an explicit operation column into a dynamic table. See [Changelog Conversion]({{< ref "docs/sql/reference/queries/changelog" >}}#from_changelog) for the full list of arguments, semantics, and usage. |
| `TO_CHANGELOG(input => TABLE t [, ...])` | Converts a dynamic table into an append-only table with an explicit operation column. See [Changelog Conversion]({{< ref "docs/sql/reference/queries/changelog" >}}#to_changelog) for the full list of arguments, semantics, and usage. |
+| `DEDUPLICATE(input => TABLE t PARTITION BY k [, ...])` | Keeps the first or the last row per key of an append-only table. See [Deduplication]({{< ref "docs/sql/reference/queries/deduplication" >}}#deduplicate) for the full list of arguments, semantics, and usage. |
| `SNAPSHOT(input => TABLE t [, ...])` | Returns the current state of a dynamic table `t`. `SNAPSHOT` can only be used in a `LATERAL` context and not as a stand-alone table function. See [LATERAL SNAPSHOT join]({{< ref "docs/sql/reference/queries/joins" >}}#lateral-snapshot-join) for the full list of arguments, the join semantics, and usage. |
| `ML_PREDICT(TABLE t, MODEL m, DESCRIPTOR(...) [, ...])` | Applies the model `m` to the columns of table `t` that the `DESCRIPTOR` names, and returns the columns of `t` plus the prediction columns of the model. See [Model Inference]({{< ref "docs/sql/reference/queries/model-inference" >}}#ml_predict-function) for the full list of arguments, semantics, and usage. |
diff --git a/docs/content/docs/sql/reference/queries/deduplication.md b/docs/content/docs/sql/reference/queries/deduplication.md
index 819893d19913bf..73c638d356e5a3 100644
--- a/docs/content/docs/sql/reference/queries/deduplication.md
+++ b/docs/content/docs/sql/reference/queries/deduplication.md
@@ -69,4 +69,143 @@ FROM (
WHERE row_num = 1
```
+## DEDUPLICATE
+
+{{< label Streaming >}}
+
+The `DEDUPLICATE` process table function (PTF) keeps the first or the last row per key of an append-only table, as long as the state of the key is retained. Rows are ordered by arrival or by `sequence_by` columns.
+
+### Syntax
+
+```sql
+SELECT * FROM DEDUPLICATE(
+ input => TABLE source_table PARTITION BY key_col[, ...],
+ [keep => 'first' | 'last',]
+ [sequence_by => DESCRIPTOR(col[, ...]),]
+ [state_ttl => INTERVAL '1' HOUR,]
+ [reset_ttl_on_duplicate => true | false,]
+ [uid => 'my-id']
+)
+```
+
+### Parameters
+
+| Parameter | Required | Description |
+|:----------|:---------|:------------|
+| `input` | Yes | The append-only input table. `PARTITION BY` is required and its columns are the key of the output.|
+| `keep` | No | `'first'` (default) keeps the earliest arrival or the smallest `sequence_by` value, `'last'` keeps the latest arrival or the greatest. Must be a literal. Arrival order is not deterministic across reruns, use `sequence_by` for a repeatable result. |
+| `sequence_by` | No | One or more columns that order the records of a key instead of their arrival. The first column decides, later columns break ties, and `NULL` orders below every other value. On a tie, the later arrival wins for `'last'` and the earlier one for `'first'`. Columns must be numeric, string, binary, boolean or date-time, and not part of `PARTITION BY`. |
+| `state_ttl` | No | How long the state of an idle key is retained, as an `INTERVAL` literal. Defaults to `table.exec.state.ttl`, and `INTERVAL '0' SECOND` disables expiry. After the state of a key expires, its next record is emitted as `+I` again. |
+| `reset_ttl_on_duplicate` | No | Whether a record that is dropped refreshes the TTL of its key. Defaults to `true`. Has no effect without a TTL. |
+
+
+### Output
+
+The output columns are the `PARTITION BY` columns, in the order listed, followed by the remaining input columns. Time attributes are emitted as regular timestamps.
+
+The result has one row per key. When a later row replaces the kept one, the row of that key is updated, so the result must be written to a table that accepts updates, for example one with a primary key on the `PARTITION BY` columns. Currently this holds for every `keep` mode, including `keep => 'first'` without `sequence_by`, which never replaces a row.
+
+{{< hint info >}}
+The first record of a key is emitted as `+I`. A record that replaces the kept one is emitted as `+U`, preceded by `-U` if the consumer needs retractions. A record that equals the kept one, or does not replace it, emits nothing. See [Updates and Changelogs]({{< ref "docs/dev/table/functions/ptfs" >}}#updates-and-changelogs).
+{{< /hint >}}
+
+### Examples
+
+#### Keep the first or the last record
+
+Rows of `user_events` arrive in this order:
+
+| `user_events` |
+|:----------------|
+| `(u1, login)` |
+| `(u2, login)` |
+| `(u1, click)` |
+
+`keep` defaults to `'first'`, which keeps the first row of each user:
+
+```sql
+SELECT * FROM DEDUPLICATE(
+ input => TABLE user_events PARTITION BY user_id)
+```
+
+| Result |
+|:----------------|
+| `(u1, login)` |
+| `(u2, login)` |
+
+With `keep => 'last'`, a later row replaces the kept one:
+
+```sql
+SELECT * FROM DEDUPLICATE(
+ input => TABLE user_events PARTITION BY user_id,
+ keep => 'last')
+```
+
+| Result |
+|:----------------|
+| `(u1, click)` |
+| `(u2, login)` |
+
+#### Order by a column
+
+`sequence_by` ranks the rows of a key by a column instead of by arrival, so a late, older row does not replace a newer one:
+
+| `customer_changes` |
+|:----------------------------|
+| `(c1, Berlin, 100)` |
+| `(c1, Munich, 120)` |
+| `(c1, Hamburg, 110)` |
+
+```sql
+SELECT * FROM DEDUPLICATE(
+ input => TABLE customer_changes PARTITION BY customer_id,
+ keep => 'last',
+ sequence_by => DESCRIPTOR(lsn),
+ state_ttl => INTERVAL '1' DAY)
+```
+
+| Result |
+|:----------------------|
+| `(c1, Munich, 120)` |
+
+`(c1, Hamburg, 110)` is dropped because its `lsn` is smaller than that of the kept row.
+
+#### Composite key
+
+The columns of `PARTITION BY` come first in the output, in the order listed:
+
+| `sessions` (`user_id`, `page`, `device`) |
+|:-----------------------------------------|
+| `(u1, a, web)` |
+| `(u1, b, mobile)` |
+| `(u1, c, web)` |
+
+```sql
+SELECT * FROM DEDUPLICATE(
+ input => TABLE sessions PARTITION BY (device, user_id),
+ keep => 'last')
+```
+
+| Result (`device`, `user_id`, `page`) |
+|:-------------------------------------|
+| `(web, u1, c)` |
+| `(mobile, u1, b)` |
+
+#### Table API
+
+```java
+// Keep the first record of each payment
+Table result = payments
+ .partitionBy($("payment_id"))
+ .deduplicate();
+
+// Keep the record with the greatest lsn per customer
+Table result = customerChanges
+ .partitionBy($("customer_id"))
+ .deduplicate(
+ lit("last").asArgument("keep"),
+ descriptor("lsn").asArgument("sequence_by"),
+ lit(Duration.ofDays(1)).asArgument("state_ttl"));
+```
+
{{< top >}}
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java
index ecf58fbe9b3914..31371edd0e72ad 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java
@@ -290,4 +290,35 @@ public interface PartitionedTable {
* @see #orderBy(Expression...)
*/
Table fromChangelog(Expression... arguments);
+
+ /**
+ * Keeps one row per partition key using the built-in {@code DEDUPLICATE} process table
+ * function. The result is an updating table with the partition columns as its upsert key.
+ *
+ *
{@code keep => 'first'} (default) keeps the earliest arrival or the smallest {@code
+ * sequence_by} value, {@code keep => 'last'} keeps the latest arrival or the greatest value.
+ *
+ *
Examples:
+ *
+ *
{@code
+ * // Keep the first delivery of each payment
+ * Table result = payments
+ * .partitionBy($("payment_id"))
+ * .deduplicate();
+ *
+ * // Keep the version with the greatest 'lsn' per customer, with bounded state retention
+ * Table result = customerChanges
+ * .partitionBy($("customer_id"))
+ * .deduplicate(
+ * lit("last").asArgument("keep"),
+ * descriptor("lsn").asArgument("sequence_by"),
+ * lit(Duration.ofDays(1)).asArgument("state_ttl"));
+ * }
+ *
+ * @param arguments optional named arguments for {@code keep}, {@code sequence_by}, {@code
+ * state_ttl}, {@code reset_ttl_on_duplicate}, and {@code uid}
+ * @return an updating {@link Table} with output schema {@code [partition_keys,
+ * non_partition_input_columns]}
+ */
+ Table deduplicate(Expression... arguments);
}
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableImpl.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableImpl.java
index c3c9cfa7f9db83..31a00a1cd503a2 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableImpl.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableImpl.java
@@ -962,6 +962,11 @@ public Table fromChangelog(final Expression... arguments) {
BuiltInFunctionDefinitions.FROM_CHANGELOG.getName(), (Object[]) arguments);
}
+ @Override
+ public Table deduplicate(final Expression... arguments) {
+ return process(BuiltInFunctionDefinitions.DEDUPLICATE.getName(), (Object[]) arguments);
+ }
+
private QueryOperation createQueryOperation() {
if (orderKeys.isEmpty()) {
return table.operationTreeBuilder.partition(partitionKeys, table.operationTree);
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
index d70f1c50725cd8..b2d0dc228abd2c 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
@@ -38,6 +38,7 @@
import org.apache.flink.table.types.inference.TraitCondition;
import org.apache.flink.table.types.inference.TypeStrategies;
import org.apache.flink.table.types.inference.strategies.ArrayOfStringArgumentTypeStrategy;
+import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy;
import org.apache.flink.table.types.inference.strategies.FromChangelogTypeStrategy;
import org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies;
import org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies;
@@ -939,6 +940,30 @@ ANY, and(logical(LogicalTypeRoot.BOOLEAN), LITERAL)
"org.apache.flink.table.runtime.functions.ptf.FromChangelogFunction")
.build();
+ public static final BuiltInFunctionDefinition DEDUPLICATE =
+ BuiltInFunctionDefinition.newBuilder()
+ .name("DEDUPLICATE")
+ .kind(PROCESS_TABLE)
+ .staticArguments(
+ StaticArgument.table(
+ "input",
+ Row.class,
+ false,
+ EnumSet.of(
+ StaticArgumentTrait.TABLE,
+ StaticArgumentTrait.SET_SEMANTIC_TABLE)),
+ StaticArgument.scalar("keep", DataTypes.STRING(), true),
+ StaticArgument.scalar("sequence_by", DataTypes.DESCRIPTOR(), true),
+ StaticArgument.scalar(
+ "state_ttl", DataTypes.INTERVAL(DataTypes.SECOND()), true),
+ StaticArgument.scalar(
+ "reset_ttl_on_duplicate", DataTypes.BOOLEAN(), true))
+ .changelogModeStrategy(DeduplicateTypeStrategy.CHANGELOG_MODE_STRATEGY)
+ .inputTypeStrategy(DeduplicateTypeStrategy.INPUT_TYPE_STRATEGY)
+ .outputTypeStrategy(DeduplicateTypeStrategy.OUTPUT_TYPE_STRATEGY)
+ .runtimeClass("org.apache.flink.table.runtime.functions.ptf.Deduplicate")
+ .build();
+
/**
* Built-in proxy function for the LATERAL SNAPSHOT temporal join.
*
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java
new file mode 100644
index 00000000000000..49483bf60b765f
--- /dev/null
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.types.inference.strategies;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.functions.ChangelogFunction.ChangelogContext;
+import org.apache.flink.table.functions.ChangelogModeStrategy;
+import org.apache.flink.table.functions.TableSemantics;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.inference.CallContext;
+import org.apache.flink.table.types.inference.InputTypeStrategy;
+import org.apache.flink.table.types.inference.TypeStrategy;
+import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep;
+import org.apache.flink.types.RowKind;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * Type strategies for the {@code DEDUPLICATE} process table function.
+ *
+ * The output is updating and keyed by the PARTITION BY columns: the PARTITION BY columns
+ * followed by the remaining input columns, without time attributes.
+ */
+@Internal
+public final class DeduplicateTypeStrategy {
+
+ public static final int ARG_INPUT = 0;
+ public static final int ARG_KEEP = 1;
+ public static final int ARG_SEQUENCE_BY = 2;
+ public static final int ARG_STATE_TTL = 3;
+ public static final int ARG_RESET_TTL_ON_DUPLICATE = 4;
+ private static final String NAME = "DEDUPLICATE";
+
+ public static final InputTypeStrategy INPUT_TYPE_STRATEGY =
+ new ValidationOnlyInputTypeStrategy() {
+ @Override
+ public Optional> inferInputTypes(
+ final CallContext callContext, final boolean throwOnFailure) {
+ final Optional error = validate(callContext);
+ if (error.isPresent()) {
+ return callContext.fail(throwOnFailure, "%s", error.get());
+ }
+ return Optional.of(callContext.getArgumentDataTypes());
+ }
+ };
+
+ public static final TypeStrategy OUTPUT_TYPE_STRATEGY =
+ callContext ->
+ Optional.of(
+ DataTypes.ROW(
+ DeduplicateTypeStrategyUtils.outputFields(
+ tableSemantics(callContext)))
+ .notNull());
+
+ public static final ChangelogModeStrategy CHANGELOG_MODE_STRATEGY =
+ DeduplicateTypeStrategy::inferChangelogMode;
+
+ /** Returns the {@code keep} argument, or {@link Keep#FIRST} when it is omitted. */
+ public static Keep keep(final CallContext callContext) {
+ return DeduplicateTypeStrategyUtils.keep(callContext, ARG_KEEP);
+ }
+
+ /**
+ * Returns the positions of the {@code sequence_by} columns, in the listed order, within the
+ * output row without the PARTITION BY columns.
+ */
+ public static int[] sequenceByOutputIndices(final CallContext callContext) {
+ return DeduplicateTypeStrategyUtils.sequenceByOutputIndices(
+ callContext, tableSemantics(callContext), ARG_SEQUENCE_BY);
+ }
+
+ private static Optional validate(final CallContext callContext) {
+ final TableSemantics tableSemantics = tableSemantics(callContext);
+ return validateNoOrderBy(tableSemantics)
+ .or(
+ () ->
+ DeduplicateTypeStrategyUtils.validatePartitionByNoTimeAttribute(
+ tableSemantics, NAME))
+ .or(() -> DeduplicateTypeStrategyUtils.validateKeep(callContext, ARG_KEEP, NAME))
+ .or(
+ () ->
+ DeduplicateTypeStrategyUtils.validateSequenceBy(
+ callContext, tableSemantics, ARG_SEQUENCE_BY, NAME))
+ .or(() -> DeduplicateTypeStrategyUtils.validateStateTtl(callContext, ARG_STATE_TTL))
+ .or(
+ () ->
+ DeduplicateTypeStrategyUtils.validateResetTtlOnDuplicate(
+ callContext, ARG_RESET_TTL_ON_DUPLICATE));
+ }
+
+ private static Optional validateNoOrderBy(final TableSemantics tableSemantics) {
+ if (tableSemantics.orderByColumns().length == 0) {
+ return Optional.empty();
+ }
+ final List fieldNames = DataType.getFieldNames(tableSemantics.dataType());
+ final String columns =
+ Arrays.stream(tableSemantics.orderByColumns())
+ .mapToObj(fieldNames::get)
+ .collect(Collectors.joining(", "));
+ return Optional.of(
+ String.format(
+ "DEDUPLICATE does not support ORDER BY, but received ORDER BY %s. "
+ + "Use sequence_by => DESCRIPTOR(%s) instead.",
+ columns, columns));
+ }
+
+ private static ChangelogMode inferChangelogMode(final ChangelogContext changelogContext) {
+ final ChangelogMode requiredMode = changelogContext.getRequiredChangelogMode();
+ final ChangelogMode.Builder builder =
+ ChangelogMode.newBuilder()
+ .addContainedKind(RowKind.INSERT)
+ .addContainedKind(RowKind.UPDATE_AFTER);
+ if (requiredMode != null && requiredMode.contains(RowKind.UPDATE_BEFORE)) {
+ builder.addContainedKind(RowKind.UPDATE_BEFORE);
+ }
+ return builder.build();
+ }
+
+ private static TableSemantics tableSemantics(final CallContext callContext) {
+ return callContext
+ .getTableSemantics(ARG_INPUT)
+ .orElseThrow(
+ () ->
+ new ValidationException(
+ "First argument must be a table for DEDUPLICATE."));
+ }
+
+ private DeduplicateTypeStrategy() {}
+}
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java
new file mode 100644
index 00000000000000..50893b82af5f03
--- /dev/null
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.types.inference.strategies;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.api.DataTypes.Field;
+import org.apache.flink.table.functions.TableSemantics;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.inference.CallContext;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeFamily;
+import org.apache.flink.table.types.logical.LogicalTypeRoot;
+import org.apache.flink.table.types.logical.utils.LogicalTypeChecks;
+import org.apache.flink.table.types.logical.utils.LogicalTypeUtils;
+import org.apache.flink.table.types.utils.DataTypeUtils;
+import org.apache.flink.types.ColumnList;
+import org.apache.flink.util.TimeUtils;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/** Argument validation and output type helpers shared by the deduplication PTFs. */
+@Internal
+public final class DeduplicateTypeStrategyUtils {
+
+ /** Which record of a key is kept. */
+ @Internal
+ public enum Keep {
+ FIRST("first"),
+ LAST("last");
+
+ private final String value;
+
+ Keep(final String value) {
+ this.value = value;
+ }
+
+ /** Returns the mode for an argument value, or empty if the value names no mode. */
+ public static Optional fromValue(final String value) {
+ return Arrays.stream(values()).filter(keep -> keep.value.equals(value)).findFirst();
+ }
+ }
+
+ /** Returns the {@code keep} argument, or {@link Keep#FIRST} when it is omitted. */
+ public static Keep keep(final CallContext callContext, final int argPos) {
+ return callContext
+ .getArgumentValue(argPos, String.class)
+ .map(
+ value ->
+ Keep.fromValue(value)
+ .orElseThrow(
+ () ->
+ new IllegalStateException(
+ "Validated keep argument expected.")))
+ .orElse(Keep.FIRST);
+ }
+
+ /**
+ * Returns the positions of the {@code sequence_by} columns, in the listed order, within the
+ * output row without the PARTITION BY columns.
+ */
+ public static int[] sequenceByOutputIndices(
+ final CallContext callContext, final TableSemantics tableSemantics, final int argPos) {
+ final List fieldNames = DataType.getFieldNames(tableSemantics.dataType());
+ final List outputIndices =
+ Arrays.stream(ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics))
+ .boxed()
+ .collect(Collectors.toList());
+ return sequenceBy(callContext, argPos).stream()
+ .mapToInt(column -> outputIndices.indexOf(fieldNames.indexOf(column)))
+ .toArray();
+ }
+
+ /** Returns the input columns without time attributes. */
+ public static List inputFields(final TableSemantics tableSemantics) {
+ final DataType inputType = tableSemantics.dataType();
+ return DataType.getFields(
+ DataTypeUtils.replaceLogicalType(
+ inputType,
+ LogicalTypeUtils.removeTimeAttributes(inputType.getLogicalType())));
+ }
+
+ /** Returns the output columns after the PARTITION BY columns, without time attributes. */
+ public static List outputFields(final TableSemantics tableSemantics) {
+ final List inputFields = inputFields(tableSemantics);
+ return Arrays.stream(ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics))
+ .mapToObj(inputFields::get)
+ .collect(Collectors.toList());
+ }
+
+ public static Optional validatePartitionByNoTimeAttribute(
+ final TableSemantics tableSemantics, final String functionName) {
+ final List fields = DataType.getFields(tableSemantics.dataType());
+ return Arrays.stream(tableSemantics.partitionByColumns())
+ .mapToObj(fields::get)
+ .filter(
+ field ->
+ LogicalTypeChecks.isTimeAttribute(
+ field.getDataType().getLogicalType()))
+ .findFirst()
+ .map(
+ field ->
+ String.format(
+ "%s does not support a time attribute as a "
+ + "PARTITION BY column, but '%s' is one. Use "
+ + "CAST(%s AS %s) in a subquery first.",
+ functionName,
+ field.getName(),
+ field.getName(),
+ DataTypeUtils.removeTimeAttribute(field.getDataType())
+ .getLogicalType()
+ .asSummaryString()));
+ }
+
+ public static Optional validateKeep(
+ final CallContext callContext, final int argPos, final String functionName) {
+ if (callContext.isArgumentNull(argPos)) {
+ return Optional.empty();
+ }
+ final String received;
+ if (!callContext.isArgumentLiteral(argPos)) {
+ received = "a non-literal expression";
+ } else {
+ final String value = callContext.getArgumentValue(argPos, String.class).orElse(null);
+ if (value == null || Keep.fromValue(value).isPresent()) {
+ return Optional.empty();
+ }
+ received = String.format("keep => '%s'", value);
+ }
+ return Optional.of(
+ String.format(
+ "%s requires keep => 'first' or keep => 'last', but received %s.",
+ functionName, received));
+ }
+
+ public static Optional validateSequenceBy(
+ final CallContext callContext,
+ final TableSemantics tableSemantics,
+ final int argPos,
+ final String functionName) {
+ final List fieldNames = DataType.getFieldNames(tableSemantics.dataType());
+ final List fieldTypes = DataType.getFieldDataTypes(tableSemantics.dataType());
+ final Set seen = new HashSet<>();
+ for (final String column : sequenceBy(callContext, argPos)) {
+ final int index = fieldNames.indexOf(column);
+ if (index == -1) {
+ return Optional.of(
+ String.format(
+ "The sequence_by column '%s' does not exist in the input of "
+ + "%s. Available columns are: %s.",
+ column, functionName, String.join(", ", fieldNames)));
+ }
+ if (!seen.add(column)) {
+ return Optional.of(
+ String.format(
+ "The sequence_by column '%s' is listed more than once. List each "
+ + "ordering column once.",
+ column));
+ }
+ if (Arrays.stream(tableSemantics.partitionByColumns()).anyMatch(c -> c == index)) {
+ return Optional.of(
+ String.format(
+ "The sequence_by column '%s' is a PARTITION BY column, which has "
+ + "the same value for every record of a key. Use a "
+ + "non-PARTITION BY column.",
+ column));
+ }
+ final LogicalType type = fieldTypes.get(index).getLogicalType();
+ if (!isOrderable(type)) {
+ return Optional.of(
+ String.format(
+ "The sequence_by column '%s' has type '%s', which cannot order "
+ + "records. Use a numeric, string, boolean, or date-time "
+ + "column.",
+ column, type.asSummaryString()));
+ }
+ }
+ return Optional.empty();
+ }
+
+ public static Optional validateStateTtl(
+ final CallContext callContext, final int argPos) {
+ if (callContext.isArgumentNull(argPos)) {
+ return Optional.empty();
+ }
+ if (!callContext.isArgumentLiteral(argPos)) {
+ return Optional.of(
+ "The 'state_ttl' argument must be a constant INTERVAL literal, but received a "
+ + "non-literal expression. Use a literal such as INTERVAL '1' HOUR.");
+ }
+ return callContext
+ .getArgumentValue(argPos, Duration.class)
+ .filter(Duration::isNegative)
+ .map(
+ stateTtl ->
+ String.format(
+ "The 'state_ttl' argument must not be negative, but was "
+ + "'%s'. Use INTERVAL '0' SECOND to retain state "
+ + "indefinitely.",
+ TimeUtils.formatWithHighestUnit(stateTtl)));
+ }
+
+ public static Optional validateResetTtlOnDuplicate(
+ final CallContext callContext, final int argPos) {
+ if (callContext.isArgumentNull(argPos) || callContext.isArgumentLiteral(argPos)) {
+ return Optional.empty();
+ }
+ return Optional.of(
+ "The 'reset_ttl_on_duplicate' argument must be a constant BOOLEAN literal, but "
+ + "received a non-literal expression. Use TRUE or FALSE.");
+ }
+
+ private static List sequenceBy(final CallContext callContext, final int argPos) {
+ return callContext
+ .getArgumentValue(argPos, ColumnList.class)
+ .map(ColumnList::getNames)
+ .orElse(List.of());
+ }
+
+ private static boolean isOrderable(final LogicalType type) {
+ return type.is(LogicalTypeFamily.NUMERIC)
+ || type.is(LogicalTypeFamily.CHARACTER_STRING)
+ || type.is(LogicalTypeFamily.BINARY_STRING)
+ || type.is(LogicalTypeFamily.DATETIME)
+ || type.is(LogicalTypeRoot.BOOLEAN);
+ }
+
+ private DeduplicateTypeStrategyUtils() {}
+}
diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java
new file mode 100644
index 00000000000000..70a01030b2a157
--- /dev/null
+++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.types.inference.strategies;
+
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.functions.ChangelogFunction.ChangelogContext;
+import org.apache.flink.table.functions.TableSemantics;
+import org.apache.flink.table.types.AtomicDataType;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.inference.InputTypeStrategiesTestBase;
+import org.apache.flink.table.types.inference.utils.CallContextMock;
+import org.apache.flink.table.types.inference.utils.TableSemanticsMock;
+import org.apache.flink.table.types.logical.LocalZonedTimestampType;
+import org.apache.flink.table.types.logical.TimestampKind;
+import org.apache.flink.types.ColumnList;
+import org.apache.flink.types.RowKind;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_INPUT;
+import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_KEEP;
+import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_RESET_TTL_ON_DUPLICATE;
+import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_SEQUENCE_BY;
+import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL;
+import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.CHANGELOG_MODE_STRATEGY;
+import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.INPUT_TYPE_STRATEGY;
+import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.OUTPUT_TYPE_STRATEGY;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link DeduplicateTypeStrategy}. */
+class DeduplicateTypeStrategyTest extends InputTypeStrategiesTestBase {
+
+ private static final DataType ROWTIME =
+ new AtomicDataType(new LocalZonedTimestampType(true, TimestampKind.ROWTIME, 3));
+
+ private static final DataType INPUT_TYPE =
+ DataTypes.ROW(
+ DataTypes.FIELD("user_id", DataTypes.STRING()),
+ DataTypes.FIELD("event", DataTypes.STRING()),
+ DataTypes.FIELD("ts", ROWTIME),
+ DataTypes.FIELD("lsn", DataTypes.BIGINT()),
+ DataTypes.FIELD("tags", DataTypes.ARRAY(DataTypes.STRING())));
+
+ private static final DataType[] ARGUMENT_TYPES = {
+ INPUT_TYPE,
+ DataTypes.STRING(),
+ DataTypes.DESCRIPTOR(),
+ DataTypes.INTERVAL(DataTypes.SECOND()),
+ DataTypes.BOOLEAN()
+ };
+
+ private static final TableSemanticsMock PARTITIONED_BY_USER =
+ new TableSemanticsMock(
+ INPUT_TYPE, new int[] {0}, new int[0], -1, ChangelogMode.insertOnly());
+
+ @Override
+ protected Stream testData() {
+ return Stream.of(
+ withDefaults("All optional arguments omitted", PARTITIONED_BY_USER)
+ .expectArgumentTypes(ARGUMENT_TYPES),
+ withDefaults("All arguments set", PARTITIONED_BY_USER)
+ .calledWithLiteralAt(ARG_KEEP, "last")
+ .calledWithLiteralAt(ARG_SEQUENCE_BY, ColumnList.of("lsn", "ts"))
+ .calledWithLiteralAt(ARG_STATE_TTL, Duration.ofHours(1))
+ .calledWithLiteralAt(ARG_RESET_TTL_ON_DUPLICATE, false)
+ .expectArgumentTypes(ARGUMENT_TYPES),
+ withDefaults("Invalid keep", PARTITIONED_BY_USER)
+ .calledWithLiteralAt(ARG_KEEP, "FIRST")
+ .expectErrorMessage(
+ "DEDUPLICATE requires keep => 'first' or keep => 'last', but "
+ + "received keep => 'FIRST'."),
+ nonLiteral("Non-literal keep", ARG_KEEP)
+ .expectErrorMessage(
+ "DEDUPLICATE requires keep => 'first' or keep => 'last', but "
+ + "received a non-literal expression."),
+ withDefaults(
+ "ORDER BY",
+ new TableSemanticsMock(
+ INPUT_TYPE,
+ new int[] {0},
+ new int[] {3},
+ -1,
+ ChangelogMode.insertOnly()))
+ .expectErrorMessage(
+ "DEDUPLICATE does not support ORDER BY, but received ORDER BY "
+ + "lsn. Use sequence_by => DESCRIPTOR(lsn) instead."),
+ withDefaults(
+ "Time attribute in PARTITION BY",
+ new TableSemanticsMock(
+ INPUT_TYPE,
+ new int[] {2},
+ new int[0],
+ -1,
+ ChangelogMode.insertOnly()))
+ .expectErrorMessage(
+ "DEDUPLICATE does not support a time attribute as a PARTITION BY "
+ + "column, but 'ts' is one. Use CAST(ts AS "
+ + "TIMESTAMP_LTZ(3)) in a subquery first."),
+ sequenceBy("Unknown sequence_by column", "nope")
+ .expectErrorMessage(
+ "The sequence_by column 'nope' does not exist in the input of "
+ + "DEDUPLICATE. Available columns are: user_id, event, "
+ + "ts, lsn, tags."),
+ sequenceBy("Repeated sequence_by column", "lsn", "lsn")
+ .expectErrorMessage(
+ "The sequence_by column 'lsn' is listed more than once. List "
+ + "each ordering column once."),
+ sequenceBy("PARTITION BY column in sequence_by", "user_id")
+ .expectErrorMessage(
+ "The sequence_by column 'user_id' is a PARTITION BY column, "
+ + "which has the same value for every record of a key."),
+ sequenceBy("Non-orderable sequence_by column", "tags")
+ .expectErrorMessage(
+ "The sequence_by column 'tags' has type 'ARRAY', which "
+ + "cannot order records. Use a numeric, string, boolean, "
+ + "or date-time column."),
+ withDefaults("Negative state_ttl", PARTITIONED_BY_USER)
+ .calledWithLiteralAt(ARG_STATE_TTL, Duration.ofSeconds(-1))
+ .expectErrorMessage(
+ "The 'state_ttl' argument must not be negative, but was '-1 s'. "
+ + "Use INTERVAL '0' SECOND to retain state indefinitely."),
+ nonLiteral("Non-literal state_ttl", ARG_STATE_TTL)
+ .expectErrorMessage(
+ "The 'state_ttl' argument must be a constant INTERVAL literal, "
+ + "but received a non-literal expression."),
+ nonLiteral("Non-literal reset_ttl_on_duplicate", ARG_RESET_TTL_ON_DUPLICATE)
+ .expectErrorMessage(
+ "The 'reset_ttl_on_duplicate' argument must be a constant "
+ + "BOOLEAN literal, but received a non-literal "
+ + "expression."));
+ }
+
+ @Test
+ void testOutputExcludesPartitionKeysAndTimeAttributes() {
+ assertThat(OUTPUT_TYPE_STRATEGY.inferType(callContext(ColumnList.of())))
+ .contains(
+ DataTypes.ROW(
+ DataTypes.FIELD("event", DataTypes.STRING()),
+ DataTypes.FIELD("ts", DataTypes.TIMESTAMP_LTZ(3)),
+ DataTypes.FIELD("lsn", DataTypes.BIGINT()),
+ DataTypes.FIELD(
+ "tags", DataTypes.ARRAY(DataTypes.STRING())))
+ .notNull());
+ }
+
+ @Test
+ void testSequenceByIndicesPointIntoPayload() {
+ assertThat(
+ DeduplicateTypeStrategy.sequenceByOutputIndices(
+ callContext(ColumnList.of("lsn", "event"))))
+ .containsExactly(2, 0);
+ }
+
+ @Test
+ void testChangelogModeEmitsUpdateBeforeOnlyWhenRequired() {
+ assertThat(inferChangelogMode(ChangelogMode.upsert()))
+ .isEqualTo(mode(RowKind.INSERT, RowKind.UPDATE_AFTER));
+ assertThat(inferChangelogMode(ChangelogMode.all()))
+ .isEqualTo(mode(RowKind.INSERT, RowKind.UPDATE_BEFORE, RowKind.UPDATE_AFTER));
+ }
+
+ private static TestSpec withDefaults(
+ final String description, final TableSemanticsMock tableSemantics) {
+ return TestSpec.forStrategy(description, INPUT_TYPE_STRATEGY)
+ .calledWithArgumentTypes(ARGUMENT_TYPES)
+ .calledWithTableSemanticsAt(ARG_INPUT, tableSemantics)
+ .calledWithLiteralAt(ARG_KEEP, null)
+ .calledWithLiteralAt(ARG_SEQUENCE_BY, null)
+ .calledWithLiteralAt(ARG_STATE_TTL, null)
+ .calledWithLiteralAt(ARG_RESET_TTL_ON_DUPLICATE, null);
+ }
+
+ private static TestSpec nonLiteral(final String description, final int argument) {
+ final TestSpec spec =
+ TestSpec.forStrategy(description, INPUT_TYPE_STRATEGY)
+ .calledWithArgumentTypes(ARGUMENT_TYPES)
+ .calledWithTableSemanticsAt(ARG_INPUT, PARTITIONED_BY_USER);
+ for (int pos = ARG_KEEP; pos <= ARG_RESET_TTL_ON_DUPLICATE; pos++) {
+ if (pos != argument) {
+ spec.calledWithLiteralAt(pos, null);
+ }
+ }
+ return spec;
+ }
+
+ private static TestSpec sequenceBy(final String description, final String... columns) {
+ return withDefaults(description, PARTITIONED_BY_USER)
+ .calledWithLiteralAt(ARG_SEQUENCE_BY, ColumnList.of(List.of(columns)));
+ }
+
+ private static CallContextMock callContext(final ColumnList sequenceBy) {
+ final CallContextMock callContext = new CallContextMock();
+ callContext.argumentValues =
+ List.of(
+ Optional.empty(),
+ Optional.empty(),
+ Optional.of(sequenceBy),
+ Optional.empty(),
+ Optional.empty());
+ callContext.tableSemantics = Map.of(ARG_INPUT, PARTITIONED_BY_USER);
+ return callContext;
+ }
+
+ private static ChangelogMode inferChangelogMode(final ChangelogMode requiredMode) {
+ return CHANGELOG_MODE_STRATEGY.inferChangelogMode(
+ new ChangelogContext() {
+ @Override
+ public ChangelogMode getTableChangelogMode(final int pos) {
+ return ChangelogMode.insertOnly();
+ }
+
+ @Override
+ public ChangelogMode getRequiredChangelogMode() {
+ return requiredMode;
+ }
+
+ @Override
+ public Optional getTableSemantics(final int pos) {
+ return Optional.empty();
+ }
+ });
+ }
+
+ private static ChangelogMode mode(final RowKind... kinds) {
+ final ChangelogMode.Builder builder = ChangelogMode.newBuilder();
+ for (final RowKind kind : kinds) {
+ builder.addContainedKind(kind);
+ }
+ return builder.build();
+ }
+}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTest.java
new file mode 100644
index 00000000000000..e9a6d2d9b15dcb
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.plan.nodes.exec.stream;
+
+import org.apache.flink.table.planner.plan.nodes.exec.testutils.RestoreTestBase;
+import org.apache.flink.table.test.program.TableTestProgram;
+
+import java.util.List;
+
+/** Restore tests for the built-in DEDUPLICATE PTF. */
+public class DeduplicateRestoreTest extends RestoreTestBase {
+
+ public DeduplicateRestoreTest() {
+ super(StreamExecProcessTableFunction.class);
+ }
+
+ @Override
+ public List programs() {
+ return List.of(
+ DeduplicateRestoreTestPrograms.FIRST_ARRIVAL_RESTORE,
+ DeduplicateRestoreTestPrograms.SEQUENCE_BY_RESTORE);
+ }
+}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTestPrograms.java
new file mode 100644
index 00000000000000..133d9ad95284ae
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTestPrograms.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.plan.nodes.exec.stream;
+
+import org.apache.flink.table.test.program.SinkTestStep;
+import org.apache.flink.table.test.program.SourceTestStep;
+import org.apache.flink.table.test.program.TableTestProgram;
+import org.apache.flink.types.Row;
+
+/** {@link TableTestProgram} definitions for restore tests of the DEDUPLICATE PTF. */
+public class DeduplicateRestoreTestPrograms {
+
+ public static final TableTestProgram FIRST_ARRIVAL_RESTORE =
+ TableTestProgram.of(
+ "deduplicate-first-arrival-restore",
+ "with the default keep => 'first', a key emitted before the restore "
+ + "stays dropped after it, and a key first seen after it is "
+ + "inserted")
+ .setupTableSource(
+ SourceTestStep.newBuilder("payments")
+ .addSchema("payment_id STRING", "amount INT")
+ .producedBeforeRestore(
+ Row.of("p1", 50),
+ Row.of("p2", 20),
+ Row.of("p1", 60),
+ Row.of("p3", 30))
+ .producedAfterRestore(
+ Row.of("p1", 70),
+ Row.of("p2", 20),
+ Row.of("p4", 40),
+ Row.of("p3", 35))
+ .build())
+ .setupTableSink(
+ SinkTestStep.newBuilder("sink")
+ .addSchema(
+ "payment_id STRING PRIMARY KEY NOT ENFORCED",
+ "amount INT")
+ .addOption("sink-changelog-mode-enforced", "I,UA,D")
+ .consumedBeforeRestore("+I[p1, 50]", "+I[p2, 20]", "+I[p3, 30]")
+ .consumedAfterRestore("+I[p4, 40]")
+ .build())
+ .runSql(
+ "INSERT INTO sink SELECT * FROM DEDUPLICATE("
+ + "input => TABLE payments PARTITION BY payment_id)")
+ .build();
+
+ public static final TableTestProgram SEQUENCE_BY_RESTORE =
+ TableTestProgram.of(
+ "deduplicate-sequence-by-restore",
+ "the sequence_by value of the kept record survives a restore, so "
+ + "with 'first' a greater or equal value arriving after the "
+ + "restore is dropped")
+ .setupTableSource(
+ SourceTestStep.newBuilder("customer_changes")
+ .addSchema("customer_id STRING", "city STRING", "lsn BIGINT")
+ .producedBeforeRestore(
+ Row.of("c1", "Berlin", 100L),
+ Row.of("c2", "Paris", 105L),
+ Row.of("c1", "Hamburg", 90L),
+ Row.of("c3", "Rome", 50L))
+ .producedAfterRestore(
+ Row.of("c1", "Munich", 95L),
+ Row.of("c2", "Lyon", 100L),
+ Row.of("c3", "Milan", 50L),
+ Row.of("c4", "Oslo", 1L))
+ .build())
+ .setupTableSink(
+ SinkTestStep.newBuilder("sink")
+ .addSchema(
+ "customer_id STRING PRIMARY KEY NOT ENFORCED",
+ "city STRING",
+ "lsn BIGINT")
+ .addOption("sink-changelog-mode-enforced", "I,UA,D")
+ .consumedBeforeRestore(
+ "+I[c1, Berlin, 100]",
+ "+I[c2, Paris, 105]",
+ "+U[c1, Hamburg, 90]",
+ "+I[c3, Rome, 50]")
+ .consumedAfterRestore("+U[c2, Lyon, 100]", "+I[c4, Oslo, 1]")
+ .build())
+ .runSql(
+ "INSERT INTO sink SELECT * FROM DEDUPLICATE("
+ + "input => TABLE customer_changes PARTITION BY customer_id, "
+ + "keep => 'first', "
+ + "sequence_by => DESCRIPTOR(lsn))")
+ .build();
+}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateSemanticTests.java
new file mode 100644
index 00000000000000..f8c5c8d3669f90
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateSemanticTests.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.plan.nodes.exec.stream;
+
+import org.apache.flink.table.planner.plan.nodes.exec.testutils.SemanticTestBase;
+import org.apache.flink.table.test.program.TableTestProgram;
+
+import java.util.List;
+
+/** Semantic tests for the built-in DEDUPLICATE process table function. */
+public class DeduplicateSemanticTests extends SemanticTestBase {
+
+ @Override
+ public List programs() {
+ return List.of(
+ DeduplicateTestPrograms.ARRIVAL_ORDER,
+ DeduplicateTestPrograms.COMPOSITE_AND_NULL_KEY,
+ DeduplicateTestPrograms.SEQUENCE_BY);
+ }
+}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateTestPrograms.java
new file mode 100644
index 00000000000000..6a1550eb3a7085
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateTestPrograms.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.plan.nodes.exec.stream;
+
+import org.apache.flink.table.test.program.SinkTestStep;
+import org.apache.flink.table.test.program.SourceTestStep;
+import org.apache.flink.table.test.program.TableTestProgram;
+import org.apache.flink.types.Row;
+
+import javax.annotation.Nullable;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+
+/** {@link TableTestProgram} definitions for testing the built-in DEDUPLICATE PTF. */
+public class DeduplicateTestPrograms {
+
+ private static final String[] USER_EVENTS_SCHEMA = {"user_id STRING", "event STRING"};
+
+ private static final String[] USER_EVENTS_UPSERT_SCHEMA = {
+ "user_id STRING PRIMARY KEY NOT ENFORCED", "event STRING"
+ };
+
+ private static final String[] CUSTOMER_UPSERT_SCHEMA = {
+ "customer_id STRING PRIMARY KEY NOT ENFORCED", "city STRING", "lsn BIGINT"
+ };
+
+ private static final String[] LABEL_UPSERT_SCHEMA = {
+ "id STRING PRIMARY KEY NOT ENFORCED", "label STRING"
+ };
+
+ private static final String[] PRICES_UPSERT_SCHEMA = {
+ "ticker STRING PRIMARY KEY NOT ENFORCED", "price INT", "ts TIMESTAMP_LTZ(3)"
+ };
+
+ public static final TableTestProgram ARRIVAL_ORDER =
+ TableTestProgram.of(
+ "deduplicate-arrival-order",
+ "'last' replaces the kept record with every new one, 'first' drops "
+ + "every record after the first, an identical record emits "
+ + "nothing, and a consumer that needs the previous record "
+ + "receives -U before +U")
+ .setupTableSource(
+ SourceTestStep.newBuilder("user_events")
+ .addSchema(USER_EVENTS_SCHEMA)
+ .producedValues(
+ Row.of("u1", "login"),
+ Row.of("u2", "login"),
+ Row.of("u1", "click"))
+ .build())
+ .setupTableSource(
+ SourceTestStep.newBuilder("profile_updates")
+ .addSchema("user_id STRING", "city STRING", "phone STRING")
+ .producedValues(
+ Row.of("u1", "Berlin", "111"),
+ Row.of("u1", "Berlin", "111"),
+ Row.of("u1", "Munich", null))
+ .build())
+ .setupTableSink(
+ upsertSink(
+ "sink_last",
+ USER_EVENTS_UPSERT_SCHEMA,
+ "+I[u1, login]",
+ "+I[u2, login]",
+ "+U[u1, click]"))
+ .setupTableSink(
+ upsertSink(
+ "sink_first",
+ USER_EVENTS_UPSERT_SCHEMA,
+ "+I[u1, login]",
+ "+I[u2, login]"))
+ .setupTableSink(
+ SinkTestStep.newBuilder("retract_sink")
+ .addSchema(USER_EVENTS_SCHEMA)
+ .consumedValues(
+ "+I[u1, login]",
+ "+I[u2, login]",
+ "-U[u1, login]",
+ "+U[u1, click]")
+ .build())
+ .setupTableSink(
+ upsertSink(
+ "latest_profiles",
+ new String[] {
+ "user_id STRING PRIMARY KEY NOT ENFORCED",
+ "city STRING",
+ "phone STRING"
+ },
+ "+I[u1, Berlin, 111]",
+ "+U[u1, Munich, null]"))
+ .runSql(
+ "INSERT INTO sink_last SELECT * FROM DEDUPLICATE("
+ + "input => TABLE user_events PARTITION BY user_id, "
+ + "keep => 'last')")
+ .runSql(
+ "INSERT INTO sink_first SELECT * FROM DEDUPLICATE("
+ + "input => TABLE user_events PARTITION BY user_id)")
+ .runSql(
+ "INSERT INTO retract_sink SELECT * FROM DEDUPLICATE("
+ + "input => TABLE user_events PARTITION BY user_id, "
+ + "keep => 'last')")
+ .runSql(
+ "INSERT INTO latest_profiles SELECT * FROM DEDUPLICATE("
+ + "input => TABLE profile_updates PARTITION BY user_id, "
+ + "keep => 'last')")
+ .build();
+
+ public static final TableTestProgram COMPOSITE_AND_NULL_KEY =
+ TableTestProgram.of(
+ "deduplicate-composite-and-null-key",
+ "a PARTITION BY of non-adjacent columns is the leading output columns "
+ + "in its own order, and a NULL key column forms its own key")
+ .setupTableSource(
+ SourceTestStep.newBuilder("sessions")
+ .addSchema("user_id STRING", "page STRING", "device STRING")
+ .producedValues(
+ Row.of("u1", "a", "web"),
+ Row.of("u1", "b", "mobile"),
+ Row.of("u1", "c", "web"),
+ Row.of(null, "d", "web"),
+ Row.of(null, "e", "web"))
+ .build())
+ .setupTableSink(
+ SinkTestStep.newBuilder("sink")
+ .addSchema("device STRING", "user_id STRING", "page STRING")
+ .consumedValues(
+ "+I[web, u1, a]",
+ "+I[mobile, u1, b]",
+ "-U[web, u1, a]",
+ "+U[web, u1, c]",
+ "+I[web, null, d]",
+ "-U[web, null, d]",
+ "+U[web, null, e]")
+ .build())
+ .runSql(
+ "INSERT INTO sink SELECT * FROM DEDUPLICATE("
+ + "input => TABLE sessions PARTITION BY (device, user_id), "
+ + "keep => 'last')")
+ .build();
+
+ public static final TableTestProgram SEQUENCE_BY =
+ TableTestProgram.of(
+ "deduplicate-sequence-by",
+ "with sequence_by, 'last' keeps the greatest and 'first' the smallest "
+ + "value, a tie goes to the later arrival for 'last' and to "
+ + "the earlier one for 'first', later columns break ties, a "
+ + "NULL orders below every other value, NaN ties with every "
+ + "value, and a watermarked column orders without watermark "
+ + "semantics and is emitted as a regular timestamp")
+ .setupTableSource(
+ SourceTestStep.newBuilder("customer_changes")
+ .addSchema("customer_id STRING", "city STRING", "lsn BIGINT")
+ .producedValues(
+ Row.of("c1", "Berlin", 100L),
+ Row.of("c1", "Munich", 120L),
+ Row.of("c1", "Hamburg", 110L))
+ .build())
+ .setupTableSource(
+ SourceTestStep.newBuilder("prices")
+ .addSchema(
+ "ticker STRING",
+ "price INT",
+ "ts TIMESTAMP_LTZ(3)",
+ "WATERMARK FOR ts AS ts - INTERVAL '1' HOUR")
+ .producedValues(
+ Row.of("AAPL", 10, at("10:02")),
+ Row.of("MSFT", 20, at("10:01")),
+ Row.of("AAPL", 12, at("10:08")),
+ Row.of("AAPL", 11, at("10:05")),
+ Row.of("AAPL", 9, at("10:00")),
+ Row.of("AAPL", 14, at("10:08")),
+ Row.of("AAPL", 8, at("10:00")))
+ .build())
+ .setupTableSource(
+ SourceTestStep.newBuilder("measurements")
+ .addSchema("id STRING", "v DOUBLE", "label STRING")
+ .producedValues(
+ Row.of("m1", 1.0, "r1"),
+ Row.of("m1", Double.NaN, "r2"),
+ Row.of("m1", 0.5, "r3"))
+ .build())
+ .setupTableSource(
+ SourceTestStep.newBuilder("versions")
+ .addSchema(
+ "id STRING",
+ "s STRING",
+ "b BOOLEAN",
+ "t TIMESTAMP(3)",
+ "y BYTES",
+ "label STRING")
+ .producedValues(
+ versionRow("a", false, 1, 1, "r1"),
+ versionRow("b", false, 1, 1, "r2"),
+ versionRow("a", true, 1, 1, "r3"),
+ versionRow("b", true, 1, 1, "r4"),
+ versionRow("b", true, 0, 9, "r5"),
+ versionRow("b", true, 2, 1, "r6"),
+ versionRow("b", true, 2, 0, "r7"),
+ versionRow("b", true, 2, 2, "r8"),
+ versionRow(null, true, 9, 9, "r9"),
+ versionRow(null, true, 9, 9, "r10"),
+ versionRow("b", true, 2, 2, "r11"),
+ versionRow("b", true, 2, null, "r12"))
+ .build())
+ .setupTableSink(
+ upsertSink(
+ "customers_last",
+ CUSTOMER_UPSERT_SCHEMA,
+ "+I[c1, Berlin, 100]",
+ "+U[c1, Munich, 120]"))
+ .setupTableSink(
+ upsertSink(
+ "prices_last",
+ PRICES_UPSERT_SCHEMA,
+ "+I[AAPL, 10, 1970-01-01T10:02:00Z]",
+ "+I[MSFT, 20, 1970-01-01T10:01:00Z]",
+ "+U[AAPL, 12, 1970-01-01T10:08:00Z]",
+ "+U[AAPL, 14, 1970-01-01T10:08:00Z]"))
+ .setupTableSink(
+ upsertSink(
+ "prices_first",
+ PRICES_UPSERT_SCHEMA,
+ "+I[AAPL, 10, 1970-01-01T10:02:00Z]",
+ "+I[MSFT, 20, 1970-01-01T10:01:00Z]",
+ "+U[AAPL, 9, 1970-01-01T10:00:00Z]"))
+ .setupTableSink(
+ upsertSink(
+ "measurements_last",
+ LABEL_UPSERT_SCHEMA,
+ "+I[m1, r1]",
+ "+U[m1, r2]",
+ "+U[m1, r3]"))
+ .setupTableSink(
+ upsertSink(
+ "versions_last",
+ LABEL_UPSERT_SCHEMA,
+ "+I[c1, r1]",
+ "+U[c1, r2]",
+ "+U[c1, r4]",
+ "+U[c1, r6]",
+ "+U[c1, r8]",
+ "+U[c1, r11]"))
+ .setupTableSink(
+ upsertSink(
+ "versions_first",
+ LABEL_UPSERT_SCHEMA,
+ "+I[c1, r1]",
+ "+U[c1, r9]"))
+ .runSql(
+ "INSERT INTO customers_last SELECT * FROM DEDUPLICATE("
+ + "input => TABLE customer_changes PARTITION BY customer_id, "
+ + "keep => 'last', "
+ + "sequence_by => DESCRIPTOR(lsn), "
+ + "state_ttl => INTERVAL '1' DAY, "
+ + "reset_ttl_on_duplicate => true, "
+ + "uid => 'customer-changes-dedup')")
+ .runSql(
+ "INSERT INTO prices_last SELECT * FROM DEDUPLICATE("
+ + "input => TABLE prices PARTITION BY ticker, "
+ + "keep => 'last', "
+ + "sequence_by => DESCRIPTOR(ts))")
+ .runSql(
+ "INSERT INTO prices_first SELECT * FROM DEDUPLICATE("
+ + "input => TABLE prices PARTITION BY ticker, "
+ + "keep => 'first', "
+ + "sequence_by => DESCRIPTOR(ts))")
+ .runSql(
+ "INSERT INTO measurements_last SELECT id, label FROM DEDUPLICATE("
+ + "input => TABLE measurements PARTITION BY id, "
+ + "keep => 'last', "
+ + "sequence_by => DESCRIPTOR(v))")
+ .runSql(
+ "INSERT INTO versions_last SELECT id, label FROM DEDUPLICATE("
+ + "input => TABLE versions PARTITION BY id, "
+ + "keep => 'last', "
+ + "sequence_by => DESCRIPTOR(s, b, t, y))")
+ .runSql(
+ "INSERT INTO versions_first SELECT id, label FROM DEDUPLICATE("
+ + "input => TABLE versions PARTITION BY id, "
+ + "keep => 'first', "
+ + "sequence_by => DESCRIPTOR(s, b, t, y))")
+ .build();
+
+ private static SinkTestStep upsertSink(
+ final String name, final String[] schema, final String... expected) {
+ return SinkTestStep.newBuilder(name)
+ .addSchema(schema)
+ .addOption("sink-changelog-mode-enforced", "I,UA,D")
+ .consumedValues(expected)
+ .build();
+ }
+
+ private static Row versionRow(
+ @Nullable final String s,
+ final boolean b,
+ final int minute,
+ @Nullable final Integer y,
+ final String label) {
+ return Row.of(
+ "c1",
+ s,
+ b,
+ LocalDateTime.ofEpochSecond(minute * 60L, 0, ZoneOffset.UTC),
+ y == null ? null : new byte[] {y.byteValue()},
+ label);
+ }
+
+ private static Instant at(final String time) {
+ return Instant.parse("1970-01-01T" + time + ":00Z");
+ }
+}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.java
new file mode 100644
index 00000000000000..a71dd9bc472b13
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.plan.stream.sql;
+
+import org.apache.flink.table.api.ExplainDetail;
+import org.apache.flink.table.api.Table;
+import org.apache.flink.table.api.TableConfig;
+import org.apache.flink.table.planner.utils.PlanKind;
+import org.apache.flink.table.planner.utils.TableTestBase;
+import org.apache.flink.table.planner.utils.TableTestUtil;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+
+import scala.Enumeration;
+
+import static org.apache.flink.core.testutils.FlinkAssertions.anyCauseMatches;
+import static org.apache.flink.table.api.Expressions.$;
+import static org.apache.flink.table.api.Expressions.descriptor;
+import static org.apache.flink.table.api.Expressions.lit;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Plan tests for the changelog mode and upsert key of the DEDUPLICATE built-in PTF. */
+class DeduplicateTableFunctionTest extends TableTestBase {
+
+ private TableTestUtil util;
+
+ @BeforeEach
+ void setup() {
+ util = streamTestUtil(TableConfig.getDefault());
+ util.tableEnv()
+ .executeSql(
+ "CREATE TABLE user_events (user_id STRING, event STRING, "
+ + "ts TIMESTAMP_LTZ(3), WATERMARK FOR ts AS ts) "
+ + "WITH ('connector' = 'values')");
+ util.tableEnv()
+ .executeSql(
+ "CREATE TABLE customer_changes (customer_id STRING, city STRING, "
+ + "lsn BIGINT) WITH ('connector' = 'values')");
+ }
+
+ @Test
+ void testUpsertKeyIsPartitionKey() {
+ verifyPlan(
+ "SELECT * FROM DEDUPLICATE("
+ + "input => TABLE user_events PARTITION BY user_id, keep => 'last')");
+ }
+
+ @Test
+ void testRetractingConsumer() {
+ verifyPlan(
+ "SELECT event, COUNT(*) AS users FROM DEDUPLICATE("
+ + "input => TABLE user_events PARTITION BY user_id, keep => 'last') "
+ + "GROUP BY event");
+ }
+
+ @Test
+ void testOutputHasNoTimeAttribute() {
+ verifyPlan(
+ "SELECT * FROM DEDUPLICATE("
+ + "input => TABLE user_events PARTITION BY event, "
+ + "sequence_by => DESCRIPTOR(ts))");
+ }
+
+ @Test
+ void testOnTimeIsRejectedForUpdatingOutput() {
+ assertThatThrownBy(
+ () ->
+ util.verifyExecPlan(
+ "SELECT * FROM DEDUPLICATE("
+ + "input => TABLE user_events PARTITION BY user_id, "
+ + "on_time => DESCRIPTOR(ts))"))
+ .satisfies(
+ anyCauseMatches(
+ "Time operations using the `on_time` argument are currently not "
+ + "supported for PTFs that consume or produce updates."));
+ }
+
+ @Test
+ void testTableApiWithArguments() {
+ final Table result =
+ util.tableEnv()
+ .from("customer_changes")
+ .partitionBy($("customer_id"))
+ .deduplicate(
+ lit("last").asArgument("keep"),
+ descriptor("lsn").asArgument("sequence_by"),
+ lit(Duration.ofDays(1)).asArgument("state_ttl"));
+ util.verifyRelPlan(result);
+ }
+
+ private void verifyPlan(final String query) {
+ util.doVerifyPlan(
+ query,
+ new ExplainDetail[] {ExplainDetail.CHANGELOG_MODE},
+ false,
+ new Enumeration.Value[] {PlanKind.AST(), PlanKind.OPT_REL()},
+ false,
+ true);
+ }
+}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/harness/DeduplicateHarnessTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/harness/DeduplicateHarnessTest.java
new file mode 100644
index 00000000000000..7cae31f1c4c160
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/harness/DeduplicateHarnessTest.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.runtime.harness;
+
+import org.apache.flink.api.dag.Transformation;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.operators.StreamOperatorFactory;
+import org.apache.flink.streaming.api.transformations.KeyedMultipleInputTransformation;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.util.KeyedMultiInputStreamOperatorTestHarness;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.runtime.keyselector.RowDataKeySelector;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Harness tests for {@code reset_ttl_on_duplicate} of the built-in DEDUPLICATE PTF. */
+class DeduplicateHarnessTest {
+
+ private static final Function PAYMENT =
+ row -> row.getString(0) + ", " + row.getInt(1);
+
+ private static Stream resetTtlOnDuplicate() {
+ final List refreshed = List.of("+I[p1, 50]@10:00", "+I[p1, 50]@10:20");
+ final List notRefreshed =
+ List.of("+I[p1, 50]@10:00", "+I[p1, 50]@10:07", "+I[p1, 50]@10:20");
+ return Stream.of(
+ Arguments.of("first", true, refreshed),
+ Arguments.of("first", false, notRefreshed),
+ Arguments.of("last", true, refreshed),
+ Arguments.of("last", false, notRefreshed));
+ }
+
+ @ParameterizedTest(name = "keep={0}, reset_ttl_on_duplicate={1}")
+ @MethodSource("resetTtlOnDuplicate")
+ void testTimeBoundedDeduplication(
+ final String keep, final boolean resetTtlOnDuplicate, final List expected)
+ throws Exception {
+ try (KeyedMultiInputStreamOperatorTestHarness harness =
+ createHarness(
+ "SELECT * FROM DEDUPLICATE("
+ + "input => TABLE payments PARTITION BY payment_id, "
+ + "keep => '"
+ + keep
+ + "', "
+ + "state_ttl => INTERVAL '5' MINUTES, "
+ + "reset_ttl_on_duplicate => "
+ + resetTtlOnDuplicate
+ + ")")) {
+ final List output = new ArrayList<>();
+ for (final String time : List.of("10:00", "10:04", "10:07", "10:20")) {
+ output.addAll(process(harness, time, PAYMENT, payment(50)));
+ }
+
+ assertThat(output).isEqualTo(expected);
+ }
+ }
+
+ private static List process(
+ final KeyedMultiInputStreamOperatorTestHarness harness,
+ final String time,
+ final Function describe,
+ final RowData input)
+ throws Exception {
+ harness.setStateTtlProcessingTime(toMillis(time));
+ harness.processElement(0, new StreamRecord<>(input));
+ final List output = new ArrayList<>();
+ for (final Object record : harness.getOutput()) {
+ final RowData row = (RowData) ((StreamRecord>) record).getValue();
+ output.add(
+ String.format(
+ "%s[%s]@%s",
+ row.getRowKind().shortString(), describe.apply(row), time));
+ }
+ harness.getOutput().clear();
+ return output;
+ }
+
+ private static RowData payment(final int amount) {
+ return GenericRowData.of(StringData.fromString("p1"), amount);
+ }
+
+ private static KeyedMultiInputStreamOperatorTestHarness createHarness(
+ final String query) throws Exception {
+ final StreamTableEnvironment tEnv =
+ StreamTableEnvironment.create(StreamExecutionEnvironment.getExecutionEnvironment());
+ tEnv.executeSql(
+ "CREATE TABLE payments (payment_id STRING, amount INT) "
+ + "WITH ('connector' = 'values')");
+ final KeyedMultipleInputTransformation> transformation =
+ findPtf(tEnv.toChangelogStream(tEnv.sqlQuery(query)).getTransformation())
+ .orElseThrow(() -> new AssertionError("No PTF transformation found."));
+ return openHarness(
+ transformation.getOperatorFactory(),
+ (RowDataKeySelector) transformation.getStateKeySelectors().get(0));
+ }
+
+ private static KeyedMultiInputStreamOperatorTestHarness openHarness(
+ final StreamOperatorFactory operatorFactory, final RowDataKeySelector keySelector)
+ throws Exception {
+ final KeyedMultiInputStreamOperatorTestHarness harness =
+ new KeyedMultiInputStreamOperatorTestHarness<>(
+ operatorFactory, keySelector.getProducedType());
+ harness.setKeySelector(0, keySelector);
+ harness.open();
+ return harness;
+ }
+
+ private static Optional> findPtf(
+ final Transformation> transformation) {
+ if (transformation instanceof KeyedMultipleInputTransformation) {
+ return Optional.of((KeyedMultipleInputTransformation>) transformation);
+ }
+ return transformation.getInputs().stream()
+ .map(DeduplicateHarnessTest::findPtf)
+ .flatMap(Optional::stream)
+ .findFirst();
+ }
+
+ private static long toMillis(final String time) {
+ final String[] parts = time.split(":");
+ return Duration.ofHours(Long.parseLong(parts[0]))
+ .plusMinutes(Long.parseLong(parts[1]))
+ .toMillis();
+ }
+}
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.xml
new file mode 100644
index 00000000000000..e2217ca87819a3
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.xml
@@ -0,0 +1,104 @@
+
+
+
+
+
+ TABLE user_events PARTITION BY event, sequence_by => DESCRIPTOR(ts))]]>
+
+
+
+
+
+
+
+
+
+
+ TABLE user_events PARTITION BY user_id, keep => 'last') GROUP BY event]]>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ TABLE user_events PARTITION BY user_id, keep => 'last')]]>
+
+
+
+
+
+
+
+
+
diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/plan/deduplicate-first-arrival-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/plan/deduplicate-first-arrival-restore.json
new file mode 100644
index 00000000000000..533b425602f2c2
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/plan/deduplicate-first-arrival-restore.json
@@ -0,0 +1,161 @@
+{
+ "flinkVersion" : "2.5",
+ "nodes" : [ {
+ "id" : 1,
+ "type" : "stream-exec-table-source-scan_2",
+ "scanTableSource" : {
+ "table" : {
+ "identifier" : "`default_catalog`.`default_database`.`payments`",
+ "resolvedTable" : {
+ "schema" : {
+ "columns" : [ {
+ "name" : "payment_id",
+ "dataType" : "VARCHAR(2147483647)"
+ }, {
+ "name" : "amount",
+ "dataType" : "INT"
+ } ]
+ }
+ }
+ }
+ },
+ "outputType" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT>",
+ "description" : "TableSourceScan(table=[[default_catalog, default_database, payments]], fields=[payment_id, amount])"
+ }, {
+ "id" : 2,
+ "type" : "stream-exec-exchange_1",
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "HASH",
+ "keys" : [ 0 ]
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT>",
+ "description" : "Exchange(distribution=[hash[payment_id]])"
+ }, {
+ "id" : 3,
+ "type" : "stream-exec-process-table-function_1",
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "HASH",
+ "keys" : [ 0 ]
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT>",
+ "description" : "ProcessTableFunction(invocation=[DEDUPLICATE(TABLE(#0) PARTITION BY($0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[DEDUPLICATE], select=[payment_id,amount], rowType=[RecordType(VARCHAR(2147483647) payment_id, INTEGER amount)])",
+ "uid" : "DEDUPLICATE",
+ "functionCall" : {
+ "kind" : "CALL",
+ "internalName" : "$DEDUPLICATE$1",
+ "operands" : [ {
+ "kind" : "TABLE_ARG_CALL",
+ "inputIndex" : 0,
+ "partitionKeys" : [ 0 ],
+ "type" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT> NOT NULL"
+ }, {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$DEFAULT$1",
+ "type" : "VARCHAR(2147483647)"
+ }, {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$DEFAULT$1",
+ "type" : "DESCRIPTOR"
+ }, {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$DEFAULT$1",
+ "type" : "INTERVAL SECOND(6)"
+ }, {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$DEFAULT$1",
+ "type" : "BOOLEAN"
+ }, {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$DEFAULT$1",
+ "type" : "DESCRIPTOR"
+ }, {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$DEFAULT$1",
+ "type" : "VARCHAR(2147483647)"
+ } ],
+ "type" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT> NOT NULL"
+ },
+ "inputChangelogModes" : [ [ "INSERT" ] ],
+ "outputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ],
+ "inputUpsertKeys" : [ [ ] ]
+ }, {
+ "id" : 4,
+ "type" : "stream-exec-sink_2",
+ "configuration" : {
+ "table.exec.sink.keyed-shuffle" : "AUTO",
+ "table.exec.sink.not-null-enforcer" : "ERROR",
+ "table.exec.sink.rowtime-inserter" : "ENABLED",
+ "table.exec.sink.type-length-enforcer" : "IGNORE",
+ "table.exec.sink.upsert-materialize" : "AUTO"
+ },
+ "dynamicTableSink" : {
+ "table" : {
+ "identifier" : "`default_catalog`.`default_database`.`sink`",
+ "resolvedTable" : {
+ "schema" : {
+ "columns" : [ {
+ "name" : "payment_id",
+ "dataType" : "VARCHAR(2147483647) NOT NULL"
+ }, {
+ "name" : "amount",
+ "dataType" : "INT"
+ } ],
+ "primaryKey" : {
+ "name" : "PK_payment_id",
+ "type" : "PRIMARY_KEY",
+ "columns" : [ "payment_id" ]
+ }
+ }
+ }
+ }
+ },
+ "inputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ],
+ "upsertMaterializeStrategy" : "MAP",
+ "inputUpsertKey" : [ 0 ],
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT>",
+ "description" : "Sink(table=[default_catalog.default_database.sink], fields=[payment_id, amount])"
+ } ],
+ "edges" : [ {
+ "source" : 1,
+ "target" : 2,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 2,
+ "target" : 3,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 3,
+ "target" : 4,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ } ]
+}
\ No newline at end of file
diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/savepoint/_metadata
new file mode 100644
index 00000000000000..df8f52de27bc53
Binary files /dev/null and b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/savepoint/_metadata differ
diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/plan/deduplicate-sequence-by-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/plan/deduplicate-sequence-by-restore.json
new file mode 100644
index 00000000000000..90d341867230bf
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/plan/deduplicate-sequence-by-restore.json
@@ -0,0 +1,169 @@
+{
+ "flinkVersion" : "2.5",
+ "nodes" : [ {
+ "id" : 5,
+ "type" : "stream-exec-table-source-scan_2",
+ "scanTableSource" : {
+ "table" : {
+ "identifier" : "`default_catalog`.`default_database`.`customer_changes`",
+ "resolvedTable" : {
+ "schema" : {
+ "columns" : [ {
+ "name" : "customer_id",
+ "dataType" : "VARCHAR(2147483647)"
+ }, {
+ "name" : "city",
+ "dataType" : "VARCHAR(2147483647)"
+ }, {
+ "name" : "lsn",
+ "dataType" : "BIGINT"
+ } ]
+ }
+ }
+ }
+ },
+ "outputType" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT>",
+ "description" : "TableSourceScan(table=[[default_catalog, default_database, customer_changes]], fields=[customer_id, city, lsn])"
+ }, {
+ "id" : 6,
+ "type" : "stream-exec-exchange_1",
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "HASH",
+ "keys" : [ 0 ]
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT>",
+ "description" : "Exchange(distribution=[hash[customer_id]])"
+ }, {
+ "id" : 7,
+ "type" : "stream-exec-process-table-function_1",
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT>",
+ "description" : "ProcessTableFunction(invocation=[DEDUPLICATE(TABLE(#0) PARTITION BY($0), _UTF-16LE'first', DESCRIPTOR(_UTF-16LE'lsn'), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[DEDUPLICATE], select=[customer_id,city,lsn], rowType=[RecordType(VARCHAR(2147483647) customer_id, VARCHAR(2147483647) city, BIGINT lsn)])",
+ "uid" : "DEDUPLICATE",
+ "functionCall" : {
+ "kind" : "CALL",
+ "internalName" : "$DEDUPLICATE$1",
+ "operands" : [ {
+ "kind" : "TABLE_ARG_CALL",
+ "inputIndex" : 0,
+ "partitionKeys" : [ 0 ],
+ "type" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT> NOT NULL"
+ }, {
+ "kind" : "LITERAL",
+ "value" : "first",
+ "type" : "CHAR(5) NOT NULL"
+ }, {
+ "kind" : "CALL",
+ "internalName" : "$DESCRIPTOR$1",
+ "operands" : [ {
+ "kind" : "LITERAL",
+ "value" : "lsn",
+ "type" : "CHAR(3) NOT NULL"
+ } ],
+ "type" : "DESCRIPTOR NOT NULL"
+ }, {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$DEFAULT$1",
+ "type" : "INTERVAL SECOND(6)"
+ }, {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$DEFAULT$1",
+ "type" : "BOOLEAN"
+ }, {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$DEFAULT$1",
+ "type" : "DESCRIPTOR"
+ }, {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$DEFAULT$1",
+ "type" : "VARCHAR(2147483647)"
+ } ],
+ "type" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT> NOT NULL"
+ },
+ "inputChangelogModes" : [ [ "INSERT" ] ],
+ "outputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ],
+ "inputUpsertKeys" : [ [ ] ]
+ }, {
+ "id" : 8,
+ "type" : "stream-exec-sink_2",
+ "configuration" : {
+ "table.exec.sink.keyed-shuffle" : "AUTO",
+ "table.exec.sink.not-null-enforcer" : "ERROR",
+ "table.exec.sink.rowtime-inserter" : "ENABLED",
+ "table.exec.sink.type-length-enforcer" : "IGNORE",
+ "table.exec.sink.upsert-materialize" : "AUTO"
+ },
+ "dynamicTableSink" : {
+ "table" : {
+ "identifier" : "`default_catalog`.`default_database`.`sink`",
+ "resolvedTable" : {
+ "schema" : {
+ "columns" : [ {
+ "name" : "customer_id",
+ "dataType" : "VARCHAR(2147483647) NOT NULL"
+ }, {
+ "name" : "city",
+ "dataType" : "VARCHAR(2147483647)"
+ }, {
+ "name" : "lsn",
+ "dataType" : "BIGINT"
+ } ],
+ "primaryKey" : {
+ "name" : "PK_customer_id",
+ "type" : "PRIMARY_KEY",
+ "columns" : [ "customer_id" ]
+ }
+ }
+ }
+ }
+ },
+ "inputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ],
+ "upsertMaterializeStrategy" : "MAP",
+ "inputUpsertKey" : [ 0 ],
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT>",
+ "description" : "Sink(table=[default_catalog.default_database.sink], fields=[customer_id, city, lsn])"
+ } ],
+ "edges" : [ {
+ "source" : 5,
+ "target" : 6,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 6,
+ "target" : 7,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 7,
+ "target" : 8,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ } ]
+}
\ No newline at end of file
diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/savepoint/_metadata
new file mode 100644
index 00000000000000..9d3219ab67d1f9
Binary files /dev/null and b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/savepoint/_metadata differ
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java
index 5cecc79866f657..bc57fe3181a53d 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java
@@ -104,6 +104,26 @@ protected BuiltInProcessTableFunction(
: toInternalStateStrategies(definitionTypeInference);
}
+ /**
+ * Declares a function with one value state of the given type that expires after the interval of
+ * the argument at {@code ttlArgPos}, or never if that argument is NULL.
+ */
+ protected static LinkedHashMap valueState(
+ final SpecializedContext context,
+ final String name,
+ final DataType valueType,
+ final int ttlArgPos) {
+ return internalState(
+ Map.entry(
+ name,
+ StateTypeStrategy.of(
+ callContext ->
+ Optional.of(ValueView.newValueViewDataType(valueType)),
+ context.getCallContext()
+ .getArgumentValue(ttlArgPos, Duration.class)
+ .orElse(null))));
+ }
+
/**
* Declares the state entries of a specialized function in order. The strategies may infer
* external data types, they are converted to internal ones.
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java
new file mode 100644
index 00000000000000..268e98fbe36c6f
--- /dev/null
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.functions.ptf;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.functions.BuiltInFunctionDefinitions;
+import org.apache.flink.table.functions.UserDefinedFunction;
+import org.apache.flink.table.runtime.functions.BuiltInSpecializedFunction;
+import org.apache.flink.table.types.inference.CallContext;
+import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy;
+import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep;
+
+/**
+ * Runtime implementation of {@link BuiltInFunctionDefinitions#DEDUPLICATE}.
+ *
+ *
+ * keep sequence_by implementation
+ * first absent {@link DeduplicateEmitOnceFunction}
+ * first present {@link DeduplicateCurrentRowFunction}
+ * last any {@link DeduplicateCurrentRowFunction}
+ *
+ */
+@Internal
+public class Deduplicate extends BuiltInSpecializedFunction {
+
+ public Deduplicate() {
+ super(BuiltInFunctionDefinitions.DEDUPLICATE);
+ }
+
+ @Override
+ public UserDefinedFunction specialize(final SpecializedContext context) {
+ final CallContext callContext = context.getCallContext();
+ final boolean isKeepFirst = DeduplicateTypeStrategy.keep(callContext) == Keep.FIRST;
+
+ if (isKeepFirst) {
+ final boolean hasSequenceBy =
+ DeduplicateTypeStrategy.sequenceByOutputIndices(callContext).length > 0;
+ if (!hasSequenceBy) {
+ return new DeduplicateEmitOnceFunction(context);
+ } else {
+ return new DeduplicateCurrentRowFunction(context);
+ }
+ } else {
+ return new DeduplicateCurrentRowFunction(context);
+ }
+ }
+}
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateCurrentRowFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateCurrentRowFunction.java
new file mode 100644
index 00000000000000..f131b9f1db3930
--- /dev/null
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateCurrentRowFunction.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.functions.ptf;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.dataview.ValueView;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.binary.BinaryRowData;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.data.utils.ProjectedRowData;
+import org.apache.flink.table.functions.FunctionContext;
+import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext;
+import org.apache.flink.table.runtime.functions.ptf.DeduplicateUtils.SequenceComparator;
+import org.apache.flink.table.runtime.typeutils.RowDataSerializer;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.inference.strategies.ChangelogTypeStrategyUtils;
+import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy;
+import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils;
+import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.types.ColumnList;
+import org.apache.flink.types.RowKind;
+
+import javax.annotation.Nullable;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL;
+
+/**
+ * {@link Deduplicate} with {@code keep => 'last'}, or {@code keep => 'first'} with {@code
+ * sequence_by}: a later record can replace the current one, so the current input row is held in the
+ * {@code kept} state.
+ */
+@Internal
+public class DeduplicateCurrentRowFunction extends DeduplicateFunctionBase {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final String CURRENT_STATE = "kept";
+
+ private final Keep keepMode;
+
+ private final LogicalType[] inputTypes;
+
+ private final boolean isUpdateBeforeEmitted;
+
+ private final @Nullable SequenceComparator sequenceComparator;
+
+ private transient RowDataSerializer inputSerializer;
+
+ private transient ProjectedRowData projectedCandidate;
+
+ private transient ProjectedRowData projectedCurrent;
+
+ private transient JoinedRowData retraction;
+
+ private transient RowData emptyRow;
+
+ DeduplicateCurrentRowFunction(final SpecializedContext context) {
+ super(
+ context,
+ valueState(
+ context,
+ CURRENT_STATE,
+ DataTypes.ROW(
+ DeduplicateTypeStrategyUtils.inputFields(
+ DeduplicateUtils.tableSemantics(context))),
+ ARG_STATE_TTL));
+ this.keepMode = DeduplicateTypeStrategy.keep(context.getCallContext());
+ final List inputFields =
+ DeduplicateTypeStrategyUtils.inputFields(DeduplicateUtils.tableSemantics(context));
+ this.inputTypes =
+ inputFields.stream()
+ .map(field -> field.getDataType().getLogicalType())
+ .toArray(LogicalType[]::new);
+ this.isUpdateBeforeEmitted =
+ context.getCallContext()
+ .getOutputChangelogMode()
+ .map(mode -> mode.contains(RowKind.UPDATE_BEFORE))
+ .orElse(false);
+ this.sequenceComparator = sequenceComparator(context, inputFields);
+ }
+
+ @Override
+ public void open(final FunctionContext context) throws Exception {
+ super.open(context);
+ if (sequenceComparator != null) {
+ sequenceComparator.open(context);
+ }
+ inputSerializer = new RowDataSerializer(inputTypes);
+ projectedCandidate = ProjectedRowData.from(outputIndices);
+ projectedCurrent = ProjectedRowData.from(outputIndices);
+ retraction = new JoinedRowData();
+ emptyRow = GenericRowData.of();
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (sequenceComparator != null) {
+ sequenceComparator.close();
+ }
+ super.close();
+ }
+
+ public void eval(
+ final Context ctx,
+ final ValueView currentState,
+ final RowData input,
+ @Nullable final StringData keep,
+ @Nullable final ColumnList sequenceBy,
+ @Nullable final Long stateTtl,
+ @Nullable final Boolean resetTtlOnDuplicate)
+ throws Exception {
+
+ final BinaryRowData current = (BinaryRowData) currentState.getValue();
+ if (current == null) {
+ final BinaryRowData candidate = inputSerializer.toBinaryRow(input);
+ currentState.setValue(candidate.copy());
+ emitCandidate(RowKind.INSERT, candidate);
+ return;
+ }
+ if (replacesCurrent(input, current)) {
+ final BinaryRowData candidate = inputSerializer.toBinaryRow(input);
+ if (!candidate.equals(current)) {
+ currentState.setValue(candidate.copy());
+ if (isUpdateBeforeEmitted) {
+ emitRetraction(current);
+ }
+ emitCandidate(RowKind.UPDATE_AFTER, candidate);
+ } else if (isResetTtlOnDuplicate) {
+ currentState.setValue(current);
+ }
+ } else if (isResetTtlOnDuplicate) {
+ currentState.setValue(current);
+ }
+ }
+
+ /**
+ * Without sequence_by only keep => 'last' reaches this class, so the latest arrival replaces.
+ */
+ private boolean replacesCurrent(final RowData candidate, final RowData current) {
+ return sequenceComparator == null
+ || DeduplicateUtils.replaces(
+ keepMode, sequenceComparator.compare(candidate, current));
+ }
+
+ private void emitCandidate(final RowKind kind, final BinaryRowData candidate) {
+ projectedCandidate.replaceRow(candidate);
+ projectedCandidate.setRowKind(kind);
+ collect(projectedCandidate);
+ }
+
+ /** Wraps the current row so that the row owned by state never changes its row kind. */
+ private void emitRetraction(final BinaryRowData current) {
+ projectedCurrent.replaceRow(current);
+ retraction.replace(projectedCurrent, emptyRow);
+ retraction.setRowKind(RowKind.UPDATE_BEFORE);
+ collect(retraction);
+ }
+
+ private static @Nullable SequenceComparator sequenceComparator(
+ final SpecializedContext context, final List inputFields) {
+ final int[] outputIndices =
+ ChangelogTypeStrategyUtils.computeOutputIndices(
+ DeduplicateUtils.tableSemantics(context));
+ final int[] sequenceIndices =
+ Arrays.stream(
+ DeduplicateTypeStrategy.sequenceByOutputIndices(
+ context.getCallContext()))
+ .map(i -> outputIndices[i])
+ .toArray();
+ if (sequenceIndices.length == 0) {
+ return null;
+ }
+ final DataType[] types =
+ Arrays.stream(sequenceIndices)
+ .mapToObj(i -> inputFields.get(i).getDataType())
+ .toArray(DataType[]::new);
+ return new SequenceComparator(context, sequenceIndices, types);
+ }
+}
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateEmitOnceFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateEmitOnceFunction.java
new file mode 100644
index 00000000000000..9c23e7562b0241
--- /dev/null
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateEmitOnceFunction.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.functions.ptf;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.dataview.ValueView;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.utils.ProjectedRowData;
+import org.apache.flink.table.functions.FunctionContext;
+import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext;
+import org.apache.flink.types.ColumnList;
+import org.apache.flink.types.RowKind;
+
+import javax.annotation.Nullable;
+
+import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL;
+
+/**
+ * {@link Deduplicate} with {@code keep => 'first'} and no {@code sequence_by}: no later record can
+ * replace the first one, so only whether the key has emitted is kept in the {@code emitted} state.
+ */
+@Internal
+public class DeduplicateEmitOnceFunction extends DeduplicateFunctionBase {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final String EMITTED_STATE = "emitted";
+
+ private transient ProjectedRowData projectedOutput;
+
+ DeduplicateEmitOnceFunction(final SpecializedContext context) {
+ super(
+ context,
+ valueState(context, EMITTED_STATE, DataTypes.BOOLEAN().notNull(), ARG_STATE_TTL));
+ }
+
+ @Override
+ public void open(final FunctionContext context) throws Exception {
+ super.open(context);
+ projectedOutput = ProjectedRowData.from(outputIndices);
+ }
+
+ public void eval(
+ final Context ctx,
+ final ValueView emitted,
+ final RowData input,
+ @Nullable final StringData keep,
+ @Nullable final ColumnList sequenceBy,
+ @Nullable final Long stateTtl,
+ @Nullable final Boolean resetTtlOnDuplicate)
+ throws Exception {
+ if (emitted.getValue() == null) {
+ projectedOutput.replaceRow(input);
+ projectedOutput.setRowKind(RowKind.INSERT);
+ collect(projectedOutput);
+ emitted.setValue(true);
+ } else if (isResetTtlOnDuplicate) {
+ emitted.setValue(true);
+ }
+ }
+}
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFunctionBase.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFunctionBase.java
new file mode 100644
index 00000000000000..f22d583805a9d9
--- /dev/null
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFunctionBase.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.functions.ptf;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.functions.BuiltInFunctionDefinitions;
+import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext;
+import org.apache.flink.table.types.inference.CallContext;
+import org.apache.flink.table.types.inference.StateTypeStrategy;
+import org.apache.flink.table.types.inference.strategies.ChangelogTypeStrategyUtils;
+
+import java.time.Duration;
+import java.util.LinkedHashMap;
+
+import static org.apache.flink.table.api.config.ExecutionConfigOptions.IDLE_STATE_RETENTION;
+import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_RESET_TTL_ON_DUPLICATE;
+import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL;
+
+/** Base class of the {@link Deduplicate} implementations. */
+@Internal
+abstract class DeduplicateFunctionBase extends BuiltInProcessTableFunction {
+
+ private static final long serialVersionUID = 1L;
+
+ protected final int[] outputIndices;
+
+ protected final boolean isResetTtlOnDuplicate;
+
+ DeduplicateFunctionBase(
+ final SpecializedContext context,
+ final LinkedHashMap stateStrategies) {
+ super(BuiltInFunctionDefinitions.DEDUPLICATE, context, stateStrategies);
+ final CallContext callContext = context.getCallContext();
+ this.outputIndices =
+ ChangelogTypeStrategyUtils.computeOutputIndices(
+ DeduplicateUtils.tableSemantics(context));
+ final Duration effectiveTtl =
+ callContext
+ .getArgumentValue(ARG_STATE_TTL, Duration.class)
+ .orElseGet(() -> context.getConfiguration().get(IDLE_STATE_RETENTION));
+ this.isResetTtlOnDuplicate =
+ callContext.getArgumentValue(ARG_RESET_TTL_ON_DUPLICATE, Boolean.class).orElse(true)
+ && !effectiveTtl.isZero();
+ }
+}
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java
new file mode 100644
index 00000000000000..3e06149db3e87a
--- /dev/null
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.functions.ptf;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.functions.FunctionContext;
+import org.apache.flink.table.functions.SpecializedFunction.ExpressionEvaluator;
+import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext;
+import org.apache.flink.table.functions.TableSemantics;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep;
+import org.apache.flink.util.FlinkRuntimeException;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+import java.lang.invoke.MethodHandle;
+
+import static org.apache.flink.table.api.Expressions.$;
+import static org.apache.flink.table.api.Expressions.ifThenElse;
+import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_INPUT;
+
+/** Record ordering of the built-in deduplication PTFs. */
+@Internal
+public final class DeduplicateUtils {
+
+ public static TableSemantics tableSemantics(final SpecializedContext context) {
+ return context.getCallContext()
+ .getTableSemantics(ARG_INPUT)
+ .orElseThrow(() -> new IllegalStateException("Table argument expected."));
+ }
+
+ /**
+ * Returns whether a new record replaces the current one, given the order of the new record
+ * relative to the current one: with {@code last} the later arrival wins on a tie, with {@code
+ * first} only a strictly smaller record replaces.
+ */
+ public static boolean replaces(final Keep keep, final int order) {
+ return keep == Keep.LAST ? order >= 0 : order < 0;
+ }
+
+ /**
+ * Compares two records by their {@code sequence_by} values in ascending order. The first column
+ * decides, later ones break ties, and NULL orders below every other value.
+ */
+ @Internal
+ public static final class SequenceComparator implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int[] indices;
+
+ private final DataType[] types;
+
+ private final ExpressionEvaluator[] evaluators;
+
+ private transient MethodHandle[] compareHandles;
+
+ private transient RowData.FieldGetter[] getters;
+
+ /** Creates a comparator of the columns at {@code indices}, which have the given types. */
+ public SequenceComparator(
+ final SpecializedContext context, final int[] indices, final DataType[] types) {
+ this.indices = indices;
+ this.types = types;
+ this.evaluators = new ExpressionEvaluator[types.length];
+ for (int i = 0; i < types.length; i++) {
+ final DataType type = types[i].notNull().toInternal();
+ evaluators[i] =
+ context.createEvaluator(
+ ifThenElse(
+ $("a").isLess($("b")),
+ -1,
+ ifThenElse($("a").isGreater($("b")), 1, 0)),
+ DataTypes.INT().notNull(),
+ DataTypes.FIELD("a", type),
+ DataTypes.FIELD("b", type));
+ }
+ }
+
+ public void open(final FunctionContext context) throws Exception {
+ compareHandles = new MethodHandle[types.length];
+ getters = new RowData.FieldGetter[types.length];
+ for (int i = 0; i < types.length; i++) {
+ compareHandles[i] = evaluators[i].open(context);
+ getters[i] = RowData.createFieldGetter(types[i].getLogicalType(), indices[i]);
+ }
+ }
+
+ public void close() throws Exception {
+ for (final ExpressionEvaluator evaluator : evaluators) {
+ evaluator.close();
+ }
+ }
+
+ /**
+ * Returns a negative number, zero, or a positive number if the left record orders before,
+ * equal to, or after the right one.
+ */
+ public int compare(final RowData left, final RowData right) {
+ for (int i = 0; i < types.length; i++) {
+ final int order =
+ compare(
+ i,
+ getters[i].getFieldOrNull(left),
+ getters[i].getFieldOrNull(right));
+ if (order != 0) {
+ return order;
+ }
+ }
+ return 0;
+ }
+
+ private int compare(
+ final int column, @Nullable final Object left, @Nullable final Object right) {
+ if (left == null || right == null) {
+ return left == null ? (right == null ? 0 : -1) : 1;
+ }
+ try {
+ return (int) compareHandles[column].invoke(left, right);
+ } catch (Throwable t) {
+ throw new FlinkRuntimeException(t);
+ }
+ }
+ }
+
+ private DeduplicateUtils() {}
+}