diff --git a/docs/content.zh/docs/sql/built-in-functions/process-table-functions.md b/docs/content.zh/docs/sql/built-in-functions/process-table-functions.md index c89fa07b7f3872..de05defee76b8c 100644 --- a/docs/content.zh/docs/sql/built-in-functions/process-table-functions.md +++ b/docs/content.zh/docs/sql/built-in-functions/process-table-functions.md @@ -31,6 +31,7 @@ The built-in process table functions (PTFs) take a table as input and return a t |---------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `FROM_CHANGELOG(input => TABLE t [, ...])` | Converts an append-only table with an explicit operation column into a dynamic table. See Changelog Conversion for the full list of arguments, semantics, and usage. | | `TO_CHANGELOG(input => TABLE t [, ...])` | Converts a dynamic table into an append-only table with an explicit operation column. See Changelog Conversion for the full list of arguments, semantics, and usage. | +| `DEDUPLICATE(input => TABLE t PARTITION BY k [, ...])` | Keeps the first or the last row per key of an append-only table. See [Deduplication]({{< ref "docs/sql/reference/queries/deduplication" >}}#deduplicate) for the full list of arguments, semantics, and usage. | | `SNAPSHOT(input => TABLE t [, ...])` | Returns the current state of a dynamic table `t`. `SNAPSHOT` can only be used in a `LATERAL` context and not as a stand-alone table function. See [LATERAL SNAPSHOT join]({{< ref "docs/sql/reference/queries/joins" >}}#lateral-snapshot-join) for the full list of arguments, the join semantics, and usage. | | `ML_PREDICT(TABLE t, MODEL m, DESCRIPTOR(...) [, ...])` | Applies the model `m` to the columns of table `t` that the `DESCRIPTOR` names, and returns the columns of `t` plus the prediction columns of the model. See [Model Inference]({{< ref "docs/sql/reference/queries/model-inference" >}}#ml_predict-函数) for the full list of arguments, semantics, and usage. | diff --git a/docs/content.zh/docs/sql/reference/queries/deduplication.md b/docs/content.zh/docs/sql/reference/queries/deduplication.md index d580f62e431905..0403f51f6dfa3e 100644 --- a/docs/content.zh/docs/sql/reference/queries/deduplication.md +++ b/docs/content.zh/docs/sql/reference/queries/deduplication.md @@ -69,4 +69,143 @@ FROM ( WHERE row_num = 1 ``` +## DEDUPLICATE + +{{< label Streaming >}} + +The `DEDUPLICATE` process table function (PTF) keeps the first or the last row per key of an append-only table, as long as the state of the key is retained. Rows are ordered by arrival or by `sequence_by` columns. + +### Syntax + +```sql +SELECT * FROM DEDUPLICATE( + input => TABLE source_table PARTITION BY key_col[, ...], + [keep => 'first' | 'last',] + [sequence_by => DESCRIPTOR(col[, ...]),] + [state_ttl => INTERVAL '1' HOUR,] + [reset_ttl_on_duplicate => true | false,] + [uid => 'my-id'] +) +``` + +### Parameters + +| Parameter | Required | Description | +|:----------|:---------|:------------| +| `input` | Yes | The append-only input table. `PARTITION BY` is required and its columns are the key of the output.| +| `keep` | No | `'first'` (default) keeps the earliest arrival or the smallest `sequence_by` value, `'last'` keeps the latest arrival or the greatest. Must be a literal. Arrival order is not deterministic across reruns, use `sequence_by` for a repeatable result. | +| `sequence_by` | No | One or more columns that order the records of a key instead of their arrival. The first column decides, later columns break ties, and `NULL` orders below every other value. On a tie, the later arrival wins for `'last'` and the earlier one for `'first'`. Columns must be numeric, string, binary, boolean or date-time, and not part of `PARTITION BY`. | +| `state_ttl` | No | How long the state of an idle key is retained, as an `INTERVAL` literal. Defaults to `table.exec.state.ttl`, and `INTERVAL '0' SECOND` disables expiry. After the state of a key expires, its next record is emitted as `+I` again. | +| `reset_ttl_on_duplicate` | No | Whether a record that is dropped refreshes the TTL of its key. Defaults to `true`. Has no effect without a TTL. | + + +### Output + +The output columns are the `PARTITION BY` columns, in the order listed, followed by the remaining input columns. Time attributes are emitted as regular timestamps. + +The result has one row per key. When a later row replaces the kept one, the row of that key is updated, so the result must be written to a table that accepts updates, for example one with a primary key on the `PARTITION BY` columns. Currently this holds for every `keep` mode, including `keep => 'first'` without `sequence_by`, which never replaces a row. + +{{< hint info >}} +The first record of a key is emitted as `+I`. A record that replaces the kept one is emitted as `+U`, preceded by `-U` if the consumer needs retractions. A record that equals the kept one, or does not replace it, emits nothing. See [Updates and Changelogs]({{< ref "docs/dev/table/functions/ptfs" >}}#updates-and-changelogs). +{{< /hint >}} + +### Examples + +#### Keep the first or the last record + +Rows of `user_events` arrive in this order: + +| `user_events` | +|:----------------| +| `(u1, login)` | +| `(u2, login)` | +| `(u1, click)` | + +`keep` defaults to `'first'`, which keeps the first row of each user: + +```sql +SELECT * FROM DEDUPLICATE( + input => TABLE user_events PARTITION BY user_id) +``` + +| Result | +|:----------------| +| `(u1, login)` | +| `(u2, login)` | + +With `keep => 'last'`, a later row replaces the kept one: + +```sql +SELECT * FROM DEDUPLICATE( + input => TABLE user_events PARTITION BY user_id, + keep => 'last') +``` + +| Result | +|:----------------| +| `(u1, click)` | +| `(u2, login)` | + +#### Order by a column + +`sequence_by` ranks the rows of a key by a column instead of by arrival, so a late, older row does not replace a newer one: + +| `customer_changes` | +|:----------------------------| +| `(c1, Berlin, 100)` | +| `(c1, Munich, 120)` | +| `(c1, Hamburg, 110)` | + +```sql +SELECT * FROM DEDUPLICATE( + input => TABLE customer_changes PARTITION BY customer_id, + keep => 'last', + sequence_by => DESCRIPTOR(lsn), + state_ttl => INTERVAL '1' DAY) +``` + +| Result | +|:----------------------| +| `(c1, Munich, 120)` | + +`(c1, Hamburg, 110)` is dropped because its `lsn` is smaller than that of the kept row. + +#### Composite key + +The columns of `PARTITION BY` come first in the output, in the order listed: + +| `sessions` (`user_id`, `page`, `device`) | +|:-----------------------------------------| +| `(u1, a, web)` | +| `(u1, b, mobile)` | +| `(u1, c, web)` | + +```sql +SELECT * FROM DEDUPLICATE( + input => TABLE sessions PARTITION BY (device, user_id), + keep => 'last') +``` + +| Result (`device`, `user_id`, `page`) | +|:-------------------------------------| +| `(web, u1, c)` | +| `(mobile, u1, b)` | + +#### Table API + +```java +// Keep the first record of each payment +Table result = payments + .partitionBy($("payment_id")) + .deduplicate(); + +// Keep the record with the greatest lsn per customer +Table result = customerChanges + .partitionBy($("customer_id")) + .deduplicate( + lit("last").asArgument("keep"), + descriptor("lsn").asArgument("sequence_by"), + lit(Duration.ofDays(1)).asArgument("state_ttl")); +``` + {{< top >}} diff --git a/docs/content/docs/sql/built-in-functions/process-table-functions.md b/docs/content/docs/sql/built-in-functions/process-table-functions.md index 0c2c3c35930a49..86a21808356ced 100644 --- a/docs/content/docs/sql/built-in-functions/process-table-functions.md +++ b/docs/content/docs/sql/built-in-functions/process-table-functions.md @@ -31,6 +31,7 @@ The built-in process table functions (PTFs) take a table as input and return a t |---------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `FROM_CHANGELOG(input => TABLE t [, ...])` | Converts an append-only table with an explicit operation column into a dynamic table. See [Changelog Conversion]({{< ref "docs/sql/reference/queries/changelog" >}}#from_changelog) for the full list of arguments, semantics, and usage. | | `TO_CHANGELOG(input => TABLE t [, ...])` | Converts a dynamic table into an append-only table with an explicit operation column. See [Changelog Conversion]({{< ref "docs/sql/reference/queries/changelog" >}}#to_changelog) for the full list of arguments, semantics, and usage. | +| `DEDUPLICATE(input => TABLE t PARTITION BY k [, ...])` | Keeps the first or the last row per key of an append-only table. See [Deduplication]({{< ref "docs/sql/reference/queries/deduplication" >}}#deduplicate) for the full list of arguments, semantics, and usage. | | `SNAPSHOT(input => TABLE t [, ...])` | Returns the current state of a dynamic table `t`. `SNAPSHOT` can only be used in a `LATERAL` context and not as a stand-alone table function. See [LATERAL SNAPSHOT join]({{< ref "docs/sql/reference/queries/joins" >}}#lateral-snapshot-join) for the full list of arguments, the join semantics, and usage. | | `ML_PREDICT(TABLE t, MODEL m, DESCRIPTOR(...) [, ...])` | Applies the model `m` to the columns of table `t` that the `DESCRIPTOR` names, and returns the columns of `t` plus the prediction columns of the model. See [Model Inference]({{< ref "docs/sql/reference/queries/model-inference" >}}#ml_predict-function) for the full list of arguments, semantics, and usage. | diff --git a/docs/content/docs/sql/reference/queries/deduplication.md b/docs/content/docs/sql/reference/queries/deduplication.md index 819893d19913bf..73c638d356e5a3 100644 --- a/docs/content/docs/sql/reference/queries/deduplication.md +++ b/docs/content/docs/sql/reference/queries/deduplication.md @@ -69,4 +69,143 @@ FROM ( WHERE row_num = 1 ``` +## DEDUPLICATE + +{{< label Streaming >}} + +The `DEDUPLICATE` process table function (PTF) keeps the first or the last row per key of an append-only table, as long as the state of the key is retained. Rows are ordered by arrival or by `sequence_by` columns. + +### Syntax + +```sql +SELECT * FROM DEDUPLICATE( + input => TABLE source_table PARTITION BY key_col[, ...], + [keep => 'first' | 'last',] + [sequence_by => DESCRIPTOR(col[, ...]),] + [state_ttl => INTERVAL '1' HOUR,] + [reset_ttl_on_duplicate => true | false,] + [uid => 'my-id'] +) +``` + +### Parameters + +| Parameter | Required | Description | +|:----------|:---------|:------------| +| `input` | Yes | The append-only input table. `PARTITION BY` is required and its columns are the key of the output.| +| `keep` | No | `'first'` (default) keeps the earliest arrival or the smallest `sequence_by` value, `'last'` keeps the latest arrival or the greatest. Must be a literal. Arrival order is not deterministic across reruns, use `sequence_by` for a repeatable result. | +| `sequence_by` | No | One or more columns that order the records of a key instead of their arrival. The first column decides, later columns break ties, and `NULL` orders below every other value. On a tie, the later arrival wins for `'last'` and the earlier one for `'first'`. Columns must be numeric, string, binary, boolean or date-time, and not part of `PARTITION BY`. | +| `state_ttl` | No | How long the state of an idle key is retained, as an `INTERVAL` literal. Defaults to `table.exec.state.ttl`, and `INTERVAL '0' SECOND` disables expiry. After the state of a key expires, its next record is emitted as `+I` again. | +| `reset_ttl_on_duplicate` | No | Whether a record that is dropped refreshes the TTL of its key. Defaults to `true`. Has no effect without a TTL. | + + +### Output + +The output columns are the `PARTITION BY` columns, in the order listed, followed by the remaining input columns. Time attributes are emitted as regular timestamps. + +The result has one row per key. When a later row replaces the kept one, the row of that key is updated, so the result must be written to a table that accepts updates, for example one with a primary key on the `PARTITION BY` columns. Currently this holds for every `keep` mode, including `keep => 'first'` without `sequence_by`, which never replaces a row. + +{{< hint info >}} +The first record of a key is emitted as `+I`. A record that replaces the kept one is emitted as `+U`, preceded by `-U` if the consumer needs retractions. A record that equals the kept one, or does not replace it, emits nothing. See [Updates and Changelogs]({{< ref "docs/dev/table/functions/ptfs" >}}#updates-and-changelogs). +{{< /hint >}} + +### Examples + +#### Keep the first or the last record + +Rows of `user_events` arrive in this order: + +| `user_events` | +|:----------------| +| `(u1, login)` | +| `(u2, login)` | +| `(u1, click)` | + +`keep` defaults to `'first'`, which keeps the first row of each user: + +```sql +SELECT * FROM DEDUPLICATE( + input => TABLE user_events PARTITION BY user_id) +``` + +| Result | +|:----------------| +| `(u1, login)` | +| `(u2, login)` | + +With `keep => 'last'`, a later row replaces the kept one: + +```sql +SELECT * FROM DEDUPLICATE( + input => TABLE user_events PARTITION BY user_id, + keep => 'last') +``` + +| Result | +|:----------------| +| `(u1, click)` | +| `(u2, login)` | + +#### Order by a column + +`sequence_by` ranks the rows of a key by a column instead of by arrival, so a late, older row does not replace a newer one: + +| `customer_changes` | +|:----------------------------| +| `(c1, Berlin, 100)` | +| `(c1, Munich, 120)` | +| `(c1, Hamburg, 110)` | + +```sql +SELECT * FROM DEDUPLICATE( + input => TABLE customer_changes PARTITION BY customer_id, + keep => 'last', + sequence_by => DESCRIPTOR(lsn), + state_ttl => INTERVAL '1' DAY) +``` + +| Result | +|:----------------------| +| `(c1, Munich, 120)` | + +`(c1, Hamburg, 110)` is dropped because its `lsn` is smaller than that of the kept row. + +#### Composite key + +The columns of `PARTITION BY` come first in the output, in the order listed: + +| `sessions` (`user_id`, `page`, `device`) | +|:-----------------------------------------| +| `(u1, a, web)` | +| `(u1, b, mobile)` | +| `(u1, c, web)` | + +```sql +SELECT * FROM DEDUPLICATE( + input => TABLE sessions PARTITION BY (device, user_id), + keep => 'last') +``` + +| Result (`device`, `user_id`, `page`) | +|:-------------------------------------| +| `(web, u1, c)` | +| `(mobile, u1, b)` | + +#### Table API + +```java +// Keep the first record of each payment +Table result = payments + .partitionBy($("payment_id")) + .deduplicate(); + +// Keep the record with the greatest lsn per customer +Table result = customerChanges + .partitionBy($("customer_id")) + .deduplicate( + lit("last").asArgument("keep"), + descriptor("lsn").asArgument("sequence_by"), + lit(Duration.ofDays(1)).asArgument("state_ttl")); +``` + {{< top >}} diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java index ecf58fbe9b3914..31371edd0e72ad 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java @@ -290,4 +290,35 @@ public interface PartitionedTable { * @see #orderBy(Expression...) */ Table fromChangelog(Expression... arguments); + + /** + * Keeps one row per partition key using the built-in {@code DEDUPLICATE} process table + * function. The result is an updating table with the partition columns as its upsert key. + * + *

{@code keep => 'first'} (default) keeps the earliest arrival or the smallest {@code + * sequence_by} value, {@code keep => 'last'} keeps the latest arrival or the greatest value. + * + *

Examples: + * + *

{@code
+     * // Keep the first delivery of each payment
+     * Table result = payments
+     *     .partitionBy($("payment_id"))
+     *     .deduplicate();
+     *
+     * // Keep the version with the greatest 'lsn' per customer, with bounded state retention
+     * Table result = customerChanges
+     *     .partitionBy($("customer_id"))
+     *     .deduplicate(
+     *         lit("last").asArgument("keep"),
+     *         descriptor("lsn").asArgument("sequence_by"),
+     *         lit(Duration.ofDays(1)).asArgument("state_ttl"));
+     * }
+ * + * @param arguments optional named arguments for {@code keep}, {@code sequence_by}, {@code + * state_ttl}, {@code reset_ttl_on_duplicate}, and {@code uid} + * @return an updating {@link Table} with output schema {@code [partition_keys, + * non_partition_input_columns]} + */ + Table deduplicate(Expression... arguments); } diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableImpl.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableImpl.java index c3c9cfa7f9db83..31a00a1cd503a2 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableImpl.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableImpl.java @@ -962,6 +962,11 @@ public Table fromChangelog(final Expression... arguments) { BuiltInFunctionDefinitions.FROM_CHANGELOG.getName(), (Object[]) arguments); } + @Override + public Table deduplicate(final Expression... arguments) { + return process(BuiltInFunctionDefinitions.DEDUPLICATE.getName(), (Object[]) arguments); + } + private QueryOperation createQueryOperation() { if (orderKeys.isEmpty()) { return table.operationTreeBuilder.partition(partitionKeys, table.operationTree); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java index d70f1c50725cd8..b2d0dc228abd2c 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java @@ -38,6 +38,7 @@ import org.apache.flink.table.types.inference.TraitCondition; import org.apache.flink.table.types.inference.TypeStrategies; import org.apache.flink.table.types.inference.strategies.ArrayOfStringArgumentTypeStrategy; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy; import org.apache.flink.table.types.inference.strategies.FromChangelogTypeStrategy; import org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies; import org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies; @@ -939,6 +940,30 @@ ANY, and(logical(LogicalTypeRoot.BOOLEAN), LITERAL) "org.apache.flink.table.runtime.functions.ptf.FromChangelogFunction") .build(); + public static final BuiltInFunctionDefinition DEDUPLICATE = + BuiltInFunctionDefinition.newBuilder() + .name("DEDUPLICATE") + .kind(PROCESS_TABLE) + .staticArguments( + StaticArgument.table( + "input", + Row.class, + false, + EnumSet.of( + StaticArgumentTrait.TABLE, + StaticArgumentTrait.SET_SEMANTIC_TABLE)), + StaticArgument.scalar("keep", DataTypes.STRING(), true), + StaticArgument.scalar("sequence_by", DataTypes.DESCRIPTOR(), true), + StaticArgument.scalar( + "state_ttl", DataTypes.INTERVAL(DataTypes.SECOND()), true), + StaticArgument.scalar( + "reset_ttl_on_duplicate", DataTypes.BOOLEAN(), true)) + .changelogModeStrategy(DeduplicateTypeStrategy.CHANGELOG_MODE_STRATEGY) + .inputTypeStrategy(DeduplicateTypeStrategy.INPUT_TYPE_STRATEGY) + .outputTypeStrategy(DeduplicateTypeStrategy.OUTPUT_TYPE_STRATEGY) + .runtimeClass("org.apache.flink.table.runtime.functions.ptf.Deduplicate") + .build(); + /** * Built-in proxy function for the LATERAL SNAPSHOT temporal join. * diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java new file mode 100644 index 00000000000000..49483bf60b765f --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.inference.strategies; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.functions.ChangelogFunction.ChangelogContext; +import org.apache.flink.table.functions.ChangelogModeStrategy; +import org.apache.flink.table.functions.TableSemantics; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.CallContext; +import org.apache.flink.table.types.inference.InputTypeStrategy; +import org.apache.flink.table.types.inference.TypeStrategy; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep; +import org.apache.flink.types.RowKind; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Type strategies for the {@code DEDUPLICATE} process table function. + * + *

The output is updating and keyed by the PARTITION BY columns: the PARTITION BY columns + * followed by the remaining input columns, without time attributes. + */ +@Internal +public final class DeduplicateTypeStrategy { + + public static final int ARG_INPUT = 0; + public static final int ARG_KEEP = 1; + public static final int ARG_SEQUENCE_BY = 2; + public static final int ARG_STATE_TTL = 3; + public static final int ARG_RESET_TTL_ON_DUPLICATE = 4; + private static final String NAME = "DEDUPLICATE"; + + public static final InputTypeStrategy INPUT_TYPE_STRATEGY = + new ValidationOnlyInputTypeStrategy() { + @Override + public Optional> inferInputTypes( + final CallContext callContext, final boolean throwOnFailure) { + final Optional error = validate(callContext); + if (error.isPresent()) { + return callContext.fail(throwOnFailure, "%s", error.get()); + } + return Optional.of(callContext.getArgumentDataTypes()); + } + }; + + public static final TypeStrategy OUTPUT_TYPE_STRATEGY = + callContext -> + Optional.of( + DataTypes.ROW( + DeduplicateTypeStrategyUtils.outputFields( + tableSemantics(callContext))) + .notNull()); + + public static final ChangelogModeStrategy CHANGELOG_MODE_STRATEGY = + DeduplicateTypeStrategy::inferChangelogMode; + + /** Returns the {@code keep} argument, or {@link Keep#FIRST} when it is omitted. */ + public static Keep keep(final CallContext callContext) { + return DeduplicateTypeStrategyUtils.keep(callContext, ARG_KEEP); + } + + /** + * Returns the positions of the {@code sequence_by} columns, in the listed order, within the + * output row without the PARTITION BY columns. + */ + public static int[] sequenceByOutputIndices(final CallContext callContext) { + return DeduplicateTypeStrategyUtils.sequenceByOutputIndices( + callContext, tableSemantics(callContext), ARG_SEQUENCE_BY); + } + + private static Optional validate(final CallContext callContext) { + final TableSemantics tableSemantics = tableSemantics(callContext); + return validateNoOrderBy(tableSemantics) + .or( + () -> + DeduplicateTypeStrategyUtils.validatePartitionByNoTimeAttribute( + tableSemantics, NAME)) + .or(() -> DeduplicateTypeStrategyUtils.validateKeep(callContext, ARG_KEEP, NAME)) + .or( + () -> + DeduplicateTypeStrategyUtils.validateSequenceBy( + callContext, tableSemantics, ARG_SEQUENCE_BY, NAME)) + .or(() -> DeduplicateTypeStrategyUtils.validateStateTtl(callContext, ARG_STATE_TTL)) + .or( + () -> + DeduplicateTypeStrategyUtils.validateResetTtlOnDuplicate( + callContext, ARG_RESET_TTL_ON_DUPLICATE)); + } + + private static Optional validateNoOrderBy(final TableSemantics tableSemantics) { + if (tableSemantics.orderByColumns().length == 0) { + return Optional.empty(); + } + final List fieldNames = DataType.getFieldNames(tableSemantics.dataType()); + final String columns = + Arrays.stream(tableSemantics.orderByColumns()) + .mapToObj(fieldNames::get) + .collect(Collectors.joining(", ")); + return Optional.of( + String.format( + "DEDUPLICATE does not support ORDER BY, but received ORDER BY %s. " + + "Use sequence_by => DESCRIPTOR(%s) instead.", + columns, columns)); + } + + private static ChangelogMode inferChangelogMode(final ChangelogContext changelogContext) { + final ChangelogMode requiredMode = changelogContext.getRequiredChangelogMode(); + final ChangelogMode.Builder builder = + ChangelogMode.newBuilder() + .addContainedKind(RowKind.INSERT) + .addContainedKind(RowKind.UPDATE_AFTER); + if (requiredMode != null && requiredMode.contains(RowKind.UPDATE_BEFORE)) { + builder.addContainedKind(RowKind.UPDATE_BEFORE); + } + return builder.build(); + } + + private static TableSemantics tableSemantics(final CallContext callContext) { + return callContext + .getTableSemantics(ARG_INPUT) + .orElseThrow( + () -> + new ValidationException( + "First argument must be a table for DEDUPLICATE.")); + } + + private DeduplicateTypeStrategy() {} +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java new file mode 100644 index 00000000000000..50893b82af5f03 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.inference.strategies; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes.Field; +import org.apache.flink.table.functions.TableSemantics; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.CallContext; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.LogicalTypeFamily; +import org.apache.flink.table.types.logical.LogicalTypeRoot; +import org.apache.flink.table.types.logical.utils.LogicalTypeChecks; +import org.apache.flink.table.types.logical.utils.LogicalTypeUtils; +import org.apache.flink.table.types.utils.DataTypeUtils; +import org.apache.flink.types.ColumnList; +import org.apache.flink.util.TimeUtils; + +import java.time.Duration; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** Argument validation and output type helpers shared by the deduplication PTFs. */ +@Internal +public final class DeduplicateTypeStrategyUtils { + + /** Which record of a key is kept. */ + @Internal + public enum Keep { + FIRST("first"), + LAST("last"); + + private final String value; + + Keep(final String value) { + this.value = value; + } + + /** Returns the mode for an argument value, or empty if the value names no mode. */ + public static Optional fromValue(final String value) { + return Arrays.stream(values()).filter(keep -> keep.value.equals(value)).findFirst(); + } + } + + /** Returns the {@code keep} argument, or {@link Keep#FIRST} when it is omitted. */ + public static Keep keep(final CallContext callContext, final int argPos) { + return callContext + .getArgumentValue(argPos, String.class) + .map( + value -> + Keep.fromValue(value) + .orElseThrow( + () -> + new IllegalStateException( + "Validated keep argument expected."))) + .orElse(Keep.FIRST); + } + + /** + * Returns the positions of the {@code sequence_by} columns, in the listed order, within the + * output row without the PARTITION BY columns. + */ + public static int[] sequenceByOutputIndices( + final CallContext callContext, final TableSemantics tableSemantics, final int argPos) { + final List fieldNames = DataType.getFieldNames(tableSemantics.dataType()); + final List outputIndices = + Arrays.stream(ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics)) + .boxed() + .collect(Collectors.toList()); + return sequenceBy(callContext, argPos).stream() + .mapToInt(column -> outputIndices.indexOf(fieldNames.indexOf(column))) + .toArray(); + } + + /** Returns the input columns without time attributes. */ + public static List inputFields(final TableSemantics tableSemantics) { + final DataType inputType = tableSemantics.dataType(); + return DataType.getFields( + DataTypeUtils.replaceLogicalType( + inputType, + LogicalTypeUtils.removeTimeAttributes(inputType.getLogicalType()))); + } + + /** Returns the output columns after the PARTITION BY columns, without time attributes. */ + public static List outputFields(final TableSemantics tableSemantics) { + final List inputFields = inputFields(tableSemantics); + return Arrays.stream(ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics)) + .mapToObj(inputFields::get) + .collect(Collectors.toList()); + } + + public static Optional validatePartitionByNoTimeAttribute( + final TableSemantics tableSemantics, final String functionName) { + final List fields = DataType.getFields(tableSemantics.dataType()); + return Arrays.stream(tableSemantics.partitionByColumns()) + .mapToObj(fields::get) + .filter( + field -> + LogicalTypeChecks.isTimeAttribute( + field.getDataType().getLogicalType())) + .findFirst() + .map( + field -> + String.format( + "%s does not support a time attribute as a " + + "PARTITION BY column, but '%s' is one. Use " + + "CAST(%s AS %s) in a subquery first.", + functionName, + field.getName(), + field.getName(), + DataTypeUtils.removeTimeAttribute(field.getDataType()) + .getLogicalType() + .asSummaryString())); + } + + public static Optional validateKeep( + final CallContext callContext, final int argPos, final String functionName) { + if (callContext.isArgumentNull(argPos)) { + return Optional.empty(); + } + final String received; + if (!callContext.isArgumentLiteral(argPos)) { + received = "a non-literal expression"; + } else { + final String value = callContext.getArgumentValue(argPos, String.class).orElse(null); + if (value == null || Keep.fromValue(value).isPresent()) { + return Optional.empty(); + } + received = String.format("keep => '%s'", value); + } + return Optional.of( + String.format( + "%s requires keep => 'first' or keep => 'last', but received %s.", + functionName, received)); + } + + public static Optional validateSequenceBy( + final CallContext callContext, + final TableSemantics tableSemantics, + final int argPos, + final String functionName) { + final List fieldNames = DataType.getFieldNames(tableSemantics.dataType()); + final List fieldTypes = DataType.getFieldDataTypes(tableSemantics.dataType()); + final Set seen = new HashSet<>(); + for (final String column : sequenceBy(callContext, argPos)) { + final int index = fieldNames.indexOf(column); + if (index == -1) { + return Optional.of( + String.format( + "The sequence_by column '%s' does not exist in the input of " + + "%s. Available columns are: %s.", + column, functionName, String.join(", ", fieldNames))); + } + if (!seen.add(column)) { + return Optional.of( + String.format( + "The sequence_by column '%s' is listed more than once. List each " + + "ordering column once.", + column)); + } + if (Arrays.stream(tableSemantics.partitionByColumns()).anyMatch(c -> c == index)) { + return Optional.of( + String.format( + "The sequence_by column '%s' is a PARTITION BY column, which has " + + "the same value for every record of a key. Use a " + + "non-PARTITION BY column.", + column)); + } + final LogicalType type = fieldTypes.get(index).getLogicalType(); + if (!isOrderable(type)) { + return Optional.of( + String.format( + "The sequence_by column '%s' has type '%s', which cannot order " + + "records. Use a numeric, string, boolean, or date-time " + + "column.", + column, type.asSummaryString())); + } + } + return Optional.empty(); + } + + public static Optional validateStateTtl( + final CallContext callContext, final int argPos) { + if (callContext.isArgumentNull(argPos)) { + return Optional.empty(); + } + if (!callContext.isArgumentLiteral(argPos)) { + return Optional.of( + "The 'state_ttl' argument must be a constant INTERVAL literal, but received a " + + "non-literal expression. Use a literal such as INTERVAL '1' HOUR."); + } + return callContext + .getArgumentValue(argPos, Duration.class) + .filter(Duration::isNegative) + .map( + stateTtl -> + String.format( + "The 'state_ttl' argument must not be negative, but was " + + "'%s'. Use INTERVAL '0' SECOND to retain state " + + "indefinitely.", + TimeUtils.formatWithHighestUnit(stateTtl))); + } + + public static Optional validateResetTtlOnDuplicate( + final CallContext callContext, final int argPos) { + if (callContext.isArgumentNull(argPos) || callContext.isArgumentLiteral(argPos)) { + return Optional.empty(); + } + return Optional.of( + "The 'reset_ttl_on_duplicate' argument must be a constant BOOLEAN literal, but " + + "received a non-literal expression. Use TRUE or FALSE."); + } + + private static List sequenceBy(final CallContext callContext, final int argPos) { + return callContext + .getArgumentValue(argPos, ColumnList.class) + .map(ColumnList::getNames) + .orElse(List.of()); + } + + private static boolean isOrderable(final LogicalType type) { + return type.is(LogicalTypeFamily.NUMERIC) + || type.is(LogicalTypeFamily.CHARACTER_STRING) + || type.is(LogicalTypeFamily.BINARY_STRING) + || type.is(LogicalTypeFamily.DATETIME) + || type.is(LogicalTypeRoot.BOOLEAN); + } + + private DeduplicateTypeStrategyUtils() {} +} diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java new file mode 100644 index 00000000000000..70a01030b2a157 --- /dev/null +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.inference.strategies; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.functions.ChangelogFunction.ChangelogContext; +import org.apache.flink.table.functions.TableSemantics; +import org.apache.flink.table.types.AtomicDataType; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.InputTypeStrategiesTestBase; +import org.apache.flink.table.types.inference.utils.CallContextMock; +import org.apache.flink.table.types.inference.utils.TableSemanticsMock; +import org.apache.flink.table.types.logical.LocalZonedTimestampType; +import org.apache.flink.table.types.logical.TimestampKind; +import org.apache.flink.types.ColumnList; +import org.apache.flink.types.RowKind; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_INPUT; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_KEEP; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_RESET_TTL_ON_DUPLICATE; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_SEQUENCE_BY; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.CHANGELOG_MODE_STRATEGY; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.INPUT_TYPE_STRATEGY; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.OUTPUT_TYPE_STRATEGY; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link DeduplicateTypeStrategy}. */ +class DeduplicateTypeStrategyTest extends InputTypeStrategiesTestBase { + + private static final DataType ROWTIME = + new AtomicDataType(new LocalZonedTimestampType(true, TimestampKind.ROWTIME, 3)); + + private static final DataType INPUT_TYPE = + DataTypes.ROW( + DataTypes.FIELD("user_id", DataTypes.STRING()), + DataTypes.FIELD("event", DataTypes.STRING()), + DataTypes.FIELD("ts", ROWTIME), + DataTypes.FIELD("lsn", DataTypes.BIGINT()), + DataTypes.FIELD("tags", DataTypes.ARRAY(DataTypes.STRING()))); + + private static final DataType[] ARGUMENT_TYPES = { + INPUT_TYPE, + DataTypes.STRING(), + DataTypes.DESCRIPTOR(), + DataTypes.INTERVAL(DataTypes.SECOND()), + DataTypes.BOOLEAN() + }; + + private static final TableSemanticsMock PARTITIONED_BY_USER = + new TableSemanticsMock( + INPUT_TYPE, new int[] {0}, new int[0], -1, ChangelogMode.insertOnly()); + + @Override + protected Stream testData() { + return Stream.of( + withDefaults("All optional arguments omitted", PARTITIONED_BY_USER) + .expectArgumentTypes(ARGUMENT_TYPES), + withDefaults("All arguments set", PARTITIONED_BY_USER) + .calledWithLiteralAt(ARG_KEEP, "last") + .calledWithLiteralAt(ARG_SEQUENCE_BY, ColumnList.of("lsn", "ts")) + .calledWithLiteralAt(ARG_STATE_TTL, Duration.ofHours(1)) + .calledWithLiteralAt(ARG_RESET_TTL_ON_DUPLICATE, false) + .expectArgumentTypes(ARGUMENT_TYPES), + withDefaults("Invalid keep", PARTITIONED_BY_USER) + .calledWithLiteralAt(ARG_KEEP, "FIRST") + .expectErrorMessage( + "DEDUPLICATE requires keep => 'first' or keep => 'last', but " + + "received keep => 'FIRST'."), + nonLiteral("Non-literal keep", ARG_KEEP) + .expectErrorMessage( + "DEDUPLICATE requires keep => 'first' or keep => 'last', but " + + "received a non-literal expression."), + withDefaults( + "ORDER BY", + new TableSemanticsMock( + INPUT_TYPE, + new int[] {0}, + new int[] {3}, + -1, + ChangelogMode.insertOnly())) + .expectErrorMessage( + "DEDUPLICATE does not support ORDER BY, but received ORDER BY " + + "lsn. Use sequence_by => DESCRIPTOR(lsn) instead."), + withDefaults( + "Time attribute in PARTITION BY", + new TableSemanticsMock( + INPUT_TYPE, + new int[] {2}, + new int[0], + -1, + ChangelogMode.insertOnly())) + .expectErrorMessage( + "DEDUPLICATE does not support a time attribute as a PARTITION BY " + + "column, but 'ts' is one. Use CAST(ts AS " + + "TIMESTAMP_LTZ(3)) in a subquery first."), + sequenceBy("Unknown sequence_by column", "nope") + .expectErrorMessage( + "The sequence_by column 'nope' does not exist in the input of " + + "DEDUPLICATE. Available columns are: user_id, event, " + + "ts, lsn, tags."), + sequenceBy("Repeated sequence_by column", "lsn", "lsn") + .expectErrorMessage( + "The sequence_by column 'lsn' is listed more than once. List " + + "each ordering column once."), + sequenceBy("PARTITION BY column in sequence_by", "user_id") + .expectErrorMessage( + "The sequence_by column 'user_id' is a PARTITION BY column, " + + "which has the same value for every record of a key."), + sequenceBy("Non-orderable sequence_by column", "tags") + .expectErrorMessage( + "The sequence_by column 'tags' has type 'ARRAY', which " + + "cannot order records. Use a numeric, string, boolean, " + + "or date-time column."), + withDefaults("Negative state_ttl", PARTITIONED_BY_USER) + .calledWithLiteralAt(ARG_STATE_TTL, Duration.ofSeconds(-1)) + .expectErrorMessage( + "The 'state_ttl' argument must not be negative, but was '-1 s'. " + + "Use INTERVAL '0' SECOND to retain state indefinitely."), + nonLiteral("Non-literal state_ttl", ARG_STATE_TTL) + .expectErrorMessage( + "The 'state_ttl' argument must be a constant INTERVAL literal, " + + "but received a non-literal expression."), + nonLiteral("Non-literal reset_ttl_on_duplicate", ARG_RESET_TTL_ON_DUPLICATE) + .expectErrorMessage( + "The 'reset_ttl_on_duplicate' argument must be a constant " + + "BOOLEAN literal, but received a non-literal " + + "expression.")); + } + + @Test + void testOutputExcludesPartitionKeysAndTimeAttributes() { + assertThat(OUTPUT_TYPE_STRATEGY.inferType(callContext(ColumnList.of()))) + .contains( + DataTypes.ROW( + DataTypes.FIELD("event", DataTypes.STRING()), + DataTypes.FIELD("ts", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("lsn", DataTypes.BIGINT()), + DataTypes.FIELD( + "tags", DataTypes.ARRAY(DataTypes.STRING()))) + .notNull()); + } + + @Test + void testSequenceByIndicesPointIntoPayload() { + assertThat( + DeduplicateTypeStrategy.sequenceByOutputIndices( + callContext(ColumnList.of("lsn", "event")))) + .containsExactly(2, 0); + } + + @Test + void testChangelogModeEmitsUpdateBeforeOnlyWhenRequired() { + assertThat(inferChangelogMode(ChangelogMode.upsert())) + .isEqualTo(mode(RowKind.INSERT, RowKind.UPDATE_AFTER)); + assertThat(inferChangelogMode(ChangelogMode.all())) + .isEqualTo(mode(RowKind.INSERT, RowKind.UPDATE_BEFORE, RowKind.UPDATE_AFTER)); + } + + private static TestSpec withDefaults( + final String description, final TableSemanticsMock tableSemantics) { + return TestSpec.forStrategy(description, INPUT_TYPE_STRATEGY) + .calledWithArgumentTypes(ARGUMENT_TYPES) + .calledWithTableSemanticsAt(ARG_INPUT, tableSemantics) + .calledWithLiteralAt(ARG_KEEP, null) + .calledWithLiteralAt(ARG_SEQUENCE_BY, null) + .calledWithLiteralAt(ARG_STATE_TTL, null) + .calledWithLiteralAt(ARG_RESET_TTL_ON_DUPLICATE, null); + } + + private static TestSpec nonLiteral(final String description, final int argument) { + final TestSpec spec = + TestSpec.forStrategy(description, INPUT_TYPE_STRATEGY) + .calledWithArgumentTypes(ARGUMENT_TYPES) + .calledWithTableSemanticsAt(ARG_INPUT, PARTITIONED_BY_USER); + for (int pos = ARG_KEEP; pos <= ARG_RESET_TTL_ON_DUPLICATE; pos++) { + if (pos != argument) { + spec.calledWithLiteralAt(pos, null); + } + } + return spec; + } + + private static TestSpec sequenceBy(final String description, final String... columns) { + return withDefaults(description, PARTITIONED_BY_USER) + .calledWithLiteralAt(ARG_SEQUENCE_BY, ColumnList.of(List.of(columns))); + } + + private static CallContextMock callContext(final ColumnList sequenceBy) { + final CallContextMock callContext = new CallContextMock(); + callContext.argumentValues = + List.of( + Optional.empty(), + Optional.empty(), + Optional.of(sequenceBy), + Optional.empty(), + Optional.empty()); + callContext.tableSemantics = Map.of(ARG_INPUT, PARTITIONED_BY_USER); + return callContext; + } + + private static ChangelogMode inferChangelogMode(final ChangelogMode requiredMode) { + return CHANGELOG_MODE_STRATEGY.inferChangelogMode( + new ChangelogContext() { + @Override + public ChangelogMode getTableChangelogMode(final int pos) { + return ChangelogMode.insertOnly(); + } + + @Override + public ChangelogMode getRequiredChangelogMode() { + return requiredMode; + } + + @Override + public Optional getTableSemantics(final int pos) { + return Optional.empty(); + } + }); + } + + private static ChangelogMode mode(final RowKind... kinds) { + final ChangelogMode.Builder builder = ChangelogMode.newBuilder(); + for (final RowKind kind : kinds) { + builder.addContainedKind(kind); + } + return builder.build(); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTest.java new file mode 100644 index 00000000000000..e9a6d2d9b15dcb --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.planner.plan.nodes.exec.testutils.RestoreTestBase; +import org.apache.flink.table.test.program.TableTestProgram; + +import java.util.List; + +/** Restore tests for the built-in DEDUPLICATE PTF. */ +public class DeduplicateRestoreTest extends RestoreTestBase { + + public DeduplicateRestoreTest() { + super(StreamExecProcessTableFunction.class); + } + + @Override + public List programs() { + return List.of( + DeduplicateRestoreTestPrograms.FIRST_ARRIVAL_RESTORE, + DeduplicateRestoreTestPrograms.SEQUENCE_BY_RESTORE); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTestPrograms.java new file mode 100644 index 00000000000000..133d9ad95284ae --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTestPrograms.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; + +/** {@link TableTestProgram} definitions for restore tests of the DEDUPLICATE PTF. */ +public class DeduplicateRestoreTestPrograms { + + public static final TableTestProgram FIRST_ARRIVAL_RESTORE = + TableTestProgram.of( + "deduplicate-first-arrival-restore", + "with the default keep => 'first', a key emitted before the restore " + + "stays dropped after it, and a key first seen after it is " + + "inserted") + .setupTableSource( + SourceTestStep.newBuilder("payments") + .addSchema("payment_id STRING", "amount INT") + .producedBeforeRestore( + Row.of("p1", 50), + Row.of("p2", 20), + Row.of("p1", 60), + Row.of("p3", 30)) + .producedAfterRestore( + Row.of("p1", 70), + Row.of("p2", 20), + Row.of("p4", 40), + Row.of("p3", 35)) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema( + "payment_id STRING PRIMARY KEY NOT ENFORCED", + "amount INT") + .addOption("sink-changelog-mode-enforced", "I,UA,D") + .consumedBeforeRestore("+I[p1, 50]", "+I[p2, 20]", "+I[p3, 30]") + .consumedAfterRestore("+I[p4, 40]") + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE(" + + "input => TABLE payments PARTITION BY payment_id)") + .build(); + + public static final TableTestProgram SEQUENCE_BY_RESTORE = + TableTestProgram.of( + "deduplicate-sequence-by-restore", + "the sequence_by value of the kept record survives a restore, so " + + "with 'first' a greater or equal value arriving after the " + + "restore is dropped") + .setupTableSource( + SourceTestStep.newBuilder("customer_changes") + .addSchema("customer_id STRING", "city STRING", "lsn BIGINT") + .producedBeforeRestore( + Row.of("c1", "Berlin", 100L), + Row.of("c2", "Paris", 105L), + Row.of("c1", "Hamburg", 90L), + Row.of("c3", "Rome", 50L)) + .producedAfterRestore( + Row.of("c1", "Munich", 95L), + Row.of("c2", "Lyon", 100L), + Row.of("c3", "Milan", 50L), + Row.of("c4", "Oslo", 1L)) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema( + "customer_id STRING PRIMARY KEY NOT ENFORCED", + "city STRING", + "lsn BIGINT") + .addOption("sink-changelog-mode-enforced", "I,UA,D") + .consumedBeforeRestore( + "+I[c1, Berlin, 100]", + "+I[c2, Paris, 105]", + "+U[c1, Hamburg, 90]", + "+I[c3, Rome, 50]") + .consumedAfterRestore("+U[c2, Lyon, 100]", "+I[c4, Oslo, 1]") + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE(" + + "input => TABLE customer_changes PARTITION BY customer_id, " + + "keep => 'first', " + + "sequence_by => DESCRIPTOR(lsn))") + .build(); +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateSemanticTests.java new file mode 100644 index 00000000000000..f8c5c8d3669f90 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateSemanticTests.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.planner.plan.nodes.exec.testutils.SemanticTestBase; +import org.apache.flink.table.test.program.TableTestProgram; + +import java.util.List; + +/** Semantic tests for the built-in DEDUPLICATE process table function. */ +public class DeduplicateSemanticTests extends SemanticTestBase { + + @Override + public List programs() { + return List.of( + DeduplicateTestPrograms.ARRIVAL_ORDER, + DeduplicateTestPrograms.COMPOSITE_AND_NULL_KEY, + DeduplicateTestPrograms.SEQUENCE_BY); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateTestPrograms.java new file mode 100644 index 00000000000000..6a1550eb3a7085 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateTestPrograms.java @@ -0,0 +1,328 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; + +import javax.annotation.Nullable; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +/** {@link TableTestProgram} definitions for testing the built-in DEDUPLICATE PTF. */ +public class DeduplicateTestPrograms { + + private static final String[] USER_EVENTS_SCHEMA = {"user_id STRING", "event STRING"}; + + private static final String[] USER_EVENTS_UPSERT_SCHEMA = { + "user_id STRING PRIMARY KEY NOT ENFORCED", "event STRING" + }; + + private static final String[] CUSTOMER_UPSERT_SCHEMA = { + "customer_id STRING PRIMARY KEY NOT ENFORCED", "city STRING", "lsn BIGINT" + }; + + private static final String[] LABEL_UPSERT_SCHEMA = { + "id STRING PRIMARY KEY NOT ENFORCED", "label STRING" + }; + + private static final String[] PRICES_UPSERT_SCHEMA = { + "ticker STRING PRIMARY KEY NOT ENFORCED", "price INT", "ts TIMESTAMP_LTZ(3)" + }; + + public static final TableTestProgram ARRIVAL_ORDER = + TableTestProgram.of( + "deduplicate-arrival-order", + "'last' replaces the kept record with every new one, 'first' drops " + + "every record after the first, an identical record emits " + + "nothing, and a consumer that needs the previous record " + + "receives -U before +U") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedValues( + Row.of("u1", "login"), + Row.of("u2", "login"), + Row.of("u1", "click")) + .build()) + .setupTableSource( + SourceTestStep.newBuilder("profile_updates") + .addSchema("user_id STRING", "city STRING", "phone STRING") + .producedValues( + Row.of("u1", "Berlin", "111"), + Row.of("u1", "Berlin", "111"), + Row.of("u1", "Munich", null)) + .build()) + .setupTableSink( + upsertSink( + "sink_last", + USER_EVENTS_UPSERT_SCHEMA, + "+I[u1, login]", + "+I[u2, login]", + "+U[u1, click]")) + .setupTableSink( + upsertSink( + "sink_first", + USER_EVENTS_UPSERT_SCHEMA, + "+I[u1, login]", + "+I[u2, login]")) + .setupTableSink( + SinkTestStep.newBuilder("retract_sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues( + "+I[u1, login]", + "+I[u2, login]", + "-U[u1, login]", + "+U[u1, click]") + .build()) + .setupTableSink( + upsertSink( + "latest_profiles", + new String[] { + "user_id STRING PRIMARY KEY NOT ENFORCED", + "city STRING", + "phone STRING" + }, + "+I[u1, Berlin, 111]", + "+U[u1, Munich, null]")) + .runSql( + "INSERT INTO sink_last SELECT * FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY user_id, " + + "keep => 'last')") + .runSql( + "INSERT INTO sink_first SELECT * FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY user_id)") + .runSql( + "INSERT INTO retract_sink SELECT * FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY user_id, " + + "keep => 'last')") + .runSql( + "INSERT INTO latest_profiles SELECT * FROM DEDUPLICATE(" + + "input => TABLE profile_updates PARTITION BY user_id, " + + "keep => 'last')") + .build(); + + public static final TableTestProgram COMPOSITE_AND_NULL_KEY = + TableTestProgram.of( + "deduplicate-composite-and-null-key", + "a PARTITION BY of non-adjacent columns is the leading output columns " + + "in its own order, and a NULL key column forms its own key") + .setupTableSource( + SourceTestStep.newBuilder("sessions") + .addSchema("user_id STRING", "page STRING", "device STRING") + .producedValues( + Row.of("u1", "a", "web"), + Row.of("u1", "b", "mobile"), + Row.of("u1", "c", "web"), + Row.of(null, "d", "web"), + Row.of(null, "e", "web")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("device STRING", "user_id STRING", "page STRING") + .consumedValues( + "+I[web, u1, a]", + "+I[mobile, u1, b]", + "-U[web, u1, a]", + "+U[web, u1, c]", + "+I[web, null, d]", + "-U[web, null, d]", + "+U[web, null, e]") + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE(" + + "input => TABLE sessions PARTITION BY (device, user_id), " + + "keep => 'last')") + .build(); + + public static final TableTestProgram SEQUENCE_BY = + TableTestProgram.of( + "deduplicate-sequence-by", + "with sequence_by, 'last' keeps the greatest and 'first' the smallest " + + "value, a tie goes to the later arrival for 'last' and to " + + "the earlier one for 'first', later columns break ties, a " + + "NULL orders below every other value, NaN ties with every " + + "value, and a watermarked column orders without watermark " + + "semantics and is emitted as a regular timestamp") + .setupTableSource( + SourceTestStep.newBuilder("customer_changes") + .addSchema("customer_id STRING", "city STRING", "lsn BIGINT") + .producedValues( + Row.of("c1", "Berlin", 100L), + Row.of("c1", "Munich", 120L), + Row.of("c1", "Hamburg", 110L)) + .build()) + .setupTableSource( + SourceTestStep.newBuilder("prices") + .addSchema( + "ticker STRING", + "price INT", + "ts TIMESTAMP_LTZ(3)", + "WATERMARK FOR ts AS ts - INTERVAL '1' HOUR") + .producedValues( + Row.of("AAPL", 10, at("10:02")), + Row.of("MSFT", 20, at("10:01")), + Row.of("AAPL", 12, at("10:08")), + Row.of("AAPL", 11, at("10:05")), + Row.of("AAPL", 9, at("10:00")), + Row.of("AAPL", 14, at("10:08")), + Row.of("AAPL", 8, at("10:00"))) + .build()) + .setupTableSource( + SourceTestStep.newBuilder("measurements") + .addSchema("id STRING", "v DOUBLE", "label STRING") + .producedValues( + Row.of("m1", 1.0, "r1"), + Row.of("m1", Double.NaN, "r2"), + Row.of("m1", 0.5, "r3")) + .build()) + .setupTableSource( + SourceTestStep.newBuilder("versions") + .addSchema( + "id STRING", + "s STRING", + "b BOOLEAN", + "t TIMESTAMP(3)", + "y BYTES", + "label STRING") + .producedValues( + versionRow("a", false, 1, 1, "r1"), + versionRow("b", false, 1, 1, "r2"), + versionRow("a", true, 1, 1, "r3"), + versionRow("b", true, 1, 1, "r4"), + versionRow("b", true, 0, 9, "r5"), + versionRow("b", true, 2, 1, "r6"), + versionRow("b", true, 2, 0, "r7"), + versionRow("b", true, 2, 2, "r8"), + versionRow(null, true, 9, 9, "r9"), + versionRow(null, true, 9, 9, "r10"), + versionRow("b", true, 2, 2, "r11"), + versionRow("b", true, 2, null, "r12")) + .build()) + .setupTableSink( + upsertSink( + "customers_last", + CUSTOMER_UPSERT_SCHEMA, + "+I[c1, Berlin, 100]", + "+U[c1, Munich, 120]")) + .setupTableSink( + upsertSink( + "prices_last", + PRICES_UPSERT_SCHEMA, + "+I[AAPL, 10, 1970-01-01T10:02:00Z]", + "+I[MSFT, 20, 1970-01-01T10:01:00Z]", + "+U[AAPL, 12, 1970-01-01T10:08:00Z]", + "+U[AAPL, 14, 1970-01-01T10:08:00Z]")) + .setupTableSink( + upsertSink( + "prices_first", + PRICES_UPSERT_SCHEMA, + "+I[AAPL, 10, 1970-01-01T10:02:00Z]", + "+I[MSFT, 20, 1970-01-01T10:01:00Z]", + "+U[AAPL, 9, 1970-01-01T10:00:00Z]")) + .setupTableSink( + upsertSink( + "measurements_last", + LABEL_UPSERT_SCHEMA, + "+I[m1, r1]", + "+U[m1, r2]", + "+U[m1, r3]")) + .setupTableSink( + upsertSink( + "versions_last", + LABEL_UPSERT_SCHEMA, + "+I[c1, r1]", + "+U[c1, r2]", + "+U[c1, r4]", + "+U[c1, r6]", + "+U[c1, r8]", + "+U[c1, r11]")) + .setupTableSink( + upsertSink( + "versions_first", + LABEL_UPSERT_SCHEMA, + "+I[c1, r1]", + "+U[c1, r9]")) + .runSql( + "INSERT INTO customers_last SELECT * FROM DEDUPLICATE(" + + "input => TABLE customer_changes PARTITION BY customer_id, " + + "keep => 'last', " + + "sequence_by => DESCRIPTOR(lsn), " + + "state_ttl => INTERVAL '1' DAY, " + + "reset_ttl_on_duplicate => true, " + + "uid => 'customer-changes-dedup')") + .runSql( + "INSERT INTO prices_last SELECT * FROM DEDUPLICATE(" + + "input => TABLE prices PARTITION BY ticker, " + + "keep => 'last', " + + "sequence_by => DESCRIPTOR(ts))") + .runSql( + "INSERT INTO prices_first SELECT * FROM DEDUPLICATE(" + + "input => TABLE prices PARTITION BY ticker, " + + "keep => 'first', " + + "sequence_by => DESCRIPTOR(ts))") + .runSql( + "INSERT INTO measurements_last SELECT id, label FROM DEDUPLICATE(" + + "input => TABLE measurements PARTITION BY id, " + + "keep => 'last', " + + "sequence_by => DESCRIPTOR(v))") + .runSql( + "INSERT INTO versions_last SELECT id, label FROM DEDUPLICATE(" + + "input => TABLE versions PARTITION BY id, " + + "keep => 'last', " + + "sequence_by => DESCRIPTOR(s, b, t, y))") + .runSql( + "INSERT INTO versions_first SELECT id, label FROM DEDUPLICATE(" + + "input => TABLE versions PARTITION BY id, " + + "keep => 'first', " + + "sequence_by => DESCRIPTOR(s, b, t, y))") + .build(); + + private static SinkTestStep upsertSink( + final String name, final String[] schema, final String... expected) { + return SinkTestStep.newBuilder(name) + .addSchema(schema) + .addOption("sink-changelog-mode-enforced", "I,UA,D") + .consumedValues(expected) + .build(); + } + + private static Row versionRow( + @Nullable final String s, + final boolean b, + final int minute, + @Nullable final Integer y, + final String label) { + return Row.of( + "c1", + s, + b, + LocalDateTime.ofEpochSecond(minute * 60L, 0, ZoneOffset.UTC), + y == null ? null : new byte[] {y.byteValue()}, + label); + } + + private static Instant at(final String time) { + return Instant.parse("1970-01-01T" + time + ":00Z"); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.java new file mode 100644 index 00000000000000..a71dd9bc472b13 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.stream.sql; + +import org.apache.flink.table.api.ExplainDetail; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.planner.utils.PlanKind; +import org.apache.flink.table.planner.utils.TableTestBase; +import org.apache.flink.table.planner.utils.TableTestUtil; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import scala.Enumeration; + +import static org.apache.flink.core.testutils.FlinkAssertions.anyCauseMatches; +import static org.apache.flink.table.api.Expressions.$; +import static org.apache.flink.table.api.Expressions.descriptor; +import static org.apache.flink.table.api.Expressions.lit; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Plan tests for the changelog mode and upsert key of the DEDUPLICATE built-in PTF. */ +class DeduplicateTableFunctionTest extends TableTestBase { + + private TableTestUtil util; + + @BeforeEach + void setup() { + util = streamTestUtil(TableConfig.getDefault()); + util.tableEnv() + .executeSql( + "CREATE TABLE user_events (user_id STRING, event STRING, " + + "ts TIMESTAMP_LTZ(3), WATERMARK FOR ts AS ts) " + + "WITH ('connector' = 'values')"); + util.tableEnv() + .executeSql( + "CREATE TABLE customer_changes (customer_id STRING, city STRING, " + + "lsn BIGINT) WITH ('connector' = 'values')"); + } + + @Test + void testUpsertKeyIsPartitionKey() { + verifyPlan( + "SELECT * FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY user_id, keep => 'last')"); + } + + @Test + void testRetractingConsumer() { + verifyPlan( + "SELECT event, COUNT(*) AS users FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY user_id, keep => 'last') " + + "GROUP BY event"); + } + + @Test + void testOutputHasNoTimeAttribute() { + verifyPlan( + "SELECT * FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY event, " + + "sequence_by => DESCRIPTOR(ts))"); + } + + @Test + void testOnTimeIsRejectedForUpdatingOutput() { + assertThatThrownBy( + () -> + util.verifyExecPlan( + "SELECT * FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY user_id, " + + "on_time => DESCRIPTOR(ts))")) + .satisfies( + anyCauseMatches( + "Time operations using the `on_time` argument are currently not " + + "supported for PTFs that consume or produce updates.")); + } + + @Test + void testTableApiWithArguments() { + final Table result = + util.tableEnv() + .from("customer_changes") + .partitionBy($("customer_id")) + .deduplicate( + lit("last").asArgument("keep"), + descriptor("lsn").asArgument("sequence_by"), + lit(Duration.ofDays(1)).asArgument("state_ttl")); + util.verifyRelPlan(result); + } + + private void verifyPlan(final String query) { + util.doVerifyPlan( + query, + new ExplainDetail[] {ExplainDetail.CHANGELOG_MODE}, + false, + new Enumeration.Value[] {PlanKind.AST(), PlanKind.OPT_REL()}, + false, + true); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/harness/DeduplicateHarnessTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/harness/DeduplicateHarnessTest.java new file mode 100644 index 00000000000000..7cae31f1c4c160 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/harness/DeduplicateHarnessTest.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.runtime.harness; + +import org.apache.flink.api.dag.Transformation; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.transformations.KeyedMultipleInputTransformation; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.KeyedMultiInputStreamOperatorTestHarness; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.runtime.keyselector.RowDataKeySelector; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Harness tests for {@code reset_ttl_on_duplicate} of the built-in DEDUPLICATE PTF. */ +class DeduplicateHarnessTest { + + private static final Function PAYMENT = + row -> row.getString(0) + ", " + row.getInt(1); + + private static Stream resetTtlOnDuplicate() { + final List refreshed = List.of("+I[p1, 50]@10:00", "+I[p1, 50]@10:20"); + final List notRefreshed = + List.of("+I[p1, 50]@10:00", "+I[p1, 50]@10:07", "+I[p1, 50]@10:20"); + return Stream.of( + Arguments.of("first", true, refreshed), + Arguments.of("first", false, notRefreshed), + Arguments.of("last", true, refreshed), + Arguments.of("last", false, notRefreshed)); + } + + @ParameterizedTest(name = "keep={0}, reset_ttl_on_duplicate={1}") + @MethodSource("resetTtlOnDuplicate") + void testTimeBoundedDeduplication( + final String keep, final boolean resetTtlOnDuplicate, final List expected) + throws Exception { + try (KeyedMultiInputStreamOperatorTestHarness harness = + createHarness( + "SELECT * FROM DEDUPLICATE(" + + "input => TABLE payments PARTITION BY payment_id, " + + "keep => '" + + keep + + "', " + + "state_ttl => INTERVAL '5' MINUTES, " + + "reset_ttl_on_duplicate => " + + resetTtlOnDuplicate + + ")")) { + final List output = new ArrayList<>(); + for (final String time : List.of("10:00", "10:04", "10:07", "10:20")) { + output.addAll(process(harness, time, PAYMENT, payment(50))); + } + + assertThat(output).isEqualTo(expected); + } + } + + private static List process( + final KeyedMultiInputStreamOperatorTestHarness harness, + final String time, + final Function describe, + final RowData input) + throws Exception { + harness.setStateTtlProcessingTime(toMillis(time)); + harness.processElement(0, new StreamRecord<>(input)); + final List output = new ArrayList<>(); + for (final Object record : harness.getOutput()) { + final RowData row = (RowData) ((StreamRecord) record).getValue(); + output.add( + String.format( + "%s[%s]@%s", + row.getRowKind().shortString(), describe.apply(row), time)); + } + harness.getOutput().clear(); + return output; + } + + private static RowData payment(final int amount) { + return GenericRowData.of(StringData.fromString("p1"), amount); + } + + private static KeyedMultiInputStreamOperatorTestHarness createHarness( + final String query) throws Exception { + final StreamTableEnvironment tEnv = + StreamTableEnvironment.create(StreamExecutionEnvironment.getExecutionEnvironment()); + tEnv.executeSql( + "CREATE TABLE payments (payment_id STRING, amount INT) " + + "WITH ('connector' = 'values')"); + final KeyedMultipleInputTransformation transformation = + findPtf(tEnv.toChangelogStream(tEnv.sqlQuery(query)).getTransformation()) + .orElseThrow(() -> new AssertionError("No PTF transformation found.")); + return openHarness( + transformation.getOperatorFactory(), + (RowDataKeySelector) transformation.getStateKeySelectors().get(0)); + } + + private static KeyedMultiInputStreamOperatorTestHarness openHarness( + final StreamOperatorFactory operatorFactory, final RowDataKeySelector keySelector) + throws Exception { + final KeyedMultiInputStreamOperatorTestHarness harness = + new KeyedMultiInputStreamOperatorTestHarness<>( + operatorFactory, keySelector.getProducedType()); + harness.setKeySelector(0, keySelector); + harness.open(); + return harness; + } + + private static Optional> findPtf( + final Transformation transformation) { + if (transformation instanceof KeyedMultipleInputTransformation) { + return Optional.of((KeyedMultipleInputTransformation) transformation); + } + return transformation.getInputs().stream() + .map(DeduplicateHarnessTest::findPtf) + .flatMap(Optional::stream) + .findFirst(); + } + + private static long toMillis(final String time) { + final String[] parts = time.split(":"); + return Duration.ofHours(Long.parseLong(parts[0])) + .plusMinutes(Long.parseLong(parts[1])) + .toMillis(); + } +} diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.xml new file mode 100644 index 00000000000000..e2217ca87819a3 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.xml @@ -0,0 +1,104 @@ + + + + + + TABLE user_events PARTITION BY event, sequence_by => DESCRIPTOR(ts))]]> + + + + + + + + + + + TABLE user_events PARTITION BY user_id, keep => 'last') GROUP BY event]]> + + + + + + + + + + + + + + + + + + + TABLE user_events PARTITION BY user_id, keep => 'last')]]> + + + + + + + + + diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/plan/deduplicate-first-arrival-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/plan/deduplicate-first-arrival-restore.json new file mode 100644 index 00000000000000..533b425602f2c2 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/plan/deduplicate-first-arrival-restore.json @@ -0,0 +1,161 @@ +{ + "flinkVersion" : "2.5", + "nodes" : [ { + "id" : 1, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`payments`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "payment_id", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "amount", + "dataType" : "INT" + } ] + } + } + } + }, + "outputType" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, payments]], fields=[payment_id, amount])" + }, { + "id" : 2, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT>", + "description" : "Exchange(distribution=[hash[payment_id]])" + }, { + "id" : 3, + "type" : "stream-exec-process-table-function_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT>", + "description" : "ProcessTableFunction(invocation=[DEDUPLICATE(TABLE(#0) PARTITION BY($0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[DEDUPLICATE], select=[payment_id,amount], rowType=[RecordType(VARCHAR(2147483647) payment_id, INTEGER amount)])", + "uid" : "DEDUPLICATE", + "functionCall" : { + "kind" : "CALL", + "internalName" : "$DEDUPLICATE$1", + "operands" : [ { + "kind" : "TABLE_ARG_CALL", + "inputIndex" : 0, + "partitionKeys" : [ 0 ], + "type" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT> NOT NULL" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "DESCRIPTOR" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "INTERVAL SECOND(6)" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "BOOLEAN" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "DESCRIPTOR" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "VARCHAR(2147483647)" + } ], + "type" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT> NOT NULL" + }, + "inputChangelogModes" : [ [ "INSERT" ] ], + "outputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], + "inputUpsertKeys" : [ [ ] ] + }, { + "id" : 4, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "payment_id", + "dataType" : "VARCHAR(2147483647) NOT NULL" + }, { + "name" : "amount", + "dataType" : "INT" + } ], + "primaryKey" : { + "name" : "PK_payment_id", + "type" : "PRIMARY_KEY", + "columns" : [ "payment_id" ] + } + } + } + } + }, + "inputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], + "upsertMaterializeStrategy" : "MAP", + "inputUpsertKey" : [ 0 ], + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT>", + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[payment_id, amount])" + } ], + "edges" : [ { + "source" : 1, + "target" : 2, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 2, + "target" : 3, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 3, + "target" : 4, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/savepoint/_metadata new file mode 100644 index 00000000000000..df8f52de27bc53 Binary files /dev/null and b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/savepoint/_metadata differ diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/plan/deduplicate-sequence-by-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/plan/deduplicate-sequence-by-restore.json new file mode 100644 index 00000000000000..90d341867230bf --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/plan/deduplicate-sequence-by-restore.json @@ -0,0 +1,169 @@ +{ + "flinkVersion" : "2.5", + "nodes" : [ { + "id" : 5, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`customer_changes`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "customer_id", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "city", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "lsn", + "dataType" : "BIGINT" + } ] + } + } + } + }, + "outputType" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, customer_changes]], fields=[customer_id, city, lsn])" + }, { + "id" : 6, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT>", + "description" : "Exchange(distribution=[hash[customer_id]])" + }, { + "id" : 7, + "type" : "stream-exec-process-table-function_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT>", + "description" : "ProcessTableFunction(invocation=[DEDUPLICATE(TABLE(#0) PARTITION BY($0), _UTF-16LE'first', DESCRIPTOR(_UTF-16LE'lsn'), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[DEDUPLICATE], select=[customer_id,city,lsn], rowType=[RecordType(VARCHAR(2147483647) customer_id, VARCHAR(2147483647) city, BIGINT lsn)])", + "uid" : "DEDUPLICATE", + "functionCall" : { + "kind" : "CALL", + "internalName" : "$DEDUPLICATE$1", + "operands" : [ { + "kind" : "TABLE_ARG_CALL", + "inputIndex" : 0, + "partitionKeys" : [ 0 ], + "type" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT> NOT NULL" + }, { + "kind" : "LITERAL", + "value" : "first", + "type" : "CHAR(5) NOT NULL" + }, { + "kind" : "CALL", + "internalName" : "$DESCRIPTOR$1", + "operands" : [ { + "kind" : "LITERAL", + "value" : "lsn", + "type" : "CHAR(3) NOT NULL" + } ], + "type" : "DESCRIPTOR NOT NULL" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "INTERVAL SECOND(6)" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "BOOLEAN" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "DESCRIPTOR" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "VARCHAR(2147483647)" + } ], + "type" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT> NOT NULL" + }, + "inputChangelogModes" : [ [ "INSERT" ] ], + "outputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], + "inputUpsertKeys" : [ [ ] ] + }, { + "id" : 8, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "customer_id", + "dataType" : "VARCHAR(2147483647) NOT NULL" + }, { + "name" : "city", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "lsn", + "dataType" : "BIGINT" + } ], + "primaryKey" : { + "name" : "PK_customer_id", + "type" : "PRIMARY_KEY", + "columns" : [ "customer_id" ] + } + } + } + } + }, + "inputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], + "upsertMaterializeStrategy" : "MAP", + "inputUpsertKey" : [ 0 ], + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT>", + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[customer_id, city, lsn])" + } ], + "edges" : [ { + "source" : 5, + "target" : 6, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 6, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 7, + "target" : 8, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/savepoint/_metadata new file mode 100644 index 00000000000000..9d3219ab67d1f9 Binary files /dev/null and b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/savepoint/_metadata differ diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java index 5cecc79866f657..bc57fe3181a53d 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java @@ -104,6 +104,26 @@ protected BuiltInProcessTableFunction( : toInternalStateStrategies(definitionTypeInference); } + /** + * Declares a function with one value state of the given type that expires after the interval of + * the argument at {@code ttlArgPos}, or never if that argument is NULL. + */ + protected static LinkedHashMap valueState( + final SpecializedContext context, + final String name, + final DataType valueType, + final int ttlArgPos) { + return internalState( + Map.entry( + name, + StateTypeStrategy.of( + callContext -> + Optional.of(ValueView.newValueViewDataType(valueType)), + context.getCallContext() + .getArgumentValue(ttlArgPos, Duration.class) + .orElse(null)))); + } + /** * Declares the state entries of a specialized function in order. The strategies may infer * external data types, they are converted to internal ones. diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java new file mode 100644 index 00000000000000..268e98fbe36c6f --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.ptf; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.UserDefinedFunction; +import org.apache.flink.table.runtime.functions.BuiltInSpecializedFunction; +import org.apache.flink.table.types.inference.CallContext; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep; + +/** + * Runtime implementation of {@link BuiltInFunctionDefinitions#DEDUPLICATE}. + * + *

+ * keep    sequence_by  implementation
+ * first   absent       {@link DeduplicateEmitOnceFunction}
+ * first   present      {@link DeduplicateCurrentRowFunction}
+ * last    any          {@link DeduplicateCurrentRowFunction}
+ * 
+ */ +@Internal +public class Deduplicate extends BuiltInSpecializedFunction { + + public Deduplicate() { + super(BuiltInFunctionDefinitions.DEDUPLICATE); + } + + @Override + public UserDefinedFunction specialize(final SpecializedContext context) { + final CallContext callContext = context.getCallContext(); + final boolean isKeepFirst = DeduplicateTypeStrategy.keep(callContext) == Keep.FIRST; + + if (isKeepFirst) { + final boolean hasSequenceBy = + DeduplicateTypeStrategy.sequenceByOutputIndices(callContext).length > 0; + if (!hasSequenceBy) { + return new DeduplicateEmitOnceFunction(context); + } else { + return new DeduplicateCurrentRowFunction(context); + } + } else { + return new DeduplicateCurrentRowFunction(context); + } + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateCurrentRowFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateCurrentRowFunction.java new file mode 100644 index 00000000000000..f131b9f1db3930 --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateCurrentRowFunction.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.ptf; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.dataview.ValueView; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.binary.BinaryRowData; +import org.apache.flink.table.data.utils.JoinedRowData; +import org.apache.flink.table.data.utils.ProjectedRowData; +import org.apache.flink.table.functions.FunctionContext; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; +import org.apache.flink.table.runtime.functions.ptf.DeduplicateUtils.SequenceComparator; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.strategies.ChangelogTypeStrategyUtils; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.types.ColumnList; +import org.apache.flink.types.RowKind; + +import javax.annotation.Nullable; + +import java.util.Arrays; +import java.util.List; + +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL; + +/** + * {@link Deduplicate} with {@code keep => 'last'}, or {@code keep => 'first'} with {@code + * sequence_by}: a later record can replace the current one, so the current input row is held in the + * {@code kept} state. + */ +@Internal +public class DeduplicateCurrentRowFunction extends DeduplicateFunctionBase { + + private static final long serialVersionUID = 1L; + + private static final String CURRENT_STATE = "kept"; + + private final Keep keepMode; + + private final LogicalType[] inputTypes; + + private final boolean isUpdateBeforeEmitted; + + private final @Nullable SequenceComparator sequenceComparator; + + private transient RowDataSerializer inputSerializer; + + private transient ProjectedRowData projectedCandidate; + + private transient ProjectedRowData projectedCurrent; + + private transient JoinedRowData retraction; + + private transient RowData emptyRow; + + DeduplicateCurrentRowFunction(final SpecializedContext context) { + super( + context, + valueState( + context, + CURRENT_STATE, + DataTypes.ROW( + DeduplicateTypeStrategyUtils.inputFields( + DeduplicateUtils.tableSemantics(context))), + ARG_STATE_TTL)); + this.keepMode = DeduplicateTypeStrategy.keep(context.getCallContext()); + final List inputFields = + DeduplicateTypeStrategyUtils.inputFields(DeduplicateUtils.tableSemantics(context)); + this.inputTypes = + inputFields.stream() + .map(field -> field.getDataType().getLogicalType()) + .toArray(LogicalType[]::new); + this.isUpdateBeforeEmitted = + context.getCallContext() + .getOutputChangelogMode() + .map(mode -> mode.contains(RowKind.UPDATE_BEFORE)) + .orElse(false); + this.sequenceComparator = sequenceComparator(context, inputFields); + } + + @Override + public void open(final FunctionContext context) throws Exception { + super.open(context); + if (sequenceComparator != null) { + sequenceComparator.open(context); + } + inputSerializer = new RowDataSerializer(inputTypes); + projectedCandidate = ProjectedRowData.from(outputIndices); + projectedCurrent = ProjectedRowData.from(outputIndices); + retraction = new JoinedRowData(); + emptyRow = GenericRowData.of(); + } + + @Override + public void close() throws Exception { + if (sequenceComparator != null) { + sequenceComparator.close(); + } + super.close(); + } + + public void eval( + final Context ctx, + final ValueView currentState, + final RowData input, + @Nullable final StringData keep, + @Nullable final ColumnList sequenceBy, + @Nullable final Long stateTtl, + @Nullable final Boolean resetTtlOnDuplicate) + throws Exception { + + final BinaryRowData current = (BinaryRowData) currentState.getValue(); + if (current == null) { + final BinaryRowData candidate = inputSerializer.toBinaryRow(input); + currentState.setValue(candidate.copy()); + emitCandidate(RowKind.INSERT, candidate); + return; + } + if (replacesCurrent(input, current)) { + final BinaryRowData candidate = inputSerializer.toBinaryRow(input); + if (!candidate.equals(current)) { + currentState.setValue(candidate.copy()); + if (isUpdateBeforeEmitted) { + emitRetraction(current); + } + emitCandidate(RowKind.UPDATE_AFTER, candidate); + } else if (isResetTtlOnDuplicate) { + currentState.setValue(current); + } + } else if (isResetTtlOnDuplicate) { + currentState.setValue(current); + } + } + + /** + * Without sequence_by only keep => 'last' reaches this class, so the latest arrival replaces. + */ + private boolean replacesCurrent(final RowData candidate, final RowData current) { + return sequenceComparator == null + || DeduplicateUtils.replaces( + keepMode, sequenceComparator.compare(candidate, current)); + } + + private void emitCandidate(final RowKind kind, final BinaryRowData candidate) { + projectedCandidate.replaceRow(candidate); + projectedCandidate.setRowKind(kind); + collect(projectedCandidate); + } + + /** Wraps the current row so that the row owned by state never changes its row kind. */ + private void emitRetraction(final BinaryRowData current) { + projectedCurrent.replaceRow(current); + retraction.replace(projectedCurrent, emptyRow); + retraction.setRowKind(RowKind.UPDATE_BEFORE); + collect(retraction); + } + + private static @Nullable SequenceComparator sequenceComparator( + final SpecializedContext context, final List inputFields) { + final int[] outputIndices = + ChangelogTypeStrategyUtils.computeOutputIndices( + DeduplicateUtils.tableSemantics(context)); + final int[] sequenceIndices = + Arrays.stream( + DeduplicateTypeStrategy.sequenceByOutputIndices( + context.getCallContext())) + .map(i -> outputIndices[i]) + .toArray(); + if (sequenceIndices.length == 0) { + return null; + } + final DataType[] types = + Arrays.stream(sequenceIndices) + .mapToObj(i -> inputFields.get(i).getDataType()) + .toArray(DataType[]::new); + return new SequenceComparator(context, sequenceIndices, types); + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateEmitOnceFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateEmitOnceFunction.java new file mode 100644 index 00000000000000..9c23e7562b0241 --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateEmitOnceFunction.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.ptf; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.dataview.ValueView; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.utils.ProjectedRowData; +import org.apache.flink.table.functions.FunctionContext; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; +import org.apache.flink.types.ColumnList; +import org.apache.flink.types.RowKind; + +import javax.annotation.Nullable; + +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL; + +/** + * {@link Deduplicate} with {@code keep => 'first'} and no {@code sequence_by}: no later record can + * replace the first one, so only whether the key has emitted is kept in the {@code emitted} state. + */ +@Internal +public class DeduplicateEmitOnceFunction extends DeduplicateFunctionBase { + + private static final long serialVersionUID = 1L; + + private static final String EMITTED_STATE = "emitted"; + + private transient ProjectedRowData projectedOutput; + + DeduplicateEmitOnceFunction(final SpecializedContext context) { + super( + context, + valueState(context, EMITTED_STATE, DataTypes.BOOLEAN().notNull(), ARG_STATE_TTL)); + } + + @Override + public void open(final FunctionContext context) throws Exception { + super.open(context); + projectedOutput = ProjectedRowData.from(outputIndices); + } + + public void eval( + final Context ctx, + final ValueView emitted, + final RowData input, + @Nullable final StringData keep, + @Nullable final ColumnList sequenceBy, + @Nullable final Long stateTtl, + @Nullable final Boolean resetTtlOnDuplicate) + throws Exception { + if (emitted.getValue() == null) { + projectedOutput.replaceRow(input); + projectedOutput.setRowKind(RowKind.INSERT); + collect(projectedOutput); + emitted.setValue(true); + } else if (isResetTtlOnDuplicate) { + emitted.setValue(true); + } + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFunctionBase.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFunctionBase.java new file mode 100644 index 00000000000000..f22d583805a9d9 --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFunctionBase.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.ptf; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; +import org.apache.flink.table.types.inference.CallContext; +import org.apache.flink.table.types.inference.StateTypeStrategy; +import org.apache.flink.table.types.inference.strategies.ChangelogTypeStrategyUtils; + +import java.time.Duration; +import java.util.LinkedHashMap; + +import static org.apache.flink.table.api.config.ExecutionConfigOptions.IDLE_STATE_RETENTION; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_RESET_TTL_ON_DUPLICATE; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL; + +/** Base class of the {@link Deduplicate} implementations. */ +@Internal +abstract class DeduplicateFunctionBase extends BuiltInProcessTableFunction { + + private static final long serialVersionUID = 1L; + + protected final int[] outputIndices; + + protected final boolean isResetTtlOnDuplicate; + + DeduplicateFunctionBase( + final SpecializedContext context, + final LinkedHashMap stateStrategies) { + super(BuiltInFunctionDefinitions.DEDUPLICATE, context, stateStrategies); + final CallContext callContext = context.getCallContext(); + this.outputIndices = + ChangelogTypeStrategyUtils.computeOutputIndices( + DeduplicateUtils.tableSemantics(context)); + final Duration effectiveTtl = + callContext + .getArgumentValue(ARG_STATE_TTL, Duration.class) + .orElseGet(() -> context.getConfiguration().get(IDLE_STATE_RETENTION)); + this.isResetTtlOnDuplicate = + callContext.getArgumentValue(ARG_RESET_TTL_ON_DUPLICATE, Boolean.class).orElse(true) + && !effectiveTtl.isZero(); + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java new file mode 100644 index 00000000000000..3e06149db3e87a --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.ptf; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.functions.FunctionContext; +import org.apache.flink.table.functions.SpecializedFunction.ExpressionEvaluator; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; +import org.apache.flink.table.functions.TableSemantics; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep; +import org.apache.flink.util.FlinkRuntimeException; + +import javax.annotation.Nullable; + +import java.io.Serializable; +import java.lang.invoke.MethodHandle; + +import static org.apache.flink.table.api.Expressions.$; +import static org.apache.flink.table.api.Expressions.ifThenElse; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_INPUT; + +/** Record ordering of the built-in deduplication PTFs. */ +@Internal +public final class DeduplicateUtils { + + public static TableSemantics tableSemantics(final SpecializedContext context) { + return context.getCallContext() + .getTableSemantics(ARG_INPUT) + .orElseThrow(() -> new IllegalStateException("Table argument expected.")); + } + + /** + * Returns whether a new record replaces the current one, given the order of the new record + * relative to the current one: with {@code last} the later arrival wins on a tie, with {@code + * first} only a strictly smaller record replaces. + */ + public static boolean replaces(final Keep keep, final int order) { + return keep == Keep.LAST ? order >= 0 : order < 0; + } + + /** + * Compares two records by their {@code sequence_by} values in ascending order. The first column + * decides, later ones break ties, and NULL orders below every other value. + */ + @Internal + public static final class SequenceComparator implements Serializable { + + private static final long serialVersionUID = 1L; + + private final int[] indices; + + private final DataType[] types; + + private final ExpressionEvaluator[] evaluators; + + private transient MethodHandle[] compareHandles; + + private transient RowData.FieldGetter[] getters; + + /** Creates a comparator of the columns at {@code indices}, which have the given types. */ + public SequenceComparator( + final SpecializedContext context, final int[] indices, final DataType[] types) { + this.indices = indices; + this.types = types; + this.evaluators = new ExpressionEvaluator[types.length]; + for (int i = 0; i < types.length; i++) { + final DataType type = types[i].notNull().toInternal(); + evaluators[i] = + context.createEvaluator( + ifThenElse( + $("a").isLess($("b")), + -1, + ifThenElse($("a").isGreater($("b")), 1, 0)), + DataTypes.INT().notNull(), + DataTypes.FIELD("a", type), + DataTypes.FIELD("b", type)); + } + } + + public void open(final FunctionContext context) throws Exception { + compareHandles = new MethodHandle[types.length]; + getters = new RowData.FieldGetter[types.length]; + for (int i = 0; i < types.length; i++) { + compareHandles[i] = evaluators[i].open(context); + getters[i] = RowData.createFieldGetter(types[i].getLogicalType(), indices[i]); + } + } + + public void close() throws Exception { + for (final ExpressionEvaluator evaluator : evaluators) { + evaluator.close(); + } + } + + /** + * Returns a negative number, zero, or a positive number if the left record orders before, + * equal to, or after the right one. + */ + public int compare(final RowData left, final RowData right) { + for (int i = 0; i < types.length; i++) { + final int order = + compare( + i, + getters[i].getFieldOrNull(left), + getters[i].getFieldOrNull(right)); + if (order != 0) { + return order; + } + } + return 0; + } + + private int compare( + final int column, @Nullable final Object left, @Nullable final Object right) { + if (left == null || right == null) { + return left == null ? (right == null ? 0 : -1) : 1; + } + try { + return (int) compareHandles[column].invoke(left, right); + } catch (Throwable t) { + throw new FlinkRuntimeException(t); + } + } + } + + private DeduplicateUtils() {} +}