From e3ce94fd0840253f2a87f6b1b860d722e5c2f8e5 Mon Sep 17 00:00:00 2001 From: Vas Shabu Date: Mon, 16 Nov 2026 10:01:25 +0000 Subject: [PATCH 1/9] [FLINK-41470][table] Add built-in DEDUPLICATE PTF for insert-only input --- .../docs/reference/pyflink.table/table.rst | 14 + flink-python/pyflink/table/__init__.py | 5 +- flink-python/pyflink/table/table.py | 60 ++- .../flink/table/api/PartitionedTable.java | 31 ++ .../flink/table/api/internal/TableImpl.java | 5 + .../functions/BuiltInFunctionDefinitions.java | 26 ++ .../strategies/DeduplicateTypeStrategy.java | 373 ++++++++++++++++++ .../runtime/functions/ptf/Deduplicate.java | 316 +++++++++++++++ .../functions/ptf/DeduplicateUtils.java | 153 +++++++ 9 files changed, 980 insertions(+), 3 deletions(-) create mode 100644 flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java create mode 100644 flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java create mode 100644 flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java diff --git a/flink-python/docs/reference/pyflink.table/table.rst b/flink-python/docs/reference/pyflink.table/table.rst index c4bd4eadfac070..4fae481ff3acf7 100644 --- a/flink-python/docs/reference/pyflink.table/table.rst +++ b/flink-python/docs/reference/pyflink.table/table.rst @@ -110,6 +110,7 @@ Example: Table.offset Table.order_by Table.over_window + Table.partition_by Table.print_schema Table.rename_columns Table.right_outer_join @@ -136,6 +137,19 @@ A table that has been grouped on a set of grouping keys. GroupedTable.flat_aggregate +PartitionedTable +================ + +A table that has been partitioned on a set of partition keys. + +.. currentmodule:: pyflink.table + +.. autosummary:: + :toctree: api/ + + PartitionedTable.deduplicate + + GroupWindowedTable ================== diff --git a/flink-python/pyflink/table/__init__.py b/flink-python/pyflink/table/__init__.py index 53ad751fcd8ec4..d14073b0b782e4 100644 --- a/flink-python/pyflink/table/__init__.py +++ b/flink-python/pyflink/table/__init__.py @@ -126,8 +126,8 @@ from pyflink.table.schema import Schema from pyflink.table.sql_dialect import SqlDialect from pyflink.table.statement_set import StatementSet -from pyflink.table.table import GroupWindowedTable, GroupedTable, OverWindowedTable, Table, \ - WindowGroupedTable +from pyflink.table.table import GroupWindowedTable, GroupedTable, OverWindowedTable, \ + PartitionedTable, Table, WindowGroupedTable from pyflink.table.table_config import TableConfig from pyflink.table.table_descriptor import TableDescriptor, FormatDescriptor from pyflink.table.table_environment import (TableEnvironment, StreamTableEnvironment) @@ -148,6 +148,7 @@ 'GroupedTable', 'GroupWindowedTable', 'OverWindowedTable', + 'PartitionedTable', 'WindowGroupedTable', 'ScalarFunction', 'TableFunction', diff --git a/flink-python/pyflink/table/table.py b/flink-python/pyflink/table/table.py index 68eef113bc03f9..d0f756e054f3cd 100644 --- a/flink-python/pyflink/table/table.py +++ b/flink-python/pyflink/table/table.py @@ -46,7 +46,8 @@ from pyflink.util.java_utils import to_jarray from pyflink.util.java_utils import to_j_explain_detail_arr -__all__ = ['Table', 'GroupedTable', 'GroupWindowedTable', 'OverWindowedTable', 'WindowGroupedTable'] +__all__ = ['Table', 'GroupedTable', 'GroupWindowedTable', 'OverWindowedTable', 'PartitionedTable', + 'WindowGroupedTable'] @PublicEvolving() @@ -205,6 +206,23 @@ def group_by(self, *fields: Expression) -> 'GroupedTable': """ return GroupedTable(self._j_table.groupBy(to_expression_jarray(fields)), self._t_env) + def partition_by(self, *fields: Expression) -> 'PartitionedTable': + """ + Partitions the table by the given keys for a process table function with set semantics. + All rows of a key are processed by the same parallel instance. + + Example: + :: + + >>> tab.partition_by(col('key')).deduplicate() + + :param fields: Partition keys. + :return: The partitioned table. + """ + return PartitionedTable( + self._j_table.partitionBy(to_expression_jarray(fields)), self._t_env + ) + def distinct(self) -> 'Table': """ Removes duplicate values and returns only distinct (different) values. @@ -1426,6 +1444,46 @@ def flat_aggregate(self, func: Union[Expression, UserDefinedAggregateFunctionWra return FlatAggregateTable(self._j_table.flatAggregate(func._j_expr), self._t_env) +@PublicEvolving() +class PartitionedTable(object): + """ + A table that has been partitioned on a set of partition keys. + """ + + def __init__(self, java_table, t_env): + self._j_table = java_table + self._t_env = t_env + + def deduplicate(self, *arguments: Expression) -> 'Table': + """ + Keeps one row per partition key using the built-in ``DEDUPLICATE`` process table + function. The result is an updating table with the partition columns as its upsert key. + + ``keep => 'first'`` (default) keeps the earliest arrival or the smallest ``sequence_by`` + value, ``keep => 'last'`` keeps the latest arrival or the greatest value. + + Example: + :: + + >>> import datetime + >>> from pyflink.table.expressions import col, descriptor, lit + >>> # Keep the first delivery of each payment + >>> result = payments.partition_by(col('payment_id')).deduplicate() + >>> # Keep the version with the greatest 'lsn' per customer, with bounded retention + >>> result = customer_changes.partition_by(col('customer_id')).deduplicate( + ... lit('last').as_argument('keep'), + ... descriptor('lsn').as_argument('sequence_by'), + ... lit(datetime.timedelta(days=1)).as_argument('state_ttl') + ... ) + + :param arguments: Optional named arguments for ``keep``, ``sequence_by``, ``state_ttl``, + ``reset_ttl_on_duplicate``, and ``uid``. + :return: An updating :class:`~pyflink.table.Table` with the partition columns followed + by the remaining input columns. + """ + return Table(self._j_table.deduplicate(to_expression_jarray(arguments)), self._t_env) + + @PublicEvolving() class GroupWindowedTable(object): """ diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java index ecf58fbe9b3914..31371edd0e72ad 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java @@ -290,4 +290,35 @@ public interface PartitionedTable { * @see #orderBy(Expression...) */ Table fromChangelog(Expression... arguments); + + /** + * Keeps one row per partition key using the built-in {@code DEDUPLICATE} process table + * function. The result is an updating table with the partition columns as its upsert key. + * + *

{@code keep => 'first'} (default) keeps the earliest arrival or the smallest {@code + * sequence_by} value, {@code keep => 'last'} keeps the latest arrival or the greatest value. + * + *

Examples: + * + *

{@code
+     * // Keep the first delivery of each payment
+     * Table result = payments
+     *     .partitionBy($("payment_id"))
+     *     .deduplicate();
+     *
+     * // Keep the version with the greatest 'lsn' per customer, with bounded state retention
+     * Table result = customerChanges
+     *     .partitionBy($("customer_id"))
+     *     .deduplicate(
+     *         lit("last").asArgument("keep"),
+     *         descriptor("lsn").asArgument("sequence_by"),
+     *         lit(Duration.ofDays(1)).asArgument("state_ttl"));
+     * }
+ * + * @param arguments optional named arguments for {@code keep}, {@code sequence_by}, {@code + * state_ttl}, {@code reset_ttl_on_duplicate}, and {@code uid} + * @return an updating {@link Table} with output schema {@code [partition_keys, + * non_partition_input_columns]} + */ + Table deduplicate(Expression... arguments); } diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableImpl.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableImpl.java index c3c9cfa7f9db83..31a00a1cd503a2 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableImpl.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableImpl.java @@ -962,6 +962,11 @@ public Table fromChangelog(final Expression... arguments) { BuiltInFunctionDefinitions.FROM_CHANGELOG.getName(), (Object[]) arguments); } + @Override + public Table deduplicate(final Expression... arguments) { + return process(BuiltInFunctionDefinitions.DEDUPLICATE.getName(), (Object[]) arguments); + } + private QueryOperation createQueryOperation() { if (orderKeys.isEmpty()) { return table.operationTreeBuilder.partition(partitionKeys, table.operationTree); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java index d70f1c50725cd8..36703573e40f03 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java @@ -38,6 +38,7 @@ import org.apache.flink.table.types.inference.TraitCondition; import org.apache.flink.table.types.inference.TypeStrategies; import org.apache.flink.table.types.inference.strategies.ArrayOfStringArgumentTypeStrategy; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy; import org.apache.flink.table.types.inference.strategies.FromChangelogTypeStrategy; import org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies; import org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies; @@ -939,6 +940,31 @@ ANY, and(logical(LogicalTypeRoot.BOOLEAN), LITERAL) "org.apache.flink.table.runtime.functions.ptf.FromChangelogFunction") .build(); + public static final BuiltInFunctionDefinition DEDUPLICATE = + BuiltInFunctionDefinition.newBuilder() + .name("DEDUPLICATE") + .kind(PROCESS_TABLE) + .staticArguments( + StaticArgument.table( + "input", + Row.class, + false, + EnumSet.of( + StaticArgumentTrait.TABLE, + StaticArgumentTrait.SET_SEMANTIC_TABLE, + StaticArgumentTrait.SUPPORT_UPDATES)), + StaticArgument.scalar("keep", DataTypes.STRING(), true), + StaticArgument.scalar("sequence_by", DataTypes.DESCRIPTOR(), true), + StaticArgument.scalar( + "state_ttl", DataTypes.INTERVAL(DataTypes.SECOND()), true), + StaticArgument.scalar( + "reset_ttl_on_duplicate", DataTypes.BOOLEAN(), true)) + .changelogModeStrategy(DeduplicateTypeStrategy.CHANGELOG_MODE_STRATEGY) + .inputTypeStrategy(DeduplicateTypeStrategy.INPUT_TYPE_STRATEGY) + .outputTypeStrategy(DeduplicateTypeStrategy.OUTPUT_TYPE_STRATEGY) + .runtimeClass("org.apache.flink.table.runtime.functions.ptf.Deduplicate") + .build(); + /** * Built-in proxy function for the LATERAL SNAPSHOT temporal join. * diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java new file mode 100644 index 00000000000000..09353bab44d4b0 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java @@ -0,0 +1,373 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.inference.strategies; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.DataTypes.Field; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.functions.ChangelogFunction.ChangelogContext; +import org.apache.flink.table.functions.ChangelogModeStrategy; +import org.apache.flink.table.functions.TableSemantics; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.CallContext; +import org.apache.flink.table.types.inference.InputTypeStrategy; +import org.apache.flink.table.types.inference.TypeStrategy; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.LogicalTypeFamily; +import org.apache.flink.table.types.logical.LogicalTypeRoot; +import org.apache.flink.table.types.logical.utils.LogicalTypeChecks; +import org.apache.flink.table.types.utils.DataTypeUtils; +import org.apache.flink.types.ColumnList; +import org.apache.flink.types.RowKind; +import org.apache.flink.util.TimeUtils; + +import java.time.Duration; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.flink.table.types.inference.SystemTypeInference.PROCESS_TABLE_FUNCTION_ARG_ON_TIME_OFFSET; + +/** + * Type strategies for the {@code DEDUPLICATE} process table function. + * + *

The output is updating and keyed by the PARTITION BY columns: the PARTITION BY columns + * followed by the remaining input columns, without time attributes. + */ +@Internal +public final class DeduplicateTypeStrategy { + + public static final int ARG_INPUT = 0; + public static final int ARG_KEEP = 1; + public static final int ARG_SEQUENCE_BY = 2; + public static final int ARG_STATE_TTL = 3; + public static final int ARG_RESET_TTL_ON_DUPLICATE = 4; + private static final int DECLARED_ARG_COUNT = ARG_RESET_TTL_ON_DUPLICATE + 1; + + /** Which record of a key {@code DEDUPLICATE} keeps. */ + @Internal + public enum Keep { + FIRST("first"), + LAST("last"); + + private final String value; + + Keep(final String value) { + this.value = value; + } + + /** Returns the mode for an argument value, or empty if the value names no mode. */ + public static Optional fromValue(final String value) { + return Arrays.stream(values()).filter(keep -> keep.value.equals(value)).findFirst(); + } + } + + public static final InputTypeStrategy INPUT_TYPE_STRATEGY = + new ValidationOnlyInputTypeStrategy() { + @Override + public Optional> inferInputTypes( + final CallContext callContext, final boolean throwOnFailure) { + final Optional error = validate(callContext); + if (error.isPresent()) { + return callContext.fail(throwOnFailure, "%s", error.get()); + } + return Optional.of(callContext.getArgumentDataTypes()); + } + }; + + public static final TypeStrategy OUTPUT_TYPE_STRATEGY = + callContext -> + Optional.of(DataTypes.ROW(outputFields(tableSemantics(callContext))).notNull()); + + public static final ChangelogModeStrategy CHANGELOG_MODE_STRATEGY = + DeduplicateTypeStrategy::inferChangelogMode; + + /** Returns the {@code keep} argument, or {@link Keep#FIRST} when it is omitted. */ + public static Keep keep(final CallContext callContext) { + return callContext + .getArgumentValue(ARG_KEEP, String.class) + .map( + value -> + Keep.fromValue(value) + .orElseThrow( + () -> + new IllegalStateException( + "Validated keep argument expected."))) + .orElse(Keep.FIRST); + } + + /** + * Returns the positions of the {@code sequence_by} columns, in the listed order, within the + * output row without the PARTITION BY columns. + */ + public static int[] sequenceByOutputIndices(final CallContext callContext) { + final TableSemantics tableSemantics = tableSemantics(callContext); + final List fieldNames = DataType.getFieldNames(tableSemantics.dataType()); + final List outputIndices = + Arrays.stream(ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics)) + .boxed() + .collect(Collectors.toList()); + return sequenceBy(callContext).stream() + .mapToInt(column -> outputIndices.indexOf(fieldNames.indexOf(column))) + .toArray(); + } + + private static Optional validate(final CallContext callContext) { + final TableSemantics tableSemantics = tableSemantics(callContext); + return validatePartitionBy(tableSemantics) + .or(() -> validateNoOrderBy(tableSemantics)) + .or(() -> validatePartitionByNoTimeAttribute(tableSemantics)) + .or(() -> validateOnTime(callContext)) + .or(() -> validateKeep(callContext)) + .or(() -> validateSequenceBy(callContext, tableSemantics)) + .or(() -> validateStateTtl(callContext)) + .or(() -> validateResetTtlOnDuplicate(callContext)); + } + + private static Optional validatePartitionBy(final TableSemantics tableSemantics) { + if (tableSemantics.partitionByColumns().length > 0) { + return Optional.empty(); + } + return Optional.of( + "DEDUPLICATE requires a key, but the input has no PARTITION BY. Add PARTITION BY " + + "with the columns to deduplicate on, for example TABLE t PARTITION BY " + + "id. To drop only rows that repeat in full, partition by every column."); + } + + private static Optional validateNoOrderBy(final TableSemantics tableSemantics) { + if (tableSemantics.orderByColumns().length == 0) { + return Optional.empty(); + } + final List fieldNames = DataType.getFieldNames(tableSemantics.dataType()); + final String columns = + Arrays.stream(tableSemantics.orderByColumns()) + .mapToObj(fieldNames::get) + .collect(Collectors.joining(", ")); + return Optional.of( + String.format( + "DEDUPLICATE does not support ORDER BY on its input, but received " + + "ORDER BY %s. Use sequence_by => DESCRIPTOR(%s) to order the " + + "records of a key.", + columns, columns)); + } + + private static Optional validatePartitionByNoTimeAttribute( + final TableSemantics tableSemantics) { + final List fields = DataType.getFields(tableSemantics.dataType()); + return Arrays.stream(tableSemantics.partitionByColumns()) + .mapToObj(fields::get) + .filter( + field -> + LogicalTypeChecks.isTimeAttribute( + field.getDataType().getLogicalType())) + .findFirst() + .map( + field -> + String.format( + "DEDUPLICATE does not support a time attribute as a " + + "PARTITION BY column, but '%s' is one. Partition " + + "by a regular column, or cast the column in a " + + "subquery first, for example CAST(%s AS %s) AS %s.", + field.getName(), + field.getName(), + DataTypeUtils.removeTimeAttribute(field.getDataType()) + .getLogicalType() + .asSummaryString(), + field.getName())); + } + + private static Optional validateOnTime(final CallContext callContext) { + final int argCount = callContext.getArgumentDataTypes().size(); + if (argCount == DECLARED_ARG_COUNT) { + return Optional.empty(); + } + final int onTimePos = argCount - 1 - PROCESS_TABLE_FUNCTION_ARG_ON_TIME_OFFSET; + return callContext + .getArgumentValue(onTimePos, ColumnList.class) + .map(ColumnList::getNames) + .filter(names -> !names.isEmpty()) + .map( + names -> + String.format( + "DEDUPLICATE does not support on_time, but received " + + "on_time => DESCRIPTOR(%s). Use sequence_by => " + + "DESCRIPTOR(%s) to order by timestamp, or " + + "DEDUPLICATE_WINDOW for event-time windows." + + "ps.this function is not currently implemented", + names.get(0), names.get(0))); + } + + private static Optional validateKeep(final CallContext callContext) { + if (callContext.isArgumentNull(ARG_KEEP)) { + return Optional.empty(); + } + final String received; + if (!callContext.isArgumentLiteral(ARG_KEEP)) { + received = "a non-literal expression"; + } else { + final String value = callContext.getArgumentValue(ARG_KEEP, String.class).orElse(null); + if (value == null || Keep.fromValue(value).isPresent()) { + return Optional.empty(); + } + received = String.format("keep => '%s'", value); + } + return Optional.of( + String.format( + "DEDUPLICATE requires keep => 'first' or keep => 'last', but received %s. " + + "Use keep => 'first' to keep the first record per key, or " + + "keep => 'last' to keep the last one.", + received)); + } + + private static Optional validateSequenceBy( + final CallContext callContext, final TableSemantics tableSemantics) { + final List fieldNames = DataType.getFieldNames(tableSemantics.dataType()); + final List fieldTypes = DataType.getFieldDataTypes(tableSemantics.dataType()); + final Set seen = new HashSet<>(); + for (final String column : sequenceBy(callContext)) { + final int index = fieldNames.indexOf(column); + if (index == -1) { + return Optional.of( + String.format( + "The sequence_by column '%s' does not exist in the input of " + + "DEDUPLICATE. Available columns are: %s.", + column, String.join(", ", fieldNames))); + } + if (!seen.add(column)) { + return Optional.of( + String.format( + "The sequence_by column '%s' is listed more than once. List each " + + "ordering column once.", + column)); + } + if (Arrays.stream(tableSemantics.partitionByColumns()).anyMatch(c -> c == index)) { + return Optional.of( + String.format( + "The sequence_by column '%s' is a PARTITION BY column, which has " + + "the same value for every record of a key. Use a " + + "column that is not part of PARTITION BY, such as a " + + "version or sequence number.", + column)); + } + final LogicalType type = fieldTypes.get(index).getLogicalType(); + if (!isOrderable(type)) { + return Optional.of( + String.format( + "The sequence_by column '%s' has type '%s', which cannot order " + + "records. Use a numeric, character string, binary " + + "string, boolean, or date-time column.", + column, type.asSummaryString())); + } + } + return Optional.empty(); + } + + private static Optional validateStateTtl(final CallContext callContext) { + if (callContext.isArgumentNull(ARG_STATE_TTL)) { + return Optional.empty(); + } + if (!callContext.isArgumentLiteral(ARG_STATE_TTL)) { + return Optional.of( + "The 'state_ttl' argument must be a constant INTERVAL literal, but received a " + + "non-literal expression. Use a literal such as INTERVAL '1' HOUR, " + + "INTERVAL '0' to retain state indefinitely, or omit the argument " + + "to fall back to 'table.exec.state.ttl'."); + } + return callContext + .getArgumentValue(ARG_STATE_TTL, Duration.class) + .filter(Duration::isNegative) + .map( + stateTtl -> + String.format( + "The 'state_ttl' argument must not be negative, but was " + + "'%s'. Use a positive INTERVAL to bound state " + + "retention, INTERVAL '0' to retain state " + + "indefinitely, or omit the argument to fall " + + "back to 'table.exec.state.ttl'.", + TimeUtils.formatWithHighestUnit(stateTtl))); + } + + private static Optional validateResetTtlOnDuplicate(final CallContext callContext) { + if (callContext.isArgumentNull(ARG_RESET_TTL_ON_DUPLICATE) + || callContext.isArgumentLiteral(ARG_RESET_TTL_ON_DUPLICATE)) { + return Optional.empty(); + } + return Optional.of( + "The 'reset_ttl_on_duplicate' argument must be a constant BOOLEAN literal, but " + + "received a non-literal expression. Use TRUE to refresh the TTL on a " + + "dropped record, FALSE to keep it, or omit the argument for the default " + + "TRUE."); + } + + private static ChangelogMode inferChangelogMode(final ChangelogContext changelogContext) { + final ChangelogMode requiredMode = changelogContext.getRequiredChangelogMode(); + final ChangelogMode.Builder builder = + ChangelogMode.newBuilder() + .addContainedKind(RowKind.INSERT) + .addContainedKind(RowKind.UPDATE_AFTER); + if (requiredMode != null && requiredMode.contains(RowKind.UPDATE_BEFORE)) { + builder.addContainedKind(RowKind.UPDATE_BEFORE); + } + return builder.build(); + } + + /** Returns the output columns after the PARTITION BY columns, without time attributes. */ + public static List outputFields(final TableSemantics tableSemantics) { + final List inputFields = DataType.getFields(tableSemantics.dataType()); + return Arrays.stream(ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics)) + .mapToObj(inputFields::get) + .map( + field -> + DataTypes.FIELD( + field.getName(), + DataTypeUtils.removeTimeAttribute(field.getDataType()))) + .collect(Collectors.toList()); + } + + private static List sequenceBy(final CallContext callContext) { + return callContext + .getArgumentValue(ARG_SEQUENCE_BY, ColumnList.class) + .map(ColumnList::getNames) + .orElse(List.of()); + } + + private static boolean isOrderable(final LogicalType type) { + return type.is(LogicalTypeFamily.NUMERIC) + || type.is(LogicalTypeFamily.CHARACTER_STRING) + || type.is(LogicalTypeFamily.BINARY_STRING) + || type.is(LogicalTypeFamily.DATETIME) + || type.is(LogicalTypeRoot.BOOLEAN); + } + + private static TableSemantics tableSemantics(final CallContext callContext) { + return callContext + .getTableSemantics(ARG_INPUT) + .orElseThrow( + () -> + new ValidationException( + "First argument must be a table for DEDUPLICATE.")); + } + + private DeduplicateTypeStrategy() {} +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java new file mode 100644 index 00000000000000..ed4befdb69cbfa --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.ptf; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.dataview.ValueView; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.binary.BinaryRowData; +import org.apache.flink.table.data.utils.JoinedRowData; +import org.apache.flink.table.data.utils.ProjectedRowData; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.FunctionContext; +import org.apache.flink.table.functions.TableSemantics; +import org.apache.flink.table.functions.UserDefinedFunction; +import org.apache.flink.table.runtime.functions.BuiltInSpecializedFunction; +import org.apache.flink.table.runtime.functions.ptf.DeduplicateUtils.SequenceComparator; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.CallContext; +import org.apache.flink.table.types.inference.StateTypeStrategy; +import org.apache.flink.table.types.inference.strategies.ChangelogTypeStrategyUtils; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.Keep; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.types.ColumnList; +import org.apache.flink.types.RowKind; + +import javax.annotation.Nullable; + +import java.time.Duration; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_INPUT; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_RESET_TTL_ON_DUPLICATE; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL; + +/** Runtime implementation of {@link BuiltInFunctionDefinitions#DEDUPLICATE}. */ +@Internal +public class Deduplicate extends BuiltInSpecializedFunction { + + public Deduplicate() { + super(BuiltInFunctionDefinitions.DEDUPLICATE); + } + + @Override + public UserDefinedFunction specialize(final SpecializedContext context) { + final ChangelogMode inputMode = + tableSemantics(context) + .changelogMode() + .orElseThrow(() -> new IllegalStateException("Changelog mode expected.")); + if (!inputMode.containsOnly(RowKind.INSERT)) { + throw new IllegalStateException("Currently not implemented"); + } + final CallContext callContext = context.getCallContext(); + final boolean isFirstArrival = + DeduplicateTypeStrategy.keep(callContext) == Keep.FIRST + && DeduplicateTypeStrategy.sequenceByOutputIndices(callContext).length == 0; + return isFirstArrival ? new FirstArrival(context) : new Sequenced(context); + } + + private abstract static class Base extends BuiltInProcessTableFunction { + + private static final long serialVersionUID = 1L; + + protected final int[] outputIndices; + + protected final boolean isResetTtlOnDuplicate; + + Base( + final SpecializedContext context, + final LinkedHashMap stateStrategies) { + super(BuiltInFunctionDefinitions.DEDUPLICATE, context, stateStrategies); + final CallContext callContext = context.getCallContext(); + this.outputIndices = + ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics(context)); + this.isResetTtlOnDuplicate = + callContext + .getArgumentValue(ARG_RESET_TTL_ON_DUPLICATE, Boolean.class) + .orElse(true) + && !Duration.ZERO.equals( + callContext + .getArgumentValue(ARG_STATE_TTL, Duration.class) + .orElse(null)); + } + } + + /** + * {@code keep => 'first'} without {@code sequence_by}: no later record can replace the first + * one, so only whether the key has emitted is kept in the {@code seen} state. + */ + @Internal + public static class FirstArrival extends Base { + + private static final long serialVersionUID = 1L; + + private transient ProjectedRowData projectedOutput; + + FirstArrival(final SpecializedContext context) { + super(context, internalState(Map.entry("seen", seenState(context)))); + } + + @Override + public void open(final FunctionContext context) throws Exception { + super.open(context); + projectedOutput = ProjectedRowData.from(outputIndices); + } + + public void eval( + final Context ctx, + final ValueView seen, + final RowData input, + @Nullable final StringData keep, + @Nullable final ColumnList sequenceBy, + @Nullable final Long stateTtl, + @Nullable final Boolean resetTtlOnDuplicate) + throws Exception { + if (seen.getValue() == null) { + projectedOutput.replaceRow(input); + projectedOutput.setRowKind(RowKind.INSERT); + collect(projectedOutput); + seen.setValue(true); + } else if (isResetTtlOnDuplicate) { + seen.setValue(true); + } + } + } + + /** + * {@code keep => 'last'}, or {@code keep => 'first'} with {@code sequence_by}: a later record + * can replace the kept one, so the kept record is held in the {@code kept} state. + */ + @Internal + public static class Sequenced extends Base { + + private static final long serialVersionUID = 1L; + + private final Keep keep; + + private final LogicalType[] payloadTypes; + + private final boolean isUpdateBeforeEmitted; + + private final @Nullable SequenceComparator sequenceComparator; + + private transient RowData.FieldGetter[] payloadGetters; + + private transient RowDataSerializer candidateSerializer; + + private transient RowDataSerializer currentSerializer; + + private transient GenericRowData reusedCandidate; + + private transient JoinedRowData output; + + private transient RowData emptyRow; + + Sequenced(final SpecializedContext context) { + super(context, internalState(Map.entry("kept", keptState(context)))); + final CallContext callContext = context.getCallContext(); + this.keep = DeduplicateTypeStrategy.keep(callContext); + this.payloadTypes = + DeduplicateTypeStrategy.outputFields(tableSemantics(context)).stream() + .map(field -> field.getDataType().getLogicalType()) + .toArray(LogicalType[]::new); + this.isUpdateBeforeEmitted = + callContext + .getOutputChangelogMode() + .map(mode -> mode.contains(RowKind.UPDATE_BEFORE)) + .orElse(false); + this.sequenceComparator = sequenceComparator(context, outputIndices); + } + + @Override + public void open(final FunctionContext context) throws Exception { + super.open(context); + payloadGetters = new RowData.FieldGetter[outputIndices.length]; + for (int i = 0; i < outputIndices.length; i++) { + payloadGetters[i] = RowData.createFieldGetter(payloadTypes[i], outputIndices[i]); + } + if (sequenceComparator != null) { + sequenceComparator.open(context); + } + // toBinaryRow reuses its result row, so each side of a comparison needs its own + // serializer + candidateSerializer = new RowDataSerializer(payloadTypes); + currentSerializer = new RowDataSerializer(payloadTypes); + reusedCandidate = new GenericRowData(outputIndices.length); + output = new JoinedRowData(); + emptyRow = GenericRowData.of(); + } + + @Override + public void close() throws Exception { + if (sequenceComparator != null) { + sequenceComparator.close(); + } + super.close(); + } + + public void eval( + final Context ctx, + final ValueView keptState, + final RowData input, + @Nullable final StringData keep, + @Nullable final ColumnList sequenceBy, + @Nullable final Long stateTtl, + @Nullable final Boolean resetTtlOnDuplicate) + throws Exception { + final RowData kept = keptState.getValue(); + if (kept == null) { + final BinaryRowData candidate = toCandidate(input); + emit(RowKind.INSERT, candidate); + keptState.setValue(candidate.copy()); + return; + } + final int order = + sequenceComparator == null ? 0 : sequenceComparator.compare(input, kept); + if (DeduplicateUtils.replaces(this.keep, order)) { + final BinaryRowData candidate = toCandidate(input); + if (!candidate.equals(currentSerializer.toBinaryRow(kept))) { + if (isUpdateBeforeEmitted) { + emit(RowKind.UPDATE_BEFORE, kept); + } + emit(RowKind.UPDATE_AFTER, candidate); + keptState.setValue(candidate.copy()); + return; + } + } + if (isResetTtlOnDuplicate) { + keptState.setValue(kept); + } + } + + /** Returns the payload of the input as a reused binary row, valid until the next call. */ + private BinaryRowData toCandidate(final RowData input) { + for (int i = 0; i < outputIndices.length; i++) { + reusedCandidate.setField(i, payloadGetters[i].getFieldOrNull(input)); + } + return candidateSerializer.toBinaryRow(reusedCandidate); + } + + /** Wraps the payload so that rows owned by state never change their row kind. */ + private void emit(final RowKind kind, final RowData payload) { + output.replace(payload, emptyRow); + output.setRowKind(kind); + collect(output); + } + } + + private static TableSemantics tableSemantics(final SpecializedContext context) { + return context.getCallContext() + .getTableSemantics(ARG_INPUT) + .orElseThrow(() -> new IllegalStateException("Table argument expected.")); + } + + private static StateTypeStrategy seenState(final SpecializedContext context) { + return state(context, DataTypes.BOOLEAN().notNull()); + } + + private static StateTypeStrategy keptState(final SpecializedContext context) { + return state( + context, + DataTypes.ROW(DeduplicateTypeStrategy.outputFields(tableSemantics(context)))); + } + + private static StateTypeStrategy state( + final SpecializedContext context, final DataType valueType) { + return StateTypeStrategy.of( + callContext -> Optional.of(ValueView.newValueViewDataType(valueType)), + context.getCallContext() + .getArgumentValue(ARG_STATE_TTL, Duration.class) + .orElse(null)); + } + + private static @Nullable SequenceComparator sequenceComparator( + final SpecializedContext context, final int[] outputIndices) { + final int[] payloadIndices = + DeduplicateTypeStrategy.sequenceByOutputIndices(context.getCallContext()); + if (payloadIndices.length == 0) { + return null; + } + final List payloadFields = + DeduplicateTypeStrategy.outputFields(tableSemantics(context)); + final int[] inputIndices = + Arrays.stream(payloadIndices).map(i -> outputIndices[i]).toArray(); + final DataType[] types = + Arrays.stream(payloadIndices) + .mapToObj(i -> payloadFields.get(i).getDataType()) + .toArray(DataType[]::new); + return new SequenceComparator(context, inputIndices, payloadIndices, types); + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java new file mode 100644 index 00000000000000..cb3666679e14cc --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.ptf; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.functions.FunctionContext; +import org.apache.flink.table.functions.SpecializedFunction.ExpressionEvaluator; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.Keep; +import org.apache.flink.util.FlinkRuntimeException; + +import javax.annotation.Nullable; + +import java.io.Serializable; +import java.lang.invoke.MethodHandle; + +import static org.apache.flink.table.api.Expressions.$; +import static org.apache.flink.table.api.Expressions.ifThenElse; + +/** Record ordering of the built-in deduplication PTFs. */ +@Internal +public final class DeduplicateUtils { + + /** + * Returns whether a new record replaces the current one, given the order of the new record + * relative to the current one: with {@code last} the later arrival wins on a tie, with {@code + * first} only a strictly smaller record replaces. + */ + public static boolean replaces(final Keep keep, final int order) { + return keep == Keep.LAST ? order >= 0 : order < 0; + } + + /** + * Compares two records by their {@code sequence_by} values in ascending order. The first column + * decides, later ones break ties, and NULL orders below every other value. + */ + @Internal + public static final class SequenceComparator implements Serializable { + + private static final long serialVersionUID = 1L; + + private final int[] leftIndices; + + private final int[] rightIndices; + + private final DataType[] types; + + private final ExpressionEvaluator[] evaluators; + + private transient MethodHandle[] compareHandles; + + private transient RowData.FieldGetter[] leftGetters; + + private transient RowData.FieldGetter[] rightGetters; + + /** + * Creates a comparator of the columns at {@code leftIndices} of the left record against the + * columns at {@code rightIndices} of the right record, both of the given types. + */ + public SequenceComparator( + final SpecializedContext context, + final int[] leftIndices, + final int[] rightIndices, + final DataType[] types) { + this.leftIndices = leftIndices; + this.rightIndices = rightIndices; + this.types = types; + this.evaluators = new ExpressionEvaluator[types.length]; + for (int i = 0; i < types.length; i++) { + final DataType type = types[i].notNull().toInternal(); + evaluators[i] = + context.createEvaluator( + ifThenElse( + $("a").isLess($("b")), + -1, + ifThenElse($("a").isGreater($("b")), 1, 0)), + DataTypes.INT().notNull(), + DataTypes.FIELD("a", type), + DataTypes.FIELD("b", type)); + } + } + + public void open(final FunctionContext context) throws Exception { + compareHandles = new MethodHandle[types.length]; + leftGetters = new RowData.FieldGetter[types.length]; + rightGetters = new RowData.FieldGetter[types.length]; + for (int i = 0; i < types.length; i++) { + compareHandles[i] = evaluators[i].open(context); + leftGetters[i] = + RowData.createFieldGetter(types[i].getLogicalType(), leftIndices[i]); + rightGetters[i] = + RowData.createFieldGetter(types[i].getLogicalType(), rightIndices[i]); + } + } + + public void close() throws Exception { + for (final ExpressionEvaluator evaluator : evaluators) { + evaluator.close(); + } + } + + /** + * Returns a negative number, zero, or a positive number if the left record orders before, + * equal to, or after the right one. + */ + public int compare(final RowData left, final RowData right) { + for (int i = 0; i < types.length; i++) { + final int order = + compare( + i, + leftGetters[i].getFieldOrNull(left), + rightGetters[i].getFieldOrNull(right)); + if (order != 0) { + return order; + } + } + return 0; + } + + private int compare( + final int column, @Nullable final Object left, @Nullable final Object right) { + if (left == null || right == null) { + return left == null ? (right == null ? 0 : -1) : 1; + } + try { + return (int) compareHandles[column].invoke(left, right); + } catch (Throwable t) { + throw new FlinkRuntimeException(t); + } + } + } + + private DeduplicateUtils() {} +} From cdbb2eb970b09377c9bf6ad8440cbb6f0f9c32be Mon Sep 17 00:00:00 2001 From: Vas Shabu Date: Mon, 16 Nov 2026 10:01:25 +0000 Subject: [PATCH 2/9] [FLINK-41470][table] Add tests for the DEDUPLICATE PTF on insert-only input --- .../pyflink/table/tests/test_explain.py | 16 + .../table/tests/test_table_completeness.py | 28 +- .../DeduplicateTypeStrategyTest.java | 291 +++++++++++++++ .../exec/stream/DeduplicateRestoreTest.java | 40 ++ .../DeduplicateRestoreTestPrograms.java | 142 +++++++ .../exec/stream/DeduplicateSemanticTests.java | 37 ++ .../exec/stream/DeduplicateTestPrograms.java | 348 ++++++++++++++++++ .../sql/DeduplicateTableFunctionTest.java | 103 ++++++ .../harness/DeduplicateHarnessTest.java | 227 ++++++++++++ .../sql/DeduplicateTableFunctionTest.xml | 104 ++++++ .../deduplicate-arrival-order-restore.json | 159 ++++++++ .../savepoint/_metadata | Bin 0 -> 12401 bytes .../deduplicate-first-arrival-restore.json | 160 ++++++++ .../savepoint/_metadata | Bin 0 -> 11317 bytes .../plan/deduplicate-sequence-by-restore.json | 168 +++++++++ .../savepoint/_metadata | Bin 0 -> 13762 bytes .../functions/ptf/DeduplicateTest.java | 177 +++++++++ 17 files changed, 1998 insertions(+), 2 deletions(-) create mode 100644 flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTest.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTestPrograms.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateSemanticTests.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateTestPrograms.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/harness/DeduplicateHarnessTest.java create mode 100644 flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.xml create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-arrival-order-restore/plan/deduplicate-arrival-order-restore.json create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-arrival-order-restore/savepoint/_metadata create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/plan/deduplicate-first-arrival-restore.json create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/savepoint/_metadata create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/plan/deduplicate-sequence-by-restore.json create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/savepoint/_metadata create mode 100644 flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateTest.java diff --git a/flink-python/pyflink/table/tests/test_explain.py b/flink-python/pyflink/table/tests/test_explain.py index b2a6a601369c08..11941d617a49a2 100644 --- a/flink-python/pyflink/table/tests/test_explain.py +++ b/flink-python/pyflink/table/tests/test_explain.py @@ -15,10 +15,12 @@ # See the License for the specific language governing permissions and # limitations under the License. ################################################################################ +import datetime import json from pyflink.testing.test_case_utils import PyFlinkStreamTableTestCase from pyflink.table.explain_detail import ExplainDetail +from pyflink.table.expressions import col, descriptor, lit class StreamTableExplainTests(PyFlinkStreamTableTestCase): @@ -44,6 +46,20 @@ def test_explain(self): except: self.fail('The execution plan of explain detail is not in json format.') + def test_explain_deduplicate(self): + payments = self.t_env.from_elements([('p1', 50)], ['payment_id', 'amount']) + result = payments.partition_by(col('payment_id')).deduplicate().explain() + self.assertGreaterEqual(result.find('DEDUPLICATE'), 0) + + customer_changes = self.t_env.from_elements( + [('c1', 'Berlin', 100)], ['customer_id', 'city', 'lsn']) + result = customer_changes.partition_by(col('customer_id')).deduplicate( + lit('last').as_argument('keep'), + descriptor('lsn').as_argument('sequence_by'), + lit(datetime.timedelta(days=1)).as_argument('state_ttl') + ).explain() + self.assertGreaterEqual(result.find('DEDUPLICATE'), 0) + if __name__ == '__main__': import unittest diff --git a/flink-python/pyflink/table/tests/test_table_completeness.py b/flink-python/pyflink/table/tests/test_table_completeness.py index 16866cca8d5dda..f201e9505a64b6 100644 --- a/flink-python/pyflink/table/tests/test_table_completeness.py +++ b/flink-python/pyflink/table/tests/test_table_completeness.py @@ -17,7 +17,7 @@ ################################################################################ from pyflink.testing.test_case_utils import PythonAPICompletenessTestCase, PyFlinkTestCase -from pyflink.table import Table +from pyflink.table import PartitionedTable, Table class TableAPICompletenessTests(PythonAPICompletenessTestCase, PyFlinkTestCase): @@ -41,7 +41,6 @@ def excluded_methods(cls): 'getQueryOperation', 'asArgument', 'process', - 'partitionBy', 'materializeInto', } @@ -57,6 +56,31 @@ def java_method_name(cls, python_method_name): return {'alias': 'as'}.get(python_method_name, python_method_name) +class PartitionedTableAPICompletenessTests(PythonAPICompletenessTestCase, PyFlinkTestCase): + """ + Tests whether the Python :class:`PartitionedTable` is consistent with + Java `org.apache.flink.table.api.PartitionedTable`. + """ + + @classmethod + def python_class(cls): + return PartitionedTable + + @classmethod + def java_class(cls): + return "org.apache.flink.table.api.PartitionedTable" + + @classmethod + def excluded_methods(cls): + return { + 'asArgument', + 'fromChangelog', + 'orderBy', + 'process', + 'toChangelog', + } + + if __name__ == '__main__': import unittest diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java new file mode 100644 index 00000000000000..82591ca87f711f --- /dev/null +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.inference.strategies; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.functions.ChangelogFunction.ChangelogContext; +import org.apache.flink.table.functions.TableSemantics; +import org.apache.flink.table.types.AtomicDataType; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.InputTypeStrategiesTestBase; +import org.apache.flink.table.types.inference.utils.CallContextMock; +import org.apache.flink.table.types.inference.utils.TableSemanticsMock; +import org.apache.flink.table.types.logical.LocalZonedTimestampType; +import org.apache.flink.table.types.logical.TimestampKind; +import org.apache.flink.types.ColumnList; +import org.apache.flink.types.RowKind; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_INPUT; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_KEEP; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_RESET_TTL_ON_DUPLICATE; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_SEQUENCE_BY; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.CHANGELOG_MODE_STRATEGY; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.INPUT_TYPE_STRATEGY; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.OUTPUT_TYPE_STRATEGY; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link DeduplicateTypeStrategy}. */ +class DeduplicateTypeStrategyTest extends InputTypeStrategiesTestBase { + + private static final DataType ROWTIME = + new AtomicDataType(new LocalZonedTimestampType(true, TimestampKind.ROWTIME, 3)); + + private static final DataType INPUT_TYPE = + DataTypes.ROW( + DataTypes.FIELD("user_id", DataTypes.STRING()), + DataTypes.FIELD("event", DataTypes.STRING()), + DataTypes.FIELD("ts", ROWTIME), + DataTypes.FIELD("lsn", DataTypes.BIGINT()), + DataTypes.FIELD("tags", DataTypes.ARRAY(DataTypes.STRING()))); + + private static final DataType[] ARGUMENT_TYPES = { + INPUT_TYPE, + DataTypes.STRING(), + DataTypes.DESCRIPTOR(), + DataTypes.INTERVAL(DataTypes.SECOND()), + DataTypes.BOOLEAN() + }; + + private static final TableSemanticsMock PARTITIONED_BY_USER = + new TableSemanticsMock( + INPUT_TYPE, new int[] {0}, new int[0], -1, ChangelogMode.insertOnly()); + + @Override + protected Stream testData() { + return Stream.of( + withDefaults("All optional arguments omitted", PARTITIONED_BY_USER) + .expectArgumentTypes(ARGUMENT_TYPES), + withDefaults("All arguments set", PARTITIONED_BY_USER) + .calledWithLiteralAt(ARG_KEEP, "last") + .calledWithLiteralAt(ARG_SEQUENCE_BY, ColumnList.of("lsn", "ts")) + .calledWithLiteralAt(ARG_STATE_TTL, Duration.ofHours(1)) + .calledWithLiteralAt(ARG_RESET_TTL_ON_DUPLICATE, false) + .expectArgumentTypes(ARGUMENT_TYPES), + withDefaults("Invalid keep", PARTITIONED_BY_USER) + .calledWithLiteralAt(ARG_KEEP, "FIRST") + .expectErrorMessage( + "DEDUPLICATE requires keep => 'first' or keep => 'last', but " + + "received keep => 'FIRST'. Use keep => 'first' to keep " + + "the first record per key, or keep => 'last' to keep " + + "the last one."), + nonLiteral("Non-literal keep", ARG_KEEP) + .expectErrorMessage( + "DEDUPLICATE requires keep => 'first' or keep => 'last', but " + + "received a non-literal expression."), + withDefaults( + "No PARTITION BY", + new TableSemanticsMock( + INPUT_TYPE, + new int[0], + new int[0], + -1, + ChangelogMode.insertOnly())) + .expectErrorMessage( + "DEDUPLICATE requires a key, but the input has no PARTITION BY. " + + "Add PARTITION BY with the columns to deduplicate on, " + + "for example TABLE t PARTITION BY id. To drop only rows " + + "that repeat in full, partition by every column."), + withDefaults( + "ORDER BY", + new TableSemanticsMock( + INPUT_TYPE, + new int[] {0}, + new int[] {3}, + -1, + ChangelogMode.insertOnly())) + .expectErrorMessage( + "DEDUPLICATE does not support ORDER BY on its input, but received " + + "ORDER BY lsn. Use sequence_by => DESCRIPTOR(lsn) to " + + "order the records of a key."), + withDefaults( + "Time attribute in PARTITION BY", + new TableSemanticsMock( + INPUT_TYPE, + new int[] {2}, + new int[0], + -1, + ChangelogMode.insertOnly())) + .expectErrorMessage( + "DEDUPLICATE does not support a time attribute as a PARTITION BY " + + "column, but 'ts' is one. Partition by a regular column, " + + "or cast the column in a subquery first, for example " + + "CAST(ts AS TIMESTAMP_LTZ(3)) AS ts."), + TestSpec.forStrategy("on_time", INPUT_TYPE_STRATEGY) + .calledWithArgumentTypes( + INPUT_TYPE, + DataTypes.STRING(), + DataTypes.DESCRIPTOR(), + DataTypes.INTERVAL(DataTypes.SECOND()), + DataTypes.BOOLEAN(), + DataTypes.DESCRIPTOR(), + DataTypes.STRING()) + .calledWithTableSemanticsAt(ARG_INPUT, PARTITIONED_BY_USER) + .calledWithLiteralAt(ARG_KEEP, null) + .calledWithLiteralAt(ARG_SEQUENCE_BY, null) + .calledWithLiteralAt(ARG_STATE_TTL, null) + .calledWithLiteralAt(ARG_RESET_TTL_ON_DUPLICATE, null) + .calledWithLiteralAt(5, ColumnList.of("ts")) + .calledWithLiteralAt(6, null) + .expectErrorMessage( + "DEDUPLICATE does not support on_time, but received on_time => " + + "DESCRIPTOR(ts). Use sequence_by => DESCRIPTOR(ts) to " + + "order by timestamp, or DEDUPLICATE_WINDOW for " + + "event-time windows."), + sequenceBy("Unknown sequence_by column", "nope") + .expectErrorMessage( + "The sequence_by column 'nope' does not exist in the input of " + + "DEDUPLICATE. Available columns are: user_id, event, " + + "ts, lsn, tags."), + sequenceBy("Repeated sequence_by column", "lsn", "lsn") + .expectErrorMessage( + "The sequence_by column 'lsn' is listed more than once. List " + + "each ordering column once."), + sequenceBy("PARTITION BY column in sequence_by", "user_id") + .expectErrorMessage( + "The sequence_by column 'user_id' is a PARTITION BY column, " + + "which has the same value for every record of a key."), + sequenceBy("Non-orderable sequence_by column", "tags") + .expectErrorMessage( + "The sequence_by column 'tags' has type 'ARRAY', which " + + "cannot order records. Use a numeric, character " + + "string, binary string, boolean, or date-time column."), + withDefaults("Negative state_ttl", PARTITIONED_BY_USER) + .calledWithLiteralAt(ARG_STATE_TTL, Duration.ofSeconds(-1)) + .expectErrorMessage( + "The 'state_ttl' argument must not be negative, but was '-1 s'."), + nonLiteral("Non-literal state_ttl", ARG_STATE_TTL) + .expectErrorMessage( + "The 'state_ttl' argument must be a constant INTERVAL literal, " + + "but received a non-literal expression."), + nonLiteral("Non-literal reset_ttl_on_duplicate", ARG_RESET_TTL_ON_DUPLICATE) + .expectErrorMessage( + "The 'reset_ttl_on_duplicate' argument must be a constant " + + "BOOLEAN literal, but received a non-literal " + + "expression.")); + } + + @Test + void testOutputExcludesPartitionKeysAndTimeAttributes() { + assertThat(OUTPUT_TYPE_STRATEGY.inferType(callContext(ColumnList.of()))) + .contains( + DataTypes.ROW( + DataTypes.FIELD("event", DataTypes.STRING()), + DataTypes.FIELD("ts", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("lsn", DataTypes.BIGINT()), + DataTypes.FIELD( + "tags", DataTypes.ARRAY(DataTypes.STRING()))) + .notNull()); + } + + @Test + void testSequenceByIndicesPointIntoPayload() { + assertThat( + DeduplicateTypeStrategy.sequenceByOutputIndices( + callContext(ColumnList.of("lsn", "event")))) + .containsExactly(2, 0); + } + + @Test + void testChangelogModeEmitsUpdateBeforeOnlyWhenRequired() { + assertThat(inferChangelogMode(ChangelogMode.upsert())) + .isEqualTo(mode(RowKind.INSERT, RowKind.UPDATE_AFTER)); + assertThat(inferChangelogMode(ChangelogMode.all())) + .isEqualTo(mode(RowKind.INSERT, RowKind.UPDATE_BEFORE, RowKind.UPDATE_AFTER)); + } + + private static TestSpec withDefaults( + final String description, final TableSemanticsMock tableSemantics) { + return TestSpec.forStrategy(description, INPUT_TYPE_STRATEGY) + .calledWithArgumentTypes(ARGUMENT_TYPES) + .calledWithTableSemanticsAt(ARG_INPUT, tableSemantics) + .calledWithLiteralAt(ARG_KEEP, null) + .calledWithLiteralAt(ARG_SEQUENCE_BY, null) + .calledWithLiteralAt(ARG_STATE_TTL, null) + .calledWithLiteralAt(ARG_RESET_TTL_ON_DUPLICATE, null); + } + + private static TestSpec nonLiteral(final String description, final int argument) { + final TestSpec spec = + TestSpec.forStrategy(description, INPUT_TYPE_STRATEGY) + .calledWithArgumentTypes(ARGUMENT_TYPES) + .calledWithTableSemanticsAt(ARG_INPUT, PARTITIONED_BY_USER); + for (int pos = ARG_KEEP; pos <= ARG_RESET_TTL_ON_DUPLICATE; pos++) { + if (pos != argument) { + spec.calledWithLiteralAt(pos, null); + } + } + return spec; + } + + private static TestSpec sequenceBy(final String description, final String... columns) { + return withDefaults(description, PARTITIONED_BY_USER) + .calledWithLiteralAt(ARG_SEQUENCE_BY, ColumnList.of(Arrays.asList(columns))); + } + + private static CallContextMock callContext(final ColumnList sequenceBy) { + final CallContextMock callContext = new CallContextMock(); + callContext.argumentValues = + Arrays.asList( + Optional.empty(), + Optional.empty(), + Optional.of(sequenceBy), + Optional.empty(), + Optional.empty()); + callContext.tableSemantics = Map.of(ARG_INPUT, PARTITIONED_BY_USER); + return callContext; + } + + private static ChangelogMode inferChangelogMode(final ChangelogMode requiredMode) { + return CHANGELOG_MODE_STRATEGY.inferChangelogMode( + new ChangelogContext() { + @Override + public ChangelogMode getTableChangelogMode(final int pos) { + return ChangelogMode.insertOnly(); + } + + @Override + public ChangelogMode getRequiredChangelogMode() { + return requiredMode; + } + + @Override + public Optional getTableSemantics(final int pos) { + return Optional.empty(); + } + }); + } + + private static ChangelogMode mode(final RowKind... kinds) { + final ChangelogMode.Builder builder = ChangelogMode.newBuilder(); + for (final RowKind kind : kinds) { + builder.addContainedKind(kind); + } + return builder.build(); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTest.java new file mode 100644 index 00000000000000..7f55ac172a8557 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.planner.plan.nodes.exec.testutils.RestoreTestBase; +import org.apache.flink.table.test.program.TableTestProgram; + +import java.util.List; + +/** Restore tests for the built-in DEDUPLICATE PTF. */ +public class DeduplicateRestoreTest extends RestoreTestBase { + + public DeduplicateRestoreTest() { + super(StreamExecProcessTableFunction.class); + } + + @Override + public List programs() { + return List.of( + DeduplicateRestoreTestPrograms.ARRIVAL_ORDER_RESTORE, + DeduplicateRestoreTestPrograms.FIRST_ARRIVAL_RESTORE, + DeduplicateRestoreTestPrograms.SEQUENCE_BY_RESTORE); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTestPrograms.java new file mode 100644 index 00000000000000..0d7001655b6930 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTestPrograms.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; + +/** {@link TableTestProgram} definitions for restore tests of the DEDUPLICATE PTF. */ +public class DeduplicateRestoreTestPrograms { + + public static final TableTestProgram ARRIVAL_ORDER_RESTORE = + TableTestProgram.of( + "deduplicate-arrival-order-restore", + "the kept record survives a restore: a key seen before the restore " + + "is updated or emits nothing for an identical record, and a " + + "key first seen after it is inserted") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema("user_id STRING", "event STRING") + .producedBeforeRestore( + Row.of("u1", "login"), + Row.of("u2", "login"), + Row.of("u1", "click"), + Row.of("u3", "view")) + .producedAfterRestore( + Row.of("u1", "click"), + Row.of("u1", "logout"), + Row.of("u2", "login"), + Row.of("u4", "login")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema( + "user_id STRING PRIMARY KEY NOT ENFORCED", + "event STRING") + .addOption("sink-changelog-mode-enforced", "I,UA,D") + .consumedBeforeRestore( + "+I[u1, login]", + "+I[u2, login]", + "+U[u1, click]", + "+I[u3, view]") + .consumedAfterRestore("+U[u1, logout]", "+I[u4, login]") + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY user_id, " + + "keep => 'last')") + .build(); + + public static final TableTestProgram FIRST_ARRIVAL_RESTORE = + TableTestProgram.of( + "deduplicate-first-arrival-restore", + "with the default keep => 'first', a key emitted before the restore " + + "stays dropped after it, and a key first seen after it is " + + "inserted") + .setupTableSource( + SourceTestStep.newBuilder("payments") + .addSchema("payment_id STRING", "amount INT") + .producedBeforeRestore( + Row.of("p1", 50), + Row.of("p2", 20), + Row.of("p1", 60), + Row.of("p3", 30)) + .producedAfterRestore( + Row.of("p1", 70), + Row.of("p2", 20), + Row.of("p4", 40), + Row.of("p3", 35)) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema( + "payment_id STRING PRIMARY KEY NOT ENFORCED", + "amount INT") + .addOption("sink-changelog-mode-enforced", "I,UA,D") + .consumedBeforeRestore("+I[p1, 50]", "+I[p2, 20]", "+I[p3, 30]") + .consumedAfterRestore("+I[p4, 40]") + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE(" + + "input => TABLE payments PARTITION BY payment_id)") + .build(); + + public static final TableTestProgram SEQUENCE_BY_RESTORE = + TableTestProgram.of( + "deduplicate-sequence-by-restore", + "the sequence_by value of the kept record survives a restore, so " + + "with 'first' a greater or equal value arriving after the " + + "restore is dropped") + .setupTableSource( + SourceTestStep.newBuilder("customer_changes") + .addSchema("customer_id STRING", "city STRING", "lsn BIGINT") + .producedBeforeRestore( + Row.of("c1", "Berlin", 100L), + Row.of("c2", "Paris", 105L), + Row.of("c1", "Hamburg", 90L), + Row.of("c3", "Rome", 50L)) + .producedAfterRestore( + Row.of("c1", "Munich", 95L), + Row.of("c2", "Lyon", 100L), + Row.of("c3", "Milan", 50L), + Row.of("c4", "Oslo", 1L)) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema( + "customer_id STRING PRIMARY KEY NOT ENFORCED", + "city STRING", + "lsn BIGINT") + .addOption("sink-changelog-mode-enforced", "I,UA,D") + .consumedBeforeRestore( + "+I[c1, Berlin, 100]", + "+I[c2, Paris, 105]", + "+U[c1, Hamburg, 90]", + "+I[c3, Rome, 50]") + .consumedAfterRestore("+U[c2, Lyon, 100]", "+I[c4, Oslo, 1]") + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE(" + + "input => TABLE customer_changes PARTITION BY customer_id, " + + "keep => 'first', " + + "sequence_by => DESCRIPTOR(lsn))") + .build(); +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateSemanticTests.java new file mode 100644 index 00000000000000..6799130151160e --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateSemanticTests.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.planner.plan.nodes.exec.testutils.SemanticTestBase; +import org.apache.flink.table.test.program.TableTestProgram; + +import java.util.List; + +/** Semantic tests for the built-in DEDUPLICATE process table function. */ +public class DeduplicateSemanticTests extends SemanticTestBase { + + @Override + public List programs() { + return List.of( + DeduplicateTestPrograms.ARRIVAL_ORDER, + DeduplicateTestPrograms.COMPOSITE_AND_NULL_KEY, + DeduplicateTestPrograms.SEQUENCE_BY, + DeduplicateTestPrograms.NO_PARTITION_BY_FAILS); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateTestPrograms.java new file mode 100644 index 00000000000000..24e1d3468cb620 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateTestPrograms.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; + +import javax.annotation.Nullable; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +/** {@link TableTestProgram} definitions for testing the built-in DEDUPLICATE PTF. */ +public class DeduplicateTestPrograms { + + private static final String[] USER_EVENTS_SCHEMA = {"user_id STRING", "event STRING"}; + + private static final String[] USER_EVENTS_UPSERT_SCHEMA = { + "user_id STRING PRIMARY KEY NOT ENFORCED", "event STRING" + }; + + private static final String[] CUSTOMER_UPSERT_SCHEMA = { + "customer_id STRING PRIMARY KEY NOT ENFORCED", "city STRING", "lsn BIGINT" + }; + + private static final String[] LABEL_UPSERT_SCHEMA = { + "id STRING PRIMARY KEY NOT ENFORCED", "label STRING" + }; + + private static final String[] PRICES_UPSERT_SCHEMA = { + "ticker STRING PRIMARY KEY NOT ENFORCED", "price INT", "ts TIMESTAMP_LTZ(3)" + }; + + public static final TableTestProgram ARRIVAL_ORDER = + TableTestProgram.of( + "deduplicate-arrival-order", + "'last' replaces the kept record with every new one, 'first' drops " + + "every record after the first, an identical record emits " + + "nothing, and a consumer that needs the previous record " + + "receives -U before +U") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedValues( + Row.of("u1", "login"), + Row.of("u2", "login"), + Row.of("u1", "click")) + .build()) + .setupTableSource( + SourceTestStep.newBuilder("profile_updates") + .addSchema("user_id STRING", "city STRING", "phone STRING") + .producedValues( + Row.of("u1", "Berlin", "111"), + Row.of("u1", "Berlin", "111"), + Row.of("u1", "Munich", null)) + .build()) + .setupTableSink( + upsertSink( + "sink_last", + USER_EVENTS_UPSERT_SCHEMA, + "+I[u1, login]", + "+I[u2, login]", + "+U[u1, click]")) + .setupTableSink( + upsertSink( + "sink_first", + USER_EVENTS_UPSERT_SCHEMA, + "+I[u1, login]", + "+I[u2, login]")) + .setupTableSink( + SinkTestStep.newBuilder("retract_sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues( + "+I[u1, login]", + "+I[u2, login]", + "-U[u1, login]", + "+U[u1, click]") + .build()) + .setupTableSink( + upsertSink( + "latest_profiles", + new String[] { + "user_id STRING PRIMARY KEY NOT ENFORCED", + "city STRING", + "phone STRING" + }, + "+I[u1, Berlin, 111]", + "+U[u1, Munich, null]")) + .runSql( + "INSERT INTO sink_last SELECT * FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY user_id, " + + "keep => 'last')") + .runSql( + "INSERT INTO sink_first SELECT * FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY user_id)") + .runSql( + "INSERT INTO retract_sink SELECT * FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY user_id, " + + "keep => 'last')") + .runSql( + "INSERT INTO latest_profiles SELECT * FROM DEDUPLICATE(" + + "input => TABLE profile_updates PARTITION BY user_id, " + + "keep => 'last')") + .build(); + + public static final TableTestProgram COMPOSITE_AND_NULL_KEY = + TableTestProgram.of( + "deduplicate-composite-and-null-key", + "a PARTITION BY of non-adjacent columns is the leading output columns " + + "in its own order, and a NULL key column forms its own key") + .setupTableSource( + SourceTestStep.newBuilder("sessions") + .addSchema("user_id STRING", "page STRING", "device STRING") + .producedValues( + Row.of("u1", "a", "web"), + Row.of("u1", "b", "mobile"), + Row.of("u1", "c", "web"), + Row.of(null, "d", "web"), + Row.of(null, "e", "web")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("device STRING", "user_id STRING", "page STRING") + .consumedValues( + "+I[web, u1, a]", + "+I[mobile, u1, b]", + "-U[web, u1, a]", + "+U[web, u1, c]", + "+I[web, null, d]", + "-U[web, null, d]", + "+U[web, null, e]") + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE(" + + "input => TABLE sessions PARTITION BY (device, user_id), " + + "keep => 'last')") + .build(); + + public static final TableTestProgram SEQUENCE_BY = + TableTestProgram.of( + "deduplicate-sequence-by", + "with sequence_by, 'last' keeps the greatest and 'first' the smallest " + + "value, a tie goes to the later arrival for 'last' and to " + + "the earlier one for 'first', later columns break ties, a " + + "NULL orders below every other value, NaN ties with every " + + "value, and a watermarked column orders without watermark " + + "semantics and is emitted as a regular timestamp") + .setupTableSource( + SourceTestStep.newBuilder("customer_changes") + .addSchema("customer_id STRING", "city STRING", "lsn BIGINT") + .producedValues( + Row.of("c1", "Berlin", 100L), + Row.of("c1", "Munich", 120L), + Row.of("c1", "Hamburg", 110L)) + .build()) + .setupTableSource( + SourceTestStep.newBuilder("prices") + .addSchema( + "ticker STRING", + "price INT", + "ts TIMESTAMP_LTZ(3)", + "WATERMARK FOR ts AS ts - INTERVAL '1' HOUR") + .producedValues( + Row.of("AAPL", 10, at("10:02")), + Row.of("MSFT", 20, at("10:01")), + Row.of("AAPL", 12, at("10:08")), + Row.of("AAPL", 11, at("10:05")), + Row.of("AAPL", 9, at("10:00")), + Row.of("AAPL", 14, at("10:08")), + Row.of("AAPL", 8, at("10:00"))) + .build()) + .setupTableSource( + SourceTestStep.newBuilder("measurements") + .addSchema("id STRING", "v DOUBLE", "label STRING") + .producedValues( + Row.of("m1", 1.0, "r1"), + Row.of("m1", Double.NaN, "r2"), + Row.of("m1", 0.5, "r3")) + .build()) + .setupTableSource( + SourceTestStep.newBuilder("versions") + .addSchema( + "id STRING", + "s STRING", + "b BOOLEAN", + "t TIMESTAMP(3)", + "y BYTES", + "label STRING") + .producedValues( + versionRow("a", false, 1, 1, "r1"), + versionRow("b", false, 1, 1, "r2"), + versionRow("a", true, 1, 1, "r3"), + versionRow("b", true, 1, 1, "r4"), + versionRow("b", true, 0, 9, "r5"), + versionRow("b", true, 2, 1, "r6"), + versionRow("b", true, 2, 0, "r7"), + versionRow("b", true, 2, 2, "r8"), + versionRow(null, true, 9, 9, "r9"), + versionRow(null, true, 9, 9, "r10"), + versionRow("b", true, 2, 2, "r11"), + versionRow("b", true, 2, null, "r12")) + .build()) + .setupTableSink( + upsertSink( + "customers_last", + CUSTOMER_UPSERT_SCHEMA, + "+I[c1, Berlin, 100]", + "+U[c1, Munich, 120]")) + .setupTableSink( + upsertSink( + "prices_last", + PRICES_UPSERT_SCHEMA, + "+I[AAPL, 10, 1970-01-01T10:02:00Z]", + "+I[MSFT, 20, 1970-01-01T10:01:00Z]", + "+U[AAPL, 12, 1970-01-01T10:08:00Z]", + "+U[AAPL, 14, 1970-01-01T10:08:00Z]")) + .setupTableSink( + upsertSink( + "prices_first", + PRICES_UPSERT_SCHEMA, + "+I[AAPL, 10, 1970-01-01T10:02:00Z]", + "+I[MSFT, 20, 1970-01-01T10:01:00Z]", + "+U[AAPL, 9, 1970-01-01T10:00:00Z]")) + .setupTableSink( + upsertSink( + "measurements_last", + LABEL_UPSERT_SCHEMA, + "+I[m1, r1]", + "+U[m1, r2]", + "+U[m1, r3]")) + .setupTableSink( + upsertSink( + "versions_last", + LABEL_UPSERT_SCHEMA, + "+I[c1, r1]", + "+U[c1, r2]", + "+U[c1, r4]", + "+U[c1, r6]", + "+U[c1, r8]", + "+U[c1, r11]")) + .setupTableSink( + upsertSink( + "versions_first", + LABEL_UPSERT_SCHEMA, + "+I[c1, r1]", + "+U[c1, r9]")) + .runSql( + "INSERT INTO customers_last SELECT * FROM DEDUPLICATE(" + + "input => TABLE customer_changes PARTITION BY customer_id, " + + "keep => 'last', " + + "sequence_by => DESCRIPTOR(lsn), " + + "state_ttl => INTERVAL '1' DAY, " + + "reset_ttl_on_duplicate => true, " + + "uid => 'customer-changes-dedup')") + .runSql( + "INSERT INTO prices_last SELECT * FROM DEDUPLICATE(" + + "input => TABLE prices PARTITION BY ticker, " + + "keep => 'last', " + + "sequence_by => DESCRIPTOR(ts))") + .runSql( + "INSERT INTO prices_first SELECT * FROM DEDUPLICATE(" + + "input => TABLE prices PARTITION BY ticker, " + + "keep => 'first', " + + "sequence_by => DESCRIPTOR(ts))") + .runSql( + "INSERT INTO measurements_last SELECT id, label FROM DEDUPLICATE(" + + "input => TABLE measurements PARTITION BY id, " + + "keep => 'last', " + + "sequence_by => DESCRIPTOR(v))") + .runSql( + "INSERT INTO versions_last SELECT id, label FROM DEDUPLICATE(" + + "input => TABLE versions PARTITION BY id, " + + "keep => 'last', " + + "sequence_by => DESCRIPTOR(s, b, t, y))") + .runSql( + "INSERT INTO versions_first SELECT id, label FROM DEDUPLICATE(" + + "input => TABLE versions PARTITION BY id, " + + "keep => 'first', " + + "sequence_by => DESCRIPTOR(s, b, t, y))") + .build(); + + public static final TableTestProgram NO_PARTITION_BY_FAILS = + TableTestProgram.of("deduplicate-no-partition-by-fails", "PARTITION BY is required") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedValues(Row.of("u1", "login")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues(new Row[0]) + .build()) + .runFailingSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE(" + + "input => TABLE user_events)", + ValidationException.class, + "DEDUPLICATE requires a key, but the input has no PARTITION BY.") + .build(); + + private static SinkTestStep upsertSink( + final String name, final String[] schema, final String... expected) { + return SinkTestStep.newBuilder(name) + .addSchema(schema) + .addOption("sink-changelog-mode-enforced", "I,UA,D") + .consumedValues(expected) + .build(); + } + + private static Row versionRow( + @Nullable final String s, + final boolean b, + final int minute, + @Nullable final Integer y, + final String label) { + return Row.of( + "c1", + s, + b, + LocalDateTime.ofEpochSecond(minute * 60L, 0, ZoneOffset.UTC), + y == null ? null : new byte[] {y.byteValue()}, + label); + } + + private static Instant at(final String time) { + return Instant.parse("1970-01-01T" + time + ":00Z"); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.java new file mode 100644 index 00000000000000..1f9c5f003462ad --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.stream.sql; + +import org.apache.flink.table.api.ExplainDetail; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.planner.utils.PlanKind; +import org.apache.flink.table.planner.utils.TableTestBase; +import org.apache.flink.table.planner.utils.TableTestUtil; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import scala.Enumeration; + +import static org.apache.flink.table.api.Expressions.$; +import static org.apache.flink.table.api.Expressions.descriptor; +import static org.apache.flink.table.api.Expressions.lit; + +/** Plan tests for the changelog mode and upsert key of the DEDUPLICATE built-in PTF. */ +class DeduplicateTableFunctionTest extends TableTestBase { + + private TableTestUtil util; + + @BeforeEach + void setup() { + util = streamTestUtil(TableConfig.getDefault()); + util.tableEnv() + .executeSql( + "CREATE TABLE user_events (user_id STRING, event STRING, " + + "ts TIMESTAMP_LTZ(3), WATERMARK FOR ts AS ts) " + + "WITH ('connector' = 'values')"); + util.tableEnv() + .executeSql( + "CREATE TABLE customer_changes (customer_id STRING, city STRING, " + + "lsn BIGINT) WITH ('connector' = 'values')"); + } + + @Test + void testUpsertKeyIsPartitionKey() { + verifyPlan( + "SELECT * FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY user_id, keep => 'last')"); + } + + @Test + void testRetractingConsumer() { + verifyPlan( + "SELECT event, COUNT(*) AS users FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY user_id, keep => 'last') " + + "GROUP BY event"); + } + + @Test + void testOutputHasNoTimeAttribute() { + verifyPlan( + "SELECT * FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY event, " + + "sequence_by => DESCRIPTOR(ts))"); + } + + @Test + void testTableApiWithArguments() { + final Table result = + util.tableEnv() + .from("customer_changes") + .partitionBy($("customer_id")) + .deduplicate( + lit("last").asArgument("keep"), + descriptor("lsn").asArgument("sequence_by"), + lit(Duration.ofDays(1)).asArgument("state_ttl")); + util.verifyRelPlan(result); + } + + private void verifyPlan(final String query) { + util.doVerifyPlan( + query, + new ExplainDetail[] {ExplainDetail.CHANGELOG_MODE}, + false, + new Enumeration.Value[] {PlanKind.AST(), PlanKind.OPT_REL()}, + false, + true); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/harness/DeduplicateHarnessTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/harness/DeduplicateHarnessTest.java new file mode 100644 index 00000000000000..7ee7f90f14f02f --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/harness/DeduplicateHarnessTest.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.runtime.harness; + +import org.apache.flink.api.dag.Transformation; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.transformations.KeyedMultipleInputTransformation; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.KeyedMultiInputStreamOperatorTestHarness; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.runtime.keyselector.RowDataKeySelector; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Harness tests for the processing-time state retention of the built-in DEDUPLICATE PTF, which need + * control over the time that state TTL observes. + */ +class DeduplicateHarnessTest { + + private static final Function PAYMENT = + row -> row.getString(0) + ", " + row.getInt(1); + + private static final Function CUSTOMER_CHANGE = + row -> row.getString(1) + ", " + row.getLong(2); + + private static Stream resetTtlOnDuplicate() { + return Stream.of( + Arguments.of(true, List.of("+I[p1, 50]@10:00", "+I[p1, 50]@10:20")), + Arguments.of( + false, + List.of("+I[p1, 50]@10:00", "+I[p1, 50]@10:07", "+I[p1, 50]@10:20"))); + } + + @ParameterizedTest(name = "reset_ttl_on_duplicate={0}") + @MethodSource("resetTtlOnDuplicate") + void testTimeBoundedDeduplication( + final boolean resetTtlOnDuplicate, final List expected) throws Exception { + try (KeyedMultiInputStreamOperatorTestHarness harness = + createHarness( + "SELECT * FROM DEDUPLICATE(" + + "input => TABLE payments PARTITION BY payment_id, " + + "state_ttl => INTERVAL '5' MINUTES, " + + "reset_ttl_on_duplicate => " + + resetTtlOnDuplicate + + ")")) { + final List output = new ArrayList<>(); + for (final String time : List.of("10:00", "10:04", "10:07", "10:20")) { + output.addAll(process(harness, time, PAYMENT, payment(50))); + } + + assertThat(output).isEqualTo(expected); + } + } + + @Test + void testRecordAfterExpiryIsFirstRecord() throws Exception { + try (KeyedMultiInputStreamOperatorTestHarness harness = + createHarness( + "SELECT * FROM DEDUPLICATE(" + + "input => TABLE customer_changes PARTITION BY customer_id, " + + "keep => 'last', " + + "sequence_by => DESCRIPTOR(lsn), " + + "state_ttl => INTERVAL '5' MINUTES)")) { + final List output = new ArrayList<>(); + output.addAll( + process(harness, "10:00", CUSTOMER_CHANGE, customerChange("Berlin", 120))); + output.addAll( + process(harness, "10:02", CUSTOMER_CHANGE, customerChange("Munich", 100))); + output.addAll( + process(harness, "10:20", CUSTOMER_CHANGE, customerChange("Hamburg", 90))); + + assertThat(output).containsExactly("+I[Berlin, 120]@10:00", "+I[Hamburg, 90]@10:20"); + } + } + + @Test + void testReplacingRecordRefreshesTtl() throws Exception { + try (KeyedMultiInputStreamOperatorTestHarness harness = + createHarness( + "SELECT * FROM DEDUPLICATE(" + + "input => TABLE payments PARTITION BY payment_id, " + + "keep => 'last', " + + "state_ttl => INTERVAL '5' MINUTES, " + + "reset_ttl_on_duplicate => false)")) { + final List output = new ArrayList<>(); + output.addAll(process(harness, "10:00", PAYMENT, payment(50))); + output.addAll(process(harness, "10:04", PAYMENT, payment(60))); + output.addAll(process(harness, "10:07", PAYMENT, payment(60))); + output.addAll(process(harness, "10:08", PAYMENT, payment(70))); + output.addAll(process(harness, "10:14", PAYMENT, payment(70))); + + assertThat(output) + .containsExactly( + "+I[p1, 50]@10:00", + "-U[p1, 50]@10:04", + "+U[p1, 60]@10:04", + "-U[p1, 60]@10:08", + "+U[p1, 70]@10:08", + "+I[p1, 70]@10:14"); + } + } + + @Test + void testIdenticalRecordRefreshesTtl() throws Exception { + try (KeyedMultiInputStreamOperatorTestHarness harness = + createHarness( + "SELECT * FROM DEDUPLICATE(" + + "input => TABLE payments PARTITION BY payment_id, " + + "keep => 'last', " + + "state_ttl => INTERVAL '5' MINUTES)")) { + final List output = new ArrayList<>(); + for (final String time : List.of("10:00", "10:04", "10:07")) { + output.addAll(process(harness, time, PAYMENT, payment(50))); + } + + assertThat(output).containsExactly("+I[p1, 50]@10:00"); + } + } + + private static List process( + final KeyedMultiInputStreamOperatorTestHarness harness, + final String time, + final Function describe, + final RowData input) + throws Exception { + harness.setStateTtlProcessingTime(toMillis(time)); + harness.processElement(0, new StreamRecord<>(input)); + final List output = new ArrayList<>(); + for (final Object record : harness.getOutput()) { + final RowData row = (RowData) ((StreamRecord) record).getValue(); + output.add( + String.format( + "%s[%s]@%s", + row.getRowKind().shortString(), describe.apply(row), time)); + } + harness.getOutput().clear(); + return output; + } + + private static RowData payment(final int amount) { + return GenericRowData.of(StringData.fromString("p1"), amount); + } + + private static RowData customerChange(final String city, final long lsn) { + return GenericRowData.of(StringData.fromString("c1"), StringData.fromString(city), lsn); + } + + private static KeyedMultiInputStreamOperatorTestHarness createHarness( + final String query) throws Exception { + final StreamTableEnvironment tEnv = + StreamTableEnvironment.create(StreamExecutionEnvironment.getExecutionEnvironment()); + tEnv.executeSql( + "CREATE TABLE payments (payment_id STRING, amount INT) " + + "WITH ('connector' = 'values')"); + tEnv.executeSql( + "CREATE TABLE customer_changes (customer_id STRING, city STRING, lsn BIGINT) " + + "WITH ('connector' = 'values')"); + final KeyedMultipleInputTransformation transformation = + findPtf(tEnv.toChangelogStream(tEnv.sqlQuery(query)).getTransformation()) + .orElseThrow(() -> new AssertionError("No PTF transformation found.")); + return openHarness( + transformation.getOperatorFactory(), + (RowDataKeySelector) transformation.getStateKeySelectors().get(0)); + } + + private static KeyedMultiInputStreamOperatorTestHarness openHarness( + final StreamOperatorFactory operatorFactory, final RowDataKeySelector keySelector) + throws Exception { + final KeyedMultiInputStreamOperatorTestHarness harness = + new KeyedMultiInputStreamOperatorTestHarness<>( + operatorFactory, keySelector.getProducedType()); + harness.setKeySelector(0, keySelector); + harness.open(); + return harness; + } + + private static Optional> findPtf( + final Transformation transformation) { + if (transformation instanceof KeyedMultipleInputTransformation) { + return Optional.of((KeyedMultipleInputTransformation) transformation); + } + return transformation.getInputs().stream() + .map(DeduplicateHarnessTest::findPtf) + .flatMap(Optional::stream) + .findFirst(); + } + + private static long toMillis(final String time) { + final String[] parts = time.split(":"); + return Duration.ofHours(Long.parseLong(parts[0])) + .plusMinutes(Long.parseLong(parts[1])) + .toMillis(); + } +} diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.xml new file mode 100644 index 00000000000000..e2217ca87819a3 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.xml @@ -0,0 +1,104 @@ + + + + + + TABLE user_events PARTITION BY event, sequence_by => DESCRIPTOR(ts))]]> + + + + + + + + + + + TABLE user_events PARTITION BY user_id, keep => 'last') GROUP BY event]]> + + + + + + + + + + + + + + + + + + + TABLE user_events PARTITION BY user_id, keep => 'last')]]> + + + + + + + + + diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-arrival-order-restore/plan/deduplicate-arrival-order-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-arrival-order-restore/plan/deduplicate-arrival-order-restore.json new file mode 100644 index 00000000000000..a6e4ef1d4302c9 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-arrival-order-restore/plan/deduplicate-arrival-order-restore.json @@ -0,0 +1,159 @@ +{ + "flinkVersion" : "2.5", + "nodes" : [ { + "id" : 1, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`user_events`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "user_id", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "event", + "dataType" : "VARCHAR(2147483647)" + } ] + } + } + } + }, + "outputType" : "ROW<`user_id` VARCHAR(2147483647), `event` VARCHAR(2147483647)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, user_events]], fields=[user_id, event])" + }, { + "id" : 2, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`user_id` VARCHAR(2147483647), `event` VARCHAR(2147483647)>", + "description" : "Exchange(distribution=[hash[user_id]])" + }, { + "id" : 3, + "type" : "stream-exec-process-table-function_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`user_id` VARCHAR(2147483647), `event` VARCHAR(2147483647)>", + "description" : "ProcessTableFunction(invocation=[DEDUPLICATE(TABLE(#0) PARTITION BY($0), _UTF-16LE'last', DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[DEDUPLICATE], select=[user_id,event], rowType=[RecordType(VARCHAR(2147483647) user_id, VARCHAR(2147483647) event)])", + "uid" : "DEDUPLICATE", + "functionCall" : { + "kind" : "CALL", + "internalName" : "$DEDUPLICATE$1", + "operands" : [ { + "kind" : "TABLE_ARG_CALL", + "inputIndex" : 0, + "partitionKeys" : [ 0 ], + "type" : "ROW<`user_id` VARCHAR(2147483647), `event` VARCHAR(2147483647)> NOT NULL" + }, { + "kind" : "LITERAL", + "value" : "last", + "type" : "CHAR(4) NOT NULL" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "DESCRIPTOR" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "INTERVAL SECOND(6)" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "BOOLEAN" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "DESCRIPTOR" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "VARCHAR(2147483647)" + } ], + "type" : "ROW<`user_id` VARCHAR(2147483647), `event` VARCHAR(2147483647)> NOT NULL" + }, + "inputChangelogModes" : [ [ "INSERT" ] ], + "outputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], + "inputUpsertKeys" : [ [ ] ] + }, { + "id" : 4, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "user_id", + "dataType" : "VARCHAR(2147483647) NOT NULL" + }, { + "name" : "event", + "dataType" : "VARCHAR(2147483647)" + } ], + "primaryKey" : { + "name" : "PK_user_id", + "type" : "PRIMARY_KEY", + "columns" : [ "user_id" ] + } + } + } + } + }, + "inputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], + "upsertMaterializeStrategy" : "VALUE", + "inputUpsertKey" : [ 0 ], + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`user_id` VARCHAR(2147483647), `event` VARCHAR(2147483647)>", + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[user_id, event])" + } ], + "edges" : [ { + "source" : 1, + "target" : 2, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 2, + "target" : 3, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 3, + "target" : 4, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-arrival-order-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-arrival-order-restore/savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..7a12e418b941b44502955a54bb1fcde998051bfb GIT binary patch literal 12401 zcmeGiTWlj&^^TKlv)j!kZL?{!U8;ypS6a0*<7aGdgoNWbY*cSz*-lCmE#`XWI+y;=H7eGJ@-8BJ?G4s{qzct5IP1wdU5s*^`e1-qOD&XGvR+ZdIcSemgwWf zk?;QFmoJ_D;H{V6`)2*Or?U>o4GQMMa=q;3BpuHi#H_29N&9ItTqi!^+$HD!taE>v zLtm{ZD!C{&upw4-RVIch21}x`+NcDtR@XvmBxIG0c&sFqH8~s&!nINpi{e^cQ!GA| zhypR`%E@( z{t5M&26|$Lndl6rFryGvHm9+Li}3!z_uf4Fo#$RD1rV1-rz*71?J_MilwHFOEK;(> ztYTV|8wPrOSL`nEJ1_iw_#1&QR0AlGMT3&AH9+W!uBBCM0^$d3#46TSMAkH!Mfa=v ziXvf^l4YTA7UmIceu<125@Y}tAJDzJ)PJpt4Uv$3^K zZO-8;%r@v*=w)1+Tt$+v3i4H5KgLW1Wg@dJ`>jsMMmm~`7kcA#Z z2v&89s-{3SNXTgQs<*Rdp`QHw)4;>?Y!KC7A+4(_hzvw&jxT6cT~;azTNLQ^zXl%< z{os#Z8DiY)RiVY@-IKRQ&_X;qg>L-(Qy)IY&~TE4Mm!R34YS4jPz!U)Fz_a|uZ`z^ z^^uo;iof22xLMR|g6RXhVDq?2%{u_=iS3DRj!uljeTG`J!y8bJ9#DLln3AE?Y^8qC zqtgY;P_z}Q(;65B<-nuxOeIdAdpByp*Zb-wF-nRIwCqi-7O|0D#l|)rokhn~qODk~ z8x-{bx=wFHfPOKRPhU#qpWWKpV)}aq=IkR4qFL%TKH*{f`i)|TPby`#-e?l5*>FFQSU z5I{?NwgcPzwwXewMvpc095jBexd%EM6Wd+s7R0W4h}}L<&u0yxFTq=@FYNDwO@&=9 zoTMt!`gXX84cAZzbz;6~Lp1+Sr{PSq3=H0IMyu$CL=5WMk3>u0cR3w=13XC?Te^|{ z{Uggi`q<}&uhrh4xcS%Uj{yzcpKefeuB%U5}0@91qyD=Lv?$p2hAt7)EKbacCL015E zPAPtTAICnVm0Bl!+ac~G4-eHx&+UWqtgfxJNFQ&1ZClCkTEVoG;MA<_{4HrneW0u%2QgGl2 zt^z(t9xH-gBL?;2O`%8?s6}cUNw!D>Z6fV%P`912yqodzA#G3x9h8r^83~IZyY|ui z21IkLra)5$@cl!$Rm$|9OV4%f!xLn8%1@1VYHZuFpJ=m>?T1KR^EwT1d~}bIu9nSzJLhMeZ!;0k5z@5|nr>rTQ1=ko{j>1VwT|9(0-s9lt5Yc1$HEQh-zk{9 z3g$CrR_ZH6oDK}(C6QBx^*=NP*-!#rye$&;hPes zMJV(uouhvAcBCs<4#nR+QWY#5t6<>{zk=mqUj>U(O|sV@>Q=!Rt^7F7vHF#h?e#0H zpyV_w3K<5AXtCgbEl@eXn<=&;jW%nfWInoacm*lll(iGg|6iu8Z5u&%C!g8}iaYW= z{0+Gp?AAum_p@M0ul;7>TQ7JAxJ46NTZlow;_a+E)e}2HyJZzF^oMQGPR+)S&~Bjz zK)aXoxhCPWQ`e({8enOil`jkF<|BMNhvcrM5LQ(UxB>yxzl4Tl#gwoir?DmxC}EVB z(16o4gQ2V%-Ur=xMN`aGZ%d(GHtX=Yp@pVUH)}+bY11)QD*_M4;Y+=WA)hQmm1>JN zTC>-&5Z76T*cvLj&L4%LHJYCU@G}8og>lrQ7vDR@Is&cKC5K^pU!9th(Q*~Okm+o;W74~5Q zUO=%`#K#0mx6q_p=s-7PA^h#7+VyxkH8(#8)fHJNt#%|l*2D<}06Vr7pLbBjCIIDj U$JKagVh10bc!h;RE;Bd%Uv>=zPXGV_ literal 0 HcmV?d00001 diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/plan/deduplicate-first-arrival-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/plan/deduplicate-first-arrival-restore.json new file mode 100644 index 00000000000000..405398dd914233 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/plan/deduplicate-first-arrival-restore.json @@ -0,0 +1,160 @@ +{ + "flinkVersion" : "2.5", + "nodes" : [ { + "id" : 5, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`payments`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "payment_id", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "amount", + "dataType" : "INT" + } ] + } + } + } + }, + "outputType" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, payments]], fields=[payment_id, amount])" + }, { + "id" : 6, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT>", + "description" : "Exchange(distribution=[hash[payment_id]])" + }, { + "id" : 7, + "type" : "stream-exec-process-table-function_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT>", + "description" : "ProcessTableFunction(invocation=[DEDUPLICATE(TABLE(#0) PARTITION BY($0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[DEDUPLICATE], select=[payment_id,amount], rowType=[RecordType(VARCHAR(2147483647) payment_id, INTEGER amount)])", + "uid" : "DEDUPLICATE", + "functionCall" : { + "kind" : "CALL", + "internalName" : "$DEDUPLICATE$1", + "operands" : [ { + "kind" : "TABLE_ARG_CALL", + "inputIndex" : 0, + "partitionKeys" : [ 0 ], + "type" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT> NOT NULL" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "DESCRIPTOR" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "INTERVAL SECOND(6)" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "BOOLEAN" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "DESCRIPTOR" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "VARCHAR(2147483647)" + } ], + "type" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT> NOT NULL" + }, + "inputChangelogModes" : [ [ "INSERT" ] ], + "outputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], + "inputUpsertKeys" : [ [ ] ] + }, { + "id" : 8, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "payment_id", + "dataType" : "VARCHAR(2147483647) NOT NULL" + }, { + "name" : "amount", + "dataType" : "INT" + } ], + "primaryKey" : { + "name" : "PK_payment_id", + "type" : "PRIMARY_KEY", + "columns" : [ "payment_id" ] + } + } + } + } + }, + "inputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], + "upsertMaterializeStrategy" : "MAP", + "inputUpsertKey" : [ 0 ], + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT>", + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[payment_id, amount])" + } ], + "edges" : [ { + "source" : 5, + "target" : 6, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 6, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 7, + "target" : 8, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..79a17c17cd5e498a34a9e855f5b2c50470747f28 GIT binary patch literal 11317 zcmeGiTWlLy^^Wssl0K5QNg;);B5k%ywd)zbCb63ja2$tP>pGF`WZ6W;y`H&FCK=C| znd>y^N~K*8i&V4-0n#qZqew;KqaY+e+AsK8Pz6W`*e`z2{vgm&AwXK~3KY)09%sg} z)4J6rt-86AJu~-l-se8fJ*kUJeuPjr{8$lc57dc<5~g8O3u^{V8YRz<~)Dz zJl_$}*GrmC&Pr?8l1iqo63dnX1<6`oD+MlJLUEk#H~=jmZ&Njzpu8v0zlP@fxX^ARd2cOqCVIKVB*ZN^&ubqe?(3;jwVB6pZ>y z(E#yBB0<%U$Kj2PhoFaAibP^1gbwxqg&=zYZRdrtDxi^>*62d^|muEaZ~eR6LzpNM@ndhvrRdNyHVbEEBP$ zgX~0#i#idlszJ3f5$Q&SRH>%hVsf1l%fNb$SQ@DKGOrL;R{V!g6vp_9$zakjJOB@h)W9X`|v*f_6~ z8yQ>%Z2cZa&*RF(BIU08f!@4dhF)3+n+;MkN_GB zv<1i`Nf(%J0rhEirmE`9FWn$2RrUE(bS=KzZ63f`yW~M3BtrKI3F0V7_ zhtU=G7@+9Dd_0>t7telvb90lM@JT@1jmsv?lWluN%Ak$tPz{T)g(a2ACqJFc^6FuZ z=c5ghzWXE^r)F7G>XY5cFXzs@fBDIf4-AuW;e&Z-z%|dBTyOmIX7uvJPbXQY3P*Sf zvCbn~jGlpEN8Mq-16d;0vA1vBeD~VRQ820$>Vh@DO2GO(;4{^7kyu~4_01;^e&a9K zUGSp<1AlrQp~2_>2fWo5wfqhi)VvIV^)LoKt58#oXA$BS>MTGXj9Ij6UXRzrKwmt*$shrJJyi|hx@lwe7hKJi$t9)s&Ra>|=?v&f+ef!9)Wh%tB z^K5;bt{QyRTpYiE4o)Sf=3vX4fbH<@U#$G;TNBosw|f@vy!d1n+eoEAKK0Bk??qLkM?G9_nZzq3%sz~3%r1Qr2rVoum-zw7M$_3 zGm`3Op&QHgmoYo1u_B;>=aMfZr<|Bk{)O2j>Np>tg(e?r9U1!6ee4||I^gadyt{>Z z49Fvt-QV@h-UsDr(^zszA8q_>;AAipKcCFa#uIJC(CWt)n_^1Dc~etaK*xqK8}T69 zp}qm@nF7~kfkToiwH*+ghYMAkSPj|+RXVXwrtOx&zGju$)FQYH@E``LiDrdZEOKj$ zd1gQ^sSPC6!g{_2mF`ba4;*;<5KK_Jeasnca1ye{B0>B9vFe2KS*Xkw>yg=eg}pwc zr}KxpfbX_{Jk#JG+ZU0l;oUUsFC*->N8ERlR`(a)Fra7s-=lqQ^Ihc!x6eA=;3A$S zWZyejxQz!3_ZN}7-|juW-qC3jV!+D4N~@z}7e8)5<4%Xkb4^FGM%SfO7xPrlvGdY2 zR^SY>G0MKTat-cz0@*(B1`|17!O}YP-W9vwOM8nVEDlo@u;IZMNE-uYDYGx}G-LRMX&l4<`)L1#`=Z zufg8{{CKH27fI}+7iOjJzcu&6+rs81;$2^bTDPlnuhhEhZyUNgS65A4Sp_f0b7mD1 z-_DLWDF^nVd;#^y1)Eoku`^#;?p^dR{IU>9zVhZLVFV93_ezhRiQ?$mz}zdETAewxc!FS%M)~jZc-lz!;dGjwIqBuve~SrWoBo?$1J@?oZO&f*3!z5O|?dN z#*5bz?$X%c6kHGPpA%7!Hc?Yw^dQ?#S>=g>Vq)tRt~Vk6Mm56o76F>ihk6#!fU4OF zw$ubR6avXY>H<3KR83<@V1)1f`nP6i_OiDoS1sCAhzPr2a#g!R43$-N^At{a>4nQw z9Ye@Zg?uoVHa5k!OnGzIkle0PT^wu-%cf#QKrpEX+Yv&po;S&c-+1|tJvToy_y*4l z>H`|fn5WH9BsV8_DW*jvvQ8AprZNp_3()k|uzKbYMJ=<61|k2Ax=M)iTV;yQHKS_9 zwU_QuNXl!11hRnaf^d@%7~SF)-C_&6xd`E}oxHN%L_9N-feY|7Br!S~9@&Bk1b{fU frFph6#jOEM3oW_v(!|nudLc0?H", + "description" : "TableSourceScan(table=[[default_catalog, default_database, customer_changes]], fields=[customer_id, city, lsn])" + }, { + "id" : 10, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT>", + "description" : "Exchange(distribution=[hash[customer_id]])" + }, { + "id" : 11, + "type" : "stream-exec-process-table-function_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT>", + "description" : "ProcessTableFunction(invocation=[DEDUPLICATE(TABLE(#0) PARTITION BY($0), _UTF-16LE'first', DESCRIPTOR(_UTF-16LE'lsn'), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[DEDUPLICATE], select=[customer_id,city,lsn], rowType=[RecordType(VARCHAR(2147483647) customer_id, VARCHAR(2147483647) city, BIGINT lsn)])", + "uid" : "DEDUPLICATE", + "functionCall" : { + "kind" : "CALL", + "internalName" : "$DEDUPLICATE$1", + "operands" : [ { + "kind" : "TABLE_ARG_CALL", + "inputIndex" : 0, + "partitionKeys" : [ 0 ], + "type" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT> NOT NULL" + }, { + "kind" : "LITERAL", + "value" : "first", + "type" : "CHAR(5) NOT NULL" + }, { + "kind" : "CALL", + "internalName" : "$DESCRIPTOR$1", + "operands" : [ { + "kind" : "LITERAL", + "value" : "lsn", + "type" : "CHAR(3) NOT NULL" + } ], + "type" : "DESCRIPTOR NOT NULL" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "INTERVAL SECOND(6)" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "BOOLEAN" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "DESCRIPTOR" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "VARCHAR(2147483647)" + } ], + "type" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT> NOT NULL" + }, + "inputChangelogModes" : [ [ "INSERT" ] ], + "outputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], + "inputUpsertKeys" : [ [ ] ] + }, { + "id" : 12, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "customer_id", + "dataType" : "VARCHAR(2147483647) NOT NULL" + }, { + "name" : "city", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "lsn", + "dataType" : "BIGINT" + } ], + "primaryKey" : { + "name" : "PK_customer_id", + "type" : "PRIMARY_KEY", + "columns" : [ "customer_id" ] + } + } + } + } + }, + "inputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], + "inputUpsertKey" : [ 0 ], + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT>", + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[customer_id, city, lsn])" + } ], + "edges" : [ { + "source" : 9, + "target" : 10, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 10, + "target" : 11, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 11, + "target" : 12, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..c17b804f00ada24df1288ee3aff1682778fa07f4 GIT binary patch literal 13762 zcmeGjZERat_1a0(^gHXaZf&|MQ`54tzW4kbJGL8s%t;*L&2=o>$!MZtuHU;(Ui zdG9$*id519>qG@ZVgrG+Zb*e6qadUSiAsR>2c~LBd~AG3{1`|`{GkYG6G&(f&b{y1 zevX|sWl2hduVlY>zs^1PoOACz=iKw+pP1ngLLKm<$zwlIJKCL5w6iCAP57S{{io4Z z(VwH4Z@y&SJpJZZm)?4Ln*99rTfcKUEz*va-kx@r*-6`T1~JR3Wzu?D3~A@Lo#$`u z=dL#ADR|JA3yMlk2y@sF3c4y2!xVfu!I+&Z_|6s2h-$xR<&01;C*?KSAMin%DG6EO zOj%PbP81^p0bihhAQB1s`Um>^L!o}b#B-#iE1Jaxd?6C@aXugR%Y{gP*dLIDKtK%V zNyN|LycpsFc~RnU82dOmPr`m2^yPzM2%%lmfMJIRCb)b~bjlvoJD!TAquKFvE|ZOB zW4V!dCYz2AO=RQa$=u2Dkr?XAq!RILts=(ybxlSS6R}(-mX1dg@u^rEO1)@OH)eQT z!qP0^3o6i%w{Tu1yiwLHrAT;dp+w4-qMCec-XeyE)eJEdtSaY;(brg%(QwI})h&SC zhIaj#@#gw1w8b>g6Dy2FM=<$Og{ZPQf-Rhd{CnTKe(*aNU(R`vCxLbsXkDesv`|;# z44%URMN3F1rZqTkpd+`1Ug`YP=ilwV>ita7i@XW6N7A)9V7j7f!zwlb@B=nr6>BpB zD;i9o9jZQ~NLZz4S;(J&aRi%Q0;89}=q0dO2u@oeQR_1}Uz#(}fkyP$wA1=IAQeq* zUepewvAGT{BykZ&+vA$(Nn9G7Mthw4lx`{(Aa0=tr(H;o=g$zy0<7E5Dw=Ge-X`eM zaRnInX=nj0cb6Pcl|lEr22FrK5P1xnvvm0R60q@j<4l;q*lctNzyo|rFnemq!}!?2 za5q1RE+7wZ%(p5)E=tybd8g2J#Y~n}l@6UiopkWDu3HuwXfk8kXk`Y`elRt3!9an# zMTz!x&_a6=f?4gRqRCSU;$vF5=&meTs4YGI4AAf>YeeOjAJ$bBSO%;#`zExaE-M9v zO$y}t{XIv-@BihCU5t9|DwI@ccl^!(YJm3-p~W{p{*OZpj7PH2J{N~;&208QSc4oj z47@Ndj?xwF&<2E~4Fq2% zrer83Td4QCWIAIRiZ(-KT7yMFDbNG&l;iZheZB;A-654tOD_^5r^tZM>O^Z28^g2M zSSF`&)S(h>#+sd{;CG=5^wDLZt&`F8@K`kc>7}J5Cc_6{+;&N^7Fh0(hqmjjt8G(F z4X_bbMV%pKCazia$RHaBPX&jLq4-DR$t)|~&xU=Zjt{rrVxvGIMXGFt2cF9u`{((? zelLhEMcoU#R98)SZK}WWw`<|^gRc$IMkRZmsD7TuX;x@Tg5v7o{cy?nvnB@>@ELuGPxl{boNl! zKGKG!R=tnd+v2jVq*hZmd0Zr>cF_ zA$p`Cv;-!cnZ&8mDbM2weR$}#hhR(Yg&#|j$R3x5hyex#dbWDdL)*81^Z6mD?PV?C zPx7<3ODSw9V77fuaVjX>E5AJSBgpo`DZo+%?ieoS%f`&*DW{ya9r%6vCS-e^A|&;* zw@YbI3gGRt_Ar&&>a!=Kmko)W=&kAx(k!TM#GHP;%`^!mVdXH!>rtC-!Q1_mVv?0k7)%R zB*0($u`d8DCO{NsMI7Al>H8xo zLzjqYW~ntWTGp72FfEFw(THkwU10 zyp@(fT4ewC-Cxhu`wry|LAdAd8!}95kWE5?gUO9QSo{GJmLw5#AyEo&0U77HeBLK= zelbKuQ4)i4Fpto|ZvlznV4+UQ^|b3eEklI)GeHro`Ze#?jpa5(T#B@3y%lK>@)mq> zf3P18UgnGDs7pxZsHfU6xgw@+DVjW}>*?63*ofmI&z?%fP}|993aY%Qxg-3^yU;Sg z8Cp{)>U0Lj!ml9D^j3Y!wLHf4QzxK0lh>?G9H>mY?C-l`V5;8|LtF=C=H<$Egz~n0zQFn6k z|9?8cHuw5?qHcXGKSau!TWM$=BlL2cZ!ME%*R}U{nKXuGteu{9v~G#Gj*!;sp#C=I zG;bXuuG`xt!3>*sb+n%)aJg@%W~Ss8W^O?JO2*_^gE3jZbS+EevzE%-{i9d{8)Tu6 z=BuGqceFs($LMR^a$3<~>ktigxIsK@GQ(v7rlC{xe$f#>`q7jD5R=3iaVt0Ar{8eP51RGi9cva z)zRB6^;+OS-D;XH>x!iGGIHHkn(xIBe+ zIYpxwzEXkwo=RTP6m!;Hk}2oSGTejJz+}p1iD)t{>R|CH@UjhJNK_0L9Wn&{*Knf| z$4$R$aHb*DnhF)Gg!Z8!^E7!7C?eu&h>)p#97DUma`~;!Yai?W0So@#4mcJu3x2jx zurY#P(hVYzc_Kl`ysimVK%>=emGE#2H4Po_hL0Sq8WW-{n8re5jYBn?wHxm~`0}Cw z63_zTKl>YqK=J0Qc=Jv0W+a5a)qFg#Z8(}7PeRyF0=`snINV!>2?zi=mVL|9L=>wX XC>NT}#*GtG$v_v&Ok`5A state = stateStrategies(function); + assertThat(state).containsOnlyKeys("seen"); + assertThat(state.get("seen").inferType(callContext)) + .contains(ValueView.newValueViewDataType(DataTypes.BOOLEAN().notNull())); + assertThat(state.get("seen").getTimeToLive(callContext)).contains(Duration.ofSeconds(5)); + } + + @Test + void testSequencedDeclaresOnlyKeptState() { + final CallContextMock callContext = callContext("last", null); + final UserDefinedFunction function = specialize(callContext); + + assertThat(function).isInstanceOf(Deduplicate.Sequenced.class); + final Map state = stateStrategies(function); + assertThat(state).containsOnlyKeys("kept"); + assertThat(state.get("kept").inferType(callContext)) + .contains( + ValueView.newValueViewDataType( + DataTypeUtils.toInternalDataType( + DataTypes.ROW( + DataTypes.FIELD("event", DataTypes.STRING()))))); + assertThat(state.get("kept").getTimeToLive(callContext)).isEmpty(); + } + + private static Map stateStrategies( + final UserDefinedFunction function) { + return function.getTypeInference(new DataTypeFactoryMock()).getStateTypeStrategies(); + } + + private static UserDefinedFunction specialize(final CallContext callContext) { + return new Deduplicate() + .specialize( + new SpecializedContext() { + @Override + public CallContext getCallContext() { + return callContext; + } + + @Override + public ReadableConfig getConfiguration() { + return new Configuration(); + } + + @Override + public ClassLoader getBuiltInClassLoader() { + return DeduplicateTest.class.getClassLoader(); + } + + @Override + public ExpressionEvaluator createEvaluator( + Expression expression, + DataType outputDataType, + DataTypes.Field... args) { + throw new UnsupportedOperationException(); + } + + @Override + public ExpressionEvaluator createEvaluator( + String sqlExpression, + DataType outputDataType, + DataTypes.Field... args) { + throw new UnsupportedOperationException(); + } + + @Override + public ExpressionEvaluator createEvaluator( + BuiltInFunctionDefinition function, + DataType outputDataType, + DataType... args) { + throw new UnsupportedOperationException(); + } + }); + } + + private static CallContextMock callContext( + @Nullable final String keep, @Nullable final Duration stateTtl) { + final CallContextMock callContext = new CallContextMock(); + callContext.typeFactory = new DataTypeFactoryMock(); + callContext.functionDefinition = BuiltInFunctionDefinitions.DEDUPLICATE; + callContext.argumentDataTypes = + List.of( + INPUT_TYPE, + DataTypes.STRING(), + DataTypes.DESCRIPTOR(), + DataTypes.INTERVAL(DataTypes.SECOND()), + DataTypes.BOOLEAN()); + callContext.argumentValues = + Arrays.asList( + Optional.empty(), + Optional.ofNullable(keep), + Optional.empty(), + Optional.ofNullable(stateTtl), + Optional.empty()); + callContext.outputDataType = Optional.of(OUTPUT_TYPE); + callContext.tableSemantics = + Map.of( + 0, + new TableSemanticsMock( + INPUT_TYPE, + new int[] {0}, + new int[0], + -1, + ChangelogMode.insertOnly())); + return callContext; + } +} From 15b1bf5c239ee75136c35ef187906d5b150a9c19 Mon Sep 17 00:00:00 2001 From: Vas Shabu Date: Mon, 16 Nov 2026 16:06:44 +0000 Subject: [PATCH 3/9] [FLINK-41470][table] Split DEDUPLICATE into per-variant classes and drop redundant parts Address review feedback on the DEDUPLICATE PTF: - Make Deduplicate a small dispatcher and move its variants into DeduplicateFirstArrivalFunction and DeduplicateKeptRowFunction, with the shared state and TTL setup in DeduplicateFunctionBase and DeduplicateUtils. - Stop declaring SUPPORT_UPDATES so the framework rejects updating input at validation time, and remove the unreachable check in specialize(). - Drop the PARTITION BY check, which the framework already enforces for set-semantic tables, and fix the leftover text in the on_time error. - Remove the PyFlink partition_by/PartitionedTable additions. - Remove DeduplicateTest and the arrival-order restore program, which duplicate the semantic and remaining restore tests. --- .../docs/reference/pyflink.table/table.rst | 14 - flink-python/pyflink/table/__init__.py | 5 +- flink-python/pyflink/table/table.py | 60 +--- .../pyflink/table/tests/test_explain.py | 16 - .../table/tests/test_table_completeness.py | 28 +- .../functions/BuiltInFunctionDefinitions.java | 3 +- .../strategies/DeduplicateTypeStrategy.java | 249 ++------------- .../DeduplicateTypeStrategyUtils.java | 257 +++++++++++++++ .../DeduplicateTypeStrategyTest.java | 16 +- .../exec/stream/DeduplicateRestoreTest.java | 1 - .../DeduplicateRestoreTestPrograms.java | 39 --- .../exec/stream/DeduplicateSemanticTests.java | 3 +- .../exec/stream/DeduplicateTestPrograms.java | 20 -- .../deduplicate-arrival-order-restore.json | 159 ---------- .../savepoint/_metadata | Bin 12401 -> 0 bytes .../runtime/functions/ptf/Deduplicate.java | 293 ++---------------- .../ptf/DeduplicateFirstArrivalFunction.java | 81 +++++ .../ptf/DeduplicateFunctionBase.java | 62 ++++ .../ptf/DeduplicateKeptRowFunction.java | 196 ++++++++++++ .../functions/ptf/DeduplicateUtils.java | 25 +- .../functions/ptf/DeduplicateTest.java | 177 ----------- 21 files changed, 672 insertions(+), 1032 deletions(-) create mode 100644 flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java delete mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-arrival-order-restore/plan/deduplicate-arrival-order-restore.json delete mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-arrival-order-restore/savepoint/_metadata create mode 100644 flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFirstArrivalFunction.java create mode 100644 flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFunctionBase.java create mode 100644 flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeptRowFunction.java delete mode 100644 flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateTest.java diff --git a/flink-python/docs/reference/pyflink.table/table.rst b/flink-python/docs/reference/pyflink.table/table.rst index 4fae481ff3acf7..c4bd4eadfac070 100644 --- a/flink-python/docs/reference/pyflink.table/table.rst +++ b/flink-python/docs/reference/pyflink.table/table.rst @@ -110,7 +110,6 @@ Example: Table.offset Table.order_by Table.over_window - Table.partition_by Table.print_schema Table.rename_columns Table.right_outer_join @@ -137,19 +136,6 @@ A table that has been grouped on a set of grouping keys. GroupedTable.flat_aggregate -PartitionedTable -================ - -A table that has been partitioned on a set of partition keys. - -.. currentmodule:: pyflink.table - -.. autosummary:: - :toctree: api/ - - PartitionedTable.deduplicate - - GroupWindowedTable ================== diff --git a/flink-python/pyflink/table/__init__.py b/flink-python/pyflink/table/__init__.py index d14073b0b782e4..53ad751fcd8ec4 100644 --- a/flink-python/pyflink/table/__init__.py +++ b/flink-python/pyflink/table/__init__.py @@ -126,8 +126,8 @@ from pyflink.table.schema import Schema from pyflink.table.sql_dialect import SqlDialect from pyflink.table.statement_set import StatementSet -from pyflink.table.table import GroupWindowedTable, GroupedTable, OverWindowedTable, \ - PartitionedTable, Table, WindowGroupedTable +from pyflink.table.table import GroupWindowedTable, GroupedTable, OverWindowedTable, Table, \ + WindowGroupedTable from pyflink.table.table_config import TableConfig from pyflink.table.table_descriptor import TableDescriptor, FormatDescriptor from pyflink.table.table_environment import (TableEnvironment, StreamTableEnvironment) @@ -148,7 +148,6 @@ 'GroupedTable', 'GroupWindowedTable', 'OverWindowedTable', - 'PartitionedTable', 'WindowGroupedTable', 'ScalarFunction', 'TableFunction', diff --git a/flink-python/pyflink/table/table.py b/flink-python/pyflink/table/table.py index d0f756e054f3cd..68eef113bc03f9 100644 --- a/flink-python/pyflink/table/table.py +++ b/flink-python/pyflink/table/table.py @@ -46,8 +46,7 @@ from pyflink.util.java_utils import to_jarray from pyflink.util.java_utils import to_j_explain_detail_arr -__all__ = ['Table', 'GroupedTable', 'GroupWindowedTable', 'OverWindowedTable', 'PartitionedTable', - 'WindowGroupedTable'] +__all__ = ['Table', 'GroupedTable', 'GroupWindowedTable', 'OverWindowedTable', 'WindowGroupedTable'] @PublicEvolving() @@ -206,23 +205,6 @@ def group_by(self, *fields: Expression) -> 'GroupedTable': """ return GroupedTable(self._j_table.groupBy(to_expression_jarray(fields)), self._t_env) - def partition_by(self, *fields: Expression) -> 'PartitionedTable': - """ - Partitions the table by the given keys for a process table function with set semantics. - All rows of a key are processed by the same parallel instance. - - Example: - :: - - >>> tab.partition_by(col('key')).deduplicate() - - :param fields: Partition keys. - :return: The partitioned table. - """ - return PartitionedTable( - self._j_table.partitionBy(to_expression_jarray(fields)), self._t_env - ) - def distinct(self) -> 'Table': """ Removes duplicate values and returns only distinct (different) values. @@ -1444,46 +1426,6 @@ def flat_aggregate(self, func: Union[Expression, UserDefinedAggregateFunctionWra return FlatAggregateTable(self._j_table.flatAggregate(func._j_expr), self._t_env) -@PublicEvolving() -class PartitionedTable(object): - """ - A table that has been partitioned on a set of partition keys. - """ - - def __init__(self, java_table, t_env): - self._j_table = java_table - self._t_env = t_env - - def deduplicate(self, *arguments: Expression) -> 'Table': - """ - Keeps one row per partition key using the built-in ``DEDUPLICATE`` process table - function. The result is an updating table with the partition columns as its upsert key. - - ``keep => 'first'`` (default) keeps the earliest arrival or the smallest ``sequence_by`` - value, ``keep => 'last'`` keeps the latest arrival or the greatest value. - - Example: - :: - - >>> import datetime - >>> from pyflink.table.expressions import col, descriptor, lit - >>> # Keep the first delivery of each payment - >>> result = payments.partition_by(col('payment_id')).deduplicate() - >>> # Keep the version with the greatest 'lsn' per customer, with bounded retention - >>> result = customer_changes.partition_by(col('customer_id')).deduplicate( - ... lit('last').as_argument('keep'), - ... descriptor('lsn').as_argument('sequence_by'), - ... lit(datetime.timedelta(days=1)).as_argument('state_ttl') - ... ) - - :param arguments: Optional named arguments for ``keep``, ``sequence_by``, ``state_ttl``, - ``reset_ttl_on_duplicate``, and ``uid``. - :return: An updating :class:`~pyflink.table.Table` with the partition columns followed - by the remaining input columns. - """ - return Table(self._j_table.deduplicate(to_expression_jarray(arguments)), self._t_env) - - @PublicEvolving() class GroupWindowedTable(object): """ diff --git a/flink-python/pyflink/table/tests/test_explain.py b/flink-python/pyflink/table/tests/test_explain.py index 11941d617a49a2..b2a6a601369c08 100644 --- a/flink-python/pyflink/table/tests/test_explain.py +++ b/flink-python/pyflink/table/tests/test_explain.py @@ -15,12 +15,10 @@ # See the License for the specific language governing permissions and # limitations under the License. ################################################################################ -import datetime import json from pyflink.testing.test_case_utils import PyFlinkStreamTableTestCase from pyflink.table.explain_detail import ExplainDetail -from pyflink.table.expressions import col, descriptor, lit class StreamTableExplainTests(PyFlinkStreamTableTestCase): @@ -46,20 +44,6 @@ def test_explain(self): except: self.fail('The execution plan of explain detail is not in json format.') - def test_explain_deduplicate(self): - payments = self.t_env.from_elements([('p1', 50)], ['payment_id', 'amount']) - result = payments.partition_by(col('payment_id')).deduplicate().explain() - self.assertGreaterEqual(result.find('DEDUPLICATE'), 0) - - customer_changes = self.t_env.from_elements( - [('c1', 'Berlin', 100)], ['customer_id', 'city', 'lsn']) - result = customer_changes.partition_by(col('customer_id')).deduplicate( - lit('last').as_argument('keep'), - descriptor('lsn').as_argument('sequence_by'), - lit(datetime.timedelta(days=1)).as_argument('state_ttl') - ).explain() - self.assertGreaterEqual(result.find('DEDUPLICATE'), 0) - if __name__ == '__main__': import unittest diff --git a/flink-python/pyflink/table/tests/test_table_completeness.py b/flink-python/pyflink/table/tests/test_table_completeness.py index f201e9505a64b6..16866cca8d5dda 100644 --- a/flink-python/pyflink/table/tests/test_table_completeness.py +++ b/flink-python/pyflink/table/tests/test_table_completeness.py @@ -17,7 +17,7 @@ ################################################################################ from pyflink.testing.test_case_utils import PythonAPICompletenessTestCase, PyFlinkTestCase -from pyflink.table import PartitionedTable, Table +from pyflink.table import Table class TableAPICompletenessTests(PythonAPICompletenessTestCase, PyFlinkTestCase): @@ -41,6 +41,7 @@ def excluded_methods(cls): 'getQueryOperation', 'asArgument', 'process', + 'partitionBy', 'materializeInto', } @@ -56,31 +57,6 @@ def java_method_name(cls, python_method_name): return {'alias': 'as'}.get(python_method_name, python_method_name) -class PartitionedTableAPICompletenessTests(PythonAPICompletenessTestCase, PyFlinkTestCase): - """ - Tests whether the Python :class:`PartitionedTable` is consistent with - Java `org.apache.flink.table.api.PartitionedTable`. - """ - - @classmethod - def python_class(cls): - return PartitionedTable - - @classmethod - def java_class(cls): - return "org.apache.flink.table.api.PartitionedTable" - - @classmethod - def excluded_methods(cls): - return { - 'asArgument', - 'fromChangelog', - 'orderBy', - 'process', - 'toChangelog', - } - - if __name__ == '__main__': import unittest diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java index 36703573e40f03..b2d0dc228abd2c 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java @@ -951,8 +951,7 @@ ANY, and(logical(LogicalTypeRoot.BOOLEAN), LITERAL) false, EnumSet.of( StaticArgumentTrait.TABLE, - StaticArgumentTrait.SET_SEMANTIC_TABLE, - StaticArgumentTrait.SUPPORT_UPDATES)), + StaticArgumentTrait.SET_SEMANTIC_TABLE)), StaticArgument.scalar("keep", DataTypes.STRING(), true), StaticArgument.scalar("sequence_by", DataTypes.DESCRIPTOR(), true), StaticArgument.scalar( diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java index 09353bab44d4b0..825a7948b92b10 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java @@ -20,7 +20,6 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.table.api.DataTypes; -import org.apache.flink.table.api.DataTypes.Field; import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.connector.ChangelogMode; import org.apache.flink.table.functions.ChangelogFunction.ChangelogContext; @@ -30,21 +29,13 @@ import org.apache.flink.table.types.inference.CallContext; import org.apache.flink.table.types.inference.InputTypeStrategy; import org.apache.flink.table.types.inference.TypeStrategy; -import org.apache.flink.table.types.logical.LogicalType; -import org.apache.flink.table.types.logical.LogicalTypeFamily; -import org.apache.flink.table.types.logical.LogicalTypeRoot; -import org.apache.flink.table.types.logical.utils.LogicalTypeChecks; -import org.apache.flink.table.types.utils.DataTypeUtils; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep; import org.apache.flink.types.ColumnList; import org.apache.flink.types.RowKind; -import org.apache.flink.util.TimeUtils; -import java.time.Duration; import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import static org.apache.flink.table.types.inference.SystemTypeInference.PROCESS_TABLE_FUNCTION_ARG_ON_TIME_OFFSET; @@ -64,24 +55,7 @@ public final class DeduplicateTypeStrategy { public static final int ARG_STATE_TTL = 3; public static final int ARG_RESET_TTL_ON_DUPLICATE = 4; private static final int DECLARED_ARG_COUNT = ARG_RESET_TTL_ON_DUPLICATE + 1; - - /** Which record of a key {@code DEDUPLICATE} keeps. */ - @Internal - public enum Keep { - FIRST("first"), - LAST("last"); - - private final String value; - - Keep(final String value) { - this.value = value; - } - - /** Returns the mode for an argument value, or empty if the value names no mode. */ - public static Optional fromValue(final String value) { - return Arrays.stream(values()).filter(keep -> keep.value.equals(value)).findFirst(); - } - } + private static final String NAME = "DEDUPLICATE"; public static final InputTypeStrategy INPUT_TYPE_STRATEGY = new ValidationOnlyInputTypeStrategy() { @@ -98,23 +72,18 @@ public Optional> inferInputTypes( public static final TypeStrategy OUTPUT_TYPE_STRATEGY = callContext -> - Optional.of(DataTypes.ROW(outputFields(tableSemantics(callContext))).notNull()); + Optional.of( + DataTypes.ROW( + DeduplicateTypeStrategyUtils.outputFields( + tableSemantics(callContext))) + .notNull()); public static final ChangelogModeStrategy CHANGELOG_MODE_STRATEGY = DeduplicateTypeStrategy::inferChangelogMode; /** Returns the {@code keep} argument, or {@link Keep#FIRST} when it is omitted. */ public static Keep keep(final CallContext callContext) { - return callContext - .getArgumentValue(ARG_KEEP, String.class) - .map( - value -> - Keep.fromValue(value) - .orElseThrow( - () -> - new IllegalStateException( - "Validated keep argument expected."))) - .orElse(Keep.FIRST); + return DeduplicateTypeStrategyUtils.keep(callContext, ARG_KEEP); } /** @@ -122,37 +91,28 @@ public static Keep keep(final CallContext callContext) { * output row without the PARTITION BY columns. */ public static int[] sequenceByOutputIndices(final CallContext callContext) { - final TableSemantics tableSemantics = tableSemantics(callContext); - final List fieldNames = DataType.getFieldNames(tableSemantics.dataType()); - final List outputIndices = - Arrays.stream(ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics)) - .boxed() - .collect(Collectors.toList()); - return sequenceBy(callContext).stream() - .mapToInt(column -> outputIndices.indexOf(fieldNames.indexOf(column))) - .toArray(); + return DeduplicateTypeStrategyUtils.sequenceByOutputIndices( + callContext, tableSemantics(callContext), ARG_SEQUENCE_BY); } private static Optional validate(final CallContext callContext) { final TableSemantics tableSemantics = tableSemantics(callContext); - return validatePartitionBy(tableSemantics) - .or(() -> validateNoOrderBy(tableSemantics)) - .or(() -> validatePartitionByNoTimeAttribute(tableSemantics)) + return validateNoOrderBy(tableSemantics) + .or( + () -> + DeduplicateTypeStrategyUtils.validatePartitionByNoTimeAttribute( + tableSemantics, NAME)) .or(() -> validateOnTime(callContext)) - .or(() -> validateKeep(callContext)) - .or(() -> validateSequenceBy(callContext, tableSemantics)) - .or(() -> validateStateTtl(callContext)) - .or(() -> validateResetTtlOnDuplicate(callContext)); - } - - private static Optional validatePartitionBy(final TableSemantics tableSemantics) { - if (tableSemantics.partitionByColumns().length > 0) { - return Optional.empty(); - } - return Optional.of( - "DEDUPLICATE requires a key, but the input has no PARTITION BY. Add PARTITION BY " - + "with the columns to deduplicate on, for example TABLE t PARTITION BY " - + "id. To drop only rows that repeat in full, partition by every column."); + .or(() -> DeduplicateTypeStrategyUtils.validateKeep(callContext, ARG_KEEP, NAME)) + .or( + () -> + DeduplicateTypeStrategyUtils.validateSequenceBy( + callContext, tableSemantics, ARG_SEQUENCE_BY, NAME)) + .or(() -> DeduplicateTypeStrategyUtils.validateStateTtl(callContext, ARG_STATE_TTL)) + .or( + () -> + DeduplicateTypeStrategyUtils.validateResetTtlOnDuplicate( + callContext, ARG_RESET_TTL_ON_DUPLICATE)); } private static Optional validateNoOrderBy(final TableSemantics tableSemantics) { @@ -172,31 +132,6 @@ private static Optional validateNoOrderBy(final TableSemantics tableSema columns, columns)); } - private static Optional validatePartitionByNoTimeAttribute( - final TableSemantics tableSemantics) { - final List fields = DataType.getFields(tableSemantics.dataType()); - return Arrays.stream(tableSemantics.partitionByColumns()) - .mapToObj(fields::get) - .filter( - field -> - LogicalTypeChecks.isTimeAttribute( - field.getDataType().getLogicalType())) - .findFirst() - .map( - field -> - String.format( - "DEDUPLICATE does not support a time attribute as a " - + "PARTITION BY column, but '%s' is one. Partition " - + "by a regular column, or cast the column in a " - + "subquery first, for example CAST(%s AS %s) AS %s.", - field.getName(), - field.getName(), - DataTypeUtils.removeTimeAttribute(field.getDataType()) - .getLogicalType() - .asSummaryString(), - field.getName())); - } - private static Optional validateOnTime(final CallContext callContext) { final int argCount = callContext.getArgumentDataTypes().size(); if (argCount == DECLARED_ARG_COUNT) { @@ -212,114 +147,10 @@ private static Optional validateOnTime(final CallContext callContext) { String.format( "DEDUPLICATE does not support on_time, but received " + "on_time => DESCRIPTOR(%s). Use sequence_by => " - + "DESCRIPTOR(%s) to order by timestamp, or " - + "DEDUPLICATE_WINDOW for event-time windows." - + "ps.this function is not currently implemented", + + "DESCRIPTOR(%s) to order by a timestamp column.", names.get(0), names.get(0))); } - private static Optional validateKeep(final CallContext callContext) { - if (callContext.isArgumentNull(ARG_KEEP)) { - return Optional.empty(); - } - final String received; - if (!callContext.isArgumentLiteral(ARG_KEEP)) { - received = "a non-literal expression"; - } else { - final String value = callContext.getArgumentValue(ARG_KEEP, String.class).orElse(null); - if (value == null || Keep.fromValue(value).isPresent()) { - return Optional.empty(); - } - received = String.format("keep => '%s'", value); - } - return Optional.of( - String.format( - "DEDUPLICATE requires keep => 'first' or keep => 'last', but received %s. " - + "Use keep => 'first' to keep the first record per key, or " - + "keep => 'last' to keep the last one.", - received)); - } - - private static Optional validateSequenceBy( - final CallContext callContext, final TableSemantics tableSemantics) { - final List fieldNames = DataType.getFieldNames(tableSemantics.dataType()); - final List fieldTypes = DataType.getFieldDataTypes(tableSemantics.dataType()); - final Set seen = new HashSet<>(); - for (final String column : sequenceBy(callContext)) { - final int index = fieldNames.indexOf(column); - if (index == -1) { - return Optional.of( - String.format( - "The sequence_by column '%s' does not exist in the input of " - + "DEDUPLICATE. Available columns are: %s.", - column, String.join(", ", fieldNames))); - } - if (!seen.add(column)) { - return Optional.of( - String.format( - "The sequence_by column '%s' is listed more than once. List each " - + "ordering column once.", - column)); - } - if (Arrays.stream(tableSemantics.partitionByColumns()).anyMatch(c -> c == index)) { - return Optional.of( - String.format( - "The sequence_by column '%s' is a PARTITION BY column, which has " - + "the same value for every record of a key. Use a " - + "column that is not part of PARTITION BY, such as a " - + "version or sequence number.", - column)); - } - final LogicalType type = fieldTypes.get(index).getLogicalType(); - if (!isOrderable(type)) { - return Optional.of( - String.format( - "The sequence_by column '%s' has type '%s', which cannot order " - + "records. Use a numeric, character string, binary " - + "string, boolean, or date-time column.", - column, type.asSummaryString())); - } - } - return Optional.empty(); - } - - private static Optional validateStateTtl(final CallContext callContext) { - if (callContext.isArgumentNull(ARG_STATE_TTL)) { - return Optional.empty(); - } - if (!callContext.isArgumentLiteral(ARG_STATE_TTL)) { - return Optional.of( - "The 'state_ttl' argument must be a constant INTERVAL literal, but received a " - + "non-literal expression. Use a literal such as INTERVAL '1' HOUR, " - + "INTERVAL '0' to retain state indefinitely, or omit the argument " - + "to fall back to 'table.exec.state.ttl'."); - } - return callContext - .getArgumentValue(ARG_STATE_TTL, Duration.class) - .filter(Duration::isNegative) - .map( - stateTtl -> - String.format( - "The 'state_ttl' argument must not be negative, but was " - + "'%s'. Use a positive INTERVAL to bound state " - + "retention, INTERVAL '0' to retain state " - + "indefinitely, or omit the argument to fall " - + "back to 'table.exec.state.ttl'.", - TimeUtils.formatWithHighestUnit(stateTtl))); - } - - private static Optional validateResetTtlOnDuplicate(final CallContext callContext) { - if (callContext.isArgumentNull(ARG_RESET_TTL_ON_DUPLICATE) - || callContext.isArgumentLiteral(ARG_RESET_TTL_ON_DUPLICATE)) { - return Optional.empty(); - } - return Optional.of( - "The 'reset_ttl_on_duplicate' argument must be a constant BOOLEAN literal, but " - + "received a non-literal expression. Use TRUE to refresh the TTL on a " - + "dropped record, FALSE to keep it, or omit the argument for the default " - + "TRUE."); - } - private static ChangelogMode inferChangelogMode(final ChangelogContext changelogContext) { final ChangelogMode requiredMode = changelogContext.getRequiredChangelogMode(); final ChangelogMode.Builder builder = @@ -332,34 +163,6 @@ private static ChangelogMode inferChangelogMode(final ChangelogContext changelog return builder.build(); } - /** Returns the output columns after the PARTITION BY columns, without time attributes. */ - public static List outputFields(final TableSemantics tableSemantics) { - final List inputFields = DataType.getFields(tableSemantics.dataType()); - return Arrays.stream(ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics)) - .mapToObj(inputFields::get) - .map( - field -> - DataTypes.FIELD( - field.getName(), - DataTypeUtils.removeTimeAttribute(field.getDataType()))) - .collect(Collectors.toList()); - } - - private static List sequenceBy(final CallContext callContext) { - return callContext - .getArgumentValue(ARG_SEQUENCE_BY, ColumnList.class) - .map(ColumnList::getNames) - .orElse(List.of()); - } - - private static boolean isOrderable(final LogicalType type) { - return type.is(LogicalTypeFamily.NUMERIC) - || type.is(LogicalTypeFamily.CHARACTER_STRING) - || type.is(LogicalTypeFamily.BINARY_STRING) - || type.is(LogicalTypeFamily.DATETIME) - || type.is(LogicalTypeRoot.BOOLEAN); - } - private static TableSemantics tableSemantics(final CallContext callContext) { return callContext .getTableSemantics(ARG_INPUT) diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java new file mode 100644 index 00000000000000..e7588394636e2f --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.inference.strategies; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.DataTypes.Field; +import org.apache.flink.table.functions.TableSemantics; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.CallContext; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.LogicalTypeFamily; +import org.apache.flink.table.types.logical.LogicalTypeRoot; +import org.apache.flink.table.types.logical.utils.LogicalTypeChecks; +import org.apache.flink.table.types.utils.DataTypeUtils; +import org.apache.flink.types.ColumnList; +import org.apache.flink.util.TimeUtils; + +import java.time.Duration; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** Argument validation and output type helpers shared by the deduplication PTFs. */ +@Internal +public final class DeduplicateTypeStrategyUtils { + + /** Which record of a key is kept. */ + @Internal + public enum Keep { + FIRST("first"), + LAST("last"); + + private final String value; + + Keep(final String value) { + this.value = value; + } + + /** Returns the mode for an argument value, or empty if the value names no mode. */ + public static Optional fromValue(final String value) { + return Arrays.stream(values()).filter(keep -> keep.value.equals(value)).findFirst(); + } + } + + /** Returns the {@code keep} argument, or {@link Keep#FIRST} when it is omitted. */ + public static Keep keep(final CallContext callContext, final int argPos) { + return callContext + .getArgumentValue(argPos, String.class) + .map( + value -> + Keep.fromValue(value) + .orElseThrow( + () -> + new IllegalStateException( + "Validated keep argument expected."))) + .orElse(Keep.FIRST); + } + + /** + * Returns the positions of the {@code sequence_by} columns, in the listed order, within the + * output row without the PARTITION BY columns. + */ + public static int[] sequenceByOutputIndices( + final CallContext callContext, final TableSemantics tableSemantics, final int argPos) { + final List fieldNames = DataType.getFieldNames(tableSemantics.dataType()); + final List outputIndices = + Arrays.stream(ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics)) + .boxed() + .collect(Collectors.toList()); + return sequenceBy(callContext, argPos).stream() + .mapToInt(column -> outputIndices.indexOf(fieldNames.indexOf(column))) + .toArray(); + } + + /** Returns the output columns after the PARTITION BY columns, without time attributes. */ + public static List outputFields(final TableSemantics tableSemantics) { + final List inputFields = DataType.getFields(tableSemantics.dataType()); + return Arrays.stream(ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics)) + .mapToObj(inputFields::get) + .map( + field -> + DataTypes.FIELD( + field.getName(), + DataTypeUtils.removeTimeAttribute(field.getDataType()))) + .collect(Collectors.toList()); + } + + public static Optional validatePartitionByNoTimeAttribute( + final TableSemantics tableSemantics, final String functionName) { + final List fields = DataType.getFields(tableSemantics.dataType()); + return Arrays.stream(tableSemantics.partitionByColumns()) + .mapToObj(fields::get) + .filter( + field -> + LogicalTypeChecks.isTimeAttribute( + field.getDataType().getLogicalType())) + .findFirst() + .map( + field -> + String.format( + "%s does not support a time attribute as a " + + "PARTITION BY column, but '%s' is one. Partition " + + "by a regular column, or cast the column in a " + + "subquery first, for example CAST(%s AS %s) AS %s.", + functionName, + field.getName(), + field.getName(), + DataTypeUtils.removeTimeAttribute(field.getDataType()) + .getLogicalType() + .asSummaryString(), + field.getName())); + } + + public static Optional validateKeep( + final CallContext callContext, final int argPos, final String functionName) { + if (callContext.isArgumentNull(argPos)) { + return Optional.empty(); + } + final String received; + if (!callContext.isArgumentLiteral(argPos)) { + received = "a non-literal expression"; + } else { + final String value = callContext.getArgumentValue(argPos, String.class).orElse(null); + if (value == null || Keep.fromValue(value).isPresent()) { + return Optional.empty(); + } + received = String.format("keep => '%s'", value); + } + return Optional.of( + String.format( + "%s requires keep => 'first' or keep => 'last', but received %s. " + + "Use keep => 'first' to keep the first record per key, or " + + "keep => 'last' to keep the last one.", + functionName, received)); + } + + public static Optional validateSequenceBy( + final CallContext callContext, + final TableSemantics tableSemantics, + final int argPos, + final String functionName) { + final List fieldNames = DataType.getFieldNames(tableSemantics.dataType()); + final List fieldTypes = DataType.getFieldDataTypes(tableSemantics.dataType()); + final Set seen = new HashSet<>(); + for (final String column : sequenceBy(callContext, argPos)) { + final int index = fieldNames.indexOf(column); + if (index == -1) { + return Optional.of( + String.format( + "The sequence_by column '%s' does not exist in the input of " + + "%s. Available columns are: %s.", + column, functionName, String.join(", ", fieldNames))); + } + if (!seen.add(column)) { + return Optional.of( + String.format( + "The sequence_by column '%s' is listed more than once. List each " + + "ordering column once.", + column)); + } + if (Arrays.stream(tableSemantics.partitionByColumns()).anyMatch(c -> c == index)) { + return Optional.of( + String.format( + "The sequence_by column '%s' is a PARTITION BY column, which has " + + "the same value for every record of a key. Use a " + + "column that is not part of PARTITION BY, such as a " + + "version or sequence number.", + column)); + } + final LogicalType type = fieldTypes.get(index).getLogicalType(); + if (!isOrderable(type)) { + return Optional.of( + String.format( + "The sequence_by column '%s' has type '%s', which cannot order " + + "records. Use a numeric, character string, binary " + + "string, boolean, or date-time column.", + column, type.asSummaryString())); + } + } + return Optional.empty(); + } + + public static Optional validateStateTtl( + final CallContext callContext, final int argPos) { + if (callContext.isArgumentNull(argPos)) { + return Optional.empty(); + } + if (!callContext.isArgumentLiteral(argPos)) { + return Optional.of( + "The 'state_ttl' argument must be a constant INTERVAL literal, but received a " + + "non-literal expression. Use a literal such as INTERVAL '1' HOUR, " + + "INTERVAL '0' to retain state indefinitely, or omit the argument " + + "to fall back to 'table.exec.state.ttl'."); + } + return callContext + .getArgumentValue(argPos, Duration.class) + .filter(Duration::isNegative) + .map( + stateTtl -> + String.format( + "The 'state_ttl' argument must not be negative, but was " + + "'%s'. Use a positive INTERVAL to bound state " + + "retention, INTERVAL '0' to retain state " + + "indefinitely, or omit the argument to fall " + + "back to 'table.exec.state.ttl'.", + TimeUtils.formatWithHighestUnit(stateTtl))); + } + + public static Optional validateResetTtlOnDuplicate( + final CallContext callContext, final int argPos) { + if (callContext.isArgumentNull(argPos) || callContext.isArgumentLiteral(argPos)) { + return Optional.empty(); + } + return Optional.of( + "The 'reset_ttl_on_duplicate' argument must be a constant BOOLEAN literal, but " + + "received a non-literal expression. Use TRUE to refresh the TTL on a " + + "dropped record, FALSE to keep it, or omit the argument for the default " + + "TRUE."); + } + + private static List sequenceBy(final CallContext callContext, final int argPos) { + return callContext + .getArgumentValue(argPos, ColumnList.class) + .map(ColumnList::getNames) + .orElse(List.of()); + } + + private static boolean isOrderable(final LogicalType type) { + return type.is(LogicalTypeFamily.NUMERIC) + || type.is(LogicalTypeFamily.CHARACTER_STRING) + || type.is(LogicalTypeFamily.BINARY_STRING) + || type.is(LogicalTypeFamily.DATETIME) + || type.is(LogicalTypeRoot.BOOLEAN); + } + + private DeduplicateTypeStrategyUtils() {} +} diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java index 82591ca87f711f..d347ce34535170 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java @@ -98,19 +98,6 @@ protected Stream testData() { .expectErrorMessage( "DEDUPLICATE requires keep => 'first' or keep => 'last', but " + "received a non-literal expression."), - withDefaults( - "No PARTITION BY", - new TableSemanticsMock( - INPUT_TYPE, - new int[0], - new int[0], - -1, - ChangelogMode.insertOnly())) - .expectErrorMessage( - "DEDUPLICATE requires a key, but the input has no PARTITION BY. " - + "Add PARTITION BY with the columns to deduplicate on, " - + "for example TABLE t PARTITION BY id. To drop only rows " - + "that repeat in full, partition by every column."), withDefaults( "ORDER BY", new TableSemanticsMock( @@ -155,8 +142,7 @@ protected Stream testData() { .expectErrorMessage( "DEDUPLICATE does not support on_time, but received on_time => " + "DESCRIPTOR(ts). Use sequence_by => DESCRIPTOR(ts) to " - + "order by timestamp, or DEDUPLICATE_WINDOW for " - + "event-time windows."), + + "order by a timestamp column."), sequenceBy("Unknown sequence_by column", "nope") .expectErrorMessage( "The sequence_by column 'nope' does not exist in the input of " diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTest.java index 7f55ac172a8557..e9a6d2d9b15dcb 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTest.java @@ -33,7 +33,6 @@ public DeduplicateRestoreTest() { @Override public List programs() { return List.of( - DeduplicateRestoreTestPrograms.ARRIVAL_ORDER_RESTORE, DeduplicateRestoreTestPrograms.FIRST_ARRIVAL_RESTORE, DeduplicateRestoreTestPrograms.SEQUENCE_BY_RESTORE); } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTestPrograms.java index 0d7001655b6930..133d9ad95284ae 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTestPrograms.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateRestoreTestPrograms.java @@ -26,45 +26,6 @@ /** {@link TableTestProgram} definitions for restore tests of the DEDUPLICATE PTF. */ public class DeduplicateRestoreTestPrograms { - public static final TableTestProgram ARRIVAL_ORDER_RESTORE = - TableTestProgram.of( - "deduplicate-arrival-order-restore", - "the kept record survives a restore: a key seen before the restore " - + "is updated or emits nothing for an identical record, and a " - + "key first seen after it is inserted") - .setupTableSource( - SourceTestStep.newBuilder("user_events") - .addSchema("user_id STRING", "event STRING") - .producedBeforeRestore( - Row.of("u1", "login"), - Row.of("u2", "login"), - Row.of("u1", "click"), - Row.of("u3", "view")) - .producedAfterRestore( - Row.of("u1", "click"), - Row.of("u1", "logout"), - Row.of("u2", "login"), - Row.of("u4", "login")) - .build()) - .setupTableSink( - SinkTestStep.newBuilder("sink") - .addSchema( - "user_id STRING PRIMARY KEY NOT ENFORCED", - "event STRING") - .addOption("sink-changelog-mode-enforced", "I,UA,D") - .consumedBeforeRestore( - "+I[u1, login]", - "+I[u2, login]", - "+U[u1, click]", - "+I[u3, view]") - .consumedAfterRestore("+U[u1, logout]", "+I[u4, login]") - .build()) - .runSql( - "INSERT INTO sink SELECT * FROM DEDUPLICATE(" - + "input => TABLE user_events PARTITION BY user_id, " - + "keep => 'last')") - .build(); - public static final TableTestProgram FIRST_ARRIVAL_RESTORE = TableTestProgram.of( "deduplicate-first-arrival-restore", diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateSemanticTests.java index 6799130151160e..f8c5c8d3669f90 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateSemanticTests.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateSemanticTests.java @@ -31,7 +31,6 @@ public List programs() { return List.of( DeduplicateTestPrograms.ARRIVAL_ORDER, DeduplicateTestPrograms.COMPOSITE_AND_NULL_KEY, - DeduplicateTestPrograms.SEQUENCE_BY, - DeduplicateTestPrograms.NO_PARTITION_BY_FAILS); + DeduplicateTestPrograms.SEQUENCE_BY); } } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateTestPrograms.java index 24e1d3468cb620..6a1550eb3a7085 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateTestPrograms.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateTestPrograms.java @@ -18,7 +18,6 @@ package org.apache.flink.table.planner.plan.nodes.exec.stream; -import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.test.program.SinkTestStep; import org.apache.flink.table.test.program.SourceTestStep; import org.apache.flink.table.test.program.TableTestProgram; @@ -299,25 +298,6 @@ public class DeduplicateTestPrograms { + "sequence_by => DESCRIPTOR(s, b, t, y))") .build(); - public static final TableTestProgram NO_PARTITION_BY_FAILS = - TableTestProgram.of("deduplicate-no-partition-by-fails", "PARTITION BY is required") - .setupTableSource( - SourceTestStep.newBuilder("user_events") - .addSchema(USER_EVENTS_SCHEMA) - .producedValues(Row.of("u1", "login")) - .build()) - .setupTableSink( - SinkTestStep.newBuilder("sink") - .addSchema(USER_EVENTS_SCHEMA) - .consumedValues(new Row[0]) - .build()) - .runFailingSql( - "INSERT INTO sink SELECT * FROM DEDUPLICATE(" - + "input => TABLE user_events)", - ValidationException.class, - "DEDUPLICATE requires a key, but the input has no PARTITION BY.") - .build(); - private static SinkTestStep upsertSink( final String name, final String[] schema, final String... expected) { return SinkTestStep.newBuilder(name) diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-arrival-order-restore/plan/deduplicate-arrival-order-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-arrival-order-restore/plan/deduplicate-arrival-order-restore.json deleted file mode 100644 index a6e4ef1d4302c9..00000000000000 --- a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-arrival-order-restore/plan/deduplicate-arrival-order-restore.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "flinkVersion" : "2.5", - "nodes" : [ { - "id" : 1, - "type" : "stream-exec-table-source-scan_2", - "scanTableSource" : { - "table" : { - "identifier" : "`default_catalog`.`default_database`.`user_events`", - "resolvedTable" : { - "schema" : { - "columns" : [ { - "name" : "user_id", - "dataType" : "VARCHAR(2147483647)" - }, { - "name" : "event", - "dataType" : "VARCHAR(2147483647)" - } ] - } - } - } - }, - "outputType" : "ROW<`user_id` VARCHAR(2147483647), `event` VARCHAR(2147483647)>", - "description" : "TableSourceScan(table=[[default_catalog, default_database, user_events]], fields=[user_id, event])" - }, { - "id" : 2, - "type" : "stream-exec-exchange_1", - "inputProperties" : [ { - "requiredDistribution" : { - "type" : "HASH", - "keys" : [ 0 ] - }, - "damBehavior" : "PIPELINED", - "priority" : 0 - } ], - "outputType" : "ROW<`user_id` VARCHAR(2147483647), `event` VARCHAR(2147483647)>", - "description" : "Exchange(distribution=[hash[user_id]])" - }, { - "id" : 3, - "type" : "stream-exec-process-table-function_1", - "inputProperties" : [ { - "requiredDistribution" : { - "type" : "UNKNOWN" - }, - "damBehavior" : "PIPELINED", - "priority" : 0 - } ], - "outputType" : "ROW<`user_id` VARCHAR(2147483647), `event` VARCHAR(2147483647)>", - "description" : "ProcessTableFunction(invocation=[DEDUPLICATE(TABLE(#0) PARTITION BY($0), _UTF-16LE'last', DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[DEDUPLICATE], select=[user_id,event], rowType=[RecordType(VARCHAR(2147483647) user_id, VARCHAR(2147483647) event)])", - "uid" : "DEDUPLICATE", - "functionCall" : { - "kind" : "CALL", - "internalName" : "$DEDUPLICATE$1", - "operands" : [ { - "kind" : "TABLE_ARG_CALL", - "inputIndex" : 0, - "partitionKeys" : [ 0 ], - "type" : "ROW<`user_id` VARCHAR(2147483647), `event` VARCHAR(2147483647)> NOT NULL" - }, { - "kind" : "LITERAL", - "value" : "last", - "type" : "CHAR(4) NOT NULL" - }, { - "kind" : "CALL", - "syntax" : "SPECIAL", - "internalName" : "$DEFAULT$1", - "type" : "DESCRIPTOR" - }, { - "kind" : "CALL", - "syntax" : "SPECIAL", - "internalName" : "$DEFAULT$1", - "type" : "INTERVAL SECOND(6)" - }, { - "kind" : "CALL", - "syntax" : "SPECIAL", - "internalName" : "$DEFAULT$1", - "type" : "BOOLEAN" - }, { - "kind" : "CALL", - "syntax" : "SPECIAL", - "internalName" : "$DEFAULT$1", - "type" : "DESCRIPTOR" - }, { - "kind" : "CALL", - "syntax" : "SPECIAL", - "internalName" : "$DEFAULT$1", - "type" : "VARCHAR(2147483647)" - } ], - "type" : "ROW<`user_id` VARCHAR(2147483647), `event` VARCHAR(2147483647)> NOT NULL" - }, - "inputChangelogModes" : [ [ "INSERT" ] ], - "outputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], - "inputUpsertKeys" : [ [ ] ] - }, { - "id" : 4, - "type" : "stream-exec-sink_2", - "configuration" : { - "table.exec.sink.keyed-shuffle" : "AUTO", - "table.exec.sink.not-null-enforcer" : "ERROR", - "table.exec.sink.rowtime-inserter" : "ENABLED", - "table.exec.sink.type-length-enforcer" : "IGNORE", - "table.exec.sink.upsert-materialize" : "AUTO" - }, - "dynamicTableSink" : { - "table" : { - "identifier" : "`default_catalog`.`default_database`.`sink`", - "resolvedTable" : { - "schema" : { - "columns" : [ { - "name" : "user_id", - "dataType" : "VARCHAR(2147483647) NOT NULL" - }, { - "name" : "event", - "dataType" : "VARCHAR(2147483647)" - } ], - "primaryKey" : { - "name" : "PK_user_id", - "type" : "PRIMARY_KEY", - "columns" : [ "user_id" ] - } - } - } - } - }, - "inputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], - "upsertMaterializeStrategy" : "VALUE", - "inputUpsertKey" : [ 0 ], - "inputProperties" : [ { - "requiredDistribution" : { - "type" : "UNKNOWN" - }, - "damBehavior" : "PIPELINED", - "priority" : 0 - } ], - "outputType" : "ROW<`user_id` VARCHAR(2147483647), `event` VARCHAR(2147483647)>", - "description" : "Sink(table=[default_catalog.default_database.sink], fields=[user_id, event])" - } ], - "edges" : [ { - "source" : 1, - "target" : 2, - "shuffle" : { - "type" : "FORWARD" - }, - "shuffleMode" : "PIPELINED" - }, { - "source" : 2, - "target" : 3, - "shuffle" : { - "type" : "FORWARD" - }, - "shuffleMode" : "PIPELINED" - }, { - "source" : 3, - "target" : 4, - "shuffle" : { - "type" : "FORWARD" - }, - "shuffleMode" : "PIPELINED" - } ] -} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-arrival-order-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-arrival-order-restore/savepoint/_metadata deleted file mode 100644 index 7a12e418b941b44502955a54bb1fcde998051bfb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12401 zcmeGiTWlj&^^TKlv)j!kZL?{!U8;ypS6a0*<7aGdgoNWbY*cSz*-lCmE#`XWI+y;=H7eGJ@-8BJ?G4s{qzct5IP1wdU5s*^`e1-qOD&XGvR+ZdIcSemgwWf zk?;QFmoJ_D;H{V6`)2*Or?U>o4GQMMa=q;3BpuHi#H_29N&9ItTqi!^+$HD!taE>v zLtm{ZD!C{&upw4-RVIch21}x`+NcDtR@XvmBxIG0c&sFqH8~s&!nINpi{e^cQ!GA| zhypR`%E@( z{t5M&26|$Lndl6rFryGvHm9+Li}3!z_uf4Fo#$RD1rV1-rz*71?J_MilwHFOEK;(> ztYTV|8wPrOSL`nEJ1_iw_#1&QR0AlGMT3&AH9+W!uBBCM0^$d3#46TSMAkH!Mfa=v ziXvf^l4YTA7UmIceu<125@Y}tAJDzJ)PJpt4Uv$3^K zZO-8;%r@v*=w)1+Tt$+v3i4H5KgLW1Wg@dJ`>jsMMmm~`7kcA#Z z2v&89s-{3SNXTgQs<*Rdp`QHw)4;>?Y!KC7A+4(_hzvw&jxT6cT~;azTNLQ^zXl%< z{os#Z8DiY)RiVY@-IKRQ&_X;qg>L-(Qy)IY&~TE4Mm!R34YS4jPz!U)Fz_a|uZ`z^ z^^uo;iof22xLMR|g6RXhVDq?2%{u_=iS3DRj!uljeTG`J!y8bJ9#DLln3AE?Y^8qC zqtgY;P_z}Q(;65B<-nuxOeIdAdpByp*Zb-wF-nRIwCqi-7O|0D#l|)rokhn~qODk~ z8x-{bx=wFHfPOKRPhU#qpWWKpV)}aq=IkR4qFL%TKH*{f`i)|TPby`#-e?l5*>FFQSU z5I{?NwgcPzwwXewMvpc095jBexd%EM6Wd+s7R0W4h}}L<&u0yxFTq=@FYNDwO@&=9 zoTMt!`gXX84cAZzbz;6~Lp1+Sr{PSq3=H0IMyu$CL=5WMk3>u0cR3w=13XC?Te^|{ z{Uggi`q<}&uhrh4xcS%Uj{yzcpKefeuB%U5}0@91qyD=Lv?$p2hAt7)EKbacCL015E zPAPtTAICnVm0Bl!+ac~G4-eHx&+UWqtgfxJNFQ&1ZClCkTEVoG;MA<_{4HrneW0u%2QgGl2 zt^z(t9xH-gBL?;2O`%8?s6}cUNw!D>Z6fV%P`912yqodzA#G3x9h8r^83~IZyY|ui z21IkLra)5$@cl!$Rm$|9OV4%f!xLn8%1@1VYHZuFpJ=m>?T1KR^EwT1d~}bIu9nSzJLhMeZ!;0k5z@5|nr>rTQ1=ko{j>1VwT|9(0-s9lt5Yc1$HEQh-zk{9 z3g$CrR_ZH6oDK}(C6QBx^*=NP*-!#rye$&;hPes zMJV(uouhvAcBCs<4#nR+QWY#5t6<>{zk=mqUj>U(O|sV@>Q=!Rt^7F7vHF#h?e#0H zpyV_w3K<5AXtCgbEl@eXn<=&;jW%nfWInoacm*lll(iGg|6iu8Z5u&%C!g8}iaYW= z{0+Gp?AAum_p@M0ul;7>TQ7JAxJ46NTZlow;_a+E)e}2HyJZzF^oMQGPR+)S&~Bjz zK)aXoxhCPWQ`e({8enOil`jkF<|BMNhvcrM5LQ(UxB>yxzl4Tl#gwoir?DmxC}EVB z(16o4gQ2V%-Ur=xMN`aGZ%d(GHtX=Yp@pVUH)}+bY11)QD*_M4;Y+=WA)hQmm1>JN zTC>-&5Z76T*cvLj&L4%LHJYCU@G}8og>lrQ7vDR@Is&cKC5K^pU!9th(Q*~Okm+o;W74~5Q zUO=%`#K#0mx6q_p=s-7PA^h#7+VyxkH8(#8)fHJNt#%|l*2D<}06Vr7pLbBjCIIDj U$JKagVh10bc!h;RE;Bd%Uv>=zPXGV_ diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java index ed4befdb69cbfa..ad8c2cfd33f437 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java @@ -19,46 +19,23 @@ package org.apache.flink.table.runtime.functions.ptf; import org.apache.flink.annotation.Internal; -import org.apache.flink.table.api.DataTypes; -import org.apache.flink.table.api.dataview.ValueView; -import org.apache.flink.table.connector.ChangelogMode; -import org.apache.flink.table.data.GenericRowData; -import org.apache.flink.table.data.RowData; -import org.apache.flink.table.data.StringData; -import org.apache.flink.table.data.binary.BinaryRowData; -import org.apache.flink.table.data.utils.JoinedRowData; -import org.apache.flink.table.data.utils.ProjectedRowData; import org.apache.flink.table.functions.BuiltInFunctionDefinitions; -import org.apache.flink.table.functions.FunctionContext; -import org.apache.flink.table.functions.TableSemantics; import org.apache.flink.table.functions.UserDefinedFunction; import org.apache.flink.table.runtime.functions.BuiltInSpecializedFunction; -import org.apache.flink.table.runtime.functions.ptf.DeduplicateUtils.SequenceComparator; -import org.apache.flink.table.runtime.typeutils.RowDataSerializer; -import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.inference.CallContext; -import org.apache.flink.table.types.inference.StateTypeStrategy; -import org.apache.flink.table.types.inference.strategies.ChangelogTypeStrategyUtils; import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy; -import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.Keep; -import org.apache.flink.table.types.logical.LogicalType; -import org.apache.flink.types.ColumnList; -import org.apache.flink.types.RowKind; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep; -import javax.annotation.Nullable; - -import java.time.Duration; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_INPUT; -import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_RESET_TTL_ON_DUPLICATE; -import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL; - -/** Runtime implementation of {@link BuiltInFunctionDefinitions#DEDUPLICATE}. */ +/** + * Runtime implementation of {@link BuiltInFunctionDefinitions#DEDUPLICATE}. + * + *

+ * keep    sequence_by  implementation
+ * first   absent       {@link DeduplicateFirstArrivalFunction}
+ * first   present      {@link DeduplicateKeptRowFunction}
+ * last    any          {@link DeduplicateKeptRowFunction}
+ * 
+ */ @Internal public class Deduplicate extends BuiltInSpecializedFunction { @@ -68,249 +45,15 @@ public Deduplicate() { @Override public UserDefinedFunction specialize(final SpecializedContext context) { - final ChangelogMode inputMode = - tableSemantics(context) - .changelogMode() - .orElseThrow(() -> new IllegalStateException("Changelog mode expected.")); - if (!inputMode.containsOnly(RowKind.INSERT)) { - throw new IllegalStateException("Currently not implemented"); - } final CallContext callContext = context.getCallContext(); - final boolean isFirstArrival = - DeduplicateTypeStrategy.keep(callContext) == Keep.FIRST - && DeduplicateTypeStrategy.sequenceByOutputIndices(callContext).length == 0; - return isFirstArrival ? new FirstArrival(context) : new Sequenced(context); - } - - private abstract static class Base extends BuiltInProcessTableFunction { - - private static final long serialVersionUID = 1L; - - protected final int[] outputIndices; - - protected final boolean isResetTtlOnDuplicate; - - Base( - final SpecializedContext context, - final LinkedHashMap stateStrategies) { - super(BuiltInFunctionDefinitions.DEDUPLICATE, context, stateStrategies); - final CallContext callContext = context.getCallContext(); - this.outputIndices = - ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics(context)); - this.isResetTtlOnDuplicate = - callContext - .getArgumentValue(ARG_RESET_TTL_ON_DUPLICATE, Boolean.class) - .orElse(true) - && !Duration.ZERO.equals( - callContext - .getArgumentValue(ARG_STATE_TTL, Duration.class) - .orElse(null)); - } - } - - /** - * {@code keep => 'first'} without {@code sequence_by}: no later record can replace the first - * one, so only whether the key has emitted is kept in the {@code seen} state. - */ - @Internal - public static class FirstArrival extends Base { - - private static final long serialVersionUID = 1L; - - private transient ProjectedRowData projectedOutput; - - FirstArrival(final SpecializedContext context) { - super(context, internalState(Map.entry("seen", seenState(context)))); - } - - @Override - public void open(final FunctionContext context) throws Exception { - super.open(context); - projectedOutput = ProjectedRowData.from(outputIndices); - } - - public void eval( - final Context ctx, - final ValueView seen, - final RowData input, - @Nullable final StringData keep, - @Nullable final ColumnList sequenceBy, - @Nullable final Long stateTtl, - @Nullable final Boolean resetTtlOnDuplicate) - throws Exception { - if (seen.getValue() == null) { - projectedOutput.replaceRow(input); - projectedOutput.setRowKind(RowKind.INSERT); - collect(projectedOutput); - seen.setValue(true); - } else if (isResetTtlOnDuplicate) { - seen.setValue(true); - } - } - } - - /** - * {@code keep => 'last'}, or {@code keep => 'first'} with {@code sequence_by}: a later record - * can replace the kept one, so the kept record is held in the {@code kept} state. - */ - @Internal - public static class Sequenced extends Base { - - private static final long serialVersionUID = 1L; - - private final Keep keep; - - private final LogicalType[] payloadTypes; - - private final boolean isUpdateBeforeEmitted; - - private final @Nullable SequenceComparator sequenceComparator; - - private transient RowData.FieldGetter[] payloadGetters; - - private transient RowDataSerializer candidateSerializer; - - private transient RowDataSerializer currentSerializer; - - private transient GenericRowData reusedCandidate; - - private transient JoinedRowData output; - - private transient RowData emptyRow; - - Sequenced(final SpecializedContext context) { - super(context, internalState(Map.entry("kept", keptState(context)))); - final CallContext callContext = context.getCallContext(); - this.keep = DeduplicateTypeStrategy.keep(callContext); - this.payloadTypes = - DeduplicateTypeStrategy.outputFields(tableSemantics(context)).stream() - .map(field -> field.getDataType().getLogicalType()) - .toArray(LogicalType[]::new); - this.isUpdateBeforeEmitted = - callContext - .getOutputChangelogMode() - .map(mode -> mode.contains(RowKind.UPDATE_BEFORE)) - .orElse(false); - this.sequenceComparator = sequenceComparator(context, outputIndices); - } - - @Override - public void open(final FunctionContext context) throws Exception { - super.open(context); - payloadGetters = new RowData.FieldGetter[outputIndices.length]; - for (int i = 0; i < outputIndices.length; i++) { - payloadGetters[i] = RowData.createFieldGetter(payloadTypes[i], outputIndices[i]); - } - if (sequenceComparator != null) { - sequenceComparator.open(context); - } - // toBinaryRow reuses its result row, so each side of a comparison needs its own - // serializer - candidateSerializer = new RowDataSerializer(payloadTypes); - currentSerializer = new RowDataSerializer(payloadTypes); - reusedCandidate = new GenericRowData(outputIndices.length); - output = new JoinedRowData(); - emptyRow = GenericRowData.of(); - } - - @Override - public void close() throws Exception { - if (sequenceComparator != null) { - sequenceComparator.close(); - } - super.close(); - } - - public void eval( - final Context ctx, - final ValueView keptState, - final RowData input, - @Nullable final StringData keep, - @Nullable final ColumnList sequenceBy, - @Nullable final Long stateTtl, - @Nullable final Boolean resetTtlOnDuplicate) - throws Exception { - final RowData kept = keptState.getValue(); - if (kept == null) { - final BinaryRowData candidate = toCandidate(input); - emit(RowKind.INSERT, candidate); - keptState.setValue(candidate.copy()); - return; - } - final int order = - sequenceComparator == null ? 0 : sequenceComparator.compare(input, kept); - if (DeduplicateUtils.replaces(this.keep, order)) { - final BinaryRowData candidate = toCandidate(input); - if (!candidate.equals(currentSerializer.toBinaryRow(kept))) { - if (isUpdateBeforeEmitted) { - emit(RowKind.UPDATE_BEFORE, kept); - } - emit(RowKind.UPDATE_AFTER, candidate); - keptState.setValue(candidate.copy()); - return; - } - } - if (isResetTtlOnDuplicate) { - keptState.setValue(kept); - } - } - - /** Returns the payload of the input as a reused binary row, valid until the next call. */ - private BinaryRowData toCandidate(final RowData input) { - for (int i = 0; i < outputIndices.length; i++) { - reusedCandidate.setField(i, payloadGetters[i].getFieldOrNull(input)); - } - return candidateSerializer.toBinaryRow(reusedCandidate); - } - - /** Wraps the payload so that rows owned by state never change their row kind. */ - private void emit(final RowKind kind, final RowData payload) { - output.replace(payload, emptyRow); - output.setRowKind(kind); - collect(output); - } - } - - private static TableSemantics tableSemantics(final SpecializedContext context) { - return context.getCallContext() - .getTableSemantics(ARG_INPUT) - .orElseThrow(() -> new IllegalStateException("Table argument expected.")); - } - - private static StateTypeStrategy seenState(final SpecializedContext context) { - return state(context, DataTypes.BOOLEAN().notNull()); - } - - private static StateTypeStrategy keptState(final SpecializedContext context) { - return state( - context, - DataTypes.ROW(DeduplicateTypeStrategy.outputFields(tableSemantics(context)))); - } - - private static StateTypeStrategy state( - final SpecializedContext context, final DataType valueType) { - return StateTypeStrategy.of( - callContext -> Optional.of(ValueView.newValueViewDataType(valueType)), - context.getCallContext() - .getArgumentValue(ARG_STATE_TTL, Duration.class) - .orElse(null)); - } + final boolean isKeepFirst = DeduplicateTypeStrategy.keep(callContext) == Keep.FIRST; + final boolean hasSequenceBy = + DeduplicateTypeStrategy.sequenceByOutputIndices(callContext).length > 0; - private static @Nullable SequenceComparator sequenceComparator( - final SpecializedContext context, final int[] outputIndices) { - final int[] payloadIndices = - DeduplicateTypeStrategy.sequenceByOutputIndices(context.getCallContext()); - if (payloadIndices.length == 0) { - return null; + if (isKeepFirst && !hasSequenceBy) { + return new DeduplicateFirstArrivalFunction(context); + } else { + return new DeduplicateKeptRowFunction(context); } - final List payloadFields = - DeduplicateTypeStrategy.outputFields(tableSemantics(context)); - final int[] inputIndices = - Arrays.stream(payloadIndices).map(i -> outputIndices[i]).toArray(); - final DataType[] types = - Arrays.stream(payloadIndices) - .mapToObj(i -> payloadFields.get(i).getDataType()) - .toArray(DataType[]::new); - return new SequenceComparator(context, inputIndices, payloadIndices, types); } } diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFirstArrivalFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFirstArrivalFunction.java new file mode 100644 index 00000000000000..b0fafe6409d6b8 --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFirstArrivalFunction.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.ptf; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.dataview.ValueView; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.utils.ProjectedRowData; +import org.apache.flink.table.functions.FunctionContext; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; +import org.apache.flink.types.ColumnList; +import org.apache.flink.types.RowKind; + +import javax.annotation.Nullable; + +import java.util.Map; + +/** + * {@link Deduplicate} with {@code keep => 'first'} and no {@code sequence_by}: no later record can + * replace the first one, so only whether the key has emitted is kept in the {@code seen} state. + */ +@Internal +public class DeduplicateFirstArrivalFunction extends DeduplicateFunctionBase { + + private static final long serialVersionUID = 1L; + + private transient ProjectedRowData projectedOutput; + + DeduplicateFirstArrivalFunction(final SpecializedContext context) { + super( + context, + internalState( + Map.entry( + "seen", + DeduplicateUtils.stateStrategy( + context, DataTypes.BOOLEAN().notNull())))); + } + + @Override + public void open(final FunctionContext context) throws Exception { + super.open(context); + projectedOutput = ProjectedRowData.from(outputIndices); + } + + public void eval( + final Context ctx, + final ValueView seen, + final RowData input, + @Nullable final StringData keep, + @Nullable final ColumnList sequenceBy, + @Nullable final Long stateTtl, + @Nullable final Boolean resetTtlOnDuplicate) + throws Exception { + if (seen.getValue() == null) { + projectedOutput.replaceRow(input); + projectedOutput.setRowKind(RowKind.INSERT); + collect(projectedOutput); + seen.setValue(true); + } else if (isResetTtlOnDuplicate) { + seen.setValue(true); + } + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFunctionBase.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFunctionBase.java new file mode 100644 index 00000000000000..f22d583805a9d9 --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFunctionBase.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.ptf; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; +import org.apache.flink.table.types.inference.CallContext; +import org.apache.flink.table.types.inference.StateTypeStrategy; +import org.apache.flink.table.types.inference.strategies.ChangelogTypeStrategyUtils; + +import java.time.Duration; +import java.util.LinkedHashMap; + +import static org.apache.flink.table.api.config.ExecutionConfigOptions.IDLE_STATE_RETENTION; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_RESET_TTL_ON_DUPLICATE; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL; + +/** Base class of the {@link Deduplicate} implementations. */ +@Internal +abstract class DeduplicateFunctionBase extends BuiltInProcessTableFunction { + + private static final long serialVersionUID = 1L; + + protected final int[] outputIndices; + + protected final boolean isResetTtlOnDuplicate; + + DeduplicateFunctionBase( + final SpecializedContext context, + final LinkedHashMap stateStrategies) { + super(BuiltInFunctionDefinitions.DEDUPLICATE, context, stateStrategies); + final CallContext callContext = context.getCallContext(); + this.outputIndices = + ChangelogTypeStrategyUtils.computeOutputIndices( + DeduplicateUtils.tableSemantics(context)); + final Duration effectiveTtl = + callContext + .getArgumentValue(ARG_STATE_TTL, Duration.class) + .orElseGet(() -> context.getConfiguration().get(IDLE_STATE_RETENTION)); + this.isResetTtlOnDuplicate = + callContext.getArgumentValue(ARG_RESET_TTL_ON_DUPLICATE, Boolean.class).orElse(true) + && !effectiveTtl.isZero(); + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeptRowFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeptRowFunction.java new file mode 100644 index 00000000000000..c2cbde977ff9dd --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeptRowFunction.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.ptf; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.dataview.ValueView; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.binary.BinaryRowData; +import org.apache.flink.table.data.utils.JoinedRowData; +import org.apache.flink.table.functions.FunctionContext; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; +import org.apache.flink.table.runtime.functions.ptf.DeduplicateUtils.SequenceComparator; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.CallContext; +import org.apache.flink.table.types.inference.StateTypeStrategy; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.types.ColumnList; +import org.apache.flink.types.RowKind; + +import javax.annotation.Nullable; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * {@link Deduplicate} with {@code keep => 'last'}, or {@code keep => 'first'} with {@code + * sequence_by}: a later record can replace the kept one, so the kept record is held in the {@code + * kept} state. + */ +@Internal +public class DeduplicateKeptRowFunction extends DeduplicateFunctionBase { + + private static final long serialVersionUID = 1L; + + private final Keep keep; + + private final LogicalType[] payloadTypes; + + private final boolean isUpdateBeforeEmitted; + + private final @Nullable SequenceComparator sequenceComparator; + + private transient RowData.FieldGetter[] payloadGetters; + + private transient RowDataSerializer candidateSerializer; + + private transient RowDataSerializer currentSerializer; + + private transient GenericRowData reusedCandidate; + + private transient JoinedRowData output; + + private transient RowData emptyRow; + + DeduplicateKeptRowFunction(final SpecializedContext context) { + super(context, internalState(Map.entry("kept", keptState(context)))); + final CallContext callContext = context.getCallContext(); + this.keep = DeduplicateTypeStrategy.keep(callContext); + this.payloadTypes = + DeduplicateTypeStrategyUtils.outputFields(DeduplicateUtils.tableSemantics(context)) + .stream() + .map(field -> field.getDataType().getLogicalType()) + .toArray(LogicalType[]::new); + this.isUpdateBeforeEmitted = + callContext + .getOutputChangelogMode() + .map(mode -> mode.contains(RowKind.UPDATE_BEFORE)) + .orElse(false); + this.sequenceComparator = sequenceComparator(context, outputIndices); + } + + @Override + public void open(final FunctionContext context) throws Exception { + super.open(context); + payloadGetters = new RowData.FieldGetter[outputIndices.length]; + for (int i = 0; i < outputIndices.length; i++) { + payloadGetters[i] = RowData.createFieldGetter(payloadTypes[i], outputIndices[i]); + } + if (sequenceComparator != null) { + sequenceComparator.open(context); + } + // toBinaryRow reuses its result row, so each side of a comparison needs its own + // serializer + candidateSerializer = new RowDataSerializer(payloadTypes); + currentSerializer = new RowDataSerializer(payloadTypes); + reusedCandidate = new GenericRowData(outputIndices.length); + output = new JoinedRowData(); + emptyRow = GenericRowData.of(); + } + + @Override + public void close() throws Exception { + if (sequenceComparator != null) { + sequenceComparator.close(); + } + super.close(); + } + + public void eval( + final Context ctx, + final ValueView keptState, + final RowData input, + @Nullable final StringData keep, + @Nullable final ColumnList sequenceBy, + @Nullable final Long stateTtl, + @Nullable final Boolean resetTtlOnDuplicate) + throws Exception { + final RowData kept = keptState.getValue(); + if (kept == null) { + final BinaryRowData candidate = toCandidate(input); + emit(RowKind.INSERT, candidate); + keptState.setValue(candidate.copy()); + return; + } + final int order = sequenceComparator == null ? 0 : sequenceComparator.compare(input, kept); + if (DeduplicateUtils.replaces(this.keep, order)) { + final BinaryRowData candidate = toCandidate(input); + if (!candidate.equals(currentSerializer.toBinaryRow(kept))) { + if (isUpdateBeforeEmitted) { + emit(RowKind.UPDATE_BEFORE, kept); + } + emit(RowKind.UPDATE_AFTER, candidate); + keptState.setValue(candidate.copy()); + return; + } + } + if (isResetTtlOnDuplicate) { + keptState.setValue(kept); + } + } + + /** Returns the payload of the input as a reused binary row, valid until the next call. */ + private BinaryRowData toCandidate(final RowData input) { + for (int i = 0; i < outputIndices.length; i++) { + reusedCandidate.setField(i, payloadGetters[i].getFieldOrNull(input)); + } + return candidateSerializer.toBinaryRow(reusedCandidate); + } + + /** Wraps the payload so that rows owned by state never change their row kind. */ + private void emit(final RowKind kind, final RowData payload) { + output.replace(payload, emptyRow); + output.setRowKind(kind); + collect(output); + } + + private static StateTypeStrategy keptState(final SpecializedContext context) { + return DeduplicateUtils.stateStrategy( + context, + DataTypes.ROW( + DeduplicateTypeStrategyUtils.outputFields( + DeduplicateUtils.tableSemantics(context)))); + } + + private static @Nullable SequenceComparator sequenceComparator( + final SpecializedContext context, final int[] outputIndices) { + final int[] payloadIndices = + DeduplicateTypeStrategy.sequenceByOutputIndices(context.getCallContext()); + if (payloadIndices.length == 0) { + return null; + } + final List payloadFields = + DeduplicateTypeStrategyUtils.outputFields(DeduplicateUtils.tableSemantics(context)); + final int[] inputIndices = + Arrays.stream(payloadIndices).map(i -> outputIndices[i]).toArray(); + final DataType[] types = + Arrays.stream(payloadIndices) + .mapToObj(i -> payloadFields.get(i).getDataType()) + .toArray(DataType[]::new); + return new SequenceComparator(context, inputIndices, payloadIndices, types); + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java index cb3666679e14cc..05fb7372748b3f 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java @@ -20,26 +20,49 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.dataview.ValueView; import org.apache.flink.table.data.RowData; import org.apache.flink.table.functions.FunctionContext; import org.apache.flink.table.functions.SpecializedFunction.ExpressionEvaluator; import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; +import org.apache.flink.table.functions.TableSemantics; import org.apache.flink.table.types.DataType; -import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.Keep; +import org.apache.flink.table.types.inference.StateTypeStrategy; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep; import org.apache.flink.util.FlinkRuntimeException; import javax.annotation.Nullable; import java.io.Serializable; import java.lang.invoke.MethodHandle; +import java.time.Duration; +import java.util.Optional; import static org.apache.flink.table.api.Expressions.$; import static org.apache.flink.table.api.Expressions.ifThenElse; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_INPUT; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL; /** Record ordering of the built-in deduplication PTFs. */ @Internal public final class DeduplicateUtils { + public static TableSemantics tableSemantics(final SpecializedContext context) { + return context.getCallContext() + .getTableSemantics(ARG_INPUT) + .orElseThrow(() -> new IllegalStateException("Table argument expected.")); + } + + /** Returns a value state of the given type that expires after the {@code state_ttl}. */ + public static StateTypeStrategy stateStrategy( + final SpecializedContext context, final DataType valueType) { + return StateTypeStrategy.of( + callContext -> Optional.of(ValueView.newValueViewDataType(valueType)), + context.getCallContext() + .getArgumentValue(ARG_STATE_TTL, Duration.class) + .orElse(null)); + } + /** * Returns whether a new record replaces the current one, given the order of the new record * relative to the current one: with {@code last} the later arrival wins on a tie, with {@code diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateTest.java deleted file mode 100644 index 0a82740c0e4a35..00000000000000 --- a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateTest.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.table.runtime.functions.ptf; - -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.ReadableConfig; -import org.apache.flink.table.api.DataTypes; -import org.apache.flink.table.api.dataview.ValueView; -import org.apache.flink.table.connector.ChangelogMode; -import org.apache.flink.table.expressions.Expression; -import org.apache.flink.table.functions.BuiltInFunctionDefinition; -import org.apache.flink.table.functions.BuiltInFunctionDefinitions; -import org.apache.flink.table.functions.SpecializedFunction.ExpressionEvaluator; -import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; -import org.apache.flink.table.functions.UserDefinedFunction; -import org.apache.flink.table.types.DataType; -import org.apache.flink.table.types.inference.CallContext; -import org.apache.flink.table.types.inference.StateTypeStrategy; -import org.apache.flink.table.types.inference.utils.CallContextMock; -import org.apache.flink.table.types.inference.utils.TableSemanticsMock; -import org.apache.flink.table.types.utils.DataTypeFactoryMock; -import org.apache.flink.table.types.utils.DataTypeUtils; - -import org.junit.jupiter.api.Test; - -import javax.annotation.Nullable; - -import java.time.Duration; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static org.assertj.core.api.Assertions.assertThat; - -/** Tests for the state that each specialization of {@link Deduplicate} declares. */ -class DeduplicateTest { - - private static final DataType INPUT_TYPE = - DataTypes.ROW( - DataTypes.FIELD("user_id", DataTypes.STRING()), - DataTypes.FIELD("event", DataTypes.STRING())); - - private static final DataType OUTPUT_TYPE = - DataTypes.ROW( - DataTypes.FIELD("user_id", DataTypes.STRING()), - DataTypes.FIELD("event", DataTypes.STRING())) - .notNull(); - - @Test - void testFirstArrivalDeclaresOnlySeenState() { - final CallContextMock callContext = callContext(null, Duration.ofSeconds(5)); - final UserDefinedFunction function = specialize(callContext); - - assertThat(function).isInstanceOf(Deduplicate.FirstArrival.class); - final Map state = stateStrategies(function); - assertThat(state).containsOnlyKeys("seen"); - assertThat(state.get("seen").inferType(callContext)) - .contains(ValueView.newValueViewDataType(DataTypes.BOOLEAN().notNull())); - assertThat(state.get("seen").getTimeToLive(callContext)).contains(Duration.ofSeconds(5)); - } - - @Test - void testSequencedDeclaresOnlyKeptState() { - final CallContextMock callContext = callContext("last", null); - final UserDefinedFunction function = specialize(callContext); - - assertThat(function).isInstanceOf(Deduplicate.Sequenced.class); - final Map state = stateStrategies(function); - assertThat(state).containsOnlyKeys("kept"); - assertThat(state.get("kept").inferType(callContext)) - .contains( - ValueView.newValueViewDataType( - DataTypeUtils.toInternalDataType( - DataTypes.ROW( - DataTypes.FIELD("event", DataTypes.STRING()))))); - assertThat(state.get("kept").getTimeToLive(callContext)).isEmpty(); - } - - private static Map stateStrategies( - final UserDefinedFunction function) { - return function.getTypeInference(new DataTypeFactoryMock()).getStateTypeStrategies(); - } - - private static UserDefinedFunction specialize(final CallContext callContext) { - return new Deduplicate() - .specialize( - new SpecializedContext() { - @Override - public CallContext getCallContext() { - return callContext; - } - - @Override - public ReadableConfig getConfiguration() { - return new Configuration(); - } - - @Override - public ClassLoader getBuiltInClassLoader() { - return DeduplicateTest.class.getClassLoader(); - } - - @Override - public ExpressionEvaluator createEvaluator( - Expression expression, - DataType outputDataType, - DataTypes.Field... args) { - throw new UnsupportedOperationException(); - } - - @Override - public ExpressionEvaluator createEvaluator( - String sqlExpression, - DataType outputDataType, - DataTypes.Field... args) { - throw new UnsupportedOperationException(); - } - - @Override - public ExpressionEvaluator createEvaluator( - BuiltInFunctionDefinition function, - DataType outputDataType, - DataType... args) { - throw new UnsupportedOperationException(); - } - }); - } - - private static CallContextMock callContext( - @Nullable final String keep, @Nullable final Duration stateTtl) { - final CallContextMock callContext = new CallContextMock(); - callContext.typeFactory = new DataTypeFactoryMock(); - callContext.functionDefinition = BuiltInFunctionDefinitions.DEDUPLICATE; - callContext.argumentDataTypes = - List.of( - INPUT_TYPE, - DataTypes.STRING(), - DataTypes.DESCRIPTOR(), - DataTypes.INTERVAL(DataTypes.SECOND()), - DataTypes.BOOLEAN()); - callContext.argumentValues = - Arrays.asList( - Optional.empty(), - Optional.ofNullable(keep), - Optional.empty(), - Optional.ofNullable(stateTtl), - Optional.empty()); - callContext.outputDataType = Optional.of(OUTPUT_TYPE); - callContext.tableSemantics = - Map.of( - 0, - new TableSemanticsMock( - INPUT_TYPE, - new int[] {0}, - new int[0], - -1, - ChangelogMode.insertOnly())); - return callContext; - } -} From 44f4f52cd7e4e3af08036613d8b46f61f698f510 Mon Sep 17 00:00:00 2001 From: Vas Shabu Date: Wed, 18 Nov 2026 13:45:51 +0000 Subject: [PATCH 4/9] [FLINK-41470][table] Keep the full input row in DEDUPLICATE state to cut per-record work Compare and emit through views of the input row instead of projecting and serializing a payload row per record, and rename the variants to DeduplicateEmitOnceFunction and DeduplicateCurrentRowFunction. Reduce the TTL harness test to reset_ttl_on_duplicate and regenerate the sequence-by restore test. --- .../harness/DeduplicateHarnessTest.java | 100 ++----------- .../plan/deduplicate-sequence-by-restore.json | 21 +-- .../savepoint/_metadata | Bin 13762 -> 14169 bytes .../runtime/functions/ptf/Deduplicate.java | 20 +-- ...ava => DeduplicateCurrentRowFunction.java} | 132 ++++++++---------- ....java => DeduplicateEmitOnceFunction.java} | 4 +- .../functions/ptf/DeduplicateUtils.java | 47 +++---- 7 files changed, 122 insertions(+), 202 deletions(-) rename flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/{DeduplicateKeptRowFunction.java => DeduplicateCurrentRowFunction.java} (51%) rename flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/{DeduplicateFirstArrivalFunction.java => DeduplicateEmitOnceFunction.java} (95%) diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/harness/DeduplicateHarnessTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/harness/DeduplicateHarnessTest.java index 7ee7f90f14f02f..7cae31f1c4c160 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/harness/DeduplicateHarnessTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/harness/DeduplicateHarnessTest.java @@ -30,7 +30,6 @@ import org.apache.flink.table.data.StringData; import org.apache.flink.table.runtime.keyselector.RowDataKeySelector; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -44,34 +43,35 @@ import static org.assertj.core.api.Assertions.assertThat; -/** - * Harness tests for the processing-time state retention of the built-in DEDUPLICATE PTF, which need - * control over the time that state TTL observes. - */ +/** Harness tests for {@code reset_ttl_on_duplicate} of the built-in DEDUPLICATE PTF. */ class DeduplicateHarnessTest { private static final Function PAYMENT = row -> row.getString(0) + ", " + row.getInt(1); - private static final Function CUSTOMER_CHANGE = - row -> row.getString(1) + ", " + row.getLong(2); - private static Stream resetTtlOnDuplicate() { + final List refreshed = List.of("+I[p1, 50]@10:00", "+I[p1, 50]@10:20"); + final List notRefreshed = + List.of("+I[p1, 50]@10:00", "+I[p1, 50]@10:07", "+I[p1, 50]@10:20"); return Stream.of( - Arguments.of(true, List.of("+I[p1, 50]@10:00", "+I[p1, 50]@10:20")), - Arguments.of( - false, - List.of("+I[p1, 50]@10:00", "+I[p1, 50]@10:07", "+I[p1, 50]@10:20"))); + Arguments.of("first", true, refreshed), + Arguments.of("first", false, notRefreshed), + Arguments.of("last", true, refreshed), + Arguments.of("last", false, notRefreshed)); } - @ParameterizedTest(name = "reset_ttl_on_duplicate={0}") + @ParameterizedTest(name = "keep={0}, reset_ttl_on_duplicate={1}") @MethodSource("resetTtlOnDuplicate") void testTimeBoundedDeduplication( - final boolean resetTtlOnDuplicate, final List expected) throws Exception { + final String keep, final boolean resetTtlOnDuplicate, final List expected) + throws Exception { try (KeyedMultiInputStreamOperatorTestHarness harness = createHarness( "SELECT * FROM DEDUPLICATE(" + "input => TABLE payments PARTITION BY payment_id, " + + "keep => '" + + keep + + "', " + "state_ttl => INTERVAL '5' MINUTES, " + "reset_ttl_on_duplicate => " + resetTtlOnDuplicate @@ -85,71 +85,6 @@ void testTimeBoundedDeduplication( } } - @Test - void testRecordAfterExpiryIsFirstRecord() throws Exception { - try (KeyedMultiInputStreamOperatorTestHarness harness = - createHarness( - "SELECT * FROM DEDUPLICATE(" - + "input => TABLE customer_changes PARTITION BY customer_id, " - + "keep => 'last', " - + "sequence_by => DESCRIPTOR(lsn), " - + "state_ttl => INTERVAL '5' MINUTES)")) { - final List output = new ArrayList<>(); - output.addAll( - process(harness, "10:00", CUSTOMER_CHANGE, customerChange("Berlin", 120))); - output.addAll( - process(harness, "10:02", CUSTOMER_CHANGE, customerChange("Munich", 100))); - output.addAll( - process(harness, "10:20", CUSTOMER_CHANGE, customerChange("Hamburg", 90))); - - assertThat(output).containsExactly("+I[Berlin, 120]@10:00", "+I[Hamburg, 90]@10:20"); - } - } - - @Test - void testReplacingRecordRefreshesTtl() throws Exception { - try (KeyedMultiInputStreamOperatorTestHarness harness = - createHarness( - "SELECT * FROM DEDUPLICATE(" - + "input => TABLE payments PARTITION BY payment_id, " - + "keep => 'last', " - + "state_ttl => INTERVAL '5' MINUTES, " - + "reset_ttl_on_duplicate => false)")) { - final List output = new ArrayList<>(); - output.addAll(process(harness, "10:00", PAYMENT, payment(50))); - output.addAll(process(harness, "10:04", PAYMENT, payment(60))); - output.addAll(process(harness, "10:07", PAYMENT, payment(60))); - output.addAll(process(harness, "10:08", PAYMENT, payment(70))); - output.addAll(process(harness, "10:14", PAYMENT, payment(70))); - - assertThat(output) - .containsExactly( - "+I[p1, 50]@10:00", - "-U[p1, 50]@10:04", - "+U[p1, 60]@10:04", - "-U[p1, 60]@10:08", - "+U[p1, 70]@10:08", - "+I[p1, 70]@10:14"); - } - } - - @Test - void testIdenticalRecordRefreshesTtl() throws Exception { - try (KeyedMultiInputStreamOperatorTestHarness harness = - createHarness( - "SELECT * FROM DEDUPLICATE(" - + "input => TABLE payments PARTITION BY payment_id, " - + "keep => 'last', " - + "state_ttl => INTERVAL '5' MINUTES)")) { - final List output = new ArrayList<>(); - for (final String time : List.of("10:00", "10:04", "10:07")) { - output.addAll(process(harness, time, PAYMENT, payment(50))); - } - - assertThat(output).containsExactly("+I[p1, 50]@10:00"); - } - } - private static List process( final KeyedMultiInputStreamOperatorTestHarness harness, final String time, @@ -174,10 +109,6 @@ private static RowData payment(final int amount) { return GenericRowData.of(StringData.fromString("p1"), amount); } - private static RowData customerChange(final String city, final long lsn) { - return GenericRowData.of(StringData.fromString("c1"), StringData.fromString(city), lsn); - } - private static KeyedMultiInputStreamOperatorTestHarness createHarness( final String query) throws Exception { final StreamTableEnvironment tEnv = @@ -185,9 +116,6 @@ private static RowData customerChange(final String city, final long lsn) { tEnv.executeSql( "CREATE TABLE payments (payment_id STRING, amount INT) " + "WITH ('connector' = 'values')"); - tEnv.executeSql( - "CREATE TABLE customer_changes (customer_id STRING, city STRING, lsn BIGINT) " - + "WITH ('connector' = 'values')"); final KeyedMultipleInputTransformation transformation = findPtf(tEnv.toChangelogStream(tEnv.sqlQuery(query)).getTransformation()) .orElseThrow(() -> new AssertionError("No PTF transformation found.")); diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/plan/deduplicate-sequence-by-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/plan/deduplicate-sequence-by-restore.json index 7560104fa3a1a7..90d341867230bf 100644 --- a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/plan/deduplicate-sequence-by-restore.json +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/plan/deduplicate-sequence-by-restore.json @@ -1,7 +1,7 @@ { "flinkVersion" : "2.5", "nodes" : [ { - "id" : 9, + "id" : 5, "type" : "stream-exec-table-source-scan_2", "scanTableSource" : { "table" : { @@ -25,7 +25,7 @@ "outputType" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT>", "description" : "TableSourceScan(table=[[default_catalog, default_database, customer_changes]], fields=[customer_id, city, lsn])" }, { - "id" : 10, + "id" : 6, "type" : "stream-exec-exchange_1", "inputProperties" : [ { "requiredDistribution" : { @@ -38,7 +38,7 @@ "outputType" : "ROW<`customer_id` VARCHAR(2147483647), `city` VARCHAR(2147483647), `lsn` BIGINT>", "description" : "Exchange(distribution=[hash[customer_id]])" }, { - "id" : 11, + "id" : 7, "type" : "stream-exec-process-table-function_1", "inputProperties" : [ { "requiredDistribution" : { @@ -98,7 +98,7 @@ "outputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], "inputUpsertKeys" : [ [ ] ] }, { - "id" : 12, + "id" : 8, "type" : "stream-exec-sink_2", "configuration" : { "table.exec.sink.keyed-shuffle" : "AUTO", @@ -132,6 +132,7 @@ } }, "inputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], + "upsertMaterializeStrategy" : "MAP", "inputUpsertKey" : [ 0 ], "inputProperties" : [ { "requiredDistribution" : { @@ -144,22 +145,22 @@ "description" : "Sink(table=[default_catalog.default_database.sink], fields=[customer_id, city, lsn])" } ], "edges" : [ { - "source" : 9, - "target" : 10, + "source" : 5, + "target" : 6, "shuffle" : { "type" : "FORWARD" }, "shuffleMode" : "PIPELINED" }, { - "source" : 10, - "target" : 11, + "source" : 6, + "target" : 7, "shuffle" : { "type" : "FORWARD" }, "shuffleMode" : "PIPELINED" }, { - "source" : 11, - "target" : 12, + "source" : 7, + "target" : 8, "shuffle" : { "type" : "FORWARD" }, diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-sequence-by-restore/savepoint/_metadata index c17b804f00ada24df1288ee3aff1682778fa07f4..9d3219ab67d1f98982d263ba485d73b02995144e 100644 GIT binary patch delta 813 zcmbVKO-K|`9DlRAyGpZbo24o0uC=J)zMPLYGjA$NGe3?&T~chCc{7iewb|O)HI~2% zIs`?vy+a2(B+*eJwnMif@ED#$BB+xcBnpBKVsEsMbN%7}{q^I||4reY|DfaezR{u| zluLHrFOT`OSR!6&8qa_>Y>__+IA_o1UAtT?6&8Mc?Y}i}VWDY#`BLY#leLk(tUI;4 zFL5UC7mj2CF2n8^wnDSox7KgzFYk1W{&*&Q6gO%ksA~~yleqFrukYwYLI4kIwPTiH zC<@XLmUTmtk%VMq3h>BH0G>BD2he%ctbPNSU zOf=7-3iVt{07%GuQ4G|f@K|JPi7bmJvrOe|CDr!!)1@Nug5a#}%zEP$pOo`Of7)PU zsqc~Q`*pjuO^9^A=lVc|JxPhtp27b{OEAWiVig*iLLow?1WoF4i%^-Wl1nh+NPDU~ zXvN-xk*5>;TiI%~%3crdVNL@XrSM>ptu5BCq?5mc%Y0h~Fq}+>nd{2m@@0P4?eg2n?1VN0nlvjYR`_=v zHV@ss*gOrVURh}D-pUC+LEwBt02#$oO~u4S$C9ZzimTZU)u@V)hCQOGlzmGc6~ZSu z|L;%Y5yy3qW<%9TLi3 delta 1015 zcmbVKOH33&5bYjeS^0~C2t?TkhL01bduMiLfP=w+fC&;%6W9>w>76ATaF)z07*8UJ zA@LK1K6p?u8$Sm~yzB)M4d_8bJgD&>6Nv|p9EgOIac!0r@Zg0?x~i+I`n|5M`eHpF zxOt!=yE9}B)RsD2`w{K|X$beOI}h(Vi#@My#oyiYo39s^KTkyEk{OVY%ugFOYgf|N zK)@ze>iZDVeX=BbeY&nlUZ2;is@@uhjF>~=kQEVSNi|hTlq7-!LEWoinbycM(hO6_ zA~BFE$_An$(TF4l3{%5IkqiZ?0A%?&lMBYK!8@6GaTfqi8)+Zvf)xN%HN^5+MRu8c z-Fg4nau zC(2aTMBOlvNEK6;F*1EYjqrD@CW>da_ea-O?fh)J6dVs*PQ)hspBt@Um=|Ry+PR;% zlWvB@5i%GGkVx2G_}M-Bpy*e7+Vqn*jdRV{C!+gUZcggNP)1>4yB(&c93X~h7!ndO zQcZ*?QUZ#>86I&OSL~UmOt`<4zc)iTpTZ9W#|8?6spTxE(w~|_Ly-$?&+ZmBUwL@4 zsy{86#G>UMcB85fmQPl w&-a_S&E%XhZ1<0MC-MM@$D95I&hvC0+W?O9lr3Oy;==!d*+E diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java index ad8c2cfd33f437..268e98fbe36c6f 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/Deduplicate.java @@ -31,9 +31,9 @@ * *
  * keep    sequence_by  implementation
- * first   absent       {@link DeduplicateFirstArrivalFunction}
- * first   present      {@link DeduplicateKeptRowFunction}
- * last    any          {@link DeduplicateKeptRowFunction}
+ * first   absent       {@link DeduplicateEmitOnceFunction}
+ * first   present      {@link DeduplicateCurrentRowFunction}
+ * last    any          {@link DeduplicateCurrentRowFunction}
  * 
*/ @Internal @@ -47,13 +47,17 @@ public Deduplicate() { public UserDefinedFunction specialize(final SpecializedContext context) { final CallContext callContext = context.getCallContext(); final boolean isKeepFirst = DeduplicateTypeStrategy.keep(callContext) == Keep.FIRST; - final boolean hasSequenceBy = - DeduplicateTypeStrategy.sequenceByOutputIndices(callContext).length > 0; - if (isKeepFirst && !hasSequenceBy) { - return new DeduplicateFirstArrivalFunction(context); + if (isKeepFirst) { + final boolean hasSequenceBy = + DeduplicateTypeStrategy.sequenceByOutputIndices(callContext).length > 0; + if (!hasSequenceBy) { + return new DeduplicateEmitOnceFunction(context); + } else { + return new DeduplicateCurrentRowFunction(context); + } } else { - return new DeduplicateKeptRowFunction(context); + return new DeduplicateCurrentRowFunction(context); } } } diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeptRowFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateCurrentRowFunction.java similarity index 51% rename from flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeptRowFunction.java rename to flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateCurrentRowFunction.java index c2cbde977ff9dd..9ae82f6366d0d7 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeptRowFunction.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateCurrentRowFunction.java @@ -26,15 +26,15 @@ import org.apache.flink.table.data.StringData; import org.apache.flink.table.data.binary.BinaryRowData; import org.apache.flink.table.data.utils.JoinedRowData; +import org.apache.flink.table.data.utils.ProjectedRowData; import org.apache.flink.table.functions.FunctionContext; import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; import org.apache.flink.table.runtime.functions.ptf.DeduplicateUtils.SequenceComparator; import org.apache.flink.table.runtime.typeutils.RowDataSerializer; import org.apache.flink.table.types.DataType; -import org.apache.flink.table.types.inference.CallContext; import org.apache.flink.table.types.inference.StateTypeStrategy; +import org.apache.flink.table.types.inference.strategies.ChangelogTypeStrategyUtils; import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy; -import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils; import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.types.ColumnList; @@ -48,67 +48,58 @@ /** * {@link Deduplicate} with {@code keep => 'last'}, or {@code keep => 'first'} with {@code - * sequence_by}: a later record can replace the kept one, so the kept record is held in the {@code - * kept} state. + * sequence_by}: a later record can replace the kept one, so the kept input row is held in the + * {@code kept} state. */ @Internal -public class DeduplicateKeptRowFunction extends DeduplicateFunctionBase { +public class DeduplicateCurrentRowFunction extends DeduplicateFunctionBase { private static final long serialVersionUID = 1L; - private final Keep keep; + private final Keep keepMode; - private final LogicalType[] payloadTypes; + private final LogicalType[] inputTypes; private final boolean isUpdateBeforeEmitted; private final @Nullable SequenceComparator sequenceComparator; - private transient RowData.FieldGetter[] payloadGetters; + private transient RowDataSerializer inputSerializer; - private transient RowDataSerializer candidateSerializer; + private transient ProjectedRowData projectedCandidate; - private transient RowDataSerializer currentSerializer; + private transient ProjectedRowData projectedKept; - private transient GenericRowData reusedCandidate; - - private transient JoinedRowData output; + private transient JoinedRowData retraction; private transient RowData emptyRow; - DeduplicateKeptRowFunction(final SpecializedContext context) { + DeduplicateCurrentRowFunction(final SpecializedContext context) { super(context, internalState(Map.entry("kept", keptState(context)))); - final CallContext callContext = context.getCallContext(); - this.keep = DeduplicateTypeStrategy.keep(callContext); - this.payloadTypes = - DeduplicateTypeStrategyUtils.outputFields(DeduplicateUtils.tableSemantics(context)) - .stream() + this.keepMode = DeduplicateTypeStrategy.keep(context.getCallContext()); + final List inputFields = DeduplicateUtils.inputFields(context); + this.inputTypes = + inputFields.stream() .map(field -> field.getDataType().getLogicalType()) .toArray(LogicalType[]::new); this.isUpdateBeforeEmitted = - callContext + context.getCallContext() .getOutputChangelogMode() .map(mode -> mode.contains(RowKind.UPDATE_BEFORE)) .orElse(false); - this.sequenceComparator = sequenceComparator(context, outputIndices); + this.sequenceComparator = sequenceComparator(context, inputFields); } @Override public void open(final FunctionContext context) throws Exception { super.open(context); - payloadGetters = new RowData.FieldGetter[outputIndices.length]; - for (int i = 0; i < outputIndices.length; i++) { - payloadGetters[i] = RowData.createFieldGetter(payloadTypes[i], outputIndices[i]); - } if (sequenceComparator != null) { sequenceComparator.open(context); } - // toBinaryRow reuses its result row, so each side of a comparison needs its own - // serializer - candidateSerializer = new RowDataSerializer(payloadTypes); - currentSerializer = new RowDataSerializer(payloadTypes); - reusedCandidate = new GenericRowData(outputIndices.length); - output = new JoinedRowData(); + inputSerializer = new RowDataSerializer(inputTypes); + projectedCandidate = ProjectedRowData.from(outputIndices); + projectedKept = ProjectedRowData.from(outputIndices); + retraction = new JoinedRowData(); emptyRow = GenericRowData.of(); } @@ -129,68 +120,65 @@ public void eval( @Nullable final Long stateTtl, @Nullable final Boolean resetTtlOnDuplicate) throws Exception { - final RowData kept = keptState.getValue(); + final BinaryRowData candidate = inputSerializer.toBinaryRow(input); + final BinaryRowData kept = (BinaryRowData) keptState.getValue(); if (kept == null) { - final BinaryRowData candidate = toCandidate(input); - emit(RowKind.INSERT, candidate); keptState.setValue(candidate.copy()); + emitCandidate(RowKind.INSERT, candidate); return; } - final int order = sequenceComparator == null ? 0 : sequenceComparator.compare(input, kept); - if (DeduplicateUtils.replaces(this.keep, order)) { - final BinaryRowData candidate = toCandidate(input); - if (!candidate.equals(currentSerializer.toBinaryRow(kept))) { - if (isUpdateBeforeEmitted) { - emit(RowKind.UPDATE_BEFORE, kept); - } - emit(RowKind.UPDATE_AFTER, candidate); - keptState.setValue(candidate.copy()); - return; + final boolean isReplacing = + sequenceComparator == null + || DeduplicateUtils.replaces( + keepMode, sequenceComparator.compare(candidate, kept)); + if (isReplacing && !candidate.equals(kept)) { + keptState.setValue(candidate.copy()); + if (isUpdateBeforeEmitted) { + emitRetraction(kept); } - } - if (isResetTtlOnDuplicate) { + emitCandidate(RowKind.UPDATE_AFTER, candidate); + } else if (isResetTtlOnDuplicate) { keptState.setValue(kept); } } - /** Returns the payload of the input as a reused binary row, valid until the next call. */ - private BinaryRowData toCandidate(final RowData input) { - for (int i = 0; i < outputIndices.length; i++) { - reusedCandidate.setField(i, payloadGetters[i].getFieldOrNull(input)); - } - return candidateSerializer.toBinaryRow(reusedCandidate); + private void emitCandidate(final RowKind kind, final BinaryRowData candidate) { + projectedCandidate.replaceRow(candidate); + projectedCandidate.setRowKind(kind); + collect(projectedCandidate); } - /** Wraps the payload so that rows owned by state never change their row kind. */ - private void emit(final RowKind kind, final RowData payload) { - output.replace(payload, emptyRow); - output.setRowKind(kind); - collect(output); + /** Wraps the kept row so that the row owned by state never changes its row kind. */ + private void emitRetraction(final BinaryRowData kept) { + projectedKept.replaceRow(kept); + retraction.replace(projectedKept, emptyRow); + retraction.setRowKind(RowKind.UPDATE_BEFORE); + collect(retraction); } private static StateTypeStrategy keptState(final SpecializedContext context) { return DeduplicateUtils.stateStrategy( - context, - DataTypes.ROW( - DeduplicateTypeStrategyUtils.outputFields( - DeduplicateUtils.tableSemantics(context)))); + context, DataTypes.ROW(DeduplicateUtils.inputFields(context))); } private static @Nullable SequenceComparator sequenceComparator( - final SpecializedContext context, final int[] outputIndices) { - final int[] payloadIndices = - DeduplicateTypeStrategy.sequenceByOutputIndices(context.getCallContext()); - if (payloadIndices.length == 0) { + final SpecializedContext context, final List inputFields) { + final int[] outputIndices = + ChangelogTypeStrategyUtils.computeOutputIndices( + DeduplicateUtils.tableSemantics(context)); + final int[] sequenceIndices = + Arrays.stream( + DeduplicateTypeStrategy.sequenceByOutputIndices( + context.getCallContext())) + .map(i -> outputIndices[i]) + .toArray(); + if (sequenceIndices.length == 0) { return null; } - final List payloadFields = - DeduplicateTypeStrategyUtils.outputFields(DeduplicateUtils.tableSemantics(context)); - final int[] inputIndices = - Arrays.stream(payloadIndices).map(i -> outputIndices[i]).toArray(); final DataType[] types = - Arrays.stream(payloadIndices) - .mapToObj(i -> payloadFields.get(i).getDataType()) + Arrays.stream(sequenceIndices) + .mapToObj(i -> inputFields.get(i).getDataType()) .toArray(DataType[]::new); - return new SequenceComparator(context, inputIndices, payloadIndices, types); + return new SequenceComparator(context, sequenceIndices, types); } } diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFirstArrivalFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateEmitOnceFunction.java similarity index 95% rename from flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFirstArrivalFunction.java rename to flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateEmitOnceFunction.java index b0fafe6409d6b8..782a4aa9d95235 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateFirstArrivalFunction.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateEmitOnceFunction.java @@ -38,13 +38,13 @@ * replace the first one, so only whether the key has emitted is kept in the {@code seen} state. */ @Internal -public class DeduplicateFirstArrivalFunction extends DeduplicateFunctionBase { +public class DeduplicateEmitOnceFunction extends DeduplicateFunctionBase { private static final long serialVersionUID = 1L; private transient ProjectedRowData projectedOutput; - DeduplicateFirstArrivalFunction(final SpecializedContext context) { + DeduplicateEmitOnceFunction(final SpecializedContext context) { super( context, internalState( diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java index 05fb7372748b3f..a10ebb70df7a0e 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java @@ -29,6 +29,7 @@ import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.inference.StateTypeStrategy; import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep; +import org.apache.flink.table.types.utils.DataTypeUtils; import org.apache.flink.util.FlinkRuntimeException; import javax.annotation.Nullable; @@ -36,7 +37,9 @@ import java.io.Serializable; import java.lang.invoke.MethodHandle; import java.time.Duration; +import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import static org.apache.flink.table.api.Expressions.$; import static org.apache.flink.table.api.Expressions.ifThenElse; @@ -53,6 +56,17 @@ public static TableSemantics tableSemantics(final SpecializedContext context) { .orElseThrow(() -> new IllegalStateException("Table argument expected.")); } + /** Returns the fields of the input row without time attributes. */ + public static List inputFields(final SpecializedContext context) { + return DataType.getFields(tableSemantics(context).dataType()).stream() + .map( + field -> + DataTypes.FIELD( + field.getName(), + DataTypeUtils.removeTimeAttribute(field.getDataType()))) + .collect(Collectors.toList()); + } + /** Returns a value state of the given type that expires after the {@code state_ttl}. */ public static StateTypeStrategy stateStrategy( final SpecializedContext context, final DataType valueType) { @@ -81,9 +95,7 @@ public static final class SequenceComparator implements Serializable { private static final long serialVersionUID = 1L; - private final int[] leftIndices; - - private final int[] rightIndices; + private final int[] indices; private final DataType[] types; @@ -91,21 +103,12 @@ public static final class SequenceComparator implements Serializable { private transient MethodHandle[] compareHandles; - private transient RowData.FieldGetter[] leftGetters; + private transient RowData.FieldGetter[] getters; - private transient RowData.FieldGetter[] rightGetters; - - /** - * Creates a comparator of the columns at {@code leftIndices} of the left record against the - * columns at {@code rightIndices} of the right record, both of the given types. - */ + /** Creates a comparator of the columns at {@code indices}, which have the given types. */ public SequenceComparator( - final SpecializedContext context, - final int[] leftIndices, - final int[] rightIndices, - final DataType[] types) { - this.leftIndices = leftIndices; - this.rightIndices = rightIndices; + final SpecializedContext context, final int[] indices, final DataType[] types) { + this.indices = indices; this.types = types; this.evaluators = new ExpressionEvaluator[types.length]; for (int i = 0; i < types.length; i++) { @@ -124,14 +127,10 @@ public SequenceComparator( public void open(final FunctionContext context) throws Exception { compareHandles = new MethodHandle[types.length]; - leftGetters = new RowData.FieldGetter[types.length]; - rightGetters = new RowData.FieldGetter[types.length]; + getters = new RowData.FieldGetter[types.length]; for (int i = 0; i < types.length; i++) { compareHandles[i] = evaluators[i].open(context); - leftGetters[i] = - RowData.createFieldGetter(types[i].getLogicalType(), leftIndices[i]); - rightGetters[i] = - RowData.createFieldGetter(types[i].getLogicalType(), rightIndices[i]); + getters[i] = RowData.createFieldGetter(types[i].getLogicalType(), indices[i]); } } @@ -150,8 +149,8 @@ public int compare(final RowData left, final RowData right) { final int order = compare( i, - leftGetters[i].getFieldOrNull(left), - rightGetters[i].getFieldOrNull(right)); + getters[i].getFieldOrNull(left), + getters[i].getFieldOrNull(right)); if (order != 0) { return order; } From 09c0f4220d38f24c145bacc8d22ea14445f139b5 Mon Sep 17 00:00:00 2001 From: Vas Shabu Date: Wed, 18 Nov 2026 13:45:51 +0000 Subject: [PATCH 5/9] [FLINK-41470][table] Shorten DEDUPLICATE error messages and rely on the framework for on_time Drop the custom on_time check, since the planner already rejects on_time for PTFs that produce updates, and keep each validation message to the bad input and its fix. --- .../strategies/DeduplicateTypeStrategy.java | 29 +----------- .../DeduplicateTypeStrategyUtils.java | 33 +++++--------- .../DeduplicateTypeStrategyTest.java | 44 +++++-------------- .../sql/DeduplicateTableFunctionTest.java | 16 +++++++ 4 files changed, 39 insertions(+), 83 deletions(-) diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java index 825a7948b92b10..49483bf60b765f 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategy.java @@ -30,7 +30,6 @@ import org.apache.flink.table.types.inference.InputTypeStrategy; import org.apache.flink.table.types.inference.TypeStrategy; import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep; -import org.apache.flink.types.ColumnList; import org.apache.flink.types.RowKind; import java.util.Arrays; @@ -38,8 +37,6 @@ import java.util.Optional; import java.util.stream.Collectors; -import static org.apache.flink.table.types.inference.SystemTypeInference.PROCESS_TABLE_FUNCTION_ARG_ON_TIME_OFFSET; - /** * Type strategies for the {@code DEDUPLICATE} process table function. * @@ -54,7 +51,6 @@ public final class DeduplicateTypeStrategy { public static final int ARG_SEQUENCE_BY = 2; public static final int ARG_STATE_TTL = 3; public static final int ARG_RESET_TTL_ON_DUPLICATE = 4; - private static final int DECLARED_ARG_COUNT = ARG_RESET_TTL_ON_DUPLICATE + 1; private static final String NAME = "DEDUPLICATE"; public static final InputTypeStrategy INPUT_TYPE_STRATEGY = @@ -102,7 +98,6 @@ private static Optional validate(final CallContext callContext) { () -> DeduplicateTypeStrategyUtils.validatePartitionByNoTimeAttribute( tableSemantics, NAME)) - .or(() -> validateOnTime(callContext)) .or(() -> DeduplicateTypeStrategyUtils.validateKeep(callContext, ARG_KEEP, NAME)) .or( () -> @@ -126,31 +121,11 @@ private static Optional validateNoOrderBy(final TableSemantics tableSema .collect(Collectors.joining(", ")); return Optional.of( String.format( - "DEDUPLICATE does not support ORDER BY on its input, but received " - + "ORDER BY %s. Use sequence_by => DESCRIPTOR(%s) to order the " - + "records of a key.", + "DEDUPLICATE does not support ORDER BY, but received ORDER BY %s. " + + "Use sequence_by => DESCRIPTOR(%s) instead.", columns, columns)); } - private static Optional validateOnTime(final CallContext callContext) { - final int argCount = callContext.getArgumentDataTypes().size(); - if (argCount == DECLARED_ARG_COUNT) { - return Optional.empty(); - } - final int onTimePos = argCount - 1 - PROCESS_TABLE_FUNCTION_ARG_ON_TIME_OFFSET; - return callContext - .getArgumentValue(onTimePos, ColumnList.class) - .map(ColumnList::getNames) - .filter(names -> !names.isEmpty()) - .map( - names -> - String.format( - "DEDUPLICATE does not support on_time, but received " - + "on_time => DESCRIPTOR(%s). Use sequence_by => " - + "DESCRIPTOR(%s) to order by a timestamp column.", - names.get(0), names.get(0))); - } - private static ChangelogMode inferChangelogMode(final ChangelogContext changelogContext) { final ChangelogMode requiredMode = changelogContext.getRequiredChangelogMode(); final ChangelogMode.Builder builder = diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java index e7588394636e2f..c57326a7f57fe9 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java @@ -119,16 +119,14 @@ public static Optional validatePartitionByNoTimeAttribute( field -> String.format( "%s does not support a time attribute as a " - + "PARTITION BY column, but '%s' is one. Partition " - + "by a regular column, or cast the column in a " - + "subquery first, for example CAST(%s AS %s) AS %s.", + + "PARTITION BY column, but '%s' is one. Use " + + "CAST(%s AS %s) in a subquery first.", functionName, field.getName(), field.getName(), DataTypeUtils.removeTimeAttribute(field.getDataType()) .getLogicalType() - .asSummaryString(), - field.getName())); + .asSummaryString())); } public static Optional validateKeep( @@ -148,9 +146,7 @@ public static Optional validateKeep( } return Optional.of( String.format( - "%s requires keep => 'first' or keep => 'last', but received %s. " - + "Use keep => 'first' to keep the first record per key, or " - + "keep => 'last' to keep the last one.", + "%s requires keep => 'first' or keep => 'last', but received %s.", functionName, received)); } @@ -183,8 +179,7 @@ public static Optional validateSequenceBy( String.format( "The sequence_by column '%s' is a PARTITION BY column, which has " + "the same value for every record of a key. Use a " - + "column that is not part of PARTITION BY, such as a " - + "version or sequence number.", + + "non-PARTITION BY column.", column)); } final LogicalType type = fieldTypes.get(index).getLogicalType(); @@ -192,8 +187,8 @@ public static Optional validateSequenceBy( return Optional.of( String.format( "The sequence_by column '%s' has type '%s', which cannot order " - + "records. Use a numeric, character string, binary " - + "string, boolean, or date-time column.", + + "records. Use a numeric, string, boolean, or date-time " + + "column.", column, type.asSummaryString())); } } @@ -208,9 +203,7 @@ public static Optional validateStateTtl( if (!callContext.isArgumentLiteral(argPos)) { return Optional.of( "The 'state_ttl' argument must be a constant INTERVAL literal, but received a " - + "non-literal expression. Use a literal such as INTERVAL '1' HOUR, " - + "INTERVAL '0' to retain state indefinitely, or omit the argument " - + "to fall back to 'table.exec.state.ttl'."); + + "non-literal expression. Use a literal such as INTERVAL '1' HOUR."); } return callContext .getArgumentValue(argPos, Duration.class) @@ -219,10 +212,8 @@ public static Optional validateStateTtl( stateTtl -> String.format( "The 'state_ttl' argument must not be negative, but was " - + "'%s'. Use a positive INTERVAL to bound state " - + "retention, INTERVAL '0' to retain state " - + "indefinitely, or omit the argument to fall " - + "back to 'table.exec.state.ttl'.", + + "'%s'. Use INTERVAL '0' to retain state " + + "indefinitely.", TimeUtils.formatWithHighestUnit(stateTtl))); } @@ -233,9 +224,7 @@ public static Optional validateResetTtlOnDuplicate( } return Optional.of( "The 'reset_ttl_on_duplicate' argument must be a constant BOOLEAN literal, but " - + "received a non-literal expression. Use TRUE to refresh the TTL on a " - + "dropped record, FALSE to keep it, or omit the argument for the default " - + "TRUE."); + + "received a non-literal expression. Use TRUE or FALSE."); } private static List sequenceBy(final CallContext callContext, final int argPos) { diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java index d347ce34535170..cce104473b14da 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java @@ -35,7 +35,7 @@ import org.junit.jupiter.api.Test; import java.time.Duration; -import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Stream; @@ -91,9 +91,7 @@ protected Stream testData() { .calledWithLiteralAt(ARG_KEEP, "FIRST") .expectErrorMessage( "DEDUPLICATE requires keep => 'first' or keep => 'last', but " - + "received keep => 'FIRST'. Use keep => 'first' to keep " - + "the first record per key, or keep => 'last' to keep " - + "the last one."), + + "received keep => 'FIRST'."), nonLiteral("Non-literal keep", ARG_KEEP) .expectErrorMessage( "DEDUPLICATE requires keep => 'first' or keep => 'last', but " @@ -107,9 +105,8 @@ protected Stream testData() { -1, ChangelogMode.insertOnly())) .expectErrorMessage( - "DEDUPLICATE does not support ORDER BY on its input, but received " - + "ORDER BY lsn. Use sequence_by => DESCRIPTOR(lsn) to " - + "order the records of a key."), + "DEDUPLICATE does not support ORDER BY, but received ORDER BY " + + "lsn. Use sequence_by => DESCRIPTOR(lsn) instead."), withDefaults( "Time attribute in PARTITION BY", new TableSemanticsMock( @@ -120,29 +117,8 @@ protected Stream testData() { ChangelogMode.insertOnly())) .expectErrorMessage( "DEDUPLICATE does not support a time attribute as a PARTITION BY " - + "column, but 'ts' is one. Partition by a regular column, " - + "or cast the column in a subquery first, for example " - + "CAST(ts AS TIMESTAMP_LTZ(3)) AS ts."), - TestSpec.forStrategy("on_time", INPUT_TYPE_STRATEGY) - .calledWithArgumentTypes( - INPUT_TYPE, - DataTypes.STRING(), - DataTypes.DESCRIPTOR(), - DataTypes.INTERVAL(DataTypes.SECOND()), - DataTypes.BOOLEAN(), - DataTypes.DESCRIPTOR(), - DataTypes.STRING()) - .calledWithTableSemanticsAt(ARG_INPUT, PARTITIONED_BY_USER) - .calledWithLiteralAt(ARG_KEEP, null) - .calledWithLiteralAt(ARG_SEQUENCE_BY, null) - .calledWithLiteralAt(ARG_STATE_TTL, null) - .calledWithLiteralAt(ARG_RESET_TTL_ON_DUPLICATE, null) - .calledWithLiteralAt(5, ColumnList.of("ts")) - .calledWithLiteralAt(6, null) - .expectErrorMessage( - "DEDUPLICATE does not support on_time, but received on_time => " - + "DESCRIPTOR(ts). Use sequence_by => DESCRIPTOR(ts) to " - + "order by a timestamp column."), + + "column, but 'ts' is one. Use CAST(ts AS " + + "TIMESTAMP_LTZ(3)) in a subquery first."), sequenceBy("Unknown sequence_by column", "nope") .expectErrorMessage( "The sequence_by column 'nope' does not exist in the input of " @@ -159,8 +135,8 @@ protected Stream testData() { sequenceBy("Non-orderable sequence_by column", "tags") .expectErrorMessage( "The sequence_by column 'tags' has type 'ARRAY', which " - + "cannot order records. Use a numeric, character " - + "string, binary string, boolean, or date-time column."), + + "cannot order records. Use a numeric, string, boolean, " + + "or date-time column."), withDefaults("Negative state_ttl", PARTITIONED_BY_USER) .calledWithLiteralAt(ARG_STATE_TTL, Duration.ofSeconds(-1)) .expectErrorMessage( @@ -231,13 +207,13 @@ private static TestSpec nonLiteral(final String description, final int argument) private static TestSpec sequenceBy(final String description, final String... columns) { return withDefaults(description, PARTITIONED_BY_USER) - .calledWithLiteralAt(ARG_SEQUENCE_BY, ColumnList.of(Arrays.asList(columns))); + .calledWithLiteralAt(ARG_SEQUENCE_BY, ColumnList.of(List.of(columns))); } private static CallContextMock callContext(final ColumnList sequenceBy) { final CallContextMock callContext = new CallContextMock(); callContext.argumentValues = - Arrays.asList( + List.of( Optional.empty(), Optional.empty(), Optional.of(sequenceBy), diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.java index 1f9c5f003462ad..a71dd9bc472b13 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTableFunctionTest.java @@ -32,9 +32,11 @@ import scala.Enumeration; +import static org.apache.flink.core.testutils.FlinkAssertions.anyCauseMatches; import static org.apache.flink.table.api.Expressions.$; import static org.apache.flink.table.api.Expressions.descriptor; import static org.apache.flink.table.api.Expressions.lit; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Plan tests for the changelog mode and upsert key of the DEDUPLICATE built-in PTF. */ class DeduplicateTableFunctionTest extends TableTestBase { @@ -78,6 +80,20 @@ void testOutputHasNoTimeAttribute() { + "sequence_by => DESCRIPTOR(ts))"); } + @Test + void testOnTimeIsRejectedForUpdatingOutput() { + assertThatThrownBy( + () -> + util.verifyExecPlan( + "SELECT * FROM DEDUPLICATE(" + + "input => TABLE user_events PARTITION BY user_id, " + + "on_time => DESCRIPTOR(ts))")) + .satisfies( + anyCauseMatches( + "Time operations using the `on_time` argument are currently not " + + "supported for PTFs that consume or produce updates.")); + } + @Test void testTableApiWithArguments() { final Table result = From 5b772a27de05fb5f9dc74aa9b2f4931ff9b3e1cc Mon Sep 17 00:00:00 2001 From: Vas Shabu Date: Wed, 18 Nov 2026 17:32:52 +0000 Subject: [PATCH 6/9] [FLINK-41470][docs] Document the DEDUPLICATE built-in PTF. --- .../process-table-functions.md | 1 + .../sql/reference/queries/deduplication.md | 110 ++++++++++++++++++ .../process-table-functions.md | 1 + .../sql/reference/queries/deduplication.md | 110 ++++++++++++++++++ 4 files changed, 222 insertions(+) diff --git a/docs/content.zh/docs/sql/built-in-functions/process-table-functions.md b/docs/content.zh/docs/sql/built-in-functions/process-table-functions.md index c89fa07b7f3872..de05defee76b8c 100644 --- a/docs/content.zh/docs/sql/built-in-functions/process-table-functions.md +++ b/docs/content.zh/docs/sql/built-in-functions/process-table-functions.md @@ -31,6 +31,7 @@ The built-in process table functions (PTFs) take a table as input and return a t |---------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `FROM_CHANGELOG(input => TABLE t [, ...])` | Converts an append-only table with an explicit operation column into a dynamic table. See Changelog Conversion for the full list of arguments, semantics, and usage. | | `TO_CHANGELOG(input => TABLE t [, ...])` | Converts a dynamic table into an append-only table with an explicit operation column. See Changelog Conversion for the full list of arguments, semantics, and usage. | +| `DEDUPLICATE(input => TABLE t PARTITION BY k [, ...])` | Keeps the first or the last row per key of an append-only table. See [Deduplication]({{< ref "docs/sql/reference/queries/deduplication" >}}#deduplicate) for the full list of arguments, semantics, and usage. | | `SNAPSHOT(input => TABLE t [, ...])` | Returns the current state of a dynamic table `t`. `SNAPSHOT` can only be used in a `LATERAL` context and not as a stand-alone table function. See [LATERAL SNAPSHOT join]({{< ref "docs/sql/reference/queries/joins" >}}#lateral-snapshot-join) for the full list of arguments, the join semantics, and usage. | | `ML_PREDICT(TABLE t, MODEL m, DESCRIPTOR(...) [, ...])` | Applies the model `m` to the columns of table `t` that the `DESCRIPTOR` names, and returns the columns of `t` plus the prediction columns of the model. See [Model Inference]({{< ref "docs/sql/reference/queries/model-inference" >}}#ml_predict-函数) for the full list of arguments, semantics, and usage. | diff --git a/docs/content.zh/docs/sql/reference/queries/deduplication.md b/docs/content.zh/docs/sql/reference/queries/deduplication.md index d580f62e431905..dc833946fced49 100644 --- a/docs/content.zh/docs/sql/reference/queries/deduplication.md +++ b/docs/content.zh/docs/sql/reference/queries/deduplication.md @@ -69,4 +69,114 @@ FROM ( WHERE row_num = 1 ``` +## DEDUPLICATE + +{{< label Streaming >}} + +The `DEDUPLICATE` process table function (PTF) keeps the first or the last row per key of an append-only table, as long as the state of the key is retained. Unlike the `ROW_NUMBER()` pattern above, it needs no time attribute. Rows are ordered by arrival or by `sequence_by` columns. + +### Syntax + +```sql +SELECT * FROM DEDUPLICATE( + input => TABLE source_table PARTITION BY key_col[, ...], + [keep => 'first' | 'last',] + [sequence_by => DESCRIPTOR(col[, ...]),] + [state_ttl => INTERVAL '1' HOUR,] + [reset_ttl_on_duplicate => true | false,] + [uid => 'my-id'] +) +``` + +### Parameters + +| Parameter | Required | Description | +|:----------|:---------|:------------| +| `input` | Yes | The append-only input table. `PARTITION BY` is required and its columns are the key of the output. `ORDER BY` is not supported, use `sequence_by` instead. | +| `keep` | No | `'first'` (default) keeps the earliest arrival or the smallest `sequence_by` value, `'last'` keeps the latest arrival or the greatest. Must be a literal. Arrival order is not deterministic across reruns, use `sequence_by` for a repeatable result. | +| `sequence_by` | No | One or more columns that order the records of a key instead of their arrival. The first column decides, later columns break ties, and `NULL` orders below every other value. On a tie, the later arrival wins for `'last'` and the earlier one for `'first'`. Columns must be numeric, string, binary, boolean or date-time, and not part of `PARTITION BY`. | +| `state_ttl` | No | The minimum time the state of an idle key is retained, in processing time, as a non-negative `INTERVAL` literal. If omitted, `table.exec.state.ttl` applies, including later changes to it. `INTERVAL '0'` retains state indefinitely; the TTL is then left out of the state, so one cannot be added later without losing the state. When the state of a key expires, its next record is treated as the first one and emits `+I` again, so a sink without an upsert key can hold several rows for that key. | +| `reset_ttl_on_duplicate` | No | Whether a record that is dropped refreshes the TTL of its key. Defaults to `true`. Has no effect without a TTL. | +| `uid` | No | A unique identifier for the PTF invocation. Defaults to `DEDUPLICATE`. See [Multiple Invocations]({{< ref "docs/sql/reference/queries/changelog" >}}#multiple-invocations). | + +System arguments such as `uid` come last. + +### Output + +The output columns are the `PARTITION BY` columns, in the order listed, followed by the remaining input columns. Time attributes are emitted as regular timestamps. + +The result is an updating table whose key is the `PARTITION BY` columns. The first record of a key emits `+I`. A record that replaces the kept one emits `+U`, preceded by `-U` if the consumer needs retractions. A record that equals the kept one, or that does not replace it, emits nothing. With `keep => 'first'` and no `sequence_by`, only `+I` is emitted. The output is currently declared as updating in every case, so a sink that cannot consume updates is rejected at planning time. See [Updates and Changelogs]({{< ref "docs/dev/table/functions/ptfs" >}}#updates-and-changelogs) for the meaning of the change flags. + +### Examples + +#### Keep the first or the last record + +```sql +CREATE TABLE user_events (user_id STRING, event STRING) WITH (...); +-- (u1, login), (u2, login), (u1, click) + +SELECT * FROM DEDUPLICATE( + input => TABLE user_events PARTITION BY user_id); + +-- +I[u1, login] +-- +I[u2, login] + +SELECT * FROM DEDUPLICATE( + input => TABLE user_events PARTITION BY user_id, + keep => 'last'); + +-- +I[u1, login] +-- +I[u2, login] +-- +U[u1, click] (an upsert consumer; a retract consumer also gets -U[u1, login] before it) +``` + +#### Order by a column + +```sql +CREATE TABLE customer_changes (customer_id STRING, city STRING, lsn BIGINT) WITH (...); +-- (c1, Berlin, 100), (c1, Munich, 120), (c1, Hamburg, 110) + +SELECT * FROM DEDUPLICATE( + input => TABLE customer_changes PARTITION BY customer_id, + keep => 'last', + sequence_by => DESCRIPTOR(lsn), + state_ttl => INTERVAL '1' DAY); + +-- +I[c1, Berlin, 100] +-- +U[c1, Munich, 120] (The entry (c1, Hamburg, 110) is older than the kept, row -> dropped) +``` + +#### Composite key + +```sql +CREATE TABLE sessions (user_id STRING, page STRING, device STRING) WITH (...); +-- (u1, a, web), (u1, b, mobile), (u1, c, web) + +SELECT * FROM DEDUPLICATE( + input => TABLE sessions PARTITION BY (device, user_id), + keep => 'last'); + +-- columns: device, user_id, page +-- +I[web, u1, a] +-- +I[mobile, u1, b] +-- +U[web, u1, c] +``` + +#### Table API + +```java +// Keep the first record of each payment +Table result = payments + .partitionBy($("payment_id")) + .deduplicate(); + +// Keep the record with the greatest lsn per customer +Table result = customerChanges + .partitionBy($("customer_id")) + .deduplicate( + lit("last").asArgument("keep"), + descriptor("lsn").asArgument("sequence_by"), + lit(Duration.ofDays(1)).asArgument("state_ttl")); +``` + {{< top >}} diff --git a/docs/content/docs/sql/built-in-functions/process-table-functions.md b/docs/content/docs/sql/built-in-functions/process-table-functions.md index 0c2c3c35930a49..86a21808356ced 100644 --- a/docs/content/docs/sql/built-in-functions/process-table-functions.md +++ b/docs/content/docs/sql/built-in-functions/process-table-functions.md @@ -31,6 +31,7 @@ The built-in process table functions (PTFs) take a table as input and return a t |---------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `FROM_CHANGELOG(input => TABLE t [, ...])` | Converts an append-only table with an explicit operation column into a dynamic table. See [Changelog Conversion]({{< ref "docs/sql/reference/queries/changelog" >}}#from_changelog) for the full list of arguments, semantics, and usage. | | `TO_CHANGELOG(input => TABLE t [, ...])` | Converts a dynamic table into an append-only table with an explicit operation column. See [Changelog Conversion]({{< ref "docs/sql/reference/queries/changelog" >}}#to_changelog) for the full list of arguments, semantics, and usage. | +| `DEDUPLICATE(input => TABLE t PARTITION BY k [, ...])` | Keeps the first or the last row per key of an append-only table. See [Deduplication]({{< ref "docs/sql/reference/queries/deduplication" >}}#deduplicate) for the full list of arguments, semantics, and usage. | | `SNAPSHOT(input => TABLE t [, ...])` | Returns the current state of a dynamic table `t`. `SNAPSHOT` can only be used in a `LATERAL` context and not as a stand-alone table function. See [LATERAL SNAPSHOT join]({{< ref "docs/sql/reference/queries/joins" >}}#lateral-snapshot-join) for the full list of arguments, the join semantics, and usage. | | `ML_PREDICT(TABLE t, MODEL m, DESCRIPTOR(...) [, ...])` | Applies the model `m` to the columns of table `t` that the `DESCRIPTOR` names, and returns the columns of `t` plus the prediction columns of the model. See [Model Inference]({{< ref "docs/sql/reference/queries/model-inference" >}}#ml_predict-function) for the full list of arguments, semantics, and usage. | diff --git a/docs/content/docs/sql/reference/queries/deduplication.md b/docs/content/docs/sql/reference/queries/deduplication.md index 819893d19913bf..1c062f8156af1c 100644 --- a/docs/content/docs/sql/reference/queries/deduplication.md +++ b/docs/content/docs/sql/reference/queries/deduplication.md @@ -69,4 +69,114 @@ FROM ( WHERE row_num = 1 ``` +## DEDUPLICATE + +{{< label Streaming >}} + +The `DEDUPLICATE` process table function (PTF) keeps the first or the last row per key of an append-only table, as long as the state of the key is retained. Unlike the `ROW_NUMBER()` pattern above, it needs no time attribute. Rows are ordered by arrival or by `sequence_by` columns. + +### Syntax + +```sql +SELECT * FROM DEDUPLICATE( + input => TABLE source_table PARTITION BY key_col[, ...], + [keep => 'first' | 'last',] + [sequence_by => DESCRIPTOR(col[, ...]),] + [state_ttl => INTERVAL '1' HOUR,] + [reset_ttl_on_duplicate => true | false,] + [uid => 'my-id'] +) +``` + +### Parameters + +| Parameter | Required | Description | +|:----------|:---------|:------------| +| `input` | Yes | The append-only input table. `PARTITION BY` is required and its columns are the key of the output. `ORDER BY` is not supported, use `sequence_by` instead. | +| `keep` | No | `'first'` (default) keeps the earliest arrival or the smallest `sequence_by` value, `'last'` keeps the latest arrival or the greatest. Must be a literal. Arrival order is not deterministic across reruns, use `sequence_by` for a repeatable result. | +| `sequence_by` | No | One or more columns that order the records of a key instead of their arrival. The first column decides, later columns break ties, and `NULL` orders below every other value. On a tie, the later arrival wins for `'last'` and the earlier one for `'first'`. Columns must be numeric, string, binary, boolean or date-time, and not part of `PARTITION BY`. | +| `state_ttl` | No | The minimum time the state of an idle key is retained, in processing time, as a non-negative `INTERVAL` literal. If omitted, `table.exec.state.ttl` applies, including later changes to it. `INTERVAL '0'` retains state indefinitely; the TTL is then left out of the state, so one cannot be added later without losing the state. When the state of a key expires, its next record is treated as the first one and emits `+I` again, so a sink without an upsert key can hold several rows for that key. | +| `reset_ttl_on_duplicate` | No | Whether a record that is dropped refreshes the TTL of its key. Defaults to `true`. Has no effect without a TTL. | +| `uid` | No | A unique identifier for the PTF invocation. Defaults to `DEDUPLICATE`. See [Multiple Invocations]({{< ref "docs/sql/reference/queries/changelog" >}}#multiple-invocations). | + +System arguments such as `uid` come last. + +### Output + +The output columns are the `PARTITION BY` columns, in the order listed, followed by the remaining input columns. Time attributes are emitted as regular timestamps. + +The result is an updating table whose key is the `PARTITION BY` columns. The first record of a key emits `+I`. A record that replaces the kept one emits `+U`, preceded by `-U` if the consumer needs retractions. A record that equals the kept one, or that does not replace it, emits nothing. With `keep => 'first'` and no `sequence_by`, only `+I` is emitted. The output is currently declared as updating in every case, so a sink that cannot consume updates is rejected at planning time. See [Updates and Changelogs]({{< ref "docs/dev/table/functions/ptfs" >}}#updates-and-changelogs) for the meaning of the change flags. + +### Examples + +#### Keep the first or the last record + +```sql +CREATE TABLE user_events (user_id STRING, event STRING) WITH (...); +-- (u1, login), (u2, login), (u1, click) + +SELECT * FROM DEDUPLICATE( + input => TABLE user_events PARTITION BY user_id); + +-- +I[u1, login] +-- +I[u2, login] + +SELECT * FROM DEDUPLICATE( + input => TABLE user_events PARTITION BY user_id, + keep => 'last'); + +-- +I[u1, login] +-- +I[u2, login] +-- +U[u1, click] (an upsert consumer; a retract consumer also gets -U[u1, login] before it) +``` + +#### Order by a column + +```sql +CREATE TABLE customer_changes (customer_id STRING, city STRING, lsn BIGINT) WITH (...); +-- (c1, Berlin, 100), (c1, Munich, 120), (c1, Hamburg, 110) + +SELECT * FROM DEDUPLICATE( + input => TABLE customer_changes PARTITION BY customer_id, + keep => 'last', + sequence_by => DESCRIPTOR(lsn), + state_ttl => INTERVAL '1' DAY); + +-- +I[c1, Berlin, 100] +-- +U[c1, Munich, 120] (The entry (c1, Hamburg, 110) is older than the kept, row -> dropped) +``` + +#### Composite key + +```sql +CREATE TABLE sessions (user_id STRING, page STRING, device STRING) WITH (...); +-- (u1, a, web), (u1, b, mobile), (u1, c, web) + +SELECT * FROM DEDUPLICATE( + input => TABLE sessions PARTITION BY (device, user_id), + keep => 'last'); + +-- columns: device, user_id, page +-- +I[web, u1, a] +-- +I[mobile, u1, b] +-- +U[web, u1, c] +``` + +#### Table API + +```java +// Keep the first record of each payment +Table result = payments + .partitionBy($("payment_id")) + .deduplicate(); + +// Keep the record with the greatest lsn per customer +Table result = customerChanges + .partitionBy($("customer_id")) + .deduplicate( + lit("last").asArgument("keep"), + descriptor("lsn").asArgument("sequence_by"), + lit(Duration.ofDays(1)).asArgument("state_ttl")); +``` + {{< top >}} From 1059cd70c3139323f3a92af3d7811a9e781bc4aa Mon Sep 17 00:00:00 2001 From: Vas Shabu Date: Fri, 20 Nov 2026 02:14:03 +0000 Subject: [PATCH 7/9] [FLINK-41470][docs] Describe the DEDUPLICATE result as a table and trim the parameter docs Show the final rows in the examples, keep the change flags in a hint box, shorten the state_ttl row, and use INTERVAL '0' SECOND. --- .../sql/reference/queries/deduplication.md | 95 ++++++++++++------- .../sql/reference/queries/deduplication.md | 95 ++++++++++++------- 2 files changed, 124 insertions(+), 66 deletions(-) diff --git a/docs/content.zh/docs/sql/reference/queries/deduplication.md b/docs/content.zh/docs/sql/reference/queries/deduplication.md index dc833946fced49..0403f51f6dfa3e 100644 --- a/docs/content.zh/docs/sql/reference/queries/deduplication.md +++ b/docs/content.zh/docs/sql/reference/queries/deduplication.md @@ -73,7 +73,7 @@ WHERE row_num = 1 {{< label Streaming >}} -The `DEDUPLICATE` process table function (PTF) keeps the first or the last row per key of an append-only table, as long as the state of the key is retained. Unlike the `ROW_NUMBER()` pattern above, it needs no time attribute. Rows are ordered by arrival or by `sequence_by` columns. +The `DEDUPLICATE` process table function (PTF) keeps the first or the last row per key of an append-only table, as long as the state of the key is retained. Rows are ordered by arrival or by `sequence_by` columns. ### Syntax @@ -92,76 +92,105 @@ SELECT * FROM DEDUPLICATE( | Parameter | Required | Description | |:----------|:---------|:------------| -| `input` | Yes | The append-only input table. `PARTITION BY` is required and its columns are the key of the output. `ORDER BY` is not supported, use `sequence_by` instead. | +| `input` | Yes | The append-only input table. `PARTITION BY` is required and its columns are the key of the output.| | `keep` | No | `'first'` (default) keeps the earliest arrival or the smallest `sequence_by` value, `'last'` keeps the latest arrival or the greatest. Must be a literal. Arrival order is not deterministic across reruns, use `sequence_by` for a repeatable result. | | `sequence_by` | No | One or more columns that order the records of a key instead of their arrival. The first column decides, later columns break ties, and `NULL` orders below every other value. On a tie, the later arrival wins for `'last'` and the earlier one for `'first'`. Columns must be numeric, string, binary, boolean or date-time, and not part of `PARTITION BY`. | -| `state_ttl` | No | The minimum time the state of an idle key is retained, in processing time, as a non-negative `INTERVAL` literal. If omitted, `table.exec.state.ttl` applies, including later changes to it. `INTERVAL '0'` retains state indefinitely; the TTL is then left out of the state, so one cannot be added later without losing the state. When the state of a key expires, its next record is treated as the first one and emits `+I` again, so a sink without an upsert key can hold several rows for that key. | +| `state_ttl` | No | How long the state of an idle key is retained, as an `INTERVAL` literal. Defaults to `table.exec.state.ttl`, and `INTERVAL '0' SECOND` disables expiry. After the state of a key expires, its next record is emitted as `+I` again. | | `reset_ttl_on_duplicate` | No | Whether a record that is dropped refreshes the TTL of its key. Defaults to `true`. Has no effect without a TTL. | -| `uid` | No | A unique identifier for the PTF invocation. Defaults to `DEDUPLICATE`. See [Multiple Invocations]({{< ref "docs/sql/reference/queries/changelog" >}}#multiple-invocations). | -System arguments such as `uid` come last. ### Output The output columns are the `PARTITION BY` columns, in the order listed, followed by the remaining input columns. Time attributes are emitted as regular timestamps. -The result is an updating table whose key is the `PARTITION BY` columns. The first record of a key emits `+I`. A record that replaces the kept one emits `+U`, preceded by `-U` if the consumer needs retractions. A record that equals the kept one, or that does not replace it, emits nothing. With `keep => 'first'` and no `sequence_by`, only `+I` is emitted. The output is currently declared as updating in every case, so a sink that cannot consume updates is rejected at planning time. See [Updates and Changelogs]({{< ref "docs/dev/table/functions/ptfs" >}}#updates-and-changelogs) for the meaning of the change flags. +The result has one row per key. When a later row replaces the kept one, the row of that key is updated, so the result must be written to a table that accepts updates, for example one with a primary key on the `PARTITION BY` columns. Currently this holds for every `keep` mode, including `keep => 'first'` without `sequence_by`, which never replaces a row. + +{{< hint info >}} +The first record of a key is emitted as `+I`. A record that replaces the kept one is emitted as `+U`, preceded by `-U` if the consumer needs retractions. A record that equals the kept one, or does not replace it, emits nothing. See [Updates and Changelogs]({{< ref "docs/dev/table/functions/ptfs" >}}#updates-and-changelogs). +{{< /hint >}} ### Examples #### Keep the first or the last record -```sql -CREATE TABLE user_events (user_id STRING, event STRING) WITH (...); --- (u1, login), (u2, login), (u1, click) +Rows of `user_events` arrive in this order: + +| `user_events` | +|:----------------| +| `(u1, login)` | +| `(u2, login)` | +| `(u1, click)` | + +`keep` defaults to `'first'`, which keeps the first row of each user: +```sql SELECT * FROM DEDUPLICATE( - input => TABLE user_events PARTITION BY user_id); + input => TABLE user_events PARTITION BY user_id) +``` + +| Result | +|:----------------| +| `(u1, login)` | +| `(u2, login)` | --- +I[u1, login] --- +I[u2, login] +With `keep => 'last'`, a later row replaces the kept one: +```sql SELECT * FROM DEDUPLICATE( input => TABLE user_events PARTITION BY user_id, - keep => 'last'); - --- +I[u1, login] --- +I[u2, login] --- +U[u1, click] (an upsert consumer; a retract consumer also gets -U[u1, login] before it) + keep => 'last') ``` +| Result | +|:----------------| +| `(u1, click)` | +| `(u2, login)` | + #### Order by a column -```sql -CREATE TABLE customer_changes (customer_id STRING, city STRING, lsn BIGINT) WITH (...); --- (c1, Berlin, 100), (c1, Munich, 120), (c1, Hamburg, 110) +`sequence_by` ranks the rows of a key by a column instead of by arrival, so a late, older row does not replace a newer one: + +| `customer_changes` | +|:----------------------------| +| `(c1, Berlin, 100)` | +| `(c1, Munich, 120)` | +| `(c1, Hamburg, 110)` | +```sql SELECT * FROM DEDUPLICATE( input => TABLE customer_changes PARTITION BY customer_id, keep => 'last', sequence_by => DESCRIPTOR(lsn), - state_ttl => INTERVAL '1' DAY); - --- +I[c1, Berlin, 100] --- +U[c1, Munich, 120] (The entry (c1, Hamburg, 110) is older than the kept, row -> dropped) + state_ttl => INTERVAL '1' DAY) ``` +| Result | +|:----------------------| +| `(c1, Munich, 120)` | + +`(c1, Hamburg, 110)` is dropped because its `lsn` is smaller than that of the kept row. + #### Composite key -```sql -CREATE TABLE sessions (user_id STRING, page STRING, device STRING) WITH (...); --- (u1, a, web), (u1, b, mobile), (u1, c, web) +The columns of `PARTITION BY` come first in the output, in the order listed: + +| `sessions` (`user_id`, `page`, `device`) | +|:-----------------------------------------| +| `(u1, a, web)` | +| `(u1, b, mobile)` | +| `(u1, c, web)` | +```sql SELECT * FROM DEDUPLICATE( input => TABLE sessions PARTITION BY (device, user_id), - keep => 'last'); - --- columns: device, user_id, page --- +I[web, u1, a] --- +I[mobile, u1, b] --- +U[web, u1, c] + keep => 'last') ``` +| Result (`device`, `user_id`, `page`) | +|:-------------------------------------| +| `(web, u1, c)` | +| `(mobile, u1, b)` | + #### Table API ```java diff --git a/docs/content/docs/sql/reference/queries/deduplication.md b/docs/content/docs/sql/reference/queries/deduplication.md index 1c062f8156af1c..73c638d356e5a3 100644 --- a/docs/content/docs/sql/reference/queries/deduplication.md +++ b/docs/content/docs/sql/reference/queries/deduplication.md @@ -73,7 +73,7 @@ WHERE row_num = 1 {{< label Streaming >}} -The `DEDUPLICATE` process table function (PTF) keeps the first or the last row per key of an append-only table, as long as the state of the key is retained. Unlike the `ROW_NUMBER()` pattern above, it needs no time attribute. Rows are ordered by arrival or by `sequence_by` columns. +The `DEDUPLICATE` process table function (PTF) keeps the first or the last row per key of an append-only table, as long as the state of the key is retained. Rows are ordered by arrival or by `sequence_by` columns. ### Syntax @@ -92,76 +92,105 @@ SELECT * FROM DEDUPLICATE( | Parameter | Required | Description | |:----------|:---------|:------------| -| `input` | Yes | The append-only input table. `PARTITION BY` is required and its columns are the key of the output. `ORDER BY` is not supported, use `sequence_by` instead. | +| `input` | Yes | The append-only input table. `PARTITION BY` is required and its columns are the key of the output.| | `keep` | No | `'first'` (default) keeps the earliest arrival or the smallest `sequence_by` value, `'last'` keeps the latest arrival or the greatest. Must be a literal. Arrival order is not deterministic across reruns, use `sequence_by` for a repeatable result. | | `sequence_by` | No | One or more columns that order the records of a key instead of their arrival. The first column decides, later columns break ties, and `NULL` orders below every other value. On a tie, the later arrival wins for `'last'` and the earlier one for `'first'`. Columns must be numeric, string, binary, boolean or date-time, and not part of `PARTITION BY`. | -| `state_ttl` | No | The minimum time the state of an idle key is retained, in processing time, as a non-negative `INTERVAL` literal. If omitted, `table.exec.state.ttl` applies, including later changes to it. `INTERVAL '0'` retains state indefinitely; the TTL is then left out of the state, so one cannot be added later without losing the state. When the state of a key expires, its next record is treated as the first one and emits `+I` again, so a sink without an upsert key can hold several rows for that key. | +| `state_ttl` | No | How long the state of an idle key is retained, as an `INTERVAL` literal. Defaults to `table.exec.state.ttl`, and `INTERVAL '0' SECOND` disables expiry. After the state of a key expires, its next record is emitted as `+I` again. | | `reset_ttl_on_duplicate` | No | Whether a record that is dropped refreshes the TTL of its key. Defaults to `true`. Has no effect without a TTL. | -| `uid` | No | A unique identifier for the PTF invocation. Defaults to `DEDUPLICATE`. See [Multiple Invocations]({{< ref "docs/sql/reference/queries/changelog" >}}#multiple-invocations). | -System arguments such as `uid` come last. ### Output The output columns are the `PARTITION BY` columns, in the order listed, followed by the remaining input columns. Time attributes are emitted as regular timestamps. -The result is an updating table whose key is the `PARTITION BY` columns. The first record of a key emits `+I`. A record that replaces the kept one emits `+U`, preceded by `-U` if the consumer needs retractions. A record that equals the kept one, or that does not replace it, emits nothing. With `keep => 'first'` and no `sequence_by`, only `+I` is emitted. The output is currently declared as updating in every case, so a sink that cannot consume updates is rejected at planning time. See [Updates and Changelogs]({{< ref "docs/dev/table/functions/ptfs" >}}#updates-and-changelogs) for the meaning of the change flags. +The result has one row per key. When a later row replaces the kept one, the row of that key is updated, so the result must be written to a table that accepts updates, for example one with a primary key on the `PARTITION BY` columns. Currently this holds for every `keep` mode, including `keep => 'first'` without `sequence_by`, which never replaces a row. + +{{< hint info >}} +The first record of a key is emitted as `+I`. A record that replaces the kept one is emitted as `+U`, preceded by `-U` if the consumer needs retractions. A record that equals the kept one, or does not replace it, emits nothing. See [Updates and Changelogs]({{< ref "docs/dev/table/functions/ptfs" >}}#updates-and-changelogs). +{{< /hint >}} ### Examples #### Keep the first or the last record -```sql -CREATE TABLE user_events (user_id STRING, event STRING) WITH (...); --- (u1, login), (u2, login), (u1, click) +Rows of `user_events` arrive in this order: + +| `user_events` | +|:----------------| +| `(u1, login)` | +| `(u2, login)` | +| `(u1, click)` | + +`keep` defaults to `'first'`, which keeps the first row of each user: +```sql SELECT * FROM DEDUPLICATE( - input => TABLE user_events PARTITION BY user_id); + input => TABLE user_events PARTITION BY user_id) +``` + +| Result | +|:----------------| +| `(u1, login)` | +| `(u2, login)` | --- +I[u1, login] --- +I[u2, login] +With `keep => 'last'`, a later row replaces the kept one: +```sql SELECT * FROM DEDUPLICATE( input => TABLE user_events PARTITION BY user_id, - keep => 'last'); - --- +I[u1, login] --- +I[u2, login] --- +U[u1, click] (an upsert consumer; a retract consumer also gets -U[u1, login] before it) + keep => 'last') ``` +| Result | +|:----------------| +| `(u1, click)` | +| `(u2, login)` | + #### Order by a column -```sql -CREATE TABLE customer_changes (customer_id STRING, city STRING, lsn BIGINT) WITH (...); --- (c1, Berlin, 100), (c1, Munich, 120), (c1, Hamburg, 110) +`sequence_by` ranks the rows of a key by a column instead of by arrival, so a late, older row does not replace a newer one: + +| `customer_changes` | +|:----------------------------| +| `(c1, Berlin, 100)` | +| `(c1, Munich, 120)` | +| `(c1, Hamburg, 110)` | +```sql SELECT * FROM DEDUPLICATE( input => TABLE customer_changes PARTITION BY customer_id, keep => 'last', sequence_by => DESCRIPTOR(lsn), - state_ttl => INTERVAL '1' DAY); - --- +I[c1, Berlin, 100] --- +U[c1, Munich, 120] (The entry (c1, Hamburg, 110) is older than the kept, row -> dropped) + state_ttl => INTERVAL '1' DAY) ``` +| Result | +|:----------------------| +| `(c1, Munich, 120)` | + +`(c1, Hamburg, 110)` is dropped because its `lsn` is smaller than that of the kept row. + #### Composite key -```sql -CREATE TABLE sessions (user_id STRING, page STRING, device STRING) WITH (...); --- (u1, a, web), (u1, b, mobile), (u1, c, web) +The columns of `PARTITION BY` come first in the output, in the order listed: + +| `sessions` (`user_id`, `page`, `device`) | +|:-----------------------------------------| +| `(u1, a, web)` | +| `(u1, b, mobile)` | +| `(u1, c, web)` | +```sql SELECT * FROM DEDUPLICATE( input => TABLE sessions PARTITION BY (device, user_id), - keep => 'last'); - --- columns: device, user_id, page --- +I[web, u1, a] --- +I[mobile, u1, b] --- +U[web, u1, c] + keep => 'last') ``` +| Result (`device`, `user_id`, `page`) | +|:-------------------------------------| +| `(web, u1, c)` | +| `(mobile, u1, b)` | + #### Table API ```java From 8eccd26ebb269c53c9015f09da5dd1316f812dd2 Mon Sep 17 00:00:00 2001 From: Vas Shabu Date: Fri, 20 Nov 2026 02:14:03 +0000 Subject: [PATCH 8/9] [FLINK-41470][table] Use INTERVAL '0' SECOND in the DEDUPLICATE state_ttl hint Flink SQL cannot parse INTERVAL '0' without a unit, so the hint now names the unit and the test pins the full message. --- .../inference/strategies/DeduplicateTypeStrategyUtils.java | 2 +- .../inference/strategies/DeduplicateTypeStrategyTest.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java index c57326a7f57fe9..da6ac4b2b7afec 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java @@ -212,7 +212,7 @@ public static Optional validateStateTtl( stateTtl -> String.format( "The 'state_ttl' argument must not be negative, but was " - + "'%s'. Use INTERVAL '0' to retain state " + + "'%s'. Use INTERVAL '0' SECOND to retain state " + "indefinitely.", TimeUtils.formatWithHighestUnit(stateTtl))); } diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java index cce104473b14da..70a01030b2a157 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyTest.java @@ -140,7 +140,8 @@ protected Stream testData() { withDefaults("Negative state_ttl", PARTITIONED_BY_USER) .calledWithLiteralAt(ARG_STATE_TTL, Duration.ofSeconds(-1)) .expectErrorMessage( - "The 'state_ttl' argument must not be negative, but was '-1 s'."), + "The 'state_ttl' argument must not be negative, but was '-1 s'. " + + "Use INTERVAL '0' SECOND to retain state indefinitely."), nonLiteral("Non-literal state_ttl", ARG_STATE_TTL) .expectErrorMessage( "The 'state_ttl' argument must be a constant INTERVAL literal, " From 11431f44594bcd52f52d8261fa6916d6d0217828 Mon Sep 17 00:00:00 2001 From: Vas Shabu Date: Fri, 20 Nov 2026 02:14:15 +0000 Subject: [PATCH 9/9] [FLINK-41470][table] Simplify the DEDUPLICATE runtime after review Add a generic valueState helper to BuiltInProcessTableFunction and name the emit-once state 'emitted', strip time attributes from the input type with a single call, serialize the candidate row only when it can replace the current one, and move the replace decision into replacesCurrent. Regenerate the first-arrival restore test for the renamed state. --- .../DeduplicateTypeStrategyUtils.java | 18 ++-- .../deduplicate-first-arrival-restore.json | 23 ++--- .../savepoint/_metadata | Bin 11317 -> 11415 bytes .../ptf/BuiltInProcessTableFunction.java | 20 +++++ .../ptf/DeduplicateCurrentRowFunction.java | 80 +++++++++++------- .../ptf/DeduplicateEmitOnceFunction.java | 20 ++--- .../functions/ptf/DeduplicateUtils.java | 29 ------- 7 files changed, 101 insertions(+), 89 deletions(-) diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java index da6ac4b2b7afec..50893b82af5f03 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateTypeStrategyUtils.java @@ -19,7 +19,6 @@ package org.apache.flink.table.types.inference.strategies; import org.apache.flink.annotation.Internal; -import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.DataTypes.Field; import org.apache.flink.table.functions.TableSemantics; import org.apache.flink.table.types.DataType; @@ -28,6 +27,7 @@ import org.apache.flink.table.types.logical.LogicalTypeFamily; import org.apache.flink.table.types.logical.LogicalTypeRoot; import org.apache.flink.table.types.logical.utils.LogicalTypeChecks; +import org.apache.flink.table.types.logical.utils.LogicalTypeUtils; import org.apache.flink.table.types.utils.DataTypeUtils; import org.apache.flink.types.ColumnList; import org.apache.flink.util.TimeUtils; @@ -92,16 +92,20 @@ public static int[] sequenceByOutputIndices( .toArray(); } + /** Returns the input columns without time attributes. */ + public static List inputFields(final TableSemantics tableSemantics) { + final DataType inputType = tableSemantics.dataType(); + return DataType.getFields( + DataTypeUtils.replaceLogicalType( + inputType, + LogicalTypeUtils.removeTimeAttributes(inputType.getLogicalType()))); + } + /** Returns the output columns after the PARTITION BY columns, without time attributes. */ public static List outputFields(final TableSemantics tableSemantics) { - final List inputFields = DataType.getFields(tableSemantics.dataType()); + final List inputFields = inputFields(tableSemantics); return Arrays.stream(ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics)) .mapToObj(inputFields::get) - .map( - field -> - DataTypes.FIELD( - field.getName(), - DataTypeUtils.removeTimeAttribute(field.getDataType()))) .collect(Collectors.toList()); } diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/plan/deduplicate-first-arrival-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/plan/deduplicate-first-arrival-restore.json index 405398dd914233..533b425602f2c2 100644 --- a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/plan/deduplicate-first-arrival-restore.json +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/plan/deduplicate-first-arrival-restore.json @@ -1,7 +1,7 @@ { "flinkVersion" : "2.5", "nodes" : [ { - "id" : 5, + "id" : 1, "type" : "stream-exec-table-source-scan_2", "scanTableSource" : { "table" : { @@ -22,7 +22,7 @@ "outputType" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT>", "description" : "TableSourceScan(table=[[default_catalog, default_database, payments]], fields=[payment_id, amount])" }, { - "id" : 6, + "id" : 2, "type" : "stream-exec-exchange_1", "inputProperties" : [ { "requiredDistribution" : { @@ -35,11 +35,12 @@ "outputType" : "ROW<`payment_id` VARCHAR(2147483647), `amount` INT>", "description" : "Exchange(distribution=[hash[payment_id]])" }, { - "id" : 7, + "id" : 3, "type" : "stream-exec-process-table-function_1", "inputProperties" : [ { "requiredDistribution" : { - "type" : "UNKNOWN" + "type" : "HASH", + "keys" : [ 0 ] }, "damBehavior" : "PIPELINED", "priority" : 0 @@ -92,7 +93,7 @@ "outputChangelogMode" : [ "INSERT", "UPDATE_AFTER" ], "inputUpsertKeys" : [ [ ] ] }, { - "id" : 8, + "id" : 4, "type" : "stream-exec-sink_2", "configuration" : { "table.exec.sink.keyed-shuffle" : "AUTO", @@ -136,22 +137,22 @@ "description" : "Sink(table=[default_catalog.default_database.sink], fields=[payment_id, amount])" } ], "edges" : [ { - "source" : 5, - "target" : 6, + "source" : 1, + "target" : 2, "shuffle" : { "type" : "FORWARD" }, "shuffleMode" : "PIPELINED" }, { - "source" : 6, - "target" : 7, + "source" : 2, + "target" : 3, "shuffle" : { "type" : "FORWARD" }, "shuffleMode" : "PIPELINED" }, { - "source" : 7, - "target" : 8, + "source" : 3, + "target" : 4, "shuffle" : { "type" : "FORWARD" }, diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-first-arrival-restore/savepoint/_metadata index 79a17c17cd5e498a34a9e855f5b2c50470747f28..df8f52de27bc53b805ba3ecd5aa2dbf0163b5422 100644 GIT binary patch delta 796 zcmbVKO=uHA6rPDTt2Hl@WPq15=t(3#oU>1_SElwL$?!HYH2I6J$Gp=}b9G@xJ+ zLG;*k4pQ1n1y70J3(eD zq!=MCRgz030f-z6CWT?;7F=rOn_&P7!*V1y3gZAY?>o2NkK&aw6@@Vm3dx-kDrJ zb-n#s?hAiaWvk`IcD-&im2;8Nw)5GkpnD*mlUFCbOtFw8W1xkk3XSV`-?|^X__60> z>E%i}xz=+J000BO0YTg~eM*#c6!$&syM8}c!trckIj%#gq`GwY1pHrR-9TfXVS>1C z_Scp5kfW49%wd#M#}(A#0gsKq!%z<&`P{lKtiEF$f~J0Cyn#_&HFzj?rt&PR;cPS# zirsFcdd6tlv9#5~E6;DjPe3e&?5vzhG!5PM!{S3w=Up~~Yn zXLn1pH<>OLJy*YOJ5ZrLmv`>pCJp-ls9jmOsAm(sUA2eRU2t>rPc%H2Sk=paL#M0n Rr}fuFZ}V1fMI8Me`UyGU@7@3a delta 697 zcmbu6F=!M)6ozNdH?fEsk{n^Yo!yz;85N?2U@M5AhHy8#JA)y38*ZZ@ zHWKY3qM1T4wP0f`PQlK~!a}sMveH6O#6qx05#!wDLQ?C%%zV7};QxQ#qsuQZJ+N(S zI%%X!O+MdPnrXjF|+W zFQ=ulpmo%Eo~IfQZ7Tz?h&r_F?iBX+wu{Sy-JY5>C)L+t*$C8T@x75(XG(s?BCUb` zRYRRGnHj6zWiz|)KO*6PMa*>~7zczv919;(;xZ^>c-qMKc4k#A*sH~Ny_x}NJ+5qK{sMg}yWao+ diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java index 5cecc79866f657..bc57fe3181a53d 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java @@ -104,6 +104,26 @@ protected BuiltInProcessTableFunction( : toInternalStateStrategies(definitionTypeInference); } + /** + * Declares a function with one value state of the given type that expires after the interval of + * the argument at {@code ttlArgPos}, or never if that argument is NULL. + */ + protected static LinkedHashMap valueState( + final SpecializedContext context, + final String name, + final DataType valueType, + final int ttlArgPos) { + return internalState( + Map.entry( + name, + StateTypeStrategy.of( + callContext -> + Optional.of(ValueView.newValueViewDataType(valueType)), + context.getCallContext() + .getArgumentValue(ttlArgPos, Duration.class) + .orElse(null)))); + } + /** * Declares the state entries of a specialized function in order. The strategies may infer * external data types, they are converted to internal ones. diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateCurrentRowFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateCurrentRowFunction.java index 9ae82f6366d0d7..f131b9f1db3930 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateCurrentRowFunction.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateCurrentRowFunction.java @@ -32,9 +32,9 @@ import org.apache.flink.table.runtime.functions.ptf.DeduplicateUtils.SequenceComparator; import org.apache.flink.table.runtime.typeutils.RowDataSerializer; import org.apache.flink.table.types.DataType; -import org.apache.flink.table.types.inference.StateTypeStrategy; import org.apache.flink.table.types.inference.strategies.ChangelogTypeStrategyUtils; import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy; +import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils; import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.types.ColumnList; @@ -44,11 +44,12 @@ import java.util.Arrays; import java.util.List; -import java.util.Map; + +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL; /** * {@link Deduplicate} with {@code keep => 'last'}, or {@code keep => 'first'} with {@code - * sequence_by}: a later record can replace the kept one, so the kept input row is held in the + * sequence_by}: a later record can replace the current one, so the current input row is held in the * {@code kept} state. */ @Internal @@ -56,6 +57,8 @@ public class DeduplicateCurrentRowFunction extends DeduplicateFunctionBase { private static final long serialVersionUID = 1L; + private static final String CURRENT_STATE = "kept"; + private final Keep keepMode; private final LogicalType[] inputTypes; @@ -68,16 +71,25 @@ public class DeduplicateCurrentRowFunction extends DeduplicateFunctionBase { private transient ProjectedRowData projectedCandidate; - private transient ProjectedRowData projectedKept; + private transient ProjectedRowData projectedCurrent; private transient JoinedRowData retraction; private transient RowData emptyRow; DeduplicateCurrentRowFunction(final SpecializedContext context) { - super(context, internalState(Map.entry("kept", keptState(context)))); + super( + context, + valueState( + context, + CURRENT_STATE, + DataTypes.ROW( + DeduplicateTypeStrategyUtils.inputFields( + DeduplicateUtils.tableSemantics(context))), + ARG_STATE_TTL)); this.keepMode = DeduplicateTypeStrategy.keep(context.getCallContext()); - final List inputFields = DeduplicateUtils.inputFields(context); + final List inputFields = + DeduplicateTypeStrategyUtils.inputFields(DeduplicateUtils.tableSemantics(context)); this.inputTypes = inputFields.stream() .map(field -> field.getDataType().getLogicalType()) @@ -98,7 +110,7 @@ public void open(final FunctionContext context) throws Exception { } inputSerializer = new RowDataSerializer(inputTypes); projectedCandidate = ProjectedRowData.from(outputIndices); - projectedKept = ProjectedRowData.from(outputIndices); + projectedCurrent = ProjectedRowData.from(outputIndices); retraction = new JoinedRowData(); emptyRow = GenericRowData.of(); } @@ -113,54 +125,60 @@ public void close() throws Exception { public void eval( final Context ctx, - final ValueView keptState, + final ValueView currentState, final RowData input, @Nullable final StringData keep, @Nullable final ColumnList sequenceBy, @Nullable final Long stateTtl, @Nullable final Boolean resetTtlOnDuplicate) throws Exception { - final BinaryRowData candidate = inputSerializer.toBinaryRow(input); - final BinaryRowData kept = (BinaryRowData) keptState.getValue(); - if (kept == null) { - keptState.setValue(candidate.copy()); + + final BinaryRowData current = (BinaryRowData) currentState.getValue(); + if (current == null) { + final BinaryRowData candidate = inputSerializer.toBinaryRow(input); + currentState.setValue(candidate.copy()); emitCandidate(RowKind.INSERT, candidate); return; } - final boolean isReplacing = - sequenceComparator == null - || DeduplicateUtils.replaces( - keepMode, sequenceComparator.compare(candidate, kept)); - if (isReplacing && !candidate.equals(kept)) { - keptState.setValue(candidate.copy()); - if (isUpdateBeforeEmitted) { - emitRetraction(kept); + if (replacesCurrent(input, current)) { + final BinaryRowData candidate = inputSerializer.toBinaryRow(input); + if (!candidate.equals(current)) { + currentState.setValue(candidate.copy()); + if (isUpdateBeforeEmitted) { + emitRetraction(current); + } + emitCandidate(RowKind.UPDATE_AFTER, candidate); + } else if (isResetTtlOnDuplicate) { + currentState.setValue(current); } - emitCandidate(RowKind.UPDATE_AFTER, candidate); } else if (isResetTtlOnDuplicate) { - keptState.setValue(kept); + currentState.setValue(current); } } + /** + * Without sequence_by only keep => 'last' reaches this class, so the latest arrival replaces. + */ + private boolean replacesCurrent(final RowData candidate, final RowData current) { + return sequenceComparator == null + || DeduplicateUtils.replaces( + keepMode, sequenceComparator.compare(candidate, current)); + } + private void emitCandidate(final RowKind kind, final BinaryRowData candidate) { projectedCandidate.replaceRow(candidate); projectedCandidate.setRowKind(kind); collect(projectedCandidate); } - /** Wraps the kept row so that the row owned by state never changes its row kind. */ - private void emitRetraction(final BinaryRowData kept) { - projectedKept.replaceRow(kept); - retraction.replace(projectedKept, emptyRow); + /** Wraps the current row so that the row owned by state never changes its row kind. */ + private void emitRetraction(final BinaryRowData current) { + projectedCurrent.replaceRow(current); + retraction.replace(projectedCurrent, emptyRow); retraction.setRowKind(RowKind.UPDATE_BEFORE); collect(retraction); } - private static StateTypeStrategy keptState(final SpecializedContext context) { - return DeduplicateUtils.stateStrategy( - context, DataTypes.ROW(DeduplicateUtils.inputFields(context))); - } - private static @Nullable SequenceComparator sequenceComparator( final SpecializedContext context, final List inputFields) { final int[] outputIndices = diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateEmitOnceFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateEmitOnceFunction.java index 782a4aa9d95235..9c23e7562b0241 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateEmitOnceFunction.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateEmitOnceFunction.java @@ -31,27 +31,25 @@ import javax.annotation.Nullable; -import java.util.Map; +import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL; /** * {@link Deduplicate} with {@code keep => 'first'} and no {@code sequence_by}: no later record can - * replace the first one, so only whether the key has emitted is kept in the {@code seen} state. + * replace the first one, so only whether the key has emitted is kept in the {@code emitted} state. */ @Internal public class DeduplicateEmitOnceFunction extends DeduplicateFunctionBase { private static final long serialVersionUID = 1L; + private static final String EMITTED_STATE = "emitted"; + private transient ProjectedRowData projectedOutput; DeduplicateEmitOnceFunction(final SpecializedContext context) { super( context, - internalState( - Map.entry( - "seen", - DeduplicateUtils.stateStrategy( - context, DataTypes.BOOLEAN().notNull())))); + valueState(context, EMITTED_STATE, DataTypes.BOOLEAN().notNull(), ARG_STATE_TTL)); } @Override @@ -62,20 +60,20 @@ public void open(final FunctionContext context) throws Exception { public void eval( final Context ctx, - final ValueView seen, + final ValueView emitted, final RowData input, @Nullable final StringData keep, @Nullable final ColumnList sequenceBy, @Nullable final Long stateTtl, @Nullable final Boolean resetTtlOnDuplicate) throws Exception { - if (seen.getValue() == null) { + if (emitted.getValue() == null) { projectedOutput.replaceRow(input); projectedOutput.setRowKind(RowKind.INSERT); collect(projectedOutput); - seen.setValue(true); + emitted.setValue(true); } else if (isResetTtlOnDuplicate) { - seen.setValue(true); + emitted.setValue(true); } } } diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java index a10ebb70df7a0e..3e06149db3e87a 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateUtils.java @@ -20,31 +20,23 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.table.api.DataTypes; -import org.apache.flink.table.api.dataview.ValueView; import org.apache.flink.table.data.RowData; import org.apache.flink.table.functions.FunctionContext; import org.apache.flink.table.functions.SpecializedFunction.ExpressionEvaluator; import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; import org.apache.flink.table.functions.TableSemantics; import org.apache.flink.table.types.DataType; -import org.apache.flink.table.types.inference.StateTypeStrategy; import org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategyUtils.Keep; -import org.apache.flink.table.types.utils.DataTypeUtils; import org.apache.flink.util.FlinkRuntimeException; import javax.annotation.Nullable; import java.io.Serializable; import java.lang.invoke.MethodHandle; -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; import static org.apache.flink.table.api.Expressions.$; import static org.apache.flink.table.api.Expressions.ifThenElse; import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_INPUT; -import static org.apache.flink.table.types.inference.strategies.DeduplicateTypeStrategy.ARG_STATE_TTL; /** Record ordering of the built-in deduplication PTFs. */ @Internal @@ -56,27 +48,6 @@ public static TableSemantics tableSemantics(final SpecializedContext context) { .orElseThrow(() -> new IllegalStateException("Table argument expected.")); } - /** Returns the fields of the input row without time attributes. */ - public static List inputFields(final SpecializedContext context) { - return DataType.getFields(tableSemantics(context).dataType()).stream() - .map( - field -> - DataTypes.FIELD( - field.getName(), - DataTypeUtils.removeTimeAttribute(field.getDataType()))) - .collect(Collectors.toList()); - } - - /** Returns a value state of the given type that expires after the {@code state_ttl}. */ - public static StateTypeStrategy stateStrategy( - final SpecializedContext context, final DataType valueType) { - return StateTypeStrategy.of( - callContext -> Optional.of(ValueView.newValueViewDataType(valueType)), - context.getCallContext() - .getArgumentValue(ARG_STATE_TTL, Duration.class) - .orElse(null)); - } - /** * Returns whether a new record replaces the current one, given the order of the new record * relative to the current one: with {@code last} the later arrival wins on a tie, with {@code