From ba5486a9b24000742355daa635a6ee4f757ccaf3 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 2 Sep 2026 03:16:31 +0800 Subject: [PATCH 01/15] core: validate explicit format table partition locations --- .../FormatTablePartitionPathResolver.java | 386 ++++++++++++++++++ .../FormatTablePartitionPathResolverTest.java | 193 +++++++++ 2 files changed, 579 insertions(+) create mode 100644 paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionPathResolver.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/table/format/FormatTablePartitionPathResolverTest.java diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionPathResolver.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionPathResolver.java new file mode 100644 index 000000000000..389407ecd6dd --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionPathResolver.java @@ -0,0 +1,386 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.format; + +import org.apache.paimon.fs.Path; +import org.apache.paimon.utils.PartitionPathUtils; + +import javax.annotation.Nullable; + +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; + +/** Resolves catalog-managed Format Table partition paths and rejects ambiguous catalog metadata. */ +public final class FormatTablePartitionPathResolver { + + private final Path tablePath; + private final String tableName; + private final boolean onlyValueInPath; + private final Map, ResolvedPath> pathsBySpec = new LinkedHashMap<>(); + private final Map ownershipRoots = new HashMap<>(); + + FormatTablePartitionPathResolver(Path tablePath, String tableName, boolean onlyValueInPath) { + this.tablePath = tablePath; + this.tableName = tableName; + this.onlyValueInPath = onlyValueInPath; + } + + Path resolve(LinkedHashMap spec, @Nullable String explicitLocation) { + Path defaultPath = + new Path( + tablePath, + PartitionPathUtils.generatePartitionPathUtil(spec, onlyValueInPath)); + if (explicitLocation == null) { + return defaultPath; + } + + try { + return resolveExplicitLocation(tablePath, spec, onlyValueInPath, explicitLocation); + } catch (IllegalArgumentException e) { + throw invalidLocation(spec); + } + } + + /** + * Canonicalizes an explicit partition location and verifies that it cannot name the table, its + * ancestor, or the partition's default directory. + */ + public static Path resolveExplicitLocation( + Path tablePath, + LinkedHashMap spec, + boolean onlyValueInPath, + String explicitLocation) { + PartitionPathUtils.validatePartitionSpecForPath(spec, onlyValueInPath); + Path defaultPath = + new Path( + tablePath, + PartitionPathUtils.generatePartitionPathUtil(spec, onlyValueInPath)); + Path explicitPath = canonicalizeExplicitLocation(explicitLocation); + if (sameLocation(explicitPath, defaultPath) + || sameLocation(explicitPath, tablePath) + || isAncestor(explicitPath, tablePath)) { + throw new IllegalArgumentException("Explicit partition location overlaps table data."); + } + return explicitPath; + } + + /** + * Records a resolved path. Returns false only for an identical duplicate entry for the same + * spec, which must not duplicate every row in that partition. + */ + boolean remember(LinkedHashMap spec, Path path) { + ResolvedPath resolved = ResolvedPath.of(path); + ResolvedPath previousForSpec = pathsBySpec.get(spec); + if (previousForSpec != null) { + if (previousForSpec.equals(resolved)) { + return false; + } + throw overlappingLocations(); + } + + if (overlapsOwnedPath(resolved)) { + throw overlappingLocations(); + } + pathsBySpec.put(new LinkedHashMap<>(spec), resolved); + return true; + } + + private boolean overlapsOwnedPath(ResolvedPath path) { + OwnershipNode node = + ownershipRoots.computeIfAbsent(path.fileSystem(), ignored -> new OwnershipNode()); + String[] segments = path.pathSegments(); + for (String segment : segments) { + // A terminal node reached before the candidate ends is an existing ancestor. + if (node.owned) { + return true; + } + node = node.children.computeIfAbsent(segment, ignored -> new OwnershipNode()); + } + // A terminal final node is equality. Children below it make the candidate an ancestor. + if (node.owned || !node.children.isEmpty()) { + return true; + } + node.owned = true; + return false; + } + + /** Returns the canonical URI representation used on the partition-location wire contract. */ + public static Path canonicalizeExplicitLocation(String location) { + String decoded = location; + try { + while (true) { + validateDecodedLocation(decoded); + String next = decodePercentOnce(decoded); + if (next.equals(decoded)) { + break; + } + decoded = next; + } + + Path path = new Path(decoded); + URI uri = path.toUri(); + String scheme = uri.getScheme(); + String authority = uri.getAuthority(); + String uriPath = uri.getPath(); + if (scheme == null + || scheme.isEmpty() + || uri.getUserInfo() != null + || uriPath == null + || !uriPath.startsWith(Path.SEPARATOR) + || uriPath.equals(Path.SEPARATOR)) { + throw new IllegalArgumentException("Invalid explicit partition location."); + } + + scheme = scheme.toLowerCase(Locale.ROOT); + if (!scheme.equals("file") && (authority == null || authority.isEmpty())) { + throw new IllegalArgumentException("Invalid explicit partition location."); + } + authority = + authority == null || authority.isEmpty() + ? null + : authority.toLowerCase(Locale.ROOT); + return new Path(scheme, authority, uriPath); + } catch (IllegalArgumentException e) { + throw e; + } catch (RuntimeException e) { + throw new IllegalArgumentException("Invalid explicit partition location.", e); + } + } + + private static void validateDecodedLocation(String location) { + if (location == null + || location.isEmpty() + || isBoundaryWhitespace(location) + || location.contains("?") + || location.contains("#") + || location.contains("\\")) { + throw new IllegalArgumentException("Invalid explicit partition location."); + } + + for (int offset = 0; offset < location.length(); ) { + int codePoint = location.codePointAt(offset); + if (Character.isISOControl(codePoint)) { + throw new IllegalArgumentException("Invalid explicit partition location."); + } + offset += Character.charCount(codePoint); + } + + for (String segment : location.split(Path.SEPARATOR, -1)) { + if (segment.equals(Path.CUR_DIR) || segment.equals("..")) { + throw new IllegalArgumentException("Invalid explicit partition location."); + } + } + } + + private static boolean isBoundaryWhitespace(String value) { + int first = value.codePointAt(0); + int last = value.codePointBefore(value.length()); + return isWhitespace(first) || isWhitespace(last); + } + + private static boolean isWhitespace(int codePoint) { + return Character.isWhitespace(codePoint) || Character.isSpaceChar(codePoint); + } + + private static String decodePercentOnce(String value) { + StringBuilder decoded = new StringBuilder(value.length()); + for (int offset = 0; offset < value.length(); ) { + char current = value.charAt(offset); + if (current != '%') { + decoded.append(current); + offset++; + continue; + } + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + while (offset < value.length() && value.charAt(offset) == '%') { + if (offset + 2 >= value.length()) { + throw new IllegalArgumentException("Invalid percent encoding in location."); + } + int high = Character.digit(value.charAt(offset + 1), 16); + int low = Character.digit(value.charAt(offset + 2), 16); + if (high < 0 || low < 0) { + throw new IllegalArgumentException("Invalid percent encoding in location."); + } + bytes.write((high << 4) + low); + offset += 3; + } + try { + decoded.append( + StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes.toByteArray()))); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException("Invalid percent encoding in location.", e); + } + } + return decoded.toString(); + } + + static boolean isWithin(Path candidate, Path root) { + ResolvedPath candidatePath = ResolvedPath.of(candidate); + ResolvedPath rootPath = ResolvedPath.of(root); + return rootPath.equals(candidatePath) || rootPath.isAncestorOf(candidatePath); + } + + private static boolean sameLocation(Path left, Path right) { + return ResolvedPath.of(left).equals(ResolvedPath.of(right)); + } + + private static boolean isAncestor(Path candidateAncestor, Path candidateChild) { + return ResolvedPath.of(candidateAncestor).isAncestorOf(ResolvedPath.of(candidateChild)); + } + + private IllegalStateException invalidLocation(Map spec) { + return new IllegalStateException( + String.format( + "Catalog returned an invalid explicit location for partition %s of Format Table %s.", + spec, tableName)); + } + + private IllegalStateException overlappingLocations() { + return new IllegalStateException( + String.format( + "Catalog returned overlapping locations for different partitions of Format Table %s.", + tableName)); + } + + /** + * One trie is maintained per filesystem. Visiting each path segment once is sufficient: + * ancestors are terminal nodes on the route, equality is the terminal node at the route's end, + * and descendants are children below that node. + */ + private static final class OwnershipNode { + + private final Map children = new HashMap<>(); + private boolean owned; + } + + private static final class FileSystemKey { + + private final String scheme; + private final String authority; + + private FileSystemKey(String scheme, String authority) { + this.scheme = scheme; + this.authority = authority; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + FileSystemKey that = (FileSystemKey) o; + return scheme.equals(that.scheme) && authority.equals(that.authority); + } + + @Override + public int hashCode() { + return Objects.hash(scheme, authority); + } + } + + private static final class ResolvedPath { + + private final String scheme; + private final String authority; + private final String path; + + private ResolvedPath(String scheme, String authority, String path) { + this.scheme = scheme; + this.authority = authority; + this.path = path; + } + + private static ResolvedPath of(Path path) { + URI uri = path.toUri().normalize(); + String scheme = uri.getScheme(); + // An absolute path without a scheme and file:/ name the same local filesystem. + scheme = scheme == null ? "file" : scheme.toLowerCase(Locale.ROOT); + String authority = uri.getAuthority(); + authority = authority == null ? "" : authority.toLowerCase(Locale.ROOT); + String normalizedPath = trimTrailingSeparators(uri.getPath()); + return new ResolvedPath(scheme, authority, normalizedPath); + } + + private boolean isAncestorOf(ResolvedPath other) { + if (!sameFileSystem(other) || path.equals(other.path)) { + return false; + } + if (path.equals(Path.SEPARATOR)) { + return other.path.startsWith(Path.SEPARATOR); + } + return other.path.startsWith(path + Path.SEPARATOR); + } + + private boolean sameFileSystem(ResolvedPath other) { + return scheme.equals(other.scheme) && authority.equals(other.authority); + } + + private FileSystemKey fileSystem() { + return new FileSystemKey(scheme, authority); + } + + private String[] pathSegments() { + return path.substring(1).split(Path.SEPARATOR, -1); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ResolvedPath that = (ResolvedPath) o; + return scheme.equals(that.scheme) + && authority.equals(that.authority) + && path.equals(that.path); + } + + @Override + public int hashCode() { + return Objects.hash(scheme, authority, path); + } + + private static String trimTrailingSeparators(String path) { + int end = path.length(); + while (end > 1 && path.charAt(end - 1) == Path.SEPARATOR_CHAR) { + end--; + } + return path.substring(0, end); + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTablePartitionPathResolverTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTablePartitionPathResolverTest.java new file mode 100644 index 000000000000..7f0efaf931c8 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTablePartitionPathResolverTest.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.format; + +import org.apache.paimon.fs.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.time.Duration; +import java.util.LinkedHashMap; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; + +/** Tests for the canonical and ownership contract of explicit partition locations. */ +class FormatTablePartitionPathResolverTest { + + private static final Path TABLE_PATH = new Path("file:/warehouse/table"); + private static final String TABLE_NAME = "db.table"; + + @ParameterizedTest + @ValueSource( + strings = { + "", + " ", + " file:/archive/dt=2026", + "file:/archive/dt=2026 ", + "/tmp/archive", + "relative/path", + "oss:/archive", + "oss:///archive", + "oss://user@bucket/archive", + "oss://bucket/", + "file:/", + "oss://bucket/archive?version=1", + "oss://bucket/archive#fragment", + "oss://bucket/archive\\child", + "oss://bucket/a/./b", + "oss://bucket/a/../b", + "oss://bucket/a/%2e%2e/b", + "oss://bucket/a%2f../b", + "oss://bucket/a%2F%2e%2E/b", + "oss://bucket/a/%252e%252e/b", + "oss://bucket/a/%25252e%25252e/b", + "oss://bucket/a/%2e%252e/b", + "oss://bucket/a%252f%252e%252e%252fb", + "oss://bucket/a%25252f%25252e%25252e%25252fb", + "oss://bucket/archive%5c..%5csecret", + "oss://bucket/archive%3Fversion=1", + "oss://bucket/archive%23fragment", + "oss://bucket/archive%", + "oss://bucket/archive%2", + "oss://bucket/archive%GG" + }) + void testRejectsInvalidExplicitLocation(String location) { + FormatTablePartitionPathResolver resolver = resolver(); + + assertThatThrownBy(() -> resolver.resolve(spec(), location)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("invalid explicit location") + .hasMessageContaining(TABLE_NAME); + } + + @Test + void testRejectsRawControlCharacter() { + String location = "oss://bucket/archive" + (char) 0 + "child"; + + assertThatThrownBy(() -> resolver().resolve(spec(), location)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("invalid explicit location") + .hasMessageContaining(TABLE_NAME); + } + + @Test + void testCanonicalizesSchemeAuthoritySeparatorsAndPercentEncoding() { + String location = "OSS://BUCKET//archive///%64t%3D2026%2Fmonth%3D09/"; + + Path resolved = resolver().resolve(spec(), location); + + assertThat(resolved.toString()).isEqualTo("oss://bucket/archive/dt=2026/month=09"); + } + + @Test + void testCanonicalizesFileLocationWithoutAuthority() { + Path resolved = resolver().resolve(spec(), "FILE:///archive//dt=2026/"); + + assertThat(resolved.toString()).isEqualTo("file:/archive/dt=2026"); + } + + @ParameterizedTest + @ValueSource( + strings = { + "file:/warehouse/table", + "FILE:///warehouse//table/", + "file:/warehouse/table/year=2025/month=11", + "file:/warehouse/table/year%3D2025%2Fmonth%3D11", + "file:/warehouse" + }) + void testRejectsLocationOwnedByTableOrDefaultPartition(String location) { + assertThatThrownBy(() -> resolver().resolve(spec(), location)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("invalid explicit location") + .hasMessageContaining(TABLE_NAME); + } + + @Test + void testOwnershipValidationScalesToLargePartitionRegistries() { + assertTimeoutPreemptively( + Duration.ofSeconds(3), + () -> { + FormatTablePartitionPathResolver resolver = resolver(); + for (int partition = 0; partition < 25_000; partition++) { + LinkedHashMap spec = spec("month-" + partition); + Path path = + resolver.resolve( + spec, "oss://bucket/archive/partition-" + partition); + assertThat(resolver.remember(spec, path)).isTrue(); + } + }); + } + + @Test + void testPathSegmentBoundariesDistinguishPrefixesFromAncestors() { + FormatTablePartitionPathResolver resolver = resolver(); + LinkedHashMap first = spec("first"); + LinkedHashMap second = spec("second"); + LinkedHashMap child = spec("child"); + + assertThat(resolver.remember(first, resolver.resolve(first, "oss://bucket/archive/a-b"))) + .isTrue(); + assertThat(resolver.remember(second, resolver.resolve(second, "oss://bucket/archive/a"))) + .isTrue(); + assertThatThrownBy( + () -> + resolver.remember( + child, + resolver.resolve(child, "oss://bucket/archive/a/child"))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("overlapping locations"); + + FormatTablePartitionPathResolver reverseOrder = resolver(); + LinkedHashMap descendant = spec("descendant"); + LinkedHashMap ancestor = spec("ancestor"); + assertThat( + reverseOrder.remember( + descendant, + reverseOrder.resolve( + descendant, "oss://bucket/archive/root/child"))) + .isTrue(); + assertThatThrownBy( + () -> + reverseOrder.remember( + ancestor, + reverseOrder.resolve( + ancestor, "oss://bucket/archive/root"))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("overlapping locations"); + } + + private static FormatTablePartitionPathResolver resolver() { + return new FormatTablePartitionPathResolver(TABLE_PATH, TABLE_NAME, false); + } + + private static LinkedHashMap spec() { + return spec("11"); + } + + private static LinkedHashMap spec(String month) { + LinkedHashMap spec = new LinkedHashMap<>(); + spec.put("year", "2025"); + spec.put("month", month); + return spec; + } +} From 2cc5bd8098b93988778a3663eb2b3b482f91c6c4 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 2 Sep 2026 03:18:24 +0800 Subject: [PATCH 02/15] api: add the catalog contract for explicit partition locations --- docs/static/rest-catalog-open-api.yaml | 115 +++++ .../apache/paimon/partition/Partition.java | 63 ++- .../paimon/partition/PartitionLocation.java | 87 ++++ .../java/org/apache/paimon/rest/RESTApi.java | 120 +++++- .../rest/RESTCatalogInternalOptions.java | 6 + .../org/apache/paimon/rest/ResourcePaths.java | 13 + .../requests/CreatePartitionsRequest.java | 29 +- .../responses/CreatePartitionsResponse.java | 27 +- .../rest/responses/GetTableResponse.java | 47 +++ .../paimon/partition/PartitionTest.java | 18 + .../apache/paimon/catalog/CachingCatalog.java | 27 ++ .../org/apache/paimon/catalog/Catalog.java | 38 ++ .../paimon/catalog/DelegateCatalog.java | 26 ++ .../org/apache/paimon/rest/RESTCatalog.java | 159 ++++++- .../CatalogFormatTablePartitionManager.java | 113 ++++- .../format/FormatTablePartitionManager.java | 26 ++ .../paimon/catalog/CachingCatalogTest.java | 54 +++ .../paimon/catalog/DelegateCatalogTest.java | 16 + .../paimon/rest/MockRESTCatalogTest.java | 398 +++++++++++++++++- .../apache/paimon/rest/MockRESTMessage.java | 1 + .../apache/paimon/rest/RESTApiJsonTest.java | 31 ++ .../rest/RESTCatalogPartitionSupport.java | 116 +++++ .../apache/paimon/rest/RESTCatalogServer.java | 221 ++++++---- .../apache/paimon/rest/ResourcePathsTest.java | 12 + ...atalogFormatTablePartitionManagerTest.java | 73 ++++ .../paimon/flink/FlinkRestCatalogITCase.java | 15 + .../paimon/flink/RESTCatalogITCaseBase.java | 5 +- 27 files changed, 1743 insertions(+), 113 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/partition/PartitionLocation.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPartitionSupport.java diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 8550851f5029..c46071db8541 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -352,6 +352,7 @@ paths: in: query schema: type: string + - $ref: '#/components/parameters/PaimonCapabilities' responses: "200": description: OK @@ -359,6 +360,8 @@ paths: application/json: schema: $ref: '#/components/schemas/ListTableDetailsResponse' + "400": + $ref: '#/components/responses/BadRequestErrorResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -426,6 +429,7 @@ paths: required: true schema: type: string + - $ref: '#/components/parameters/PaimonCapabilities' responses: "200": description: OK @@ -433,6 +437,8 @@ paths: application/json: schema: $ref: '#/components/schemas/GetTableResponse' + "400": + $ref: '#/components/responses/BadRequestErrorResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -461,6 +467,7 @@ paths: required: true schema: type: string + - $ref: '#/components/parameters/PaimonCapabilities' responses: "200": description: OK @@ -468,6 +475,8 @@ paths: application/json: schema: $ref: '#/components/schemas/GetTableResponse' + "400": + $ref: '#/components/responses/BadRequestErrorResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -967,6 +976,7 @@ paths: in: query schema: type: string + - $ref: '#/components/parameters/PaimonCapabilities' responses: "200": description: OK @@ -974,6 +984,8 @@ paths: application/json: schema: $ref: '#/components/schemas/ListPartitionsResponse' + "400": + $ref: '#/components/responses/BadRequestErrorResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -1001,6 +1013,7 @@ paths: required: true schema: type: string + - $ref: '#/components/parameters/PaimonCapabilities' requestBody: required: true content: @@ -1014,6 +1027,55 @@ paths: application/json: schema: $ref: '#/components/schemas/CreatePartitionsResponse' + "400": + $ref: '#/components/responses/BadRequestErrorResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + $ref: '#/components/responses/TableNotExistErrorResponse' + "409": + $ref: '#/components/responses/ResourceAlreadyExistErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/tables/{table}/partitions/with-locations: + post: + tags: + - partition + summary: Create partitions with explicit locations + description: Creates partitions carrying explicit locations on a resource that legacy servers do not implement. Clients must use this route whenever partitionLocations is non-empty and must verify the stored locations echoed by the response. + operationId: createPartitionsWithLocations + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: table + in: path + required: true + schema: + type: string + - $ref: '#/components/parameters/RequiredPaimonCapabilities' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreatePartitionsRequest' + responses: + "200": + description: Partitions created or found to exist, including the explicit locations stored by the server + content: + application/json: + schema: + $ref: '#/components/schemas/CreatePartitionsResponse' + "400": + $ref: '#/components/responses/BadRequestErrorResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -1045,6 +1107,7 @@ paths: required: true schema: type: string + - $ref: '#/components/parameters/PaimonCapabilities' requestBody: required: true content: @@ -1058,6 +1121,8 @@ paths: application/json: schema: $ref: '#/components/schemas/DropPartitionsResponse' + "400": + $ref: '#/components/responses/BadRequestErrorResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -1122,6 +1187,7 @@ paths: required: true schema: type: string + - $ref: '#/components/parameters/PaimonCapabilities' requestBody: content: application/json: @@ -1165,6 +1231,7 @@ paths: required: true schema: type: string + - $ref: '#/components/parameters/PaimonCapabilities' requestBody: required: true content: @@ -2215,6 +2282,24 @@ paths: $ref: '#/components/responses/ServerErrorResponse' components: + ############################## + # Reusable Request Parameters # + ############################## + parameters: + PaimonCapabilities: + name: X-Paimon-Capabilities + in: header + required: false + description: Comma-separated client capability tokens. Include the exact token format-table-partition-location-v1 when the client can consume Partition.location and safely operate on tables that contain explicit partition locations. + schema: + type: string + RequiredPaimonCapabilities: + name: X-Paimon-Capabilities + in: header + required: true + description: Must contain the exact comma-separated token format-table-partition-location-v1 because this request creates explicit partition locations. + schema: + type: string ############################# # Reusable Response Objects # ############################# @@ -2497,6 +2582,11 @@ components: replaceStatistics: description: Whether partitionStatistics replace the stored values rather than add to them; required whenever partitionStatistics is present, and absent otherwise. Replacing overwrites recordCount, fileSizeInBytes, fileCount and lastFileCreationTime; adding sums the three counts and keeps the later lastFileCreationTime, since two timestamps do not add. A field reported as unknown leaves the stored one alone either way, and totalBuckets is never combined. A client that reports only the files it just wrote adds; one that reports a whole partition, such as an overwrite or a directory rescan, replaces. type: [ boolean, "null" ] + partitionLocations: + description: Explicit partition locations matched to partitionSpecs by spec rather than by position. Partitions omitted from this array use their derived default locations. + type: [ array, "null" ] + items: + $ref: '#/components/schemas/PartitionLocation' CreatePartitionsResponse: type: object required: @@ -2515,6 +2605,11 @@ components: type: object additionalProperties: type: string + partitionLocations: + description: The explicit locations actually stored for the requested partitions. A client that requested explicit locations must verify this field instead of assuming an older server accepted them. + type: [ array, "null" ] + items: + $ref: '#/components/schemas/PartitionLocation' AlterTableRequest: type: object properties: @@ -2991,6 +3086,10 @@ components: format: int64 schema: $ref: '#/components/schemas/Schema' + explicitPartitionLocationCount: + description: Fresh authoritative number of registered partitions carrying explicit locations. Omitted by servers that cannot provide this signal; clients must treat an omitted value as unknown, never as zero. + type: [ integer, "null" ] + format: int64 owner: type: string createdAt: @@ -3766,6 +3865,22 @@ components: type: object additionalProperties: type: string + location: + description: Explicit absolute location of this partition. When absent, the location is derived from the table root and partition spec. + type: string + PartitionLocation: + type: object + required: + - spec + - location + properties: + spec: + type: object + additionalProperties: + type: string + location: + type: string + description: Explicit absolute location associated with spec. PartitionStatistics: type: object properties: diff --git a/paimon-api/src/main/java/org/apache/paimon/partition/Partition.java b/paimon-api/src/main/java/org/apache/paimon/partition/Partition.java index ad8d0d5cc73c..0e581099041d 100644 --- a/paimon-api/src/main/java/org/apache/paimon/partition/Partition.java +++ b/paimon-api/src/main/java/org/apache/paimon/partition/Partition.java @@ -45,6 +45,7 @@ public class Partition extends PartitionStatistics { public static final String FIELD_DONE = "done"; public static final String FIELD_OPTIONS = "options"; + public static final String FIELD_LOCATION = "location"; @JsonProperty(FIELD_DONE) private final boolean done; @@ -74,6 +75,11 @@ public class Partition extends PartitionStatistics { @Nullable private final Map options; + @JsonProperty(FIELD_LOCATION) + @JsonInclude(JsonInclude.Include.NON_NULL) + @Nullable + private final String location; + public Partition( Map spec, long recordCount, @@ -87,6 +93,36 @@ public Partition( @Nullable Long updatedAt, @Nullable String updatedBy, @Nullable Map options) { + this( + spec, + recordCount, + fileSizeInBytes, + fileCount, + lastFileCreationTime, + totalBuckets, + done, + createdAt, + createdBy, + updatedAt, + updatedBy, + options, + null); + } + + public Partition( + Map spec, + long recordCount, + long fileSizeInBytes, + long fileCount, + long lastFileCreationTime, + int totalBuckets, + boolean done, + @Nullable Long createdAt, + @Nullable String createdBy, + @Nullable Long updatedAt, + @Nullable String updatedBy, + @Nullable Map options, + @Nullable String location) { super(spec, recordCount, fileSizeInBytes, fileCount, lastFileCreationTime, totalBuckets); this.done = done; this.createdAt = createdAt; @@ -94,6 +130,7 @@ public Partition( this.updatedAt = updatedAt; this.updatedBy = updatedBy; this.options = options; + this.location = location; } public Partition( @@ -137,7 +174,8 @@ static Partition fromJson( @JsonProperty(FIELD_CREATED_BY) @Nullable String createdBy, @JsonProperty(FIELD_UPDATED_AT) @Nullable Long updatedAt, @JsonProperty(FIELD_UPDATED_BY) @Nullable String updatedBy, - @JsonProperty(FIELD_OPTIONS) @Nullable Map options) { + @JsonProperty(FIELD_OPTIONS) @Nullable Map options, + @JsonProperty(FIELD_LOCATION) @Nullable String location) { return new Partition( spec, orUnknown(recordCount), @@ -150,7 +188,8 @@ static Partition fromJson( createdBy, updatedAt, updatedBy, - options); + options, + location); } private static long orUnknown(@Nullable Long value) { @@ -192,6 +231,12 @@ public Map options() { return options; } + @Nullable + @JsonGetter(FIELD_LOCATION) + public String location() { + return location; + } + @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { @@ -206,13 +251,21 @@ public boolean equals(Object o) { && Objects.equals(createdBy, partition.createdBy) && Objects.equals(updatedAt, partition.updatedAt) && Objects.equals(updatedBy, partition.updatedBy) - && Objects.equals(options, partition.options); + && Objects.equals(options, partition.options) + && Objects.equals(location, partition.location); } @Override public int hashCode() { return Objects.hash( - super.hashCode(), done, createdAt, createdBy, updatedAt, updatedBy, options); + super.hashCode(), + done, + createdAt, + createdBy, + updatedAt, + updatedBy, + options, + location); } @Override @@ -242,6 +295,8 @@ public String toString() { + updatedBy + ", options=" + options + + ", location=" + + location + '}'; } } diff --git a/paimon-api/src/main/java/org/apache/paimon/partition/PartitionLocation.java b/paimon-api/src/main/java/org/apache/paimon/partition/PartitionLocation.java new file mode 100644 index 000000000000..c5817f2c8539 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/partition/PartitionLocation.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.partition; + +import org.apache.paimon.annotation.Public; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.util.Map; +import java.util.Objects; + +/** An explicit data location associated with a partition spec. */ +@JsonIgnoreProperties(ignoreUnknown = true) +@Public +public class PartitionLocation implements Serializable { + + private static final long serialVersionUID = 1L; + + public static final String FIELD_SPEC = "spec"; + public static final String FIELD_LOCATION = "location"; + + @JsonProperty(FIELD_SPEC) + private final Map spec; + + @JsonProperty(FIELD_LOCATION) + private final String location; + + @JsonCreator + public PartitionLocation( + @JsonProperty(FIELD_SPEC) Map spec, + @JsonProperty(FIELD_LOCATION) String location) { + this.spec = spec; + this.location = location; + } + + @JsonGetter(FIELD_SPEC) + public Map spec() { + return spec; + } + + @JsonGetter(FIELD_LOCATION) + public String location() { + return location; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PartitionLocation that = (PartitionLocation) o; + return Objects.equals(spec, that.spec) && Objects.equals(location, that.location); + } + + @Override + public int hashCode() { + return Objects.hash(spec, location); + } + + @Override + public String toString() { + return "{" + "spec=" + spec + ", location='" + location + '\'' + '}'; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 9fb56cca26d2..d637e8e0c601 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -34,6 +34,7 @@ import org.apache.paimon.management.PolicyType; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionLocation; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.rest.auth.AuthProvider; import org.apache.paimon.rest.auth.RESTAuthFunction; @@ -119,6 +120,7 @@ import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -173,6 +175,10 @@ public class RESTApi { */ public static final String READ_VIA_HEADER = "X-Paimon-Read-Via"; + public static final String CAPABILITIES_HEADER = "X-Paimon-Capabilities"; + public static final String FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY = + "format-table-partition-location-v1"; + public static final String MAX_RESULTS = "maxResults"; public static final String PAGE_TOKEN = "pageToken"; @@ -217,28 +223,81 @@ public RESTApi(Options options) { public RESTApi(Options options, boolean configRequired) { this.client = new HttpClient(options.get(RESTCatalogOptions.URI)); AuthProvider authProvider = createAuthProvider(options); - Map baseHeaders = extractPrefixMap(options, HEADER_PREFIX); + Map clientProperties = new HashMap<>(options.toMap()); + Map clientHeaders = extractPrefixMap(clientProperties, HEADER_PREFIX); + Map baseHeaders = new HashMap<>(clientHeaders); if (configRequired) { String warehouse = options.get(WAREHOUSE); Map queryParams = StringUtils.isNotEmpty(warehouse) ? ImmutableMap.of(WAREHOUSE.key(), warehouse) : ImmutableMap.of(); - options = - new Options( - client.get( - ResourcePaths.config(), - queryParams, - ConfigResponse.class, - new RESTAuthFunction(baseHeaders, authProvider)) - .merge(options.toMap())); + ConfigResponse configResponse = + client.get( + ResourcePaths.config(), + queryParams, + ConfigResponse.class, + new RESTAuthFunction(baseHeaders, authProvider)); + Map mergedProperties = + new HashMap<>(configResponse.merge(clientProperties)); + preserveServerProperty( + mergedProperties, + configResponse, + RESTCatalogInternalOptions.SERVER_CAPABILITIES.key()); + removeCapabilityHeaders(mergedProperties, HEADER_PREFIX); + String clientCapabilitiesHeader = capabilityHeader(clientHeaders); + if (clientCapabilitiesHeader != null) { + mergedProperties.put(HEADER_PREFIX + CAPABILITIES_HEADER, clientCapabilitiesHeader); + } + options = new Options(mergedProperties); baseHeaders.putAll(extractPrefixMap(options, HEADER_PREFIX)); + removeCapabilityHeaders(baseHeaders, ""); + if (clientCapabilitiesHeader != null) { + baseHeaders.put(CAPABILITIES_HEADER, clientCapabilitiesHeader); + } } this.restAuthFunction = new RESTAuthFunction(baseHeaders, authProvider); this.options = options; this.resourcePaths = ResourcePaths.forCatalogProperties(options); } + private static void preserveServerProperty( + Map mergedProperties, ConfigResponse configResponse, String key) { + String value = null; + if (configResponse.getDefaults() != null) { + value = configResponse.getDefaults().get(key); + } + if (configResponse.getOverrides() != null + && configResponse.getOverrides().containsKey(key)) { + value = configResponse.getOverrides().get(key); + } + if (value == null) { + mergedProperties.remove(key); + } else { + mergedProperties.put(key, value); + } + } + + @Nullable + private static String capabilityHeader(Map headers) { + for (Map.Entry entry : headers.entrySet()) { + if (CAPABILITIES_HEADER.equalsIgnoreCase(entry.getKey())) { + return entry.getValue(); + } + } + return null; + } + + private static void removeCapabilityHeaders(Map properties, String prefix) { + properties + .keySet() + .removeIf( + key -> + key.regionMatches(true, 0, prefix, 0, prefix.length()) + && CAPABILITIES_HEADER.equalsIgnoreCase( + key.substring(prefix.length()))); + } + /** Get the configured options which has been merged from REST Server. */ public Options options() { return options; @@ -1004,19 +1063,58 @@ public CreatePartitionsResponse createPartitions( boolean ignoreIfExists, @Nullable List statistics, boolean replaceStatistics) { + return createPartitions( + identifier, partitions, ignoreIfExists, statistics, replaceStatistics, null); + } + + /** Create partitions with optional explicit locations matched by partition spec. */ + public CreatePartitionsResponse createPartitions( + Identifier identifier, + List> partitions, + boolean ignoreIfExists, + @Nullable List statistics, + boolean replaceStatistics, + @Nullable List partitionLocations) { + boolean hasExplicitLocations = partitionLocations != null && !partitionLocations.isEmpty(); + if (hasExplicitLocations + && !containsCapability( + options.get(RESTCatalogInternalOptions.SERVER_CAPABILITIES), + FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY)) { + throw new UnsupportedOperationException( + String.format( + "REST Catalog server does not advertise required capability '%s'.", + FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY)); + } CreatePartitionsRequest request = new CreatePartitionsRequest( partitions, ignoreIfExists, statistics, - statistics == null ? null : replaceStatistics); + statistics == null ? null : replaceStatistics, + partitionLocations); return client.post( - resourcePaths.partitions(identifier.getDatabaseName(), identifier.getObjectName()), + hasExplicitLocations + ? resourcePaths.partitionsWithLocations( + identifier.getDatabaseName(), identifier.getObjectName()) + : resourcePaths.partitions( + identifier.getDatabaseName(), identifier.getObjectName()), request, CreatePartitionsResponse.class, restAuthFunction); } + static boolean containsCapability(@Nullable String capabilities, String required) { + if (capabilities == null) { + return false; + } + for (String capability : capabilities.split(",")) { + if (required.equals(capability.trim())) { + return true; + } + } + return false; + } + /** Drop (unregister) partitions for table; the server never deletes data files. */ public DropPartitionsResponse dropPartitions( Identifier identifier, diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogInternalOptions.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogInternalOptions.java index 4749ee23e6e1..cf939a3d3996 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogInternalOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogInternalOptions.java @@ -29,4 +29,10 @@ public class RESTCatalogInternalOptions { .stringType() .noDefaultValue() .withDescription("REST Catalog uri's prefix."); + + public static final ConfigOption SERVER_CAPABILITIES = + ConfigOptions.key("rest.server-capabilities") + .stringType() + .noDefaultValue() + .withDescription("Capabilities advertised by the REST Catalog server."); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 4cef311061a5..5d5a0f911506 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -245,6 +245,19 @@ public String partitions(String databaseName, String objectName) { PARTITIONS); } + /** Create partitions carrying explicit locations on a route unknown to legacy servers. */ + public String partitionsWithLocations(String databaseName, String objectName) { + return SLASH.join( + V1, + prefix, + DATABASES, + encodeString(databaseName), + TABLES, + encodeString(objectName), + PARTITIONS, + "with-locations"); + } + public String dropPartitions(String databaseName, String objectName) { return SLASH.join( V1, diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java index dadf1f9faa62..f5c566b75f86 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java @@ -18,6 +18,7 @@ package org.apache.paimon.rest.requests; +import org.apache.paimon.partition.PartitionLocation; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.rest.RESTRequest; @@ -47,6 +48,7 @@ public class CreatePartitionsRequest implements RESTRequest { private static final String FIELD_IGNORE_IF_EXISTS = "ignoreIfExists"; private static final String FIELD_PARTITION_STATISTICS = "partitionStatistics"; private static final String FIELD_REPLACE_STATISTICS = "replaceStatistics"; + private static final String FIELD_PARTITION_LOCATIONS = "partitionLocations"; @JsonProperty(FIELD_PARTITION_SPECS) private final List> partitionSpecs; @@ -64,13 +66,26 @@ public class CreatePartitionsRequest implements RESTRequest { @Nullable private final Boolean replaceStatistics; + @JsonProperty(FIELD_PARTITION_LOCATIONS) + @JsonInclude(JsonInclude.Include.NON_NULL) + @Nullable + private final List partitionLocations; + public CreatePartitionsRequest(List> partitionSpecs) { this(partitionSpecs, true); } public CreatePartitionsRequest( List> partitionSpecs, @Nullable Boolean ignoreIfExists) { - this(partitionSpecs, ignoreIfExists, null, null); + this(partitionSpecs, ignoreIfExists, null, null, null); + } + + public CreatePartitionsRequest( + List> partitionSpecs, + @Nullable Boolean ignoreIfExists, + @Nullable List partitionStatistics, + @Nullable Boolean replaceStatistics) { + this(partitionSpecs, ignoreIfExists, partitionStatistics, replaceStatistics, null); } @JsonCreator @@ -79,11 +94,14 @@ public CreatePartitionsRequest( @JsonProperty(FIELD_IGNORE_IF_EXISTS) @Nullable Boolean ignoreIfExists, @JsonProperty(FIELD_PARTITION_STATISTICS) @Nullable List partitionStatistics, - @JsonProperty(FIELD_REPLACE_STATISTICS) @Nullable Boolean replaceStatistics) { + @JsonProperty(FIELD_REPLACE_STATISTICS) @Nullable Boolean replaceStatistics, + @JsonProperty(FIELD_PARTITION_LOCATIONS) @Nullable + List partitionLocations) { this.partitionSpecs = partitionSpecs; this.ignoreIfExists = ignoreIfExists == null || ignoreIfExists; this.partitionStatistics = partitionStatistics; this.replaceStatistics = replaceStatistics; + this.partitionLocations = partitionLocations; } @JsonGetter(FIELD_PARTITION_SPECS) @@ -113,6 +131,13 @@ public Boolean replaceStatistics() { return replaceStatistics; } + /** Explicit partition locations, matched to the partition specs by spec. */ + @JsonGetter(FIELD_PARTITION_LOCATIONS) + @Nullable + public List getPartitionLocations() { + return partitionLocations; + } + /** * Registering is an upsert and replacing lands on the same value twice, so both survive being * sent again. Adding does not: a second delivery is counted again. A request that reports no diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java index 03a923ed21eb..d774d7753b28 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java @@ -18,13 +18,17 @@ package org.apache.paimon.rest.responses; +import org.apache.paimon.partition.PartitionLocation; import org.apache.paimon.rest.RESTResponse; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; +import javax.annotation.Nullable; + import java.util.List; import java.util.Map; @@ -34,6 +38,7 @@ public class CreatePartitionsResponse implements RESTResponse { private static final String FIELD_CREATED = "created"; private static final String FIELD_EXISTED = "existed"; + private static final String FIELD_PARTITION_LOCATIONS = "partitionLocations"; @JsonProperty(FIELD_CREATED) private final List> created; @@ -41,12 +46,25 @@ public class CreatePartitionsResponse implements RESTResponse { @JsonProperty(FIELD_EXISTED) private final List> existed; + @JsonProperty(FIELD_PARTITION_LOCATIONS) + @JsonInclude(JsonInclude.Include.NON_NULL) + @Nullable + private final List partitionLocations; + + public CreatePartitionsResponse( + List> created, List> existed) { + this(created, existed, null); + } + @JsonCreator public CreatePartitionsResponse( @JsonProperty(FIELD_CREATED) List> created, - @JsonProperty(FIELD_EXISTED) List> existed) { + @JsonProperty(FIELD_EXISTED) List> existed, + @JsonProperty(FIELD_PARTITION_LOCATIONS) @Nullable + List partitionLocations) { this.created = created; this.existed = existed; + this.partitionLocations = partitionLocations; } @JsonGetter(FIELD_CREATED) @@ -58,4 +76,11 @@ public List> getCreated() { public List> getExisted() { return existed; } + + /** Actual explicit locations stored for the requested partitions. */ + @JsonGetter(FIELD_PARTITION_LOCATIONS) + @Nullable + public List getPartitionLocations() { + return partitionLocations; + } } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetTableResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetTableResponse.java index 39a6dcfe9b38..4745ff366223 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetTableResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetTableResponse.java @@ -26,6 +26,8 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; +import javax.annotation.Nullable; + /** Response for getting table. */ @JsonIgnoreProperties(ignoreUnknown = true) public class GetTableResponse extends AuditRESTResponse implements RESTResponse { @@ -37,6 +39,8 @@ public class GetTableResponse extends AuditRESTResponse implements RESTResponse private static final String FIELD_IS_EXTERNAL = "isExternal"; private static final String FIELD_SCHEMA_ID = "schemaId"; private static final String FIELD_SCHEMA = "schema"; + private static final String FIELD_EXPLICIT_PARTITION_LOCATION_COUNT = + "explicitPartitionLocationCount"; @JsonProperty(FIELD_ID) private final String id; @@ -59,6 +63,39 @@ public class GetTableResponse extends AuditRESTResponse implements RESTResponse @JsonProperty(FIELD_SCHEMA) private final Schema schema; + @JsonProperty(FIELD_EXPLICIT_PARTITION_LOCATION_COUNT) + @Nullable + private final Long explicitPartitionLocationCount; + + public GetTableResponse( + String id, + String database, + String name, + String path, + boolean isExternal, + long schemaId, + Schema schema, + String owner, + long createdAt, + String createdBy, + long updatedAt, + String updatedBy) { + this( + id, + database, + name, + path, + isExternal, + schemaId, + schema, + null, + owner, + createdAt, + createdBy, + updatedAt, + updatedBy); + } + @JsonCreator public GetTableResponse( @JsonProperty(FIELD_ID) String id, @@ -68,6 +105,8 @@ public GetTableResponse( @JsonProperty(FIELD_IS_EXTERNAL) boolean isExternal, @JsonProperty(FIELD_SCHEMA_ID) long schemaId, @JsonProperty(FIELD_SCHEMA) Schema schema, + @JsonProperty(FIELD_EXPLICIT_PARTITION_LOCATION_COUNT) @Nullable + Long explicitPartitionLocationCount, @JsonProperty(FIELD_OWNER) String owner, @JsonProperty(FIELD_CREATED_AT) long createdAt, @JsonProperty(FIELD_CREATED_BY) String createdBy, @@ -81,6 +120,7 @@ public GetTableResponse( this.isExternal = isExternal; this.schemaId = schemaId; this.schema = schema; + this.explicitPartitionLocationCount = explicitPartitionLocationCount; } @JsonGetter(FIELD_ID) @@ -117,4 +157,11 @@ public long getSchemaId() { public Schema getSchema() { return this.schema; } + + /** Number of registered partitions with explicit locations, or null when not reported. */ + @JsonGetter(FIELD_EXPLICIT_PARTITION_LOCATION_COUNT) + @Nullable + public Long getExplicitPartitionLocationCount() { + return explicitPartitionLocationCount; + } } diff --git a/paimon-api/src/test/java/org/apache/paimon/partition/PartitionTest.java b/paimon-api/src/test/java/org/apache/paimon/partition/PartitionTest.java index ce514caae991..78ffea14e66e 100644 --- a/paimon-api/src/test/java/org/apache/paimon/partition/PartitionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/partition/PartitionTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test; +import java.io.ObjectStreamClass; import java.util.Collections; import java.util.Map; @@ -30,6 +31,23 @@ /** Test for {@link Partition} JSON serialization. */ class PartitionTest { + @Test + void testJavaSerializationVersionStaysCompatible() { + assertThat(ObjectStreamClass.lookup(Partition.class).getSerialVersionUID()).isEqualTo(3L); + } + + @Test + void testLocationSurvivesJsonRoundTrip() { + String json = + "{\"spec\":{\"pt\":\"1\"},\"done\":true," + + "\"location\":\"oss://archive-bucket/table/pt=1\"}"; + + Partition partition = JsonSerdeUtil.fromJson(json, Partition.class); + + assertThat(JsonSerdeUtil.toFlatJson(partition)) + .contains("\"location\":\"oss://archive-bucket/table/pt=1\""); + } + @Test void testJsonSerializationWithNullValues() { Map spec = Collections.singletonMap("pt", "1"); diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java index e2a2558dc0d7..eb8ea172b853 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java @@ -24,6 +24,7 @@ import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionLocation; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; @@ -359,6 +360,32 @@ public void createPartitions( } } + @Override + public void createPartitions( + Identifier identifier, + List> partitions, + boolean ignoreIfExists, + @Nullable List statistics, + boolean replaceStatistics, + @Nullable List partitionLocations) + throws TableNotExistException { + try { + wrapped.createPartitions( + identifier, + partitions, + ignoreIfExists, + statistics, + replaceStatistics, + partitionLocations); + } finally { + // A REST server may have committed before response confirmation fails. Never retain a + // listing that predates an explicit-location request whose remote outcome is unknown. + if (partitionCache != null) { + partitionCache.invalidate(identifier); + } + } + } + @Override public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index 9653c67d01f8..6ffad86c254f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -26,6 +26,7 @@ import org.apache.paimon.function.Function; import org.apache.paimon.function.FunctionChange; import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionLocation; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.rest.responses.GetTagResponse; @@ -46,6 +47,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; /** * This interface is responsible for reading and writing metadata such as database/table from a @@ -464,6 +466,18 @@ List listPartitionsByNames( Identifier identifier, List> partitions) throws TableNotExistException; + /** + * Return a fresh authoritative count of partitions carrying explicit locations. + * + *

An empty result means that this catalog cannot provide the signal. Callers must then use + * the full partition registry for table-wide location validation; unknown must never be + * interpreted as zero. + */ + default OptionalLong getExplicitPartitionLocationCount(Identifier identifier) + throws TableNotExistException { + return OptionalLong.empty(); + } + // ======================= view methods =============================== /** @@ -1110,6 +1124,30 @@ default void createPartitions( createPartitions(identifier, partitions); } + /** + * Create partitions with optional explicit locations. + * + *

Locations are matched to {@code partitions} by {@link PartitionLocation#spec()} instead of + * by position and may cover only some of them. Implementations that do not understand explicit + * locations fail closed instead of registering a partition at its derived default path. + */ + default void createPartitions( + Identifier identifier, + List> partitions, + boolean ignoreIfExists, + @Nullable List statistics, + boolean replaceStatistics, + @Nullable List partitionLocations) + throws TableNotExistException { + if (partitionLocations != null && !partitionLocations.isEmpty()) { + throw new UnsupportedOperationException( + String.format( + "Catalog %s does not support explicit partition locations.", + getClass().getName())); + } + createPartitions(identifier, partitions, ignoreIfExists, statistics, replaceStatistics); + } + /** * Drop partitions of the specify table. Ignore non-existent partitions. * diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index 6c691e6cee28..ecb7f01f91a8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -23,6 +23,7 @@ import org.apache.paimon.function.Function; import org.apache.paimon.function.FunctionChange; import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionLocation; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.rest.responses.GetTagResponse; @@ -40,6 +41,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; /** A {@link Catalog} to delegate all operations to another {@link Catalog}. */ public abstract class DelegateCatalog implements Catalog { @@ -337,6 +339,24 @@ public void createPartitions( identifier, partitions, ignoreIfExists, statistics, replaceStatistics); } + @Override + public void createPartitions( + Identifier identifier, + List> partitions, + boolean ignoreIfExists, + @Nullable List statistics, + boolean replaceStatistics, + @Nullable List partitionLocations) + throws TableNotExistException { + wrapped.createPartitions( + identifier, + partitions, + ignoreIfExists, + statistics, + replaceStatistics, + partitionLocations); + } + @Override public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { @@ -482,6 +502,12 @@ public List listPartitionsByNames( return wrapped.listPartitionsByNames(identifier, partitions); } + @Override + public OptionalLong getExplicitPartitionLocationCount(Identifier identifier) + throws TableNotExistException { + return wrapped.getExplicitPartitionLocationCount(identifier); + } + @Override public TableQueryAuthResult authTableQuery(Identifier identifier, @Nullable List select) throws TableNotExistException { diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 50fe373d6a03..73fc19fff275 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -44,6 +44,7 @@ import org.apache.paimon.management.PolicyManagement; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionLocation; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.rest.exceptions.AlreadyExistsException; @@ -53,6 +54,7 @@ import org.apache.paimon.rest.exceptions.NotImplementedException; import org.apache.paimon.rest.exceptions.ServiceFailureException; import org.apache.paimon.rest.responses.AuthTableQueryResponse; +import org.apache.paimon.rest.responses.CreatePartitionsResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetFunctionResponse; @@ -67,6 +69,7 @@ import org.apache.paimon.table.Instant; import org.apache.paimon.table.Table; import org.apache.paimon.table.TableSnapshot; +import org.apache.paimon.table.format.FormatTablePartitionPathResolver; import org.apache.paimon.table.sink.BatchTableCommit; import org.apache.paimon.table.system.SystemTableLoader; import org.apache.paimon.utils.JsonSerdeUtil; @@ -88,6 +91,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import java.util.stream.Collectors; @@ -550,6 +554,7 @@ private TableMetadata loadTableMetadata(Identifier identifier) throws TableNotEx } private TableMetadata toTableMetadata(String db, GetTableResponse response) { + validateExplicitPartitionLocationCount(response); TableSchema schema = TableSchema.create(response.getSchemaId(), response.getSchema()); Map options = new HashMap<>(schema.options()); options.put(PATH.key(), response.getPath()); @@ -771,10 +776,41 @@ public void createPartitions( @Nullable List statistics, boolean replaceStatistics) throws TableNotExistException { + createPartitions( + identifier, partitions, ignoreIfExists, statistics, replaceStatistics, null); + } + + @Override + public void createPartitions( + Identifier identifier, + List> partitions, + boolean ignoreIfExists, + @Nullable List statistics, + boolean replaceStatistics, + @Nullable List partitionLocations) + throws TableNotExistException { + List canonicalLocations = + canonicalizePartitionLocations(identifier, partitionLocations); try { - api.createPartitions( - identifier, partitions, ignoreIfExists, statistics, replaceStatistics); + CreatePartitionsResponse response = + api.createPartitions( + identifier, + partitions, + ignoreIfExists, + statistics, + replaceStatistics, + canonicalLocations); + validatePartitionLocations(identifier, canonicalLocations, response); } catch (NoSuchResourceException e) { + if (canonicalLocations != null + && !canonicalLocations.isEmpty() + && !StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_TABLE)) { + throw new UnsupportedOperationException( + String.format( + "REST Catalog server advertised capability '%s' but does not provide the location-aware partition create endpoint.", + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY), + e); + } throw new TableNotExistException(identifier); } catch (ForbiddenException e) { throw new TableNoPermissionException(identifier, e); @@ -789,6 +825,90 @@ public void createPartitions( } } + @Nullable + private static List canonicalizePartitionLocations( + Identifier identifier, @Nullable List requested) { + if (requested == null || requested.isEmpty()) { + return requested; + } + List canonical = new ArrayList<>(requested.size()); + for (PartitionLocation location : requested) { + if (location == null || location.spec() == null || location.location() == null) { + throw invalidPartitionLocation(identifier, location, null); + } + try { + canonical.add( + new PartitionLocation( + location.spec(), + FormatTablePartitionPathResolver.canonicalizeExplicitLocation( + location.location()) + .toString())); + } catch (IllegalArgumentException e) { + throw invalidPartitionLocation(identifier, location, e); + } + } + return canonical; + } + + private static IllegalArgumentException invalidPartitionLocation( + Identifier identifier, + @Nullable PartitionLocation location, + @Nullable IllegalArgumentException cause) { + String message = + String.format( + "Invalid explicit partition location for partition %s of table %s.", + location == null ? null : location.spec(), identifier.getFullName()); + return cause == null + ? new IllegalArgumentException(message) + : new IllegalArgumentException(message, cause); + } + + private static void validatePartitionLocations( + Identifier identifier, + @Nullable List requested, + CreatePartitionsResponse response) { + if (requested == null || requested.isEmpty()) { + return; + } + List stored = response.getPartitionLocations(); + if (stored == null) { + throw new UnsupportedOperationException( + String.format( + "Catalog server did not confirm explicit partition locations for table %s.", + identifier.getFullName())); + } + Map, String> requestedBySpec = + indexPartitionLocations(identifier, "request", requested); + Map, String> storedBySpec = + indexPartitionLocations(identifier, "response", stored); + if (!requestedBySpec.equals(storedBySpec)) { + throw new IllegalStateException( + String.format( + "Catalog server returned partition locations that differ from the request for table %s.", + identifier.getFullName())); + } + } + + private static Map, String> indexPartitionLocations( + Identifier identifier, String source, List locations) { + Map, String> indexed = new HashMap<>(); + for (PartitionLocation location : locations) { + if (location == null || location.spec() == null || location.location() == null) { + throw new IllegalStateException( + String.format( + "Catalog partition location %s contains a null value for table %s.", + source, identifier.getFullName())); + } + if (indexed.put(location.spec(), location.location()) != null) { + throw new IllegalStateException( + String.format( + "Catalog partition location %s repeats partition %s for table %s.", + source, location.spec(), identifier.getFullName())); + } + } + return indexed; + } + @Override public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { @@ -832,6 +952,41 @@ public List listPartitions(Identifier identifier) throws TableNotExis } } + @Override + public OptionalLong getExplicitPartitionLocationCount(Identifier identifier) + throws TableNotExistException { + try { + GetTableResponse response = api.getTable(identifier); + validateExplicitPartitionLocationCount(response); + Long count = response.getExplicitPartitionLocationCount(); + return count == null || count < 0 ? OptionalLong.empty() : OptionalLong.of(count); + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } catch (ForbiddenException e) { + throw new TableNoPermissionException(identifier, e); + } + } + + private void validateExplicitPartitionLocationCount(GetTableResponse response) { + Long count = response.getExplicitPartitionLocationCount(); + if (RESTApi.containsCapability( + api.options().get(RESTCatalogInternalOptions.SERVER_CAPABILITIES), + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY) + && (count == null || count < 0)) { + String violation = + count == null + ? "omitted explicitPartitionLocationCount" + : "did not provide a non-negative explicitPartitionLocationCount"; + throw new IllegalStateException( + String.format( + "REST Catalog server advertised capability '%s' but get-table response for %s.%s %s.", + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY, + response.getDatabase(), + response.getName(), + violation)); + } + } + @Override public PagedList listPartitionsPaged( Identifier identifier, diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManager.java b/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManager.java index 1f4d4575a4ff..614d86453199 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManager.java @@ -23,6 +23,7 @@ import org.apache.paimon.catalog.CatalogLoader; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionLocation; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.utils.FunctionWithException; @@ -37,6 +38,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.OptionalLong; import java.util.Set; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -154,15 +156,34 @@ public List listPartitionsByNames(List> partition "list partitions by names"); } + @Override + public OptionalLong explicitPartitionLocationCount() { + return execute( + catalog -> catalog.getExplicitPartitionLocationCount(identifier), + "get explicit partition location count"); + } + @Override public void createPartitions( List> partitions, boolean ignoreIfExists, @Nullable List statistics, boolean replaceStatistics) { + createPartitions(partitions, ignoreIfExists, statistics, replaceStatistics, null); + } + + @Override + public void createPartitions( + List> partitions, + boolean ignoreIfExists, + @Nullable List statistics, + boolean replaceStatistics, + @Nullable List partitionLocations) { // Validated before the empty check: returning early would swallow a malformed report. Map, PartitionStatistics> statisticsBySpec = validateAndIndexStatistics(statistics, partitions); + Map, PartitionLocation> locationsBySpec = + validateAndIndexLocations(partitionLocations, partitions); if (partitions.isEmpty()) { return; } @@ -171,20 +192,40 @@ public void createPartitions( if (!ignoreIfExists) { // Rejecting the whole batch when any partition exists is only meaningful // if the batch stays one request, so a strict create is never split. - catalog.createPartitions( - identifier, partitions, false, statistics, replaceStatistics); + if (locationsBySpec == null || locationsBySpec.isEmpty()) { + catalog.createPartitions( + identifier, partitions, false, statistics, replaceStatistics); + } else { + catalog.createPartitions( + identifier, + partitions, + false, + statistics, + replaceStatistics, + locationsOf(partitions, locationsBySpec)); + } return null; } // isRetrySafe() bounds the transport retry only: a caller-level rerun of a // multi-batch ADD still double counts the batches that already landed. for (List> batch : batches(partitions)) { // A partition and its statistics travel in the same request. - catalog.createPartitions( - identifier, - batch, - true, - statisticsOf(batch, statisticsBySpec), - replaceStatistics); + if (locationsBySpec == null || locationsBySpec.isEmpty()) { + catalog.createPartitions( + identifier, + batch, + true, + statisticsOf(batch, statisticsBySpec), + replaceStatistics); + } else { + catalog.createPartitions( + identifier, + batch, + true, + statisticsOf(batch, statisticsBySpec), + replaceStatistics, + locationsOf(batch, locationsBySpec)); + } } return null; }, @@ -237,6 +278,49 @@ private Map, PartitionStatistics> validateAndIndexStatistics return bySpec; } + @Nullable + private Map, PartitionLocation> validateAndIndexLocations( + @Nullable List locations, List> partitions) { + if (locations == null) { + return null; + } + if (locations.isEmpty()) { + return Collections.emptyMap(); + } + String tableName = identifier.getFullName(); + Set> registered = capacityFor(partitions.size()); + for (Map spec : partitions) { + checkArgument( + registered.add(spec), + "Partition %s of table %s is registered twice in one request with an " + + "explicit location; register each partition once.", + spec, + tableName); + } + Map, PartitionLocation> bySpec = + new HashMap<>(hashCapacity(locations.size())); + for (PartitionLocation location : locations) { + checkArgument( + registered.contains(location.spec()), + "Location was provided for partition %s of table %s, which this request " + + "does not register.", + location.spec(), + tableName); + checkArgument( + !StringUtils.isBlank(location.location()), + "Location for partition %s of table %s is blank.", + location.spec(), + tableName); + checkArgument( + bySpec.put(location.spec(), location) == null, + "Location was provided twice for partition %s of table %s; provide one " + + "location per partition.", + location.spec(), + tableName); + } + return bySpec; + } + @Nullable private static List statisticsOf( List> batch, @@ -254,6 +338,19 @@ private static List statisticsOf( return ofBatch; } + private static List locationsOf( + List> batch, + Map, PartitionLocation> locationsBySpec) { + List ofBatch = new ArrayList<>(batch.size()); + for (Map spec : batch) { + PartitionLocation location = locationsBySpec.get(spec); + if (location != null) { + ofBatch.add(location); + } + } + return ofBatch; + } + @Override public void dropPartitions(List> partitions) { if (partitions.isEmpty()) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionManager.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionManager.java index aaacebbe7159..2712f53ca69f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionManager.java @@ -22,6 +22,7 @@ import org.apache.paimon.catalog.CatalogLoader; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionLocation; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.predicate.Predicate; @@ -30,6 +31,7 @@ import java.io.Serializable; import java.util.List; import java.util.Map; +import java.util.OptionalLong; /** * The catalog partition registrations of a single Format Table with catalog-managed partitions. @@ -61,6 +63,11 @@ public interface FormatTablePartitionManager extends Serializable { /** Return those of the given complete partition specs that are registered. */ List listPartitionsByNames(List> partitions); + /** Return a fresh authoritative explicit-location count, or empty when unavailable. */ + default OptionalLong explicitPartitionLocationCount() { + return OptionalLong.empty(); + } + /** * Register partitions, reporting no statistics for them. With {@code ignoreIfExists=false} the * whole batch is rejected when any partition already exists, so such a request is never split. @@ -90,6 +97,25 @@ void createPartitions( @Nullable List statistics, boolean replaceStatistics); + /** + * Register partitions with optional explicit locations, matched by partition spec. + * Implementations that do not support locations fail closed. + */ + default void createPartitions( + List> partitions, + boolean ignoreIfExists, + @Nullable List statistics, + boolean replaceStatistics, + @Nullable List partitionLocations) { + if (partitionLocations != null && !partitionLocations.isEmpty()) { + throw new UnsupportedOperationException( + String.format( + "%s does not support explicit partition locations.", + getClass().getName())); + } + createPartitions(partitions, ignoreIfExists, statistics, replaceStatistics); + } + /** Unregister partitions. Metadata only; missing partitions are ignored. */ void dropPartitions(List> partitions); diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java index 37b991fc6846..56a1c9b0d703 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java @@ -25,6 +25,7 @@ import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionLocation; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; @@ -79,6 +80,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; class CachingCatalogTest extends CatalogTestBase { @@ -392,6 +394,58 @@ public void testCreatePartitionsWithStatisticsForwardsAndInvalidatesPartitionCac assertThat(catalog.listPartitions(identifier)).containsExactly(created); } + @Test + public void testCreatePartitionsWithLocationForwardsAndInvalidatesPartitionCache() + throws Exception { + Catalog wrapped = Mockito.mock(Catalog.class); + TestableCachingCatalog catalog = + new TestableCachingCatalog(wrapped, EXPIRATION_TTL, ticker); + Identifier identifier = new Identifier("db", "tbl"); + Map spec = singletonMap("dt", "20260717"); + List locations = + singletonList(new PartitionLocation(spec, "file:/archive/dt=20260717")); + Partition created = new Partition(spec, 0, 0, 0, 0, -1, false); + when(wrapped.listPartitions(identifier)).thenReturn(emptyList(), singletonList(created)); + + assertThat(catalog.listPartitions(identifier)).isEmpty(); + catalog.createPartitions(identifier, singletonList(spec), true, null, false, locations); + + Mockito.verify(wrapped) + .createPartitions(identifier, singletonList(spec), true, null, false, locations); + assertThat(catalog.listPartitions(identifier)).containsExactly(created); + } + + @Test + public void testCreatePartitionsWithLocationInvalidatesCacheAfterConfirmationFailure() + throws Exception { + Catalog wrapped = Mockito.mock(Catalog.class); + TestableCachingCatalog catalog = + new TestableCachingCatalog(wrapped, EXPIRATION_TTL, ticker); + Identifier identifier = new Identifier("db", "tbl"); + Map spec = singletonMap("dt", "20260717"); + List locations = + singletonList(new PartitionLocation(spec, "file:/archive/dt=20260717")); + Partition created = new Partition(spec, 0, 0, 0, 0, -1, false); + when(wrapped.listPartitions(identifier)).thenReturn(emptyList(), singletonList(created)); + doThrow(new IllegalStateException("location confirmation failed")) + .when(wrapped) + .createPartitions(identifier, singletonList(spec), true, null, false, locations); + + assertThat(catalog.listPartitions(identifier)).isEmpty(); + assertThatThrownBy( + () -> + catalog.createPartitions( + identifier, + singletonList(spec), + true, + null, + false, + locations)) + .hasMessage("location confirmation failed"); + + assertThat(catalog.listPartitions(identifier)).containsExactly(created); + } + @Test public void testDeadlock() throws Exception { Catalog underlyCatalog = this.catalog; diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/DelegateCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/DelegateCatalogTest.java index ec230c8504bb..b3affb7c6952 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/DelegateCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/DelegateCatalogTest.java @@ -18,6 +18,7 @@ package org.apache.paimon.catalog; +import org.apache.paimon.partition.PartitionLocation; import org.apache.paimon.partition.PartitionStatistics; import org.junit.jupiter.api.Test; @@ -72,6 +73,21 @@ void testCreatePartitionsCarriesTheAbsenceOfStatisticsThrough() throws Exception verify(wrapped).createPartitions(IDENTIFIER, specs, false, null, false); } + @Test + void testCreatePartitionsCarriesLocationsToTheWrappedCatalog() throws Exception { + Catalog wrapped = mock(Catalog.class); + Catalog delegating = new TestDelegateCatalog(wrapped); + Map spec = Collections.singletonMap("dt", "20260728"); + List> specs = Collections.singletonList(spec); + List locations = + Collections.singletonList(new PartitionLocation(spec, "file:/archive/dt=20260728")); + + delegating.createPartitions(IDENTIFIER, specs, true, null, false, locations); + + verify(wrapped).createPartitions(IDENTIFIER, specs, true, null, false, locations); + verify(wrapped, never()).createPartitions(IDENTIFIER, specs, true, null, false); + } + /** {@link DelegateCatalog} forwards every operation; these tests never rebuild one. */ private static class TestDelegateCatalog extends DelegateCatalog { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index 7c5503534581..044f53cdd39f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -36,6 +36,7 @@ import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionLocation; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.predicate.FieldRef; import org.apache.paimon.predicate.Predicate; @@ -83,7 +84,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.OptionalLong; import java.util.UUID; +import java.util.function.UnaryOperator; import static org.apache.paimon.catalog.Catalog.SYSTEM_DATABASE_NAME; import static org.apache.paimon.catalog.Catalog.TABLE_DEFAULT_OPTION_PREFIX; @@ -486,6 +489,331 @@ void testCatalogManagedPartitionListingReflectsCatalogMutationsImmediately() thr assertThat(partitionManager.listPartitions(Collections.emptyMap(), null)).isEmpty(); } + @Test + void testExplicitPartitionLocationIsStoredAndEchoed() throws Exception { + Identifier identifier = createFormatTableWithCatalogManagedPartitions(); + Map spec = Collections.singletonMap("dt", "20260717"); + String location = "file:/archive/dt=20260717"; + ResourcePaths paths = ResourcePaths.forCatalogProperties(restCatalog.api().options()); + String ordinaryResource = + paths.partitions(identifier.getDatabaseName(), identifier.getObjectName()); + String locationResource = + paths.partitionsWithLocations( + identifier.getDatabaseName(), identifier.getObjectName()); + restCatalogServer.clearReceivedHeaders(); + + restCatalog.createPartitions( + identifier, + Collections.singletonList(spec), + true, + null, + false, + Collections.singletonList(new PartitionLocation(spec, location))); + + assertThat(restCatalogServer.getReceivedHeaders(locationResource)).hasSize(1); + assertThat(restCatalogServer.getReceivedHeaders(ordinaryResource)).isEmpty(); + assertThat(onlyPartition(identifier).location()).isEqualTo(location); + } + + @Test + void testExplicitPartitionLocationIsCanonicalizedBeforePost() throws Exception { + Identifier identifier = createFormatTableWithCatalogManagedPartitions(); + Map spec = Collections.singletonMap("dt", "20260717"); + String requested = "OSS://ARCHIVE-BUCKET//history///%64t%3D20260717/"; + String canonical = "oss://archive-bucket/history/dt=20260717"; + ResourcePaths paths = ResourcePaths.forCatalogProperties(restCatalog.api().options()); + String ordinaryResource = + paths.partitions(identifier.getDatabaseName(), identifier.getObjectName()); + String locationResource = + paths.partitionsWithLocations( + identifier.getDatabaseName(), identifier.getObjectName()); + restCatalogServer.clearReceivedHeaders(); + + restCatalog.createPartitions( + identifier, + Collections.singletonList(spec), + true, + null, + false, + Collections.singletonList(new PartitionLocation(spec, requested))); + + assertThat(restCatalogServer.getReceivedHeaders(locationResource)).hasSize(1); + assertThat(restCatalogServer.getReceivedHeaders(ordinaryResource)).isEmpty(); + assertThat(onlyPartition(identifier).location()).isEqualTo(canonical); + } + + @Test + void testInvalidExplicitPartitionLocationFailsBeforePost() throws Exception { + Identifier identifier = createFormatTableWithCatalogManagedPartitions(); + Map spec = Collections.singletonMap("dt", "20260717"); + ResourcePaths paths = ResourcePaths.forCatalogProperties(restCatalog.api().options()); + String ordinaryResource = + paths.partitions(identifier.getDatabaseName(), identifier.getObjectName()); + String locationResource = + paths.partitionsWithLocations( + identifier.getDatabaseName(), identifier.getObjectName()); + restCatalogServer.clearReceivedHeaders(); + + assertThatThrownBy( + () -> + restCatalog.createPartitions( + identifier, + Collections.singletonList(spec), + true, + null, + false, + Collections.singletonList( + new PartitionLocation( + spec, + "oss://archive-bucket/history/%2e%2e/secret")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid explicit partition location"); + + assertThat(restCatalogServer.getReceivedHeaders(locationResource)).isEmpty(); + assertThat(restCatalogServer.getReceivedHeaders(ordinaryResource)).isEmpty(); + } + + @Test + void testPartitionManagerReadsFreshExplicitLocationCount() throws Exception { + Identifier identifier = createFormatTableWithCatalogManagedPartitions(); + FormatTable table = (FormatTable) restCatalog.getTable(identifier); + FormatTablePartitionManager partitionManager = table.partitionManager(); + Map spec = Collections.singletonMap("dt", "20260717"); + + assertThat(partitionManager.explicitPartitionLocationCount()) + .isEqualTo(OptionalLong.of(0L)); + + restCatalog.createPartitions( + identifier, + Collections.singletonList(spec), + true, + null, + false, + Collections.singletonList( + new PartitionLocation(spec, "file:/archive/dt=20260717"))); + assertThat(partitionManager.explicitPartitionLocationCount()) + .isEqualTo(OptionalLong.of(1L)); + + restCatalog.dropPartitions(identifier, Collections.singletonList(spec)); + assertThat(partitionManager.explicitPartitionLocationCount()) + .isEqualTo(OptionalLong.of(0L)); + } + + @Test + void testCapableServerGetTableRequiresExplicitLocationCount() throws Exception { + Identifier identifier = createFormatTableWithCatalogManagedPartitions(); + restCatalogServer.setExplicitPartitionLocationCountReported(false); + + assertThatThrownBy(() -> restCatalog.getTable(identifier)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining(RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY) + .hasMessageContaining("explicitPartitionLocationCount"); + } + + @Test + void testCapableServerGetTableRequiresNonNegativeExplicitLocationCount() throws Exception { + Identifier identifier = createFormatTableWithCatalogManagedPartitions(); + restCatalogServer.setExplicitPartitionLocationCount(-1L); + + assertThatThrownBy(() -> restCatalog.getTable(identifier)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining(RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY) + .hasMessageContaining("non-negative explicitPartitionLocationCount"); + } + + @Test + void testLegacyServerMayOmitExplicitPartitionLocationCount() throws Exception { + config.getDefaults().remove(RESTCatalogInternalOptions.SERVER_CAPABILITIES.key()); + restCatalogServer.setExplicitPartitionLocationCountReported(false); + Options legacyOptions = new Options(options.toMap()); + legacyOptions.remove(RESTCatalogInternalOptions.SERVER_CAPABILITIES.key()); + RESTCatalog legacyCatalog = new RESTCatalog(CatalogContext.create(legacyOptions)); + Identifier identifier = createFormatTableWithCatalogManagedPartitions(legacyCatalog); + + assertThat(legacyCatalog.getTable(identifier)).isInstanceOf(FormatTable.class); + assertThat(legacyCatalog.getExplicitPartitionLocationCount(identifier)) + .isEqualTo(OptionalLong.empty()); + } + + @Test + void testUnsupportedLocationCreateRouteFailsWithoutLegacyMutation() throws Exception { + Identifier identifier = createFormatTableWithCatalogManagedPartitions(); + Map spec = Collections.singletonMap("dt", "20260717"); + ResourcePaths paths = ResourcePaths.forCatalogProperties(restCatalog.api().options()); + String ordinaryResource = + paths.partitions(identifier.getDatabaseName(), identifier.getObjectName()); + String locationResource = + paths.partitionsWithLocations( + identifier.getDatabaseName(), identifier.getObjectName()); + restCatalogServer.setPartitionLocationCreateSupported(false); + restCatalogServer.clearReceivedHeaders(); + + assertThatThrownBy( + () -> + restCatalog.createPartitions( + identifier, + Collections.singletonList(spec), + true, + null, + false, + Collections.singletonList( + new PartitionLocation( + spec, "file:/archive/dt=20260717")))) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("location-aware partition create endpoint"); + + assertThat(restCatalogServer.getReceivedHeaders(locationResource)).hasSize(1); + assertThat(restCatalogServer.getReceivedHeaders(ordinaryResource)).isEmpty(); + assertThat(restCatalog.listPartitions(identifier)).isEmpty(); + } + + @Test + void testOrdinaryPartitionCreateKeepsLegacyResource() throws Exception { + Identifier identifier = createFormatTableWithCatalogManagedPartitions(); + Map spec = Collections.singletonMap("dt", "20260717"); + ResourcePaths paths = ResourcePaths.forCatalogProperties(restCatalog.api().options()); + String ordinaryResource = + paths.partitions(identifier.getDatabaseName(), identifier.getObjectName()); + String locationResource = + paths.partitionsWithLocations( + identifier.getDatabaseName(), identifier.getObjectName()); + restCatalogServer.clearReceivedHeaders(); + + restCatalog.createPartitions(identifier, Collections.singletonList(spec)); + + assertThat(restCatalogServer.getReceivedHeaders(ordinaryResource)).hasSize(1); + assertThat(restCatalogServer.getReceivedHeaders(locationResource)).isEmpty(); + } + + @Test + void testExplicitPartitionLocationRequiresServerCapabilityBeforePost() throws Exception { + config.getDefaults() + .put( + RESTCatalogInternalOptions.SERVER_CAPABILITIES.key(), + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY + "0"); + Options unsupportedOptions = new Options(options.toMap()); + unsupportedOptions.set( + RESTCatalogInternalOptions.SERVER_CAPABILITIES, + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY); + RESTCatalog unsupportedCatalog = new RESTCatalog(CatalogContext.create(unsupportedOptions)); + Identifier identifier = createFormatTableWithCatalogManagedPartitions(unsupportedCatalog); + Map spec = Collections.singletonMap("dt", "20260717"); + String partitionsResource = + ResourcePaths.forCatalogProperties(unsupportedCatalog.api().options()) + .partitions(identifier.getDatabaseName(), identifier.getObjectName()); + restCatalogServer.clearReceivedHeaders(); + + assertThatThrownBy( + () -> + unsupportedCatalog.createPartitions( + identifier, + Collections.singletonList(spec), + true, + null, + false, + Collections.singletonList( + new PartitionLocation( + spec, "file:/archive/dt=20260717")))) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("format-table-partition-location-v1"); + assertThat(restCatalogServer.getReceivedHeaders(partitionsResource)).isEmpty(); + assertThat(unsupportedCatalog.listPartitions(identifier)).isEmpty(); + } + + @Test + void testExplicitPartitionLocationRejectsNullEchoEntry() throws Exception { + assertRejectedPartitionLocationEcho( + ignored -> Collections.singletonList(null), + IllegalStateException.class, + "contains a null value"); + } + + @Test + void testExplicitPartitionLocationRejectsMissingEchoEntry() throws Exception { + assertRejectedPartitionLocationEcho( + ignored -> Collections.emptyList(), + IllegalStateException.class, + "differ from the request"); + } + + @Test + void testExplicitPartitionLocationRejectsDuplicateEchoEntry() throws Exception { + assertRejectedPartitionLocationEcho( + echoed -> Arrays.asList(echoed.get(0), echoed.get(0)), + IllegalStateException.class, + "repeats partition"); + } + + @Test + void testExplicitPartitionLocationRejectsSpecLocationMismatch() throws Exception { + Identifier identifier = createFormatTableWithCatalogManagedPartitions(); + Map first = Collections.singletonMap("dt", "20260717"); + Map second = Collections.singletonMap("dt", "20260718"); + String firstLocation = "file:/archive/dt=20260717"; + String secondLocation = "file:/archive/dt=20260718"; + restCatalogServer.setPartitionLocationEcho( + echoed -> + Arrays.asList( + new PartitionLocation(first, secondLocation), + new PartitionLocation(second, firstLocation))); + + assertThatThrownBy( + () -> + restCatalog.createPartitions( + identifier, + Arrays.asList(first, second), + true, + null, + false, + Arrays.asList( + new PartitionLocation(first, firstLocation), + new PartitionLocation(second, secondLocation)))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("differ from the request"); + } + + @Test + void testExplicitPartitionLocationRejectsNullLocationInEcho() throws Exception { + assertRejectedPartitionLocationEcho( + echoed -> + Collections.singletonList( + new PartitionLocation(echoed.get(0).spec(), null)), + IllegalStateException.class, + "contains a null value"); + } + + @Test + void testExplicitPartitionLocationFailsClosedWhenOldServerOmitsEcho() throws Exception { + assertRejectedPartitionLocationEcho( + ignored -> null, + UnsupportedOperationException.class, + "did not confirm explicit partition locations"); + } + + private void assertRejectedPartitionLocationEcho( + UnaryOperator> echo, + Class exceptionType, + String message) + throws Exception { + Identifier identifier = createFormatTableWithCatalogManagedPartitions(); + Map spec = Collections.singletonMap("dt", "20260717"); + String location = "file:/archive/dt=20260717"; + restCatalogServer.setPartitionLocationEcho(echo); + + assertThatThrownBy( + () -> + restCatalog.createPartitions( + identifier, + Collections.singletonList(spec), + true, + null, + false, + Collections.singletonList( + new PartitionLocation(spec, location)))) + .isInstanceOf(exceptionType) + .hasMessageContaining(message); + } + @Test void testPartitionManagerSurvivesSerialization() throws Exception { Identifier identifier = createFormatTableWithCatalogManagedPartitions(); @@ -741,9 +1069,14 @@ void testRoundTrippedFormatTableReplacePassesClientValidation() throws Exception } private Identifier createFormatTableWithCatalogManagedPartitions() throws Exception { + return createFormatTableWithCatalogManagedPartitions(restCatalog); + } + + private Identifier createFormatTableWithCatalogManagedPartitions(RESTCatalog catalog) + throws Exception { Identifier identifier = Identifier.create("db1", "managed_partition_table"); - restCatalog.createDatabase(identifier.getDatabaseName(), true); - restCatalog.createTable( + catalog.createDatabase(identifier.getDatabaseName(), true); + catalog.createTable( identifier, Schema.newBuilder() .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString()) @@ -790,6 +1123,62 @@ void testBaseHeadersInRequests() throws Exception { checkHeader(customHeaderName, customHeaderValue); } + @Test + void testServerConfigCannotMakeGenericRestCatalogClaimClientCapability() throws Exception { + config.getDefaults() + .put( + "header.x-paimon-capabilities", + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY); + config.getOverrides() + .put( + "header.X-PaImOn-CaPaBiLiTiEs", + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY); + + RESTCatalog genericCatalog = + new RESTCatalog(CatalogContext.create(new Options(options.toMap()))); + assertThat( + genericCatalog.options().keySet().stream() + .noneMatch( + key -> + key.equalsIgnoreCase( + HEADER_PREFIX + + RESTApi.CAPABILITIES_HEADER))) + .isTrue(); + restCatalogServer.clearReceivedHeaders(); + genericCatalog.listDatabases(); + + assertThat(restCatalogServer.getReceivedHeaders()) + .allSatisfy( + headers -> + assertThat(headers) + .doesNotContainKey( + RESTApi.CAPABILITIES_HEADER.toLowerCase())); + } + + @Test + void testServerConfigCannotOverrideClientCapabilityDeclaration() throws Exception { + String clientCapabilities = + "client-v0," + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY; + String serverCapabilities = "server-v0"; + Options clientOptions = new Options(options.toMap()); + clientOptions.set("header.x-paimon-capabilities", clientCapabilities); + config.getOverrides().put("header.X-PaImOn-CaPaBiLiTiEs", serverCapabilities); + + RESTCatalog clientCatalog = new RESTCatalog(CatalogContext.create(clientOptions)); + assertThat(clientCatalog.options()) + .containsEntry(HEADER_PREFIX + RESTApi.CAPABILITIES_HEADER, clientCapabilities); + restCatalogServer.clearReceivedHeaders(); + clientCatalog.listDatabases(); + + assertThat(restCatalogServer.getReceivedHeaders()) + .allSatisfy( + headers -> + assertThat(headers) + .containsEntry( + RESTApi.CAPABILITIES_HEADER.toLowerCase(), + clientCapabilities)); + } + @Test void testReadViaHeaderOnDependencyTableAndDataTokenRequests() throws Exception { Identifier root = Identifier.create("db", "root"); @@ -1028,11 +1417,14 @@ private RESTCatalog initCatalogUtil( enableDataToken + "", CatalogOptions.WAREHOUSE.key(), restWarehouse)); + defaultConf.put( + RESTCatalogInternalOptions.SERVER_CAPABILITIES.key(), + "server-v0, " + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY + " ,server-v2"); if (createTableDefaultKey != null) { defaultConf.put( TABLE_DEFAULT_OPTION_PREFIX + createTableDefaultKey, createTableDefaultValue); } - this.config = new ConfigResponse(defaultConf, ImmutableMap.of()); + this.config = new ConfigResponse(defaultConf, new HashMap<>()); restCatalogServer = new RESTCatalogServer(dataPath, this.authProvider, this.config, restWarehouse); restCatalogServer.start(); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java index 4d4f101bd614..1f94605ddab8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java @@ -240,6 +240,7 @@ public static GetTableResponse getTableResponse() { false, 1, schema(options), + 2L, "owner", System.currentTimeMillis(), "created", diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java index 44449fcabcec..ae31763d6b92 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java @@ -233,6 +233,9 @@ public void getTableResponseParseTest() throws Exception { GetTableResponse parseData = RESTApi.fromJson(responseStr, GetTableResponse.class); assertEquals(response.getSchemaId(), parseData.getSchemaId()); assertEquals(response.getSchema(), parseData.getSchema()); + assertEquals( + response.getExplicitPartitionLocationCount(), + parseData.getExplicitPartitionLocationCount()); } @Test @@ -307,6 +310,20 @@ public void createPartitionsResponseParseTest() throws Exception { assertEquals(Collections.singletonList(existed), parsed.getExisted()); } + @Test + public void createPartitionsResponsePreservesLocationsTest() throws Exception { + String json = + "{\"created\":[{\"dt\":\"20260901\"}],\"existed\":[]," + + "\"partitionLocations\":[{\"spec\":{\"dt\":\"20260901\"}," + + "\"location\":\"oss://archive-bucket/table/dt=20260901\"}]}"; + + CreatePartitionsResponse response = RESTApi.fromJson(json, CreatePartitionsResponse.class); + + assertTrue( + RESTApi.toJson(response) + .contains("\"location\":\"oss://archive-bucket/table/dt=20260901\"")); + } + @Test public void createPartitionsRequestParseTest() throws Exception { String requestWithoutIgnoreIfExists = "{\"partitionSpecs\":[{\"dt\":\"20260715\"}]}"; @@ -337,6 +354,20 @@ public void createPartitionsRequestParseTest() throws Exception { assertNull(defaultRequest.replaceStatistics()); } + @Test + public void createPartitionsRequestPreservesLocationsTest() throws Exception { + String json = + "{\"partitionSpecs\":[{\"dt\":\"20260901\"}]," + + "\"partitionLocations\":[{\"spec\":{\"dt\":\"20260901\"}," + + "\"location\":\"oss://archive-bucket/table/dt=20260901\"}]}"; + + CreatePartitionsRequest request = RESTApi.fromJson(json, CreatePartitionsRequest.class); + + assertTrue( + RESTApi.toJson(request) + .contains("\"location\":\"oss://archive-bucket/table/dt=20260901\"")); + } + @Test public void createPartitionsRequestCarriesStatisticsTest() throws Exception { Map spec = Collections.singletonMap("dt", "20260728"); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPartitionSupport.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPartitionSupport.java new file mode 100644 index 000000000000..d3131b110d80 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPartitionSupport.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionStatistics; + +import javax.annotation.Nullable; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** Partition-state helpers kept outside the already large mock REST server. */ +final class RESTCatalogPartitionSupport { + + private RESTCatalogPartitionSupport() {} + + @Nullable + static Long explicitLocationCount( + boolean reported, @Nullable Long override, List partitions) { + if (!reported) { + return null; + } + if (override != null) { + return override; + } + return partitions.stream().filter(partition -> partition.location() != null).count(); + } + + /** + * Folds reported statistics into stored partitions. Applying is all-or-nothing when a report + * names an unknown partition, because applying only part would double-count on a retry. + */ + static void applyStatistics( + List storedPartitions, + @Nullable List statistics, + @Nullable Boolean replaceStatistics) { + if (statistics == null) { + return; + } + boolean accumulate = !Boolean.TRUE.equals(replaceStatistics); + Map, PartitionStatistics> reported = new HashMap<>(); + for (PartitionStatistics statistic : statistics) { + reported.put(statistic.spec(), statistic); + } + Set> storedSpecs = + storedPartitions.stream().map(Partition::spec).collect(Collectors.toSet()); + if (!storedSpecs.containsAll(reported.keySet())) { + return; + } + for (int i = 0; i < storedPartitions.size(); i++) { + Partition stored = storedPartitions.get(i); + PartitionStatistics update = reported.get(stored.spec()); + if (update == null) { + continue; + } + storedPartitions.set(i, mergeStatistics(stored, update, accumulate)); + } + } + + private static Partition mergeStatistics( + Partition stored, PartitionStatistics update, boolean accumulate) { + return new Partition( + stored.spec(), + combine(stored.recordCount(), update.recordCount(), accumulate), + combine(stored.fileSizeInBytes(), update.fileSizeInBytes(), accumulate), + combine(stored.fileCount(), update.fileCount(), accumulate), + combineLastFileCreationTime( + stored.lastFileCreationTime(), update.lastFileCreationTime(), accumulate), + stored.totalBuckets(), + stored.done(), + stored.createdAt(), + stored.createdBy(), + stored.updatedAt(), + stored.updatedBy(), + stored.options(), + stored.location()); + } + + private static long combine(long stored, long reported, boolean accumulate) { + if (!PartitionStatistics.isKnown(reported)) { + return stored; + } + if (!accumulate || !PartitionStatistics.isKnown(stored)) { + return reported; + } + return stored + reported; + } + + private static long combineLastFileCreationTime( + long stored, long reported, boolean accumulate) { + if (!PartitionStatistics.isKnown(reported)) { + return stored; + } + return accumulate ? Math.max(stored, reported) : reported; + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index 8e36aa384401..142d8fb14e18 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -49,6 +49,7 @@ import org.apache.paimon.operation.Lock; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionLocation; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.partition.PartitionUtils; import org.apache.paimon.predicate.Predicate; @@ -178,6 +179,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.function.Supplier; +import java.util.function.UnaryOperator; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -272,6 +274,10 @@ public class RESTCatalogServer { private final Map>> receivedHeadersByPath = new HashMap<>(); private volatile boolean partitionListingSupported = true; + private volatile boolean partitionLocationCreateSupported = true; + private volatile boolean explicitPartitionLocationCountReported = true; + @Nullable private volatile Long explicitPartitionLocationCountOverride; + private volatile UnaryOperator> partitionLocationEcho = value -> value; public RESTCatalogServer( String dataPath, AuthProvider authProvider, ConfigResponse config, String warehouse) { @@ -340,6 +346,22 @@ public void setPartitionListingSupported(boolean partitionListingSupported) { this.partitionListingSupported = partitionListingSupported; } + public void setPartitionLocationCreateSupported(boolean partitionLocationCreateSupported) { + this.partitionLocationCreateSupported = partitionLocationCreateSupported; + } + + public void setExplicitPartitionLocationCountReported(boolean reported) { + this.explicitPartitionLocationCountReported = reported; + } + + public void setExplicitPartitionLocationCount(long explicitPartitionLocationCount) { + this.explicitPartitionLocationCountOverride = explicitPartitionLocationCount; + } + + public void setPartitionLocationEcho(UnaryOperator> echo) { + this.partitionLocationEcho = echo; + } + public void clearReceivedListPartitionsByFilterRequests() { receivedListPartitionsByFilterRequests.clear(); } @@ -584,6 +606,11 @@ && isTableByIdRequest(request.getPath())) { && ResourcePaths.TABLES.equals(resources[1]) && "partitions".equals(resources[3]) && "drop".equals(resources[4]); + boolean isPartitionsWithLocations = + resources.length == 5 + && ResourcePaths.TABLES.equals(resources[1]) + && "partitions".equals(resources[3]) + && "with-locations".equals(resources[4]); boolean isBranches = resources.length >= 4 @@ -614,7 +641,10 @@ && isTableByIdRequest(request.getPath())) { } } // validate partition - if (isPartitions || isMarkDonePartitions || isDropPartitions) { + if (isPartitions + || isPartitionsWithLocations + || isMarkDonePartitions + || isDropPartitions) { String tableName = RESTUtil.decodeString(resources[2]); Optional error = checkTablePartitioned( @@ -636,6 +666,15 @@ && isTableByIdRequest(request.getPath())) { return mockResponse(new ErrorResponse(null, null, "", 501), 501); } else if (isDropPartitions) { return dropPartitionsHandle(restAuthParameter.data(), identifier); + } else if (isPartitionsWithLocations) { + if (!partitionLocationCreateSupported) { + return new MockResponse().setResponseCode(404); + } + return partitionsApiHandle( + restAuthParameter.method(), + restAuthParameter.data(), + parameters, + identifier); } else if (isPartitions) { return partitionsApiHandle( restAuthParameter.method(), @@ -1913,6 +1952,11 @@ private List listTableDetails( entry.getValue().isExternal(), entry.getValue().schema().id(), entry.getValue().schema().toSchema(), + RESTCatalogPartitionSupport.explicitLocationCount( + explicitPartitionLocationCountReported, + explicitPartitionLocationCountOverride, + tablePartitionsStore.getOrDefault( + identifier.getFullName(), Collections.emptyList())), "owner", 1L, "created", @@ -2017,6 +2061,11 @@ private MockResponse tableHandle(String method, String data, Identifier identifi tableMetadata.isExternal(), tableMetadata.schema().id(), schema, + RESTCatalogPartitionSupport.explicitLocationCount( + explicitPartitionLocationCountReported, + explicitPartitionLocationCountOverride, + tablePartitionsStore.getOrDefault( + identifier.getFullName(), Collections.emptyList())), "owner", 1L, "created", @@ -2188,8 +2237,32 @@ private MockResponse partitionsApiHandle( List storedPartitions = tablePartitionsStore.computeIfAbsent( tableIdentifier.getFullName(), ignored -> new ArrayList<>()); - Set> existingSpecs = - storedPartitions.stream().map(Partition::spec).collect(Collectors.toSet()); + Map, Partition> existingBySpec = + storedPartitions.stream() + .collect(Collectors.toMap(Partition::spec, partition -> partition)); + Set> existingSpecs = new HashSet<>(existingBySpec.keySet()); + List requestedLocationList = request.getPartitionLocations(); + Map, PartitionLocation> requestedLocations = new HashMap<>(); + if (requestedLocationList != null) { + Set> requestedSpecs = + new HashSet<>(request.getPartitionSpecs()); + for (PartitionLocation location : requestedLocationList) { + if (location == null + || location.spec() == null + || location.location() == null + || location.location().trim().isEmpty() + || !requestedSpecs.contains(location.spec()) + || requestedLocations.put(location.spec(), location) != null) { + return mockResponse( + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_PARTITION, + null, + "Invalid explicit partition locations.", + 400), + 400); + } + } + } if (!request.ignoreIfExists()) { Set> seenSpecs = new HashSet<>(existingSpecs); Optional> conflictingSpec = @@ -2209,12 +2282,44 @@ private MockResponse partitionsApiHandle( return mockResponse(response, 409); } } + if (requestedLocationList != null) { + Optional> conflictingLocation = + request.getPartitionSpecs().stream() + .filter(existingBySpec::containsKey) + .filter( + spec -> { + PartitionLocation requested = + requestedLocations.get(spec); + String requestedLocation = + requested == null + ? null + : requested.location(); + return !Objects.equals( + existingBySpec.get(spec).location(), + requestedLocation); + }) + .findFirst(); + if (conflictingLocation.isPresent()) { + String partitionName = + PartitionUtils.buildPartitionName(conflictingLocation.get()); + return mockResponse( + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_PARTITION, + partitionName, + String.format( + "Partition %s already exists at a different location.", + partitionName), + 409), + 409); + } + } List> created = new ArrayList<>(); List> existed = new ArrayList<>(); for (Map spec : request.getPartitionSpecs()) { if (existingSpecs.add(spec)) { // A registration measures nothing, so a new partition starts unknown. - storedPartitions.add( + PartitionLocation requestedLocation = requestedLocations.get(spec); + Partition stored = new Partition( spec, PartitionStatistics.UNKNOWN, @@ -2222,70 +2327,48 @@ private MockResponse partitionsApiHandle( PartitionStatistics.UNKNOWN, PartitionStatistics.UNKNOWN, PartitionStatistics.UNKNOWN_TOTAL_BUCKETS, - false)); + false, + null, + null, + null, + null, + null, + requestedLocation == null + ? null + : requestedLocation.location()); + storedPartitions.add(stored); + existingBySpec.put(spec, stored); created.add(spec); } else { existed.add(spec); } } - applyPartitionStatistics( + RESTCatalogPartitionSupport.applyStatistics( storedPartitions, request.getPartitionStatistics(), request.replaceStatistics()); - return mockResponse(new CreatePartitionsResponse(created, existed), 200); + List storedLocations = + requestedLocationList == null + ? null + : requestedLocationList.stream() + .map( + requested -> + new PartitionLocation( + requested.spec(), + existingBySpec + .get(requested.spec()) + .location())) + .collect(Collectors.toList()); + if (storedLocations != null) { + storedLocations = partitionLocationEcho.apply(storedLocations); + } + return mockResponse( + new CreatePartitionsResponse(created, existed, storedLocations), 200); default: return new MockResponse().setResponseCode(404); } } - /** - * Folds reported statistics into the stored partitions, the way a catalog server does: - * replacing overwrites, adding accumulates, a field reported as unknown leaves the stored one - * alone, and no report adds or removes a partition row. - * - *

All or nothing: if any reported spec names a partition this table does not hold, none of - * the report is applied, since a reporter sending it again would count the applied part twice. - */ - private static void applyPartitionStatistics( - List storedPartitions, - @Nullable List statistics, - @Nullable Boolean replaceStatistics) { - if (statistics == null) { - return; - } - boolean accumulate = !Boolean.TRUE.equals(replaceStatistics); - Map, PartitionStatistics> reported = new HashMap<>(); - for (PartitionStatistics statistic : statistics) { - reported.put(statistic.spec(), statistic); - } - Set> storedSpecs = - storedPartitions.stream().map(Partition::spec).collect(Collectors.toSet()); - if (!storedSpecs.containsAll(reported.keySet())) { - // Applying the half that matched would count it twice on the next report. - return; - } - for (int i = 0; i < storedPartitions.size(); i++) { - Partition stored = storedPartitions.get(i); - PartitionStatistics update = reported.get(stored.spec()); - if (update == null) { - continue; - } - storedPartitions.set( - i, - new Partition( - stored.spec(), - combine(stored.recordCount(), update.recordCount(), accumulate), - combine(stored.fileSizeInBytes(), update.fileSizeInBytes(), accumulate), - combine(stored.fileCount(), update.fileCount(), accumulate), - combineLastFileCreationTime( - stored.lastFileCreationTime(), - update.lastFileCreationTime(), - accumulate), - stored.totalBuckets(), - stored.done())); - } - } - /** * Folds a snapshot commit's report onto a stored value. That report is a delta, so a negative * is a decrement rather than an unknown, but a value nobody has measured is replaced rather @@ -2296,31 +2379,6 @@ private static long accumulateDelta(long stored, long reported) { return PartitionStatistics.isKnown(stored) ? stored + reported : reported; } - private static long combine(long stored, long reported, boolean accumulate) { - if (!PartitionStatistics.isKnown(reported)) { - return stored; - } - if (!accumulate || !PartitionStatistics.isKnown(stored)) { - return reported; - } - return stored + reported; - } - - /** - * Folds a reported creation time in: adding takes the later of the two, setting takes what the - * report says even when that moves the time backwards. - */ - private static long combineLastFileCreationTime( - long stored, long reported, boolean accumulate) { - if (!PartitionStatistics.isKnown(reported)) { - return stored; - } - if (!accumulate) { - return reported; - } - return Math.max(stored, reported); - } - private MockResponse dropPartitionsHandle(String data, Identifier tableIdentifier) throws Exception { DropPartitionsRequest request = parseRequest(data, DropPartitionsRequest.class); @@ -3304,7 +3362,8 @@ private synchronized MockResponse commitSnapshot( oldPartition.createdBy(), oldPartition.updatedAt(), oldPartition.updatedBy(), - oldPartition.options()); + oldPartition.options(), + oldPartition.location()); }) .collect(Collectors.toList()); Set> existingSpecs = diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java index b6f0f39f38f5..2965f9eec512 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java @@ -71,4 +71,16 @@ public void testPoliciesAreNestedUnderAttachmentResource() { "/v1/catalog%2Fid/databases/sales+db/tables/orders%2Fall/policies/drop", paths.dropPolicy(table)); } + + @Test + public void testExplicitPartitionLocationsUseDedicatedCreateResource() { + ResourcePaths paths = new ResourcePaths("catalog/id"); + + assertEquals( + "/v1/catalog%2Fid/databases/sales+db/tables/orders%2Fall/partitions", + paths.partitions("sales db", "orders/all")); + assertEquals( + "/v1/catalog%2Fid/databases/sales+db/tables/orders%2Fall/partitions/with-locations", + paths.partitionsWithLocations("sales db", "orders/all")); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManagerTest.java index 0c558973b7f3..7192cd0ea9c4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManagerTest.java @@ -24,6 +24,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.BinaryString; import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionLocation; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; @@ -394,6 +395,74 @@ void testStrictCreateStaysOneRequest() throws Exception { assertThat(batches.get(0)).isEqualTo(specs); } + @Test + void testStrictCreateWithLocationsStaysOneAtomicRequest() throws Exception { + Catalog catalog = mock(Catalog.class); + List> specs = specs(2500); + List locations = + specs.stream() + .map(CatalogFormatTablePartitionManagerTest::location) + .collect(Collectors.toList()); + + partitionManager(catalog).createPartitions(specs, false, null, false, locations); + + @SuppressWarnings("unchecked") + ArgumentCaptor>> specCaptor = ArgumentCaptor.forClass(List.class); + @SuppressWarnings("unchecked") + ArgumentCaptor> locationCaptor = + ArgumentCaptor.forClass(List.class); + verify(catalog) + .createPartitions( + eq(IDENTIFIER), + specCaptor.capture(), + eq(false), + isNull(), + eq(false), + locationCaptor.capture()); + assertThat(specCaptor.getValue()).isEqualTo(specs); + assertThat(locationCaptor.getValue()).containsExactlyElementsOf(locations); + verify(catalog, never()) + .createPartitions(eq(IDENTIFIER), anyList(), eq(false), isNull(), eq(false)); + } + + @Test + void testLocationsRideInTheRequestOfTheirOwnPartitions() throws Exception { + Catalog catalog = mock(Catalog.class); + List> specs = specs(2500); + List locations = + Arrays.asList( + location(specs.get(2499)), + location(specs.get(1000)), + location(specs.get(999)), + location(specs.get(0))); + + partitionManager(catalog).createPartitions(specs, true, null, false, locations); + + @SuppressWarnings("unchecked") + ArgumentCaptor>> specCaptor = ArgumentCaptor.forClass(List.class); + @SuppressWarnings("unchecked") + ArgumentCaptor> locationCaptor = + ArgumentCaptor.forClass(List.class); + verify(catalog, times(3)) + .createPartitions( + eq(IDENTIFIER), + specCaptor.capture(), + eq(true), + isNull(), + eq(false), + locationCaptor.capture()); + List>> requestedSpecs = specCaptor.getAllValues(); + List> requestedLocations = locationCaptor.getAllValues(); + assertThat(requestedSpecs).extracting(List::size).containsExactly(1000, 1000, 500); + for (int request = 0; request < requestedSpecs.size(); request++) { + for (PartitionLocation location : requestedLocations.get(request)) { + assertThat(requestedSpecs.get(request)).contains(location.spec()); + } + } + assertThat(requestedLocations.stream().flatMap(List::stream).collect(Collectors.toList())) + .containsExactlyInAnyOrderElementsOf(locations); + } + @Test void testDropIsSplitIntoRequests() throws Exception { Catalog catalog = mock(Catalog.class); @@ -759,6 +828,10 @@ private static List> flatten(List>> return batches.stream().flatMap(List::stream).collect(Collectors.toList()); } + private static PartitionLocation location(Map spec) { + return new PartitionLocation(spec, "file:/archive/" + spec.get("month")); + } + private static PredicateBuilder partitionPredicates() { return new PredicateBuilder( RowType.of( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkRestCatalogITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkRestCatalogITCase.java index be92ce163d57..515325b2bfb1 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkRestCatalogITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkRestCatalogITCase.java @@ -21,6 +21,7 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; +import org.apache.paimon.rest.RESTApi; import org.apache.paimon.rest.RESTCatalogOptions; import org.apache.paimon.rest.RESTTestFileIO; import org.apache.paimon.rest.auth.AuthProviderEnum; @@ -68,6 +69,20 @@ void testListTableReturnsView() throws Exception { assertThat(catalog.listTables("test")).containsExactlyInAnyOrder("t1", "v"); } + @Test + void testFlinkCatalogDoesNotDeclarePartitionLocationCapability() { + restCatalogServer.clearReceivedHeaders(); + catalog.listDatabases(); + + assertThat(restCatalogServer.getReceivedHeaders()) + .isNotEmpty() + .allSatisfy( + headers -> + assertThat(headers) + .doesNotContainKey( + RESTApi.CAPABILITIES_HEADER.toLowerCase())); + } + private CatalogTable createTable(Map options) { ResolvedSchema resolvedSchema = this.createSchema(); CatalogTable origin = diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCaseBase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCaseBase.java index adfa219a6583..e57d5441ca0e 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCaseBase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RESTCatalogITCaseBase.java @@ -19,6 +19,7 @@ package org.apache.paimon.flink; import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.rest.RESTApi; import org.apache.paimon.rest.RESTCatalogInternalOptions; import org.apache.paimon.rest.RESTCatalogOptions; import org.apache.paimon.rest.RESTCatalogServer; @@ -67,7 +68,9 @@ public void before() throws IOException { RESTTokenFileIO.DATA_TOKEN_ENABLED.key(), "true", CatalogOptions.WAREHOUSE.key(), - warehouse), + warehouse, + "header.x-paimon-capabilities", + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY), ImmutableMap.of()); AuthProvider authProvider = new BearTokenAuthProvider(INIT_TOKEN); restCatalogServer = new RESTCatalogServer(dataPath, authProvider, config, warehouse); From a27f7cbd7e14bec89ce1c185b47153708fd4d42a Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 2 Sep 2026 03:19:50 +0800 Subject: [PATCH 03/15] core: read format table partitions from explicit locations --- .../table/format/CatalogSplitEnumerator.java | 108 ++++++- .../paimon/table/format/FormatDataSplit.java | 21 +- .../table/format/FormatReadBuilder.java | 22 +- .../format/FormatTableFileIOResolver.java | 90 ++++++ .../paimon/table/format/SplitEnumerator.java | 11 +- .../CatalogManagedPartitionScanTest.java | 273 ++++++++++++++++-- .../table/format/FormatDataSplitTest.java | 26 ++ .../table/format/FormatReadBuilderTest.java | 93 ++++++ 8 files changed, 605 insertions(+), 39 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableFileIOResolver.java diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java b/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java index 9095df296fee..2c4c001cf1f5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java @@ -20,7 +20,6 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryRow; -import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; import org.apache.paimon.manifest.PartitionEntry; @@ -89,7 +88,9 @@ final class CatalogSplitEnumerator extends SplitEnumerator { @Override List enumeratePartitions(@Nullable PartitionPredicate partitionFilter) throws IOException { - return enumeratePartitions(findCatalogPartitions(partitionFilter), partitionFilter); + List partitions = findCatalogPartitions(partitionFilter); + validatePathOwnership(partitions, partitionFilter); + return enumeratePartitions(partitions, partitionFilter); } @Override @@ -98,6 +99,7 @@ ScanPlan plan(@Nullable PartitionPredicate partitionFilter) throws IOException { return super.plan(partitionFilter); } List partitions = findCatalogPartitions(partitionFilter); + validatePathOwnership(partitions, partitionFilter); List entries = toPartitionEntries(partitions, partitionFilter); return new ScanPlan(enumeratePartitions(partitions, partitionFilter), rowCount(entries)); } @@ -113,10 +115,18 @@ private List enumeratePartitions( return splits; } - FileIO fileIO = table.fileIO(); - // Establish the filesystem on the caller thread so listing workers reuse it under the - // caller's security context instead of creating it lazily under a shared worker. - fileIO.exists(new Path(table.location())); + FormatTableFileIOResolver fileIOResolver = new FormatTableFileIOResolver(table); + boolean tableFileIOPrepared = false; + for (Pair, Path> partition : partitions) { + boolean useCatalogContextFileIO = + fileIOResolver.useCatalogContextFileIO(partition.getValue()); + if (useCatalogContextFileIO) { + fileIOResolver.prepare(partition.getValue(), true); + } else if (!tableFileIOPrepared) { + fileIOResolver.prepare(partition.getValue(), false); + tableFileIOPrepared = true; + } + } Function, Path>, List> lister = pair -> { BinaryRow partitionRow = toPartitionRow(pair.getKey()); @@ -124,7 +134,13 @@ private List enumeratePartitions( return Collections.emptyList(); } try { - return createSplits(fileIO, pair.getValue(), partitionRow); + boolean useCatalogContextFileIO = + fileIOResolver.useCatalogContextFileIO(pair.getValue()); + return createSplits( + fileIOResolver.fileIO(useCatalogContextFileIO), + pair.getValue(), + partitionRow, + useCatalogContextFileIO); } catch (FileNotFoundException e) { warnMissingPartition(pair.getKey(), pair.getValue()); return Collections.emptyList(); @@ -146,9 +162,47 @@ private List enumeratePartitions( @Override List, Path>> findPartitions( @Nullable PartitionPredicate partitionFilter) { - return toSpecsAndPaths( - findCatalogPartitions(partitionFilter), - coreOptions.formatTablePartitionOnlyValueInPath()); + List partitions = findCatalogPartitions(partitionFilter); + validatePathOwnership(partitions, partitionFilter); + return toSpecsAndPaths(partitions, coreOptions.formatTablePartitionOnlyValueInPath()); + } + + private void validatePathOwnership( + List selected, @Nullable PartitionPredicate partitionFilter) { + OptionalLong explicitLocationCount = partitionManager.explicitPartitionLocationCount(); + List authoritative = selected; + if (partitionFilter != null + && (!explicitLocationCount.isPresent() || explicitLocationCount.getAsLong() != 0)) { + authoritative = findCatalogPartitions(null); + } + validateExplicitLocationCount(authoritative, explicitLocationCount); + if (authoritative.stream().noneMatch(partition -> partition.location() != null)) { + return; + } + // A catalog predicate is only a pruning hint. Once one partition has an explicit + // location, every registered partition participates in the ownership invariant, including + // siblings hidden by that predicate. + toSpecsAndPaths(authoritative, coreOptions.formatTablePartitionOnlyValueInPath()); + } + + private void validateExplicitLocationCount( + List authoritative, OptionalLong explicitLocationCount) { + if (!explicitLocationCount.isPresent()) { + return; + } + long observed = + authoritative.stream() + .filter( + partition -> + partition.location() != null + && !partition.location().isEmpty()) + .count(); + if (observed != explicitLocationCount.getAsLong()) { + throw new IllegalStateException( + String.format( + "Catalog reported %d explicit partition locations for format table %s, but its complete listing contained %d.", + explicitLocationCount.getAsLong(), table.fullName(), observed)); + } } private List findCatalogPartitions(@Nullable PartitionPredicate partitionFilter) { @@ -219,16 +273,38 @@ private OptionalLong rowCount(List entries) { private List, Path>> toSpecsAndPaths( List partitions, boolean onlyValueInPath) { + if (partitions.stream().noneMatch(partition -> partition.location() != null)) { + return toDefaultSpecsAndPaths(partitions, onlyValueInPath); + } + List, Path>> result = new ArrayList<>(partitions.size()); + Path tablePath = new Path(table.location()); + FormatTablePartitionPathResolver pathResolver = + new FormatTablePartitionPathResolver(tablePath, table.fullName(), onlyValueInPath); + for (Partition partition : partitions) { + LinkedHashMap spec = normalizeSpec(partition.spec(), onlyValueInPath); + Path partitionPath = pathResolver.resolve(spec, partition.location()); + if (pathResolver.remember(spec, partitionPath)) { + result.add(Pair.of(spec, partitionPath)); + } + } + return result; + } + + private List, Path>> toDefaultSpecsAndPaths( + List partitions, boolean onlyValueInPath) { List, Path>> result = new ArrayList<>(partitions.size()); + Set> seen = new HashSet<>(partitions.size()); Path tablePath = new Path(table.location()); - // A duplicate catalog entry must not duplicate all records in that partition. - Set seenPartitionPaths = new HashSet<>(partitions.size()); for (Partition partition : partitions) { LinkedHashMap spec = normalizeSpec(partition.spec(), onlyValueInPath); - String partitionPath = - PartitionPathUtils.generatePartitionPathUtil(spec, onlyValueInPath); - if (seenPartitionPaths.add(partitionPath)) { - result.add(Pair.of(spec, new Path(tablePath, partitionPath))); + if (seen.add(spec)) { + result.add( + Pair.of( + spec, + new Path( + tablePath, + PartitionPathUtils.generatePartitionPathUtil( + spec, onlyValueInPath)))); } } return result; diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatDataSplit.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatDataSplit.java index 78f2f9532e8a..614780e42521 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatDataSplit.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatDataSplit.java @@ -39,10 +39,17 @@ public class FormatDataSplit implements Split { private final List files; @Nullable private final BinaryRow partition; + private final boolean useCatalogContextFileIO; public FormatDataSplit(List files, @Nullable BinaryRow partition) { + this(files, partition, false); + } + + public FormatDataSplit( + List files, @Nullable BinaryRow partition, boolean useCatalogContextFileIO) { this.files = files; this.partition = partition; + this.useCatalogContextFileIO = useCatalogContextFileIO; } public List files() { @@ -54,6 +61,14 @@ public BinaryRow partition() { return partition; } + /** + * Whether readers must resolve this split through the client {@code CatalogContext} instead of + * the FileIO bound to the table root. + */ + public boolean useCatalogContextFileIO() { + return useCatalogContextFileIO; + } + /** Total bytes to read for this split, i.e. the sum of {@link FileMeta#readSize()}. */ public long totalSize() { return files.stream().mapToLong(FileMeta::readSize).sum(); @@ -83,12 +98,14 @@ public boolean equals(Object o) { return false; } FormatDataSplit that = (FormatDataSplit) o; - return Objects.equals(files, that.files) && Objects.equals(partition, that.partition); + return useCatalogContextFileIO == that.useCatalogContextFileIO + && Objects.equals(files, that.files) + && Objects.equals(partition, that.partition); } @Override public int hashCode() { - return Objects.hash(files, partition); + return Objects.hash(files, partition, useCatalogContextFileIO); } /** diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java index ff5d73e48f1e..168cb0c50106 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java @@ -25,6 +25,7 @@ import org.apache.paimon.format.FileFormatDiscover; import org.apache.paimon.format.FormatReaderContext; import org.apache.paimon.format.FormatReaderFactory; +import org.apache.paimon.fs.FileIO; import org.apache.paimon.io.DataFileRecordReader; import org.apache.paimon.mergetree.compact.ConcatRecordReader; import org.apache.paimon.options.CatalogOptions; @@ -78,6 +79,7 @@ public class FormatReadBuilder implements ReadBuilder { @Nullable private Predicate filter; @Nullable private PartitionPredicate partitionFilter; @Nullable private Integer limit; + @Nullable private transient volatile FormatTableFileIOResolver fileIOResolver; public FormatReadBuilder(FormatTable table) { this.table = table; @@ -204,11 +206,13 @@ protected RecordReader createReader( table.partitionKeys(), readType().getFields(), table.partitionType()); BinaryRow partition = dataSplit.partition(); + FileIO fileIO = fileIOResolver().fileIO(dataSplit.useCatalogContextFileIO()); List> suppliers = new ArrayList<>(); for (FormatDataSplit.FileMeta file : dataSplit.files()) { suppliers.add( () -> createFileReader( + fileIO, file, partition, readerFactory, @@ -219,6 +223,7 @@ protected RecordReader createReader( } private RecordReader createFileReader( + FileIO fileIO, FormatDataSplit.FileMeta file, @Nullable BinaryRow partition, FormatReaderFactory readerFactory, @@ -227,7 +232,7 @@ private RecordReader createFileReader( throws IOException { FormatReaderContext formatReaderContext = new FormatReaderContext( - table.fileIO(), file.filePath(), file.fileSize(), null, readBatchSizer); + fileIO, file.filePath(), file.fileSize(), null, readBatchSizer); try { FileRecordReader reader; Long length = file.length(); @@ -271,6 +276,21 @@ private static RowType getRowTypeWithoutPartition(RowType rowType, List .collect(Collectors.toList())); } + private FormatTableFileIOResolver fileIOResolver() { + FormatTableFileIOResolver result = fileIOResolver; + if (result != null) { + return result; + } + synchronized (this) { + result = fileIOResolver; + if (result == null) { + result = new FormatTableFileIOResolver(table); + fileIOResolver = result; + } + return result; + } + } + // ===================== Unsupported =============================== @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableFileIOResolver.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableFileIOResolver.java new file mode 100644 index 000000000000..4397ff83fdd2 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableFileIOResolver.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.format; + +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.ResolvingFileIO; +import org.apache.paimon.table.FormatTable; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.io.Serializable; + +/** + * Chooses the credential boundary for Format Table data without deriving it from a listed file's + * URI. + */ +final class FormatTableFileIOResolver implements Serializable { + + private static final long serialVersionUID = 1L; + + private final Path tableRoot; + private final FileIO tableFileIO; + @Nullable private final CatalogContext catalogContext; + @Nullable private transient volatile ResolvingFileIO catalogContextFileIO; + + FormatTableFileIOResolver(FormatTable table) { + this.tableRoot = new Path(table.location()); + this.tableFileIO = table.fileIO(); + this.catalogContext = table.catalogContext(); + } + + boolean useCatalogContextFileIO(Path partitionPath) { + return !FormatTablePartitionPathResolver.isWithin(partitionPath, tableRoot); + } + + /** + * Resolves an external filesystem on the caller thread before parallel listing starts. The + * underlying resolver caches the result by scheme and authority. + */ + void prepare(Path path, boolean useCatalogContextFileIO) throws IOException { + if (useCatalogContextFileIO) { + catalogContextFileIO().fileIO(path); + } else { + tableFileIO.exists(tableRoot); + } + } + + FileIO fileIO(boolean useCatalogContextFileIO) { + return useCatalogContextFileIO ? catalogContextFileIO() : tableFileIO; + } + + private ResolvingFileIO catalogContextFileIO() { + ResolvingFileIO result = catalogContextFileIO; + if (result != null) { + return result; + } + synchronized (this) { + result = catalogContextFileIO; + if (result == null) { + if (catalogContext == null) { + throw new IllegalStateException( + "A CatalogContext is required to access a Format Table partition outside the table root."); + } + result = new ResolvingFileIO(); + result.configure(catalogContext); + catalogContextFileIO = result; + } + return result; + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/SplitEnumerator.java b/paimon-core/src/main/java/org/apache/paimon/table/format/SplitEnumerator.java index c46919576d9d..b6578fd85905 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/SplitEnumerator.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/SplitEnumerator.java @@ -145,6 +145,15 @@ BinaryRow toPartitionRow(LinkedHashMap partitionSpec) { List createSplits(FileIO fileIO, Path path, @Nullable BinaryRow partition) throws IOException { + return createSplits(fileIO, path, partition, false); + } + + List createSplits( + FileIO fileIO, + Path path, + @Nullable BinaryRow partition, + boolean useCatalogContextFileIO) + throws IOException { List segments = new ArrayList<>(); // The listed directory is a single partition, or the table itself when unpartitioned. List files = FormatTableScan.listDataFiles(fileIO, path); @@ -159,7 +168,7 @@ List createSplits(FileIO fileIO, Path path, @Nullable BinaryRow partition segments, file -> Math.max(file.readSize(), openFileCost), targetSplitSize)) { - splits.add(new FormatDataSplit(bin, partition)); + splits.add(new FormatDataSplit(bin, partition, useCatalogContextFileIO)); } return splits; } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java index c1f644febc96..14f177c57f19 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java @@ -20,14 +20,17 @@ import org.apache.paimon.PagedList; import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.fs.FileIOLoader; import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.partition.PartitionStatistics; @@ -96,6 +99,11 @@ void testLeadingPatternResidualFilterAndUnregisteredDirectory() throws Exception new PagedList<>( Arrays.asList(partition("2025", "10"), partition("2025", "11")), null)); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenReturn( + new PagedList<>( + Arrays.asList(partition("2025", "10"), partition("2025", "11")), + null)); TrackingLocalFileIO fileIO = new TrackingLocalFileIO(); Path tablePath = new Path(tempDir.toUri()); Path octoberFile = writeDataFile(fileIO, tablePath, "year=2025/month=10"); @@ -149,12 +157,14 @@ void testPushesFilterWithoutLeadingPrefixAndAppliesResidualFilter() throws Excep assertThat(plannedFiles).containsExactly(novemberFile); assertThat(plannedFiles).doesNotContain(octoberFile); assertThat(fileIO.listedPaths).containsExactly(new Path(tablePath, "year=2025/month=11")); - assertThat(requestedPrefixes).containsExactly(Collections.emptyMap()); - assertThat(requestedFilters).hasSize(1); + assertThat(requestedPrefixes) + .containsExactly(Collections.emptyMap(), Collections.emptyMap()); + assertThat(requestedFilters).hasSize(2); Predicate pushedFilter = requestedFilters.get(0); assertThat(pushedFilter).isNotNull(); assertThat(pushedFilter.test(GenericRow.of(2025, 11))).isTrue(); assertThat(pushedFilter.test(GenericRow.of(2025, 10))).isFalse(); + assertThat(requestedFilters.get(1)).isNull(); } @Test @@ -300,9 +310,12 @@ void testUnderscoreInPartitionNameRemainsLiteralPrefix() { List plannedFiles = plannedFiles(new FormatTableScan(table, filter, null).plan().splits()); - // '_' has no special meaning in the prefix contract; it must not widen the match. + // '_' has no special meaning in the prefix contract; it must not widen the match. An + // unknown explicit-location count requires one unfiltered registry read to prove that no + // sibling hidden by the prefix owns an overlapping path. assertThat(plannedFiles).isEmpty(); - assertThat(requestedPrefixes).containsExactly(Collections.singletonMap("year", "a_b")); + assertThat(requestedPrefixes) + .containsExactly(Collections.singletonMap("year", "a_b"), Collections.emptyMap()); } @Test @@ -341,6 +354,184 @@ void testDuplicateCatalogPartitionPlansSplitsOnce() throws Exception { assertThat(plannedFiles).containsExactly(dataFile); } + @Test + void testDifferentPartitionsCannotShareAnExplicitLocation() throws Exception { + Path externalPath = new Path(tempDir.resolve("external").toUri()); + Catalog catalog = mock(Catalog.class); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenReturn( + new PagedList<>( + Arrays.asList( + partition("2025", "10", externalPath.toString()), + partition("2025", "11", externalPath.toString())), + null)); + LocalFileIO fileIO = LocalFileIO.create(); + writeDataFile(fileIO, externalPath, "files"); + Path tablePath = new Path(tempDir.resolve("table").toUri()); + FormatTable table = createTable(fileIO, tablePath, partitionManager(catalog), false); + + assertThatThrownBy(() -> new FormatTableScan(table, null, null).plan().splits()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("overlapping locations") + .hasMessageContaining(IDENTIFIER.getFullName()); + } + + @Test + void testPredicateCannotHideSiblingWithConflictingExplicitLocation() throws Exception { + Path tablePath = new Path(tempDir.resolve("table").toUri()); + Path selectedDefaultPath = new Path(tablePath, "year=2025/month=11"); + Partition hidden = partition("2025", "10", selectedDefaultPath.toString()); + Partition selected = partition("2025", "11"); + Catalog catalog = mock(Catalog.class); + when(catalog.listPartitionsByFilterPaged( + eq(IDENTIFIER), any(Predicate.class), eq(1000), isNull(), isNull())) + .thenReturn(new PagedList<>(Collections.singletonList(selected), null)); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenReturn(new PagedList<>(Arrays.asList(hidden, selected), null)); + when(catalog.getExplicitPartitionLocationCount(IDENTIFIER)).thenReturn(OptionalLong.of(1L)); + TrackingLocalFileIO fileIO = new TrackingLocalFileIO(); + writeDataFile(fileIO, tablePath, "year=2025/month=11"); + FileIOLoader clientLoader = + new FileIOLoader() { + @Override + public String getScheme() { + return "file"; + } + + @Override + public LocalFileIO load(Path path) { + return fileIO; + } + }; + FormatTable table = + createTable( + fileIO, + tablePath, + partitionManager(catalog), + false, + CatalogContext.create(new Options(), clientLoader, null)); + Predicate predicate = new PredicateBuilder(table.partitionType()).equal(1, 11); + PartitionPredicate filter = + PartitionPredicate.fromPredicate(table.partitionType(), predicate); + + assertThatThrownBy(() -> new FormatTableScan(table, filter, null).plan().splits()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("overlapping locations") + .hasMessageContaining(IDENTIFIER.getFullName()); + assertThat(fileIO.listedPaths).isEmpty(); + } + + @Test + void testZeroExplicitLocationCountKeepsPredicateScanPruned() throws Exception { + TrackingLocalFileIO fileIO = new TrackingLocalFileIO(); + Path tablePath = new Path(tempDir.resolve("table").toUri()); + Path selectedFile = writeDataFile(fileIO, tablePath, "year=2025/month=11"); + List partitions = + Arrays.asList(partition("2025", "10"), partition("2025", "11")); + FormatTable table = + createTable( + fileIO, + tablePath, + recordingCatalog(partitions, OptionalLong.of(0L)), + false); + PredicateBuilder builder = new PredicateBuilder(table.partitionType()); + Predicate predicate = PredicateBuilder.and(builder.equal(0, 2025), builder.equal(1, 11)); + PartitionPredicate filter = + PartitionPredicate.fromPredicate(table.partitionType(), predicate); + + List plannedFiles = + plannedFiles(new FormatTableScan(table, filter, null).plan().splits()); + + assertThat(plannedFiles).containsExactly(selectedFile); + assertThat(requestedPrefixes).containsExactly(partition("2025", "11").spec()); + } + + @Test + void testFullScanFailsWhenExplicitLocationCountExceedsObservedLocations() { + TrackingLocalFileIO fileIO = new TrackingLocalFileIO(); + Path tablePath = new Path(tempDir.resolve("table").toUri()); + List partitions = + Arrays.asList(partition("2025", "10"), partition("2025", "11")); + FormatTable table = + createTable( + fileIO, + tablePath, + recordingCatalog(partitions, OptionalLong.of(1L)), + false); + + assertThatThrownBy(() -> new FormatTableScan(table, null, null).plan().splits()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("reported 1 explicit partition locations") + .hasMessageContaining("complete listing contained 0"); + assertThat(fileIO.listedPaths).isEmpty(); + } + + @Test + void testFilteredScanDoesNotCountEmptyLocationAsObserved() { + TrackingLocalFileIO fileIO = new TrackingLocalFileIO(); + Path tablePath = new Path(tempDir.resolve("table").toUri()); + List partitions = + Arrays.asList(partition("2025", "10", ""), partition("2025", "11")); + FormatTable table = + createTable( + fileIO, + tablePath, + recordingCatalog(partitions, OptionalLong.of(1L)), + false); + Predicate predicate = new PredicateBuilder(table.partitionType()).equal(1, 11); + PartitionPredicate filter = + PartitionPredicate.fromPredicate(table.partitionType(), predicate); + + assertThatThrownBy(() -> new FormatTableScan(table, filter, null).plan().splits()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("reported 1 explicit partition locations") + .hasMessageContaining("complete listing contained 0"); + assertThat(fileIO.listedPaths).isEmpty(); + } + + @Test + void testExplicitLocationOutsideTableUsesCatalogContextFileIOForPlanning() throws Exception { + LocalFileIO clientFileIO = LocalFileIO.create(); + Path externalPath = new Path(tempDir.resolve("external").toUri()); + Path dataFile = writeDataFile(clientFileIO, externalPath, "files"); + Catalog catalog = mock(Catalog.class); + when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull())) + .thenReturn( + new PagedList<>( + Collections.singletonList( + partition("2025", "11", externalPath.toString())), + null)); + Path tablePath = new Path(tempDir.resolve("table").toUri()); + TableRootOnlyLocalFileIO tableFileIO = new TableRootOnlyLocalFileIO(tablePath); + FileIOLoader clientLoader = + new FileIOLoader() { + @Override + public String getScheme() { + return "file"; + } + + @Override + public LocalFileIO load(Path path) { + return clientFileIO; + } + }; + CatalogContext catalogContext = CatalogContext.create(new Options(), clientLoader, null); + FormatTable table = + createTable( + tableFileIO, tablePath, partitionManager(catalog), false, catalogContext); + + List splits = new FormatTableScan(table, null, null).plan().splits(); + List plannedFiles = plannedFiles(splits); + + assertThat(plannedFiles).containsExactly(dataFile); + assertThat(splits) + .allSatisfy( + split -> + assertThat(((FormatDataSplit) split).useCatalogContextFileIO()) + .isTrue()); + assertThat(tableFileIO.listedPaths).isEmpty(); + } + @Test void testWhitespacePartitionValueIsVisible() throws Exception { Catalog catalog = mock(Catalog.class); @@ -471,9 +662,19 @@ private FormatTablePartitionManager partitionManager(Catalog catalog) { /** Records the prefix the scan pushes down and answers from a fixed partition list. */ private FormatTablePartitionManager recordingCatalog(List partitions) { + return recordingCatalog(partitions, OptionalLong.empty()); + } + + private FormatTablePartitionManager recordingCatalog( + List partitions, OptionalLong explicitPartitionLocationCount) { List> prefixes = requestedPrefixes; List filters = requestedFilters; return new FormatTablePartitionManager() { + @Override + public OptionalLong explicitPartitionLocationCount() { + return explicitPartitionLocationCount; + } + @Override public List listPartitions( Map prefix, @Nullable Predicate filter) { @@ -521,25 +722,38 @@ private FormatTable createTable( Path tablePath, FormatTablePartitionManager partitionManager, boolean valueOnlyPath) { + return createTable(fileIO, tablePath, partitionManager, valueOnlyPath, null); + } + + private FormatTable createTable( + LocalFileIO fileIO, + Path tablePath, + FormatTablePartitionManager partitionManager, + boolean valueOnlyPath, + @Nullable CatalogContext catalogContext) { RowType rowType = RowType.builder() .field("year", DataTypes.INT()) .field("month", DataTypes.INT()) .field("id", DataTypes.INT()) .build(); - return FormatTable.builder() - .fileIO(fileIO) - .identifier(IDENTIFIER) - .rowType(rowType) - .partitionKeys(Arrays.asList("year", "month")) - .location(tablePath.toString()) - .format(FormatTable.Format.CSV) - .options( - Collections.singletonMap( - FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(), - Boolean.toString(valueOnlyPath))) - .partitionManager(partitionManager) - .build(); + FormatTable.Builder builder = + FormatTable.builder() + .fileIO(fileIO) + .identifier(IDENTIFIER) + .rowType(rowType) + .partitionKeys(Arrays.asList("year", "month")) + .location(tablePath.toString()) + .format(FormatTable.Format.CSV) + .options( + Collections.singletonMap( + FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(), + Boolean.toString(valueOnlyPath))) + .partitionManager(partitionManager); + if (catalogContext != null) { + builder.catalogContext(catalogContext); + } + return builder.build(); } private FormatTable createStringPartitionTable( @@ -584,10 +798,14 @@ private Path writeDataFile(LocalFileIO fileIO, Path tablePath, String partitionP } private static Partition partition(String year, String month) { + return partition(year, month, null); + } + + private static Partition partition(String year, String month, @Nullable String location) { Map spec = new LinkedHashMap<>(); spec.put("year", year); spec.put("month", month); - return new Partition(spec, 0, 0, 0, 0, -1, false); + return new Partition(spec, 0, 0, 0, 0, -1, false, null, null, null, null, null, location); } private static List plannedFiles(List splits) { @@ -600,7 +818,7 @@ private static List plannedFiles(List splits) { private static class TrackingLocalFileIO extends LocalFileIO { - private final List listedPaths = new ArrayList<>(); + protected final List listedPaths = new ArrayList<>(); @Override public FileStatus[] listStatus(Path path) throws IOException { @@ -609,6 +827,23 @@ public FileStatus[] listStatus(Path path) throws IOException { } } + private static class TableRootOnlyLocalFileIO extends TrackingLocalFileIO { + + private final Path tableRoot; + + private TableRootOnlyLocalFileIO(Path tableRoot) { + this.tableRoot = tableRoot; + } + + @Override + public FileStatus[] listStatus(Path path) throws IOException { + if (!FormatTablePartitionPathResolver.isWithin(path, tableRoot)) { + throw new AssertionError("The table FileIO must not list an external location."); + } + return super.listStatus(path); + } + } + @Test void testCatalogPartitionListingRunsInParallelAndPreservesOrder() throws Exception { ParallelTrackingLocalFileIO fileIO = new ParallelTrackingLocalFileIO(); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatDataSplitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatDataSplitTest.java index 73a1bf6f2232..45211c29a1eb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatDataSplitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatDataSplitTest.java @@ -25,6 +25,7 @@ import org.junit.jupiter.api.Test; import java.io.IOException; +import java.io.ObjectStreamClass; import java.util.Arrays; import static org.assertj.core.api.Assertions.assertThat; @@ -46,6 +47,7 @@ public void testSerializeAndDeserialize() throws IOException, ClassNotFoundExcep assertThat(deserialized).isEqualTo(split); assertThat(deserialized.files()).isEqualTo(split.files()); assertThat(deserialized.partition()).isEqualTo(split.partition()); + assertThat(deserialized.useCatalogContextFileIO()).isFalse(); assertThat(deserialized.fileCount()).isEqualTo(2); // readSize: whole file -> fileSize (1024), range -> length (512). assertThat(deserialized.totalSize()).isEqualTo(1024L + 512L); @@ -63,4 +65,28 @@ public void testSerializeAndDeserialize() throws IOException, ClassNotFoundExcep assertThat(f1.length()).isEqualTo(512L); assertThat(f1.readSize()).isEqualTo(512L); } + + @Test + public void testCatalogContextFileIORouteSurvivesSerialization() + throws IOException, ClassNotFoundException { + FormatDataSplit split = + new FormatDataSplit( + Arrays.asList(new FileMeta(new Path("oss://archive/data.csv"), 10L)), + null, + true); + + FormatDataSplit deserialized = + InstantiationUtil.deserializeObject( + InstantiationUtil.serializeObject(split), getClass().getClassLoader()); + + assertThat(deserialized).isEqualTo(split); + assertThat(deserialized.useCatalogContextFileIO()).isTrue(); + } + + @Test + public void testSerialVersionUIDsStayCompatible() { + assertThat(ObjectStreamClass.lookup(FormatDataSplit.class).getSerialVersionUID()) + .isEqualTo(3L); + assertThat(ObjectStreamClass.lookup(FileMeta.class).getSerialVersionUID()).isEqualTo(1L); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatReadBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatReadBuilderTest.java index 172d7a0501e5..f49327b9c2ac 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatReadBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatReadBuilderTest.java @@ -18,6 +18,7 @@ package org.apache.paimon.table.format; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; @@ -27,8 +28,10 @@ import org.apache.paimon.format.FormatWriter; import org.apache.paimon.format.FormatWriterFactory; import org.apache.paimon.format.csv.CsvFileFormat; +import org.apache.paimon.fs.FileIOLoader; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; import org.apache.paimon.predicate.Predicate; @@ -50,8 +53,10 @@ import org.junit.jupiter.api.io.TempDir; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -257,6 +262,58 @@ public void testCreateReaderWithCsvSplit() throws IOException { assertThat(partialResult.get(0).getString(1).toString()).isEqualTo("Alice"); } + @Test + public void testExternalSplitUsesCatalogContextFileIOAfterSerialization() throws Exception { + RowType rowType = + RowType.builder() + .field("id", DataTypes.INT()) + .field("name", DataTypes.STRING()) + .build(); + LocalFileIO clientFileIO = LocalFileIO.create(); + Path externalPath = new Path(tempPath.resolve("external").toUri()); + Path csvFile = new Path(externalPath, "data.csv"); + clientFileIO.mkdirs(externalPath); + try (PositionOutputStream out = clientFileIO.newOutputStream(csvFile, false)) { + out.write("1,Alice\n".getBytes(StandardCharsets.UTF_8)); + } + + Path tablePath = new Path(tempPath.resolve("table").toUri()); + FileIOLoader clientLoader = new LocalFileIOLoader(clientFileIO); + FormatTable table = + FormatTable.builder() + .fileIO(new TableRootOnlyLocalFileIO(tablePath)) + .identifier(Identifier.create("test_db", "external_csv")) + .rowType(rowType) + .partitionKeys(Collections.emptyList()) + .location(tablePath.toString()) + .format(FormatTable.Format.CSV) + .options(Collections.singletonMap("file.format", "csv")) + .catalogContext(CatalogContext.create(new Options(), clientLoader, null)) + .build(); + FormatReadBuilder readBuilder = + InstantiationUtil.deserializeObject( + InstantiationUtil.serializeObject(new FormatReadBuilder(table)), + getClass().getClassLoader()); + FormatDataSplit split = + InstantiationUtil.deserializeObject( + InstantiationUtil.serializeObject( + new FormatDataSplit( + Collections.singletonList( + new FormatDataSplit.FileMeta( + csvFile, + clientFileIO.getFileSize(csvFile))), + null, + true)), + getClass().getClassLoader()); + + List rows = readAllRows(readBuilder.createReader(split), rowType); + + assertThat(rows).hasSize(1); + assertThat(rows.get(0).getInt(0)).isEqualTo(1); + assertThat(rows.get(0).getString(1).toString()).isEqualTo("Alice"); + assertThat(split.useCatalogContextFileIO()).isTrue(); + } + private List readAllRows(RecordReader reader, RowType rowType) throws IOException { InternalRowSerializer serializer = new InternalRowSerializer(rowType); @@ -311,4 +368,40 @@ private static int readBatchSize(RecordReader reader) throws IOExce } return size; } + + private static class TableRootOnlyLocalFileIO extends LocalFileIO { + + private final Path tableRoot; + + private TableRootOnlyLocalFileIO(Path tableRoot) { + this.tableRoot = tableRoot; + } + + @Override + public SeekableInputStream newInputStream(Path path) throws IOException { + if (!FormatTablePartitionPathResolver.isWithin(path, tableRoot)) { + throw new AssertionError("The table FileIO must not read an external split."); + } + return super.newInputStream(path); + } + } + + private static class LocalFileIOLoader implements FileIOLoader { + + private final LocalFileIO fileIO; + + private LocalFileIOLoader(LocalFileIO fileIO) { + this.fileIO = fileIO; + } + + @Override + public String getScheme() { + return "file"; + } + + @Override + public LocalFileIO load(Path path) { + return fileIO; + } + } } From 8aaa56b2528240d33d7a64cdd1ec17ae4b91e4d6 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 2 Sep 2026 03:20:20 +0800 Subject: [PATCH 04/15] core: reject unsafe writes to explicitly located partitions --- .../table/format/FormatTableCommit.java | 193 +++++- .../table/format/FormatTableCommitTest.java | 648 ++++++++++++++++++ 2 files changed, 831 insertions(+), 10 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java index 76de97dcfb39..78f268833f66 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java @@ -63,6 +63,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -203,6 +204,8 @@ public void commit(List commitMessages) { } } + rejectWritesToExplicitLocationPartitions(messages); + Set> partitionSpecs = new HashSet<>(); Set clearedPartitionPaths = new HashSet<>(); Path staticPartitionPath = null; @@ -388,6 +391,157 @@ private void publishMessages(List messages) throws IOExce } } + /** Rejects writes whose files would belong to a catalog partition outside the table root. */ + private void rejectWritesToExplicitLocationPartitions(List messages) { + if (partitionManager == null || partitionKeys == null || partitionKeys.isEmpty()) { + return; + } + + try { + rejectWritesToExplicitLocationPartitionsBeforeMutation(messages); + } catch (RuntimeException failure) { + // Nothing has been published yet. Abort should clean staging only: the target may be + // a pre-existing file in a directory owned by another partition. + markPublishedTargetsToPreserveOnAbort(messages); + throw failure; + } + } + + private void rejectWritesToExplicitLocationPartitionsBeforeMutation( + List messages) { + List affectedPartitions; + Set> affectedDefaultPaths = new LinkedHashSet<>(); + LinkedHashMap overwritePrefix = null; + if (overwrite && staticPartitions != null && !staticPartitions.isEmpty()) { + affectedPartitions = registeredPartitionMetadata(staticPartitions); + LinkedHashMap staticSpec = orderedPartitionPrefix(staticPartitions); + if (staticSpec.size() == partitionKeys.size()) { + affectedDefaultPaths.add(staticSpec); + } else { + overwritePrefix = staticSpec; + } + } else if (overwrite && !replacesOnlyWrittenPartitions()) { + affectedPartitions = partitionManager.listPartitions(Collections.emptyMap(), null); + } else { + Set> affectedSpecs = new LinkedHashSet<>(); + for (TwoPhaseCommitMessage message : messages) { + Path targetPath = message.getCommitter().targetPath(); + if (targetPath == null) { + // Preserve the established failure order for a malformed committer. The + // publish or registration path will report its own contract violation. + continue; + } + affectedSpecs.add( + extractPartitionSpecFromPath(targetPath.getParent(), partitionKeys)); + } + if (!overwrite + && affectedSpecs.isEmpty() + && staticPartitions != null + && staticPartitions.size() == partitionKeys.size()) { + affectedSpecs.add(staticPartitions); + } + if (affectedSpecs.isEmpty()) { + return; + } + affectedPartitions = + partitionManager.listPartitionsByNames(new ArrayList<>(affectedSpecs)); + for (Map affectedSpec : affectedSpecs) { + affectedDefaultPaths.add(orderedPartitionPrefix(affectedSpec)); + } + } + for (Partition partition : affectedPartitions) { + if (partition.location() != null) { + throw unsupportedExplicitLocation(overwrite ? "Overwriting" : "Writing", partition); + } + } + rejectOverlappingDefaultPaths(affectedDefaultPaths, overwritePrefix); + } + + private void rejectOverlappingDefaultPaths( + Set> affectedDefaultPaths, + @Nullable LinkedHashMap overwritePrefix) { + if (affectedDefaultPaths.isEmpty() && overwritePrefix == null) { + return; + } + + OptionalLong explicitLocationCount = partitionManager.explicitPartitionLocationCount(); + if (explicitLocationCount.isPresent() && explicitLocationCount.getAsLong() == 0) { + return; + } + + FormatTablePartitionPathResolver ownership = + new FormatTablePartitionPathResolver( + new Path(location), + tableIdentifier.getFullName(), + formatTablePartitionOnlyValueInPath); + FormatTablePartitionPathResolver explicitOwnership = + overwritePrefix == null + ? null + : new FormatTablePartitionPathResolver( + new Path(location), + tableIdentifier.getFullName(), + formatTablePartitionOnlyValueInPath); + for (Partition partition : partitionManager.listPartitions(Collections.emptyMap(), null)) { + LinkedHashMap spec = orderedRegisteredPartitionSpec(partition.spec()); + Path resolved = ownership.resolve(spec, partition.location()); + ownership.remember(spec, resolved); + if (explicitOwnership != null && partition.location() != null) { + explicitOwnership.remember(spec, resolved); + } + } + + for (LinkedHashMap affectedDefaultPath : affectedDefaultPaths) { + ownership.remember(affectedDefaultPath, ownership.resolve(affectedDefaultPath, null)); + } + if (overwritePrefix != null) { + explicitOwnership.remember( + overwritePrefix, explicitOwnership.resolve(overwritePrefix, null)); + } + } + + private LinkedHashMap orderedRegisteredPartitionSpec( + Map partitionSpec) { + if (partitionSpec.size() != partitionKeys.size()) { + throw new IllegalStateException( + String.format( + "Catalog returned incomplete partition spec %s for Format Table %s.", + partitionSpec, tableIdentifier.getFullName())); + } + return orderedPartitionPrefix(partitionSpec); + } + + private LinkedHashMap orderedPartitionPrefix( + Map partitionSpec) { + LinkedHashMap orderedSpec = new LinkedHashMap<>(); + for (int i = 0; i < partitionSpec.size(); i++) { + String key = partitionKeys.get(i); + if (!partitionSpec.containsKey(key)) { + throw new IllegalArgumentException( + String.format( + "Partition spec %s is not a leading prefix of partition keys %s.", + partitionSpec, partitionKeys)); + } + orderedSpec.put(key, partitionSpec.get(key)); + } + return orderedSpec; + } + + private List registeredPartitionMetadata(Map partitionSpec) { + if (partitionSpec.size() == partitionKeys.size()) { + return partitionManager.listPartitionsByNames(Collections.singletonList(partitionSpec)); + } + return partitionManager.listPartitions(partitionSpec, null); + } + + private UnsupportedOperationException unsupportedExplicitLocation( + String operation, Partition partition) { + return new UnsupportedOperationException( + String.format( + "%s catalog-managed Format Table partition %s with explicit location " + + "'%s' is not supported.", + operation, partition.spec(), partition.location())); + } + private List publishMessage(TwoPhaseCommitMessage message) { try { message.getCommitter().commit(fileIO); @@ -1070,7 +1224,14 @@ public void truncateTable() { // Emptying the table is emptying every partition it has, and which those are is answered // by whatever the table reads its partitions from. if (partitionManager != null) { - truncate(registeredPartitions(Collections.emptyMap())); + List partitions = + partitionManager.listPartitions(Collections.emptyMap(), null); + for (Partition partition : partitions) { + if (partition.location() != null) { + throw unsupportedExplicitLocation("Truncating", partition); + } + } + truncate(partitions.stream().map(Partition::spec).collect(Collectors.toList())); return; } // Filesystem partition discovery: the partition directories the scan reads are the table. @@ -1102,23 +1263,35 @@ public void truncatePartitions(List> partitionSpecs) { complete.add(partitionSpec); } } - Set> registered = + Map, Partition> registered = complete.isEmpty() - ? Collections.emptySet() + ? Collections.emptyMap() : partitionManager.listPartitionsByNames(complete).stream() - .map(Partition::spec) - .collect(Collectors.toSet()); - List> partitions = new ArrayList<>(); + .collect( + Collectors.toMap( + Partition::spec, + Function.identity(), + (left, right) -> left, + LinkedHashMap::new)); + Map, Partition> partitions = new LinkedHashMap<>(); for (Map partitionSpec : partitionSpecs) { if (partitionSpec.size() == partitionKeys.size()) { - if (registered.contains(partitionSpec)) { - partitions.add(partitionSpec); + Partition partition = registered.get(partitionSpec); + if (partition != null) { + partitions.put(partition.spec(), partition); } } else { - partitions.addAll(registeredPartitions(partitionSpec)); + for (Partition partition : partitionManager.listPartitions(partitionSpec, null)) { + partitions.putIfAbsent(partition.spec(), partition); + } + } + } + for (Partition partition : partitions.values()) { + if (partition.location() != null) { + throw unsupportedExplicitLocation("Truncating", partition); } } - truncate(partitions); + truncate(partitions.values().stream().map(Partition::spec).collect(Collectors.toList())); } /** diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java index 737fd65adbfa..f0251bb713e5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java @@ -19,6 +19,7 @@ package org.apache.paimon.table.format; import org.apache.paimon.CoreOptions; +import org.apache.paimon.PagedList; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.FileSystemCatalog; import org.apache.paimon.catalog.Identifier; @@ -63,6 +64,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.OptionalLong; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; @@ -87,8 +89,10 @@ import static org.assertj.core.api.Assertions.entry; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -101,6 +105,496 @@ class FormatTableCommitTest { @TempDir java.nio.file.Path tempDir; + @Test + void testAppendRejectsRegisteredExplicitLocationBeforeAnyTableMutation() throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "append-explicit-location"); + Map spec = Collections.singletonMap("part", "external"); + Path defaultPartitionPath = new Path(tablePath, "part=external"); + Path targetPath = new Path(defaultPartitionPath, "data-new.csv"); + TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class); + when(committer.targetPath()).thenReturn(targetPath); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.listPartitionsByNames(Collections.singletonList(spec))) + .thenReturn( + Collections.singletonList( + partitionAt(spec, "file:/external/part=external"))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Collections.singletonList("part"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + false, + Identifier.create("location_db", "location_table"), + null, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ true); + fileIO.startTrackingMutations(); + + assertThatThrownBy( + () -> + commit.commit( + Collections.singletonList( + new TwoPhaseCommitMessage(committer)))) + .isInstanceOf(RuntimeException.class) + .hasRootCauseInstanceOf(UnsupportedOperationException.class) + .hasRootCauseMessage( + "Writing catalog-managed Format Table partition {part=external} with " + + "explicit location 'file:/external/part=external' is not " + + "supported."); + + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + assertThat(fileIO.exists(defaultPartitionPath)).isFalse(); + verify(committer, never()).commit(fileIO); + verify(committer, never()).clean(fileIO); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + + @Test + void testStaticOverwriteRejectsExplicitLocationBeforeDeletingOldData() throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "overwrite-explicit-location"); + Map spec = Collections.singletonMap("part", "external"); + Path oldData = new Path(tablePath, "part=external/data-old.csv"); + fileIO.writeFile(oldData, "old", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.listPartitionsByNames(Collections.singletonList(spec))) + .thenReturn( + Collections.singletonList( + partitionAt(spec, "file:/external/part=external"))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Collections.singletonList("part"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + true, + Identifier.create("location_db", "location_table"), + spec, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ true); + fileIO.startTrackingMutations(); + + assertThatThrownBy(() -> commit.commit(Collections.emptyList())) + .isInstanceOf(RuntimeException.class) + .hasRootCauseInstanceOf(UnsupportedOperationException.class) + .hasRootCauseMessage( + "Overwriting catalog-managed Format Table partition {part=external} with " + + "explicit location 'file:/external/part=external' is not " + + "supported."); + + assertThat(fileIO.exists(oldData)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + + @Test + void testStaticOverwriteRejectsFutureDefaultPathOwnedByExplicitPartition() throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "overwrite-future-default-location"); + Map explicitSpec = Collections.singletonMap("part", "external"); + Map targetSpec = Collections.singletonMap("part", "future"); + Path targetPartitionPath = new Path(tablePath, "part=future"); + Path explicitData = new Path(targetPartitionPath, "data-explicit.csv"); + fileIO.writeFile(explicitData, "explicit", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(1)); + when(partitionManager.listPartitionsByNames(Collections.singletonList(targetSpec))) + .thenReturn(Collections.emptyList()); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Collections.singletonList( + partitionAt(explicitSpec, targetPartitionPath.toString()))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Collections.singletonList("part"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + true, + Identifier.create("location_db", "location_table"), + targetSpec, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ true); + fileIO.startTrackingMutations(); + + assertThatThrownBy(() -> commit.commit(Collections.emptyList())) + .isInstanceOf(RuntimeException.class) + .hasRootCauseInstanceOf(IllegalStateException.class) + .hasRootCauseMessage( + "Catalog returned overlapping locations for different partitions of Format Table location_db.location_table."); + + assertThat(fileIO.exists(explicitData)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + + @Test + void testAppendRejectsFutureDefaultPathOwnedByExplicitPartition() throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "append-future-default-location"); + Map explicitSpec = Collections.singletonMap("part", "external"); + Map targetSpec = Collections.singletonMap("part", "future"); + Path targetPartitionPath = new Path(tablePath, "part=future"); + Path explicitData = new Path(targetPartitionPath, "data-explicit.csv"); + Path targetPath = new Path(targetPartitionPath, "data-new.csv"); + fileIO.writeFile(explicitData, "explicit", false); + TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class); + when(committer.targetPath()).thenReturn(targetPath); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(1)); + when(partitionManager.listPartitionsByNames(Collections.singletonList(targetSpec))) + .thenReturn(Collections.emptyList()); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Collections.singletonList( + partitionAt(explicitSpec, targetPartitionPath.toString()))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Collections.singletonList("part"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + false, + Identifier.create("location_db", "location_table"), + null, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ true); + fileIO.startTrackingMutations(); + + assertThatThrownBy( + () -> + commit.commit( + Collections.singletonList( + new TwoPhaseCommitMessage(committer)))) + .isInstanceOf(RuntimeException.class) + .hasRootCauseInstanceOf(IllegalStateException.class) + .hasRootCauseMessage( + "Catalog returned overlapping locations for different partitions of Format Table location_db.location_table."); + + assertThat(fileIO.exists(explicitData)).isTrue(); + assertThat(fileIO.exists(targetPath)).isFalse(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(committer, never()).commit(fileIO); + verify(committer, never()).clean(fileIO); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + + @Test + void testDynamicOverwriteRejectsAffectedExplicitLocationBeforeReplacingData() throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "dynamic-overwrite-explicit-location"); + Map spec = Collections.singletonMap("part", "external"); + Path oldData = new Path(tablePath, "part=external/data-old.csv"); + fileIO.writeFile(oldData, "old", false); + TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class); + when(committer.targetPath()).thenReturn(new Path(tablePath, "part=external/data-new.csv")); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.listPartitionsByNames(Collections.singletonList(spec))) + .thenReturn( + Collections.singletonList( + partitionAt(spec, "file:/external/part=external"))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Collections.singletonList("part"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + true, + Identifier.create("location_db", "location_table"), + null, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ true); + fileIO.startTrackingMutations(); + + assertThatThrownBy( + () -> + commit.commit( + Collections.singletonList( + new TwoPhaseCommitMessage(committer)))) + .isInstanceOf(RuntimeException.class) + .hasRootCauseInstanceOf(UnsupportedOperationException.class) + .hasRootCauseMessage( + "Overwriting catalog-managed Format Table partition {part=external} with " + + "explicit location 'file:/external/part=external' is not " + + "supported."); + + assertThat(fileIO.exists(oldData)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(committer, never()).commit(fileIO); + verify(committer, never()).clean(fileIO); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + + @Test + void testDynamicOverwriteRejectsFutureDefaultPathOwnedByExplicitPartition() throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = + new Path(new Path(tempDir.toUri()), "dynamic-overwrite-future-default-location"); + Map explicitSpec = Collections.singletonMap("part", "external"); + Map targetSpec = Collections.singletonMap("part", "future"); + Path targetPartitionPath = new Path(tablePath, "part=future"); + Path explicitData = new Path(targetPartitionPath, "data-explicit.csv"); + Path targetPath = new Path(targetPartitionPath, "data-new.csv"); + fileIO.writeFile(explicitData, "explicit", false); + TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class); + when(committer.targetPath()).thenReturn(targetPath); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.empty()); + when(partitionManager.listPartitionsByNames(Collections.singletonList(targetSpec))) + .thenReturn(Collections.emptyList()); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Collections.singletonList( + partitionAt(explicitSpec, targetPartitionPath.toString()))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Collections.singletonList("part"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + true, + Identifier.create("location_db", "location_table"), + null, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ true); + fileIO.startTrackingMutations(); + + assertThatThrownBy( + () -> + commit.commit( + Collections.singletonList( + new TwoPhaseCommitMessage(committer)))) + .isInstanceOf(RuntimeException.class) + .hasRootCauseInstanceOf(IllegalStateException.class) + .hasRootCauseMessage( + "Catalog returned overlapping locations for different partitions of Format Table location_db.location_table."); + + assertThat(fileIO.exists(explicitData)).isTrue(); + assertThat(fileIO.exists(targetPath)).isFalse(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(committer, never()).commit(fileIO); + verify(committer, never()).clean(fileIO); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + + @Test + void testStaticPrefixOverwriteRejectsExplicitDescendantBeforeDeletingAnyPartition() + throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "prefix-overwrite-explicit-location"); + Map prefix = Collections.singletonMap("year", "2025"); + Map defaultSpec = partitionSpec("2025", "10"); + Map explicitSpec = partitionSpec("2025", "11"); + Path defaultData = new Path(tablePath, "year=2025/month=10/data-old.csv"); + Path explicitResidue = new Path(tablePath, "year=2025/month=11/data-old.csv"); + fileIO.writeFile(defaultData, "default", false); + fileIO.writeFile(explicitResidue, "residue", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.listPartitions(prefix, null)) + .thenReturn( + Arrays.asList( + partitionAt(defaultSpec, null), + partitionAt(explicitSpec, "file:/external/year=2025/month=11"))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Arrays.asList("year", "month"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + true, + Identifier.create("location_db", "location_table"), + prefix, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ true); + fileIO.startTrackingMutations(); + + assertThatThrownBy(() -> commit.commit(Collections.emptyList())) + .isInstanceOf(RuntimeException.class) + .hasRootCauseInstanceOf(UnsupportedOperationException.class) + .hasRootCauseMessage( + "Overwriting catalog-managed Format Table partition " + + "{year=2025, month=11} with explicit location " + + "'file:/external/year=2025/month=11' is not supported."); + + assertThat(fileIO.exists(defaultData)).isTrue(); + assertThat(fileIO.exists(explicitResidue)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + + @Test + void testStaticPrefixOverwriteRejectsLocationOwnedByPartitionOutsidePrefix() throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = + new Path(new Path(tempDir.toUri()), "prefix-overwrite-future-default-location"); + Map prefix = Collections.singletonMap("year", "2025"); + Map explicitSpec = partitionSpec("2024", "external"); + Path targetPrefixPath = new Path(tablePath, "year=2025"); + Path explicitLocation = new Path(targetPrefixPath, "month=future"); + Path explicitData = new Path(explicitLocation, "data-explicit.csv"); + fileIO.writeFile(explicitData, "explicit", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(1)); + when(partitionManager.listPartitions(prefix, null)).thenReturn(Collections.emptyList()); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Collections.singletonList( + partitionAt(explicitSpec, explicitLocation.toString()))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Arrays.asList("year", "month"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + true, + Identifier.create("location_db", "location_table"), + prefix, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ true); + fileIO.startTrackingMutations(); + + assertThatThrownBy(() -> commit.commit(Collections.emptyList())) + .isInstanceOf(RuntimeException.class) + .hasRootCauseInstanceOf(IllegalStateException.class) + .hasRootCauseMessage( + "Catalog returned overlapping locations for different partitions of Format Table location_db.location_table."); + + assertThat(fileIO.exists(explicitData)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + + @Test + void testStaticOverwriteAllowsRegisteredDefaultPathWithOtherExplicitPartition() + throws Exception { + LocalFileIO fileIO = LocalFileIO.create(); + Path tablePath = new Path(new Path(tempDir.toUri()), "overwrite-registered-default"); + Map targetSpec = Collections.singletonMap("part", "default"); + Map explicitSpec = Collections.singletonMap("part", "external"); + Path oldData = new Path(tablePath, "part=default/data-old.csv"); + fileIO.writeFile(oldData, "old", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(1)); + when(partitionManager.listPartitionsByNames(Collections.singletonList(targetSpec))) + .thenReturn(Collections.singletonList(partitionAt(targetSpec, null))); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Arrays.asList( + partitionAt(targetSpec, null), + partitionAt(explicitSpec, "file:/external/part=external"))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Collections.singletonList("part"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + true, + Identifier.create("location_db", "location_table"), + targetSpec, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ true); + + commit.commit(Collections.emptyList()); + + assertThat(fileIO.exists(oldData)).isFalse(); + assertThat(fileIO.exists(new Path(tablePath, "part=default"))).isTrue(); + verify(partitionManager).createPartitions(anyList(), eq(true), anyList(), eq(true)); + } + + @Test + void testWholeTableOverwriteRejectsExplicitLocationBeforeDeletingAnyPartition() + throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "whole-overwrite-explicit-location"); + Map defaultSpec = Collections.singletonMap("part", "default"); + Map explicitSpec = Collections.singletonMap("part", "external"); + Path defaultData = new Path(tablePath, "part=default/data-old.csv"); + Path explicitResidue = new Path(tablePath, "part=external/data-old.csv"); + fileIO.writeFile(defaultData, "default", false); + fileIO.writeFile(explicitResidue, "residue", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Arrays.asList( + partitionAt(defaultSpec, null), + partitionAt(explicitSpec, "file:/external/part=external"))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Collections.singletonList("part"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + true, + Identifier.create("location_db", "location_table"), + null, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ false); + fileIO.startTrackingMutations(); + + assertThatThrownBy(() -> commit.commit(Collections.emptyList())) + .isInstanceOf(RuntimeException.class) + .hasRootCauseInstanceOf(UnsupportedOperationException.class) + .hasRootCauseMessage( + "Overwriting catalog-managed Format Table partition {part=external} with " + + "explicit location 'file:/external/part=external' is not " + + "supported."); + + assertThat(fileIO.exists(defaultData)).isTrue(); + assertThat(fileIO.exists(explicitResidue)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + @Test void testPartitionRegistrationFailureDeletesPublishedTarget() throws Exception { LocalFileIO fileIO = LocalFileIO.create(); @@ -152,6 +646,8 @@ void testRegistrationResponseLossStillDeletesPublishedTarget() throws Exception Catalog catalog = mock(Catalog.class); List> registeredPartitions = new ArrayList<>(); RuntimeException registrationFailure = new RuntimeException("registration response lost"); + when(catalog.listPartitionsPaged(eq(identifier), anyInt(), isNull(), isNull())) + .thenReturn(new PagedList<>(Collections.emptyList(), null)); doAnswer( invocation -> { List> batch = invocation.getArgument(1); @@ -846,6 +1342,36 @@ void testTruncateTableEmptiesTheRegisteredPartitionsOfACatalogManagedTable() thr assertThat(fileIO.exists(awaitingRepairData)).isTrue(); } + @Test + void testTruncateTableRejectsExplicitLocationBeforeDeletingOrReporting() throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "truncate-explicit-location"); + Map spec = Collections.singletonMap("part", "external"); + Path defaultData = new Path(tablePath, "part=external/data-old.csv"); + fileIO.writeFile(defaultData, "old", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Collections.singletonList( + partitionAt(spec, "file:/external/part=external"))); + FormatTableCommit commit = + truncatingCommit(tablePath, fileIO, false, partitionManager, "part"); + fileIO.startTrackingMutations(); + + assertThatThrownBy(commit::truncateTable) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage( + "Truncating catalog-managed Format Table partition {part=external} with " + + "explicit location 'file:/external/part=external' is not " + + "supported."); + + assertThat(fileIO.exists(defaultData)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + @Test void testTruncateTableOnlyEmptiesTheDirectoriesThatAreItsPartitions() throws Exception { LocalFileIO fileIO = LocalFileIO.create(); @@ -931,6 +1457,74 @@ void testTruncatePartitionsStaysInsideThePartitionsItNames() throws Exception { assertThat(fileIO.exists(novemberData)).isTrue(); } + @Test + void testTruncateNamedPartitionRejectsExplicitLocationBeforeDeletingOrReporting() + throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "truncate-named-explicit-location"); + Map spec = partitionSpec("2025", "10"); + Path defaultData = new Path(tablePath, "year=2025/month=10/data-old.csv"); + fileIO.writeFile(defaultData, "old", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.listPartitionsByNames(Collections.singletonList(spec))) + .thenReturn( + Collections.singletonList( + partitionAt(spec, "file:/external/year=2025/month=10"))); + FormatTableCommit commit = + truncatingCommit(tablePath, fileIO, false, partitionManager, "year", "month"); + fileIO.startTrackingMutations(); + + assertThatThrownBy(() -> commit.truncatePartitions(Collections.singletonList(spec))) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage( + "Truncating catalog-managed Format Table partition " + + "{year=2025, month=10} with explicit location " + + "'file:/external/year=2025/month=10' is not supported."); + + assertThat(fileIO.exists(defaultData)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + + @Test + void testTruncatePrefixRejectsExplicitDescendantBeforeMutatingDefaultDescendant() + throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "truncate-prefix-explicit-location"); + Map prefix = Collections.singletonMap("year", "2025"); + Map defaultSpec = partitionSpec("2025", "10"); + Map explicitSpec = partitionSpec("2025", "11"); + Path defaultData = new Path(tablePath, "year=2025/month=10/data-old.csv"); + Path explicitResidue = new Path(tablePath, "year=2025/month=11/data-old.csv"); + fileIO.writeFile(defaultData, "default", false); + fileIO.writeFile(explicitResidue, "residue", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.listPartitions(prefix, null)) + .thenReturn( + Arrays.asList( + partitionAt(defaultSpec, null), + partitionAt(explicitSpec, "file:/external/year=2025/month=11"))); + FormatTableCommit commit = + truncatingCommit(tablePath, fileIO, false, partitionManager, "year", "month"); + fileIO.startTrackingMutations(); + + assertThatThrownBy(() -> commit.truncatePartitions(Collections.singletonList(prefix))) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage( + "Truncating catalog-managed Format Table partition " + + "{year=2025, month=11} with explicit location " + + "'file:/external/year=2025/month=11' is not supported."); + + assertThat(fileIO.exists(defaultData)).isTrue(); + assertThat(fileIO.exists(explicitResidue)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + @Test void testTruncatingAPrefixClearsThePartitionsBelowItButNotStagingTrees() throws Exception { LocalFileIO fileIO = LocalFileIO.create(); @@ -3023,6 +3617,60 @@ public boolean delete(Path path, boolean recursive) { } } + private static class MutationTrackingLocalFileIO extends LocalFileIO { + + private final AtomicInteger deleteCalls = new AtomicInteger(); + private final AtomicInteger mkdirsCalls = new AtomicInteger(); + private boolean tracking; + + private void startTrackingMutations() { + deleteCalls.set(0); + mkdirsCalls.set(0); + tracking = true; + } + + @Override + public boolean delete(Path path, boolean recursive) throws IOException { + if (tracking) { + deleteCalls.incrementAndGet(); + } + return super.delete(path, recursive); + } + + @Override + public boolean mkdirs(Path path) throws IOException { + if (tracking) { + mkdirsCalls.incrementAndGet(); + } + return super.mkdirs(path); + } + + private int deleteCalls() { + return deleteCalls.get(); + } + + private int mkdirsCalls() { + return mkdirsCalls.get(); + } + } + + private static Partition partitionAt(Map spec, String location) { + return new Partition( + spec, + 0, + 0, + 0, + 0, + PartitionStatistics.UNKNOWN_TOTAL_BUCKETS, + false, + null, + null, + null, + null, + null, + location); + } + private static Map partitionSpec(String year, String month) { LinkedHashMap spec = new LinkedHashMap<>(); spec.put("year", year); From 5f8ef5ee26b09ba3fda3bbb2e5e6bac4ab9d683c Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 2 Sep 2026 03:20:56 +0800 Subject: [PATCH 05/15] spark: support location-aware format table partition DDL --- .../spark/format/PaimonFormatTable.scala | 182 +++++++++++--- .../FormatTablePartitionManagementTest.scala | 228 +++++++++++++++++- 2 files changed, 364 insertions(+), 46 deletions(-) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala index 08de9785674b..a410f22e2fbc 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala @@ -21,10 +21,11 @@ package org.apache.paimon.spark.format import org.apache.paimon.CoreOptions import org.apache.paimon.format.csv.CsvOptions import org.apache.paimon.fs.Path +import org.apache.paimon.partition.{Partition, PartitionLocation} import org.apache.paimon.spark.{BaseTable, FormatTableScanBuilder} import org.apache.paimon.spark.write.{BaseV2WriteBuilder, PaimonWriteRequirement} import org.apache.paimon.table.FormatTable -import org.apache.paimon.table.format.FormatTablePartitionManager +import org.apache.paimon.table.format.{FormatTablePartitionManager, FormatTablePartitionPathResolver} import org.apache.paimon.table.sink.BatchTableCommit import org.apache.paimon.types.RowType import org.apache.paimon.utils.{PartitionPathUtils, StringUtils} @@ -47,7 +48,8 @@ import java.util import java.util.{Collections, Locale, Map => JMap, Objects} import scala.collection.JavaConverters._ -import scala.collection.mutable.{ArrayBuffer, HashSet} +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer case class PaimonFormatTable(table: FormatTable) extends BaseTable @@ -198,14 +200,37 @@ case class PaimonFormatTable(table: FormatTable) private[spark] def formatTablePartitionsRegistered( partitionNames: Array[Array[String]], rows: Array[InternalRow]): Array[Boolean] = { + formatTablePartitions(partitionNames, rows).map(_ != null) + } + + /** + * Resolves complete specs to their catalog metadata, aligned with the input arrays. A missing + * partition is represented by null so callers can preserve Spark's positional error reporting. + */ + private[spark] def formatTablePartitions( + partitionNames: Array[Array[String]], + rows: Array[InternalRow]): Array[Partition] = { if (rows.isEmpty) { return Array.empty } val requested = rows.zip(partitionNames).map { case (row, names) => toPaimonPartition(row, names.toSeq) } val registered = requirePartitionManager().listPartitionsByNames(requested.toSeq.asJava) - val registeredSpecs = registered.asScala.map(_.spec().asScala.toMap).toSet - requested.map(spec => registeredSpecs.contains(spec.asScala.toMap)) + val bySpec = mutable.LinkedHashMap.empty[Map[String, String], Partition] + registered.asScala.foreach { + partition => + val spec = validateCatalogRegisteredPartition(partition.spec()).asScala.toMap + bySpec.get(spec).foreach { + previous => + if (!Objects.equals(previous.location(), partition.location())) { + throw new IllegalStateException( + s"Catalog returned conflicting locations for partition $spec of Format Table " + + s"${table.fullName()}.") + } + } + bySpec.put(spec, partition) + } + requested.map(spec => bySpec.getOrElse(spec.asScala.toMap, null)) } /** @@ -241,19 +266,38 @@ case class PaimonFormatTable(table: FormatTable) rows: Array[InternalRow], maps: Array[JMap[String, String]], ignoreIfExists: Boolean): Unit = { - if (maps.exists(_.keySet().asScala.exists(_.equalsIgnoreCase("location")))) { - throw new UnsupportedOperationException( - s"ADD PARTITION with LOCATION is not supported for Format Table ${table.fullName()}.") - } val onlyValueInPath = CoreOptions.fromMap(table.options()).formatTablePartitionOnlyValueInPath() val partitionKeys = table.partitionKeys().asScala.toSeq rows.foreach(row => requireNameablePartitionValues("ADD PARTITION", row, partitionKeys)) - val specs = rows.map(row => toPaimonPartition(row, partitionKeys.take(row.numFields))).toSeq - // Resolve (and path-safety validate) every directory before mutating anything. + val partitions = rows + .zip(maps) + .map { + case (row, properties) => + val spec = toPaimonPartition(row, partitionKeys.take(row.numFields)) + val location = properties.asScala.collectFirst { + case (key, value) if key.equalsIgnoreCase("location") => + normalizeExplicitPartitionLocation(value, spec, onlyValueInPath) + } + spec -> location + } + .toSeq + val specs = partitions.map(_._1) + // Resolve (and path-safety validate) every default directory before mutating anything. val partitionPaths = - specs.map(spec => resolvePartitionPathWithinTable(orderedSpec(spec), onlyValueInPath)) - requirePartitionManager().createPartitions(specs.asJava, ignoreIfExists) + partitions.collect { + case (spec, None) => + resolvePartitionPathWithinTable(orderedSpec(spec), onlyValueInPath) + } + val locations = partitions.collect { + case (spec, Some(location)) => new PartitionLocation(spec, location) + } + if (locations.isEmpty) { + requirePartitionManager().createPartitions(specs.asJava, ignoreIfExists) + } else { + requirePartitionManager() + .createPartitions(specs.asJava, ignoreIfExists, null, false, locations.asJava) + } // Create the partition directories client-side (symmetric with DROP deleting them), so an // added partition exists on the filesystem and a subsequent scan returns an empty partition // rather than depending on lazy directory creation, matching Hive ADD PARTITION semantics. @@ -261,6 +305,27 @@ case class PaimonFormatTable(table: FormatTable) partitionPaths.foreach(partitionPath => fileIO.mkdirs(partitionPath)) } + private def normalizeExplicitPartitionLocation( + location: String, + spec: JMap[String, String], + onlyValueInPath: Boolean): String = { + try { + FormatTablePartitionPathResolver + .resolveExplicitLocation( + new Path(table.location()), + orderedSpec(spec), + onlyValueInPath, + location) + .toString + } catch { + case error: IllegalArgumentException => + throw new IllegalArgumentException( + s"ADD PARTITION LOCATION is invalid for partition $spec of Format Table " + + s"${table.fullName()}.", + error) + } + } + /** * Drops the given partitions: complete specs are unregistered and their directories deleted * as-is, partial specs are expanded to the registered leaf partitions they cover. Callers are @@ -276,17 +341,40 @@ case class PaimonFormatTable(table: FormatTable) } val requested = rows.zip(partitionNames).map { case (row, names) => toPaimonPartition(row, names.toSeq) } - val partitions = ArrayBuffer.empty[JMap[String, String]] - val seenPartitions = HashSet.empty[Map[String, String]] - - def addPartition(partition: JMap[String, String]): Unit = { - if (seenPartitions.add(partition.asScala.toMap)) { - partitions += partition + val onlyValueInPath = + CoreOptions.fromMap(table.options()).formatTablePartitionOnlyValueInPath() + // Validate every user-supplied spec before asking the catalog for its authoritative metadata. + // This preserves the path-safety boundary even when an invalid spec is not registered. + requested.foreach(spec => resolvePartitionPathWithinTable(orderedSpec(spec), onlyValueInPath)) + val partitions = ArrayBuffer.empty[Partition] + val seenPartitions = mutable.LinkedHashMap.empty[Map[String, String], Partition] + + def addPartition(partition: Partition): Unit = { + val validated = validateCatalogRegisteredPartition(partition.spec()) + val key = validated.asScala.toMap + seenPartitions.get(key) match { + case Some(previous) if !Objects.equals(previous.location(), partition.location()) => + throw new IllegalStateException( + s"Catalog returned conflicting locations for partition $key of Format Table " + + s"${table.fullName()}.") + case Some(_) => + case None => + seenPartitions.put(key, partition) + partitions += partition } } - // Preserve exact requests as-is and let discovery add only missing complete leaves. - requested.filter(_.size() == partitionKeyCount).foreach(addPartition) + // Resolve exact requests back to their full metadata. Location is authoritative catalog state, + // so DROP must not infer it from the request or silently treat an unknown response as default. + val completeRequests = requested.filter(_.size() == partitionKeyCount) + if (completeRequests.nonEmpty) { + val requestedKeys = completeRequests.map(_.asScala.toMap).toSet + requirePartitionManager() + .listPartitionsByNames(completeRequests.toSeq.asJava) + .asScala + .filter(partition => requestedKeys.contains(partition.spec().asScala.toMap)) + .foreach(addPartition) + } val partialSpecs = requested.filter(_.size() < partitionKeyCount).toSeq.distinct if (partialSpecs.nonEmpty) { def matchesRequestedPartial(partition: JMap[String, String]): Boolean = { @@ -304,38 +392,58 @@ case class PaimonFormatTable(table: FormatTable) partition => val validated = validateCatalogRegisteredPartition(partition.spec()) if (matchesRequestedPartial(validated)) { - addPartition(validated) + addPartition(partition) } } } dropCatalogRegisteredPartitions(partitions.toSeq) } - private def dropCatalogRegisteredPartitions(partitions: Seq[JMap[String, String]]): Boolean = { - // Unregister first so new queries stop seeing the partition, then delete the data directory - // with the table FileIO (client-side; the server never deletes data). A deletion failure leaves - // the possibly incomplete directory invisible; it must not be registered again automatically. + private def dropCatalogRegisteredPartitions(partitions: Seq[Partition]): Boolean = { if (partitions.isEmpty) { return true } val onlyValueInPath = CoreOptions.fromMap(table.options()).formatTablePartitionOnlyValueInPath() - // Resolve (and path-safety validate) every partition directory before any mutation, so a - // traversal attempt ('.'/'..') fails the whole DROP before unregistering anything. - val partitionPaths = - partitions.map(spec => resolvePartitionPathWithinTable(orderedSpec(spec), onlyValueInPath)) - logInfo("Try to drop catalog-registered partitions: " + partitions.mkString(",")) - requirePartitionManager().dropPartitions(partitions.asJava) val fileIO = table.fileIO() - partitionPaths.foreach { - partitionPath => - val deleted = fileIO.delete(partitionPath, true) - if (!deleted && fileIO.exists(partitionPath)) { - throw new java.io.IOException( - s"FileIO reported that partition directory $partitionPath was not deleted.") + val resolved = partitions.map { + partition => + val spec = validateCatalogRegisteredPartition(partition.spec()) + val defaultPath = + resolvePartitionPathWithinTable(orderedSpec(spec), onlyValueInPath) + (partition, spec, defaultPath) + } + // An explicit-location partition is metadata-only on DROP. A default directory at the same + // spec would become visible again through MSCK, so reject the whole batch before unregistering. + resolved.foreach { + case (partition, spec, defaultPath) if partition.location() != null => + if (fileIO.exists(defaultPath)) { + throw new IllegalStateException( + s"Cannot drop explicit-location partition $spec of Format Table " + + s"${table.fullName()} because its default partition directory still exists.") } + case _ => } + + val specs = resolved.map(_._2) + logInfo("Try to drop catalog-registered partitions: " + specs.mkString(",")) + requirePartitionManager().dropPartitions(specs.asJava) + // Default-location partitions keep the existing unregister-then-delete ordering. A deletion + // failure leaves the incomplete directory invisible. Explicit locations are never probed or + // deleted. + resolved + .collect { + case (partition, _, defaultPath) if partition.location() == null => defaultPath + } + .foreach { + partitionPath => + val deleted = fileIO.delete(partitionPath, true) + if (!deleted && fileIO.exists(partitionPath)) { + throw new java.io.IOException( + s"FileIO reported that partition directory $partitionPath was not deleted.") + } + } true } diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala index 0488594fcc93..61baa9396a58 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala @@ -22,7 +22,7 @@ import org.apache.paimon.catalog.{CatalogContext, Identifier} import org.apache.paimon.fs.{FileIO, Path} import org.apache.paimon.fs.local.LocalFileIO import org.apache.paimon.options.Options -import org.apache.paimon.partition.{Partition, PartitionStatistics} +import org.apache.paimon.partition.{Partition, PartitionLocation, PartitionStatistics} import org.apache.paimon.predicate.Predicate import org.apache.paimon.table.FormatTable import org.apache.paimon.table.format.FormatTablePartitionManager @@ -141,8 +141,9 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { assert(dropCalls == 0) } - test("catalog-managed ADD with LOCATION is rejected before any catalog RPC") { + test("catalog-managed ADD with LOCATION forwards metadata without creating a default directory") { var createCalls = 0 + var forwardedLocations = Seq.empty[PartitionLocation] val gateway = new FormatTablePartitionManager { override def createPartitions( partitions: JList[JMap[String, String]], @@ -150,6 +151,16 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { statistics: JList[PartitionStatistics], replaceStatistics: Boolean): Unit = createCalls += 1 + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean, + statistics: JList[PartitionStatistics], + replaceStatistics: Boolean, + partitionLocations: JList[PartitionLocation]): Unit = { + createCalls += 1 + forwardedLocations = partitionLocations.asScala.toSeq + } + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {} override def listPartitionsByNames( @@ -162,17 +173,86 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { Collections.emptyList() } + val tablePath = + new Path(Files.createTempDirectory("catalog-partition-format-add-location").toUri) + val externalPath = + new Path(Files.createTempDirectory("catalog-partition-format-add-external").toUri) + val requestedLocation = + "FILE://" + externalPath.toUri.getPath.replace("/", "//") + "/" val sparkTable = - new PaimonFormatTable(formatTableWithCatalogManagedPartitions(partitionManager = gateway)) - val error = intercept[UnsupportedOperationException] { + new PaimonFormatTable( + formatTableWithCatalogManagedPartitions( + location = tablePath.toString, + partitionManager = gateway)) + try { sparkTable.createFormatTablePartitions( Array(new GenericInternalRow(Array[Any](20260715, 10))), - Array(Map("location" -> "file:/tmp/custom").asJava), + Array(Map("location" -> requestedLocation).asJava), ignoreIfExists = true) + + assert(createCalls == 1) + assert( + forwardedLocations.map(location => location.spec().asScala.toMap -> location.location()) == + Seq(partitionSpec(20260715, 10) -> + externalPath.toString.stripSuffix("/"))) + assert(!sparkTable.table.fileIO().exists(new Path(tablePath, "dt=20260715/hh=10"))) + } finally { + sparkTable.table.fileIO().delete(tablePath, true) + sparkTable.table.fileIO().delete(externalPath, true) } + } - assert(error.getMessage.contains("LOCATION")) - assert(createCalls == 0) + test("catalog-managed ADD rejects locations owned by the table before catalog RPC") { + var createCalls = 0 + val gateway = new FormatTablePartitionManager { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean, + statistics: JList[PartitionStatistics], + replaceStatistics: Boolean): Unit = createCalls += 1 + + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean, + statistics: JList[PartitionStatistics], + replaceStatistics: Boolean, + partitionLocations: JList[PartitionLocation]): Unit = createCalls += 1 + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {} + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[Partition] = + Collections.emptyList() + + override def listPartitions( + prefix: JMap[String, String], + filter: Predicate): JList[Partition] = + Collections.emptyList() + } + + val tablePath = + new Path(Files.createTempDirectory("catalog-partition-format-add-owned-location").toUri) + val sparkTable = + new PaimonFormatTable( + formatTableWithCatalogManagedPartitions( + location = tablePath.toString, + partitionManager = gateway)) + val defaultPath = new Path(tablePath, "dt=20260715/hh=10") + try { + Seq(tablePath, defaultPath, tablePath.getParent).foreach { + location => + intercept[IllegalArgumentException] { + sparkTable.createFormatTablePartitions( + Array(new GenericInternalRow(Array[Any](20260715, 10))), + Array(Map("location" -> location.toString).asJava), + ignoreIfExists = true) + } + } + + assert(createCalls == 0) + } finally { + sparkTable.table.fileIO().delete(tablePath, true) + } } test("catalog-managed ADD rejects a partition value that would escape the table location") { @@ -264,7 +344,7 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { override def listPartitionsByNames( partitions: JList[JMap[String, String]]): JList[Partition] = - Collections.emptyList() + registeredPartitions(partitions.asScala.map(_.asScala.toMap).toSeq: _*) override def listPartitions( prefix: JMap[String, String], @@ -292,6 +372,136 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { } } + test("catalog-managed DROP of an explicit-location partition rejects default-directory residue") { + val fileIO = LocalFileIO.create() + val tablePath = + new Path(Files.createTempDirectory("catalog-partition-format-drop-location-residue").toUri) + val defaultDir = new Path(tablePath, "dt=20260715/hh=10") + val externalDir = + new Path(Files.createTempDirectory("catalog-partition-format-drop-location-external").toUri) + val spec = partitionSpec(20260715, 10) + var dropCalls = 0 + val explicitPartition = + new Partition( + spec.asJava, + 0L, + 0L, + 0L, + 0L, + 0, + false, + null, + null, + null, + null, + null, + externalDir.toString) + val gateway = new FormatTablePartitionManager { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean, + statistics: JList[PartitionStatistics], + replaceStatistics: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1 + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[Partition] = + Collections.singletonList(explicitPartition) + + override def listPartitions( + prefix: JMap[String, String], + filter: Predicate): JList[Partition] = + Collections.singletonList(explicitPartition) + } + + try { + fileIO.writeFile(new Path(defaultDir, "residue.csv"), "default", false) + fileIO.writeFile(new Path(externalDir, "external.csv"), "external", false) + val sparkTable = + new PaimonFormatTable( + formatTableWithCatalogManagedPartitions(fileIO, tablePath.toString, gateway)) + + val error = intercept[IllegalStateException] { + sparkTable.dropFormatTablePartitions( + Array(Array("dt", "hh")), + Array(partitionRow(20260715, 10))) + } + + assert(error.getMessage.contains("default partition directory")) + assert(dropCalls == 0) + assert(fileIO.exists(defaultDir)) + assert(fileIO.exists(externalDir)) + } finally { + fileIO.delete(tablePath, true) + fileIO.delete(externalDir, true) + } + } + + test("catalog-managed DROP unregisters an explicit location without deleting its data") { + val fileIO = LocalFileIO.create() + val tablePath = + new Path(Files.createTempDirectory("catalog-partition-format-drop-location-table").toUri) + val externalDir = + new Path(Files.createTempDirectory("catalog-partition-format-drop-location-data").toUri) + val externalFile = new Path(externalDir, "external.csv") + val spec = partitionSpec(20260715, 10) + val explicitPartition = + new Partition( + spec.asJava, + 0L, + 0L, + 0L, + 0L, + 0, + false, + null, + null, + null, + null, + null, + externalDir.toString) + var dropped = Seq.empty[Map[String, String]] + val gateway = new FormatTablePartitionManager { + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean, + statistics: JList[PartitionStatistics], + replaceStatistics: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + dropped = partitions.asScala.map(_.asScala.toMap).toSeq + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[Partition] = + Collections.singletonList(explicitPartition) + + override def listPartitions( + prefix: JMap[String, String], + filter: Predicate): JList[Partition] = + Collections.singletonList(explicitPartition) + } + + try { + fileIO.writeFile(externalFile, "external", false) + val sparkTable = + new PaimonFormatTable( + formatTableWithCatalogManagedPartitions(fileIO, tablePath.toString, gateway)) + + assert( + sparkTable + .dropFormatTablePartitions(Array(Array("dt", "hh")), Array(partitionRow(20260715, 10)))) + + assert(dropped == Seq(spec)) + assert(fileIO.exists(externalFile)) + assert(!fileIO.exists(new Path(tablePath, "dt=20260715/hh=10"))) + } finally { + fileIO.delete(tablePath, true) + fileIO.delete(externalDir, true) + } + } + test("catalog-managed direct partial DROP expands leading values to exact leaf partitions") { val fileIO = LocalFileIO.create() val tablePath = @@ -809,7 +1019,7 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1 override def listPartitionsByNames(partitions: JList[JMap[String, String]]): JList[Partition] = - Collections.emptyList() + registeredPartitions(partitions.asScala.map(_.asScala.toMap).toSeq: _*) override def listPartitions(prefix: JMap[String, String], filter: Predicate): JList[Partition] = Collections.emptyList() From 133727709b3d53b95470a6c2957a9f9f6abc33e0 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 2 Sep 2026 03:23:34 +0800 Subject: [PATCH 06/15] spark: protect maintenance operations for explicit partition locations --- .../format/FormatTablePartitionRepair.java | 10 ++- ...nAnalyzeFormatTablePartitionsCommand.scala | 10 ++- .../FormatTablePartitionRepairTest.java | 70 ++++++++++++++++++- .../CatalogManagedPartitionAnalyzeTest.scala | 26 +++++++ 4 files changed, 111 insertions(+), 5 deletions(-) diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/format/FormatTablePartitionRepair.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/format/FormatTablePartitionRepair.java index d213f4d86b29..39aab11f55f5 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/format/FormatTablePartitionRepair.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/format/FormatTablePartitionRepair.java @@ -142,9 +142,13 @@ static int apply( boolean dropPartitions, @Nullable FormatTablePartitionStatsCollector statsCollector) { Set> registeredPartitions = new HashSet<>(); + Set> explicitlyLocatedPartitions = new HashSet<>(); for (Partition partition : partitionManager.listPartitions(Collections.emptyMap(), null)) { registeredPartitions.add(partition.spec()); + if (partition.location() != null) { + explicitlyLocatedPartitions.add(partition.spec()); + } } Set> filesystemSet = new HashSet<>(filesystemPartitions); @@ -161,7 +165,8 @@ static int apply( List> dropDiff = new ArrayList<>(); if (dropPartitions) { for (Map partition : registeredPartitions) { - if (!filesystemSet.contains(partition)) { + if (!filesystemSet.contains(partition) + && !explicitlyLocatedPartitions.contains(partition)) { dropDiff.add(partition); } } @@ -177,7 +182,8 @@ static int apply( // exists to correct. Without ADD it stays inside the already registered set. List> measured = new ArrayList<>(); for (Map partition : filesystemPartitions) { - if (addPartitions || registeredPartitions.contains(partition)) { + if ((addPartitions || registeredPartitions.contains(partition)) + && !explicitlyLocatedPartitions.contains(partition)) { measured.add(partition); } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonAnalyzeFormatTablePartitionsCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonAnalyzeFormatTablePartitionsCommand.scala index 0f68c63773e7..da8b337dcda6 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonAnalyzeFormatTablePartitionsCommand.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonAnalyzeFormatTablePartitionsCommand.scala @@ -62,11 +62,17 @@ case class PaimonAnalyzeFormatTablePartitionsCommand( override def run(sparkSession: SparkSession): Seq[Row] = { val prefix = leadingPrefix(sparkSession) - val partitions = v2Table.partitionManager + val registeredPartitions = v2Table.partitionManager .listPartitions(prefix.asJava, null) .asScala - .map(_.spec()) .toList + val explicitlyLocated = registeredPartitions.filter(_.location() != null) + if (explicitlyLocated.nonEmpty) { + throw new UnsupportedOperationException( + s"ANALYZE TABLE cannot measure partitions with an explicit location in Format Table " + + s"${v2Table.name()}: " + explicitlyLocated.map(_.spec()).mkString("[", ", ", "]")) + } + val partitions = registeredPartitions.map(_.spec()) if (partitions.isEmpty && prefix.nonEmpty) { throw new NoSuchPartitionException( diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/format/FormatTablePartitionRepairTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/format/FormatTablePartitionRepairTest.java index 1ee208f2b9ca..8cd9efc78677 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/format/FormatTablePartitionRepairTest.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/format/FormatTablePartitionRepairTest.java @@ -171,6 +171,26 @@ void dropOnlyUnregistersOnlyMissingDirectories() { assertThat(catalog.createdPartitions).isEmpty(); } + @Test + void dropRepairNeverUnregistersAnExplicitLocation() { + RecordingPartitionManager catalog = new RecordingPartitionManager(); + catalog.registerAtLocation( + spec("dt", "20260714"), tempDir.resolve("external").toUri().toString()); + + int applied = + FormatTablePartitionRepair.apply( + catalog, + Collections.emptyList(), + Collections.singletonList("dt"), + false, + true); + + // Its absence below the table root says nothing about an explicitly located partition. + assertThat(applied).isZero(); + assertThat(catalog.droppedPartitions).isEmpty(); + assertThat(catalog.createdPartitions).isEmpty(); + } + @Test void addOnlyNeverDropsStaleCatalogPartitions() { RecordingPartitionManager catalog = new RecordingPartitionManager(); @@ -409,6 +429,34 @@ void repairMeasuresEveryPartitionOnDiskAndReplacesTheirStatistics() throws Excep assertThat(catalog.droppedPartitions).isEmpty(); } + @Test + void repairNeverMeasuresAnExplicitLocationFromTheDefaultDirectory() throws Exception { + java.nio.file.Path defaultDirectory = + Files.createDirectories(tempDir.resolve("dt=20260701")); + Files.write( + defaultDirectory.resolve("stale.csv"), + Collections.singletonList("1"), + StandardCharsets.UTF_8); + + RecordingPartitionManager catalog = new RecordingPartitionManager(); + catalog.registerAtLocation( + spec("dt", "20260701"), tempDir.resolve("external").toUri().toString()); + FormatTable table = formatTable(tempDir.toUri().toString(), catalog); + + int applied = + FormatTablePartitionRepair.repair( + new PaimonFormatTable(table), + true, + true, + new FormatTablePartitionStatsCollector(table, 1)); + + // The directory below the table root is residue, not the explicit partition's data. + assertThat(applied).isZero(); + assertThat(catalog.createdPartitions).isEmpty(); + assertThat(catalog.reportedStatistics).isEmpty(); + assertThat(catalog.droppedPartitions).isEmpty(); + } + @Test void repairWritesNothingWhenMeasuringAPartitionFailsToList() throws Exception { Files.write( @@ -576,6 +624,7 @@ private static class RecordingPartitionManager implements FormatTablePartitionMa private static final long serialVersionUID = 1L; private final List> registered = new ArrayList<>(); + private final Map, String> locations = new LinkedHashMap<>(); private final List> requestedPrefixes = new ArrayList<>(); private final List>> createdPartitions = new ArrayList<>(); private final List createIgnoreFlags = new ArrayList<>(); @@ -587,6 +636,11 @@ private void register(List> partitions) { registered.addAll(partitions); } + private void registerAtLocation(Map partition, String location) { + registered.add(partition); + locations.put(partition, location); + } + @Override public void createPartitions( List> partitions, @@ -615,7 +669,21 @@ public List listPartitions( requestedPrefixes.add(prefix); List partitions = new ArrayList<>(registered.size()); for (Map spec : registered) { - partitions.add(new Partition(spec, 0L, 0L, 0L, 0L, 0, false)); + partitions.add( + new Partition( + spec, + 0L, + 0L, + 0L, + 0L, + 0, + false, + null, + null, + null, + null, + null, + locations.get(spec))); } return partitions; } diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionAnalyzeTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionAnalyzeTest.scala index e915b6fedce9..8c7dd945f277 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionAnalyzeTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionAnalyzeTest.scala @@ -663,6 +663,32 @@ class CatalogManagedPartitionAnalyzeTest extends PaimonSparkTestWithRestCatalogB } } + test("ANALYZE fails before updating an explicitly located partition") { + val tableName = "analyze_explicit_location" + withTable(tableName) { + createTable(tableName) + val external = new Path(new Path(tempDBDir.toURI), "analyze-explicit-location") + formatTable(tableName) + .fileIO() + .writeFile(new Path(external, "data.csv"), "1,payload\n", false) + sql( + s"ALTER TABLE ${qualified(tableName)} ADD PARTITION " + + s"(dt = '20260101', hour = '00') LOCATION '${external.toString}'") + + val before = statisticsOf(tableName, "20260101", "00") + val error = intercept[Exception] { + sql( + s"ANALYZE TABLE ${qualified(tableName)} PARTITION " + + s"(dt = '20260101', hour = '00') COMPUTE STATISTICS NOSCAN").collect() + } + + val messages = causeMessages(error) + assert(messages.contains("explicit location"), messages) + assert(statisticsOf(tableName, "20260101", "00") == before) + assert(formatTable(tableName).fileIO().exists(new Path(external, "data.csv"))) + } + } + test("ANALYZE is rejected for a format table discovering partitions from the filesystem") { val tableName = "analyze_filesystem_partitions" withTable(tableName) { From 6748340df0e9d5a8f27ec3e0c8dcf486ab815a50 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 2 Sep 2026 03:26:07 +0800 Subject: [PATCH 07/15] spark: advertise explicit partition location capability --- .../org/apache/paimon/spark/SparkCatalog.java | 33 ++++++++- .../spark/SparkCatalogWithRestTest.java | 69 ++++++++++++++++++- .../PaimonSparkTestWithRestCatalogBase.scala | 7 +- ...CatalogManagedPartitionDdlParityTest.scala | 36 ++++++++++ 4 files changed, 139 insertions(+), 6 deletions(-) diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java index 6fe7ea50335e..5b62922b871f 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java @@ -26,8 +26,11 @@ import org.apache.paimon.catalog.PropertyChange; import org.apache.paimon.function.Function; import org.apache.paimon.function.FunctionDefinition; +import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; +import org.apache.paimon.rest.RESTApi; import org.apache.paimon.rest.RESTCatalog; +import org.apache.paimon.rest.RESTCatalogFactory; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.spark.catalog.FormatTableCatalog; @@ -89,6 +92,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -137,10 +141,13 @@ public void initialize(String name, CaseInsensitiveStringMap options) { SparkSession sparkSession = PaimonSparkSession$.MODULE$.active(); checkRequiredConfigurations(sparkSession); this.catalogName = name; + Options catalogOptions = Options.fromMap(options.asCaseSensitiveMap()); + if (RESTCatalogFactory.IDENTIFIER.equalsIgnoreCase( + catalogOptions.get(CatalogOptions.METASTORE))) { + addCapability(catalogOptions, RESTApi.HEADER_PREFIX + RESTApi.CAPABILITIES_HEADER); + } CatalogContext catalogContext = - CatalogContext.create( - Options.fromMap(options.asCaseSensitiveMap()), - sparkSession.sessionState().newHadoopConf()); + CatalogContext.create(catalogOptions, sparkSession.sessionState().newHadoopConf()); this.catalog = CatalogFactory.createCatalog(catalogContext); this.defaultDatabase = options.getOrDefault(DEFAULT_DATABASE.key(), DEFAULT_DATABASE.defaultValue()); @@ -164,6 +171,26 @@ public void initialize(String name, CaseInsensitiveStringMap options) { } } + private static void addCapability(Options options, String key) { + List equivalentKeys = + options.keySet().stream() + .filter(optionKey -> key.equalsIgnoreCase(optionKey)) + .collect(Collectors.toList()); + Set capabilities = new LinkedHashSet<>(); + for (String equivalentKey : equivalentKeys) { + String configuredCapabilities = options.get(equivalentKey); + if (!StringUtils.isBlank(configuredCapabilities)) { + Arrays.stream(configuredCapabilities.split(",")) + .map(String::trim) + .filter(capability -> !capability.isEmpty()) + .forEach(capabilities::add); + } + } + capabilities.add(RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY); + equivalentKeys.forEach(options::remove); + options.setString(key, String.join(",", capabilities)); + } + @Override public Catalog paimonCatalog() { return catalog; diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java index c9ff2b5fd34a..cdaa18f15593 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithRestTest.java @@ -40,6 +40,7 @@ import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.predicate.Transform; import org.apache.paimon.predicate.UpperTransform; +import org.apache.paimon.rest.RESTApi; import org.apache.paimon.rest.RESTCatalogInternalOptions; import org.apache.paimon.rest.RESTCatalogServer; import org.apache.paimon.rest.auth.AuthProvider; @@ -56,6 +57,7 @@ import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.connector.catalog.CatalogManager; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -95,7 +97,9 @@ public void before() throws IOException { RESTCatalogInternalOptions.PREFIX.key(), "paimon", CatalogOptions.WAREHOUSE.key(), - warehouse), + warehouse, + RESTCatalogInternalOptions.SERVER_CAPABILITIES.key(), + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY), ImmutableMap.of()); AuthProvider authProvider = new BearTokenAuthProvider(initToken); restCatalogServer = new RESTCatalogServer(dataPath, authProvider, config, warehouse); @@ -108,6 +112,9 @@ public void before() throws IOException { .config("spark.sql.catalog.paimon.uri", serverUrl) .config("spark.sql.catalog.paimon.token", initToken) .config("spark.sql.catalog.paimon.warehouse", warehouse) + .config( + "spark.sql.catalog.paimon.header.X-Paimon-Capabilities", + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY + "0") .config( "spark.sql.catalog.paimon.token.provider", AuthProviderEnum.BEAR.identifier()) @@ -140,6 +147,66 @@ public void testTable() { assertThat(spark.sql("SHOW TABLES").collectAsList().size() == 0); } + @Test + public void testSparkCatalogDeclaresPartitionLocationCapability() { + String expectedHeader = + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY + + "0," + + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY; + Catalog paimonCatalog = getPaimonCatalog(); + assertThat(paimonCatalog.options()) + .containsEntry(RESTApi.HEADER_PREFIX + RESTApi.CAPABILITIES_HEADER, expectedHeader); + + restCatalogServer.clearReceivedHeaders(); + paimonCatalog.listDatabases(); + + assertThat(restCatalogServer.getReceivedHeaders()) + .isNotEmpty() + .allSatisfy( + headers -> + assertThat(headers) + .containsEntry( + RESTApi.CAPABILITIES_HEADER.toLowerCase(), + expectedHeader)); + } + + @Test + public void testSparkCatalogMergesCaseVariantCapabilityHeader() throws Exception { + Map options = new HashMap<>(); + options.put(CatalogOptions.METASTORE.key(), "rest"); + options.put("uri", serverUrl); + options.put("token", initToken); + options.put(CatalogOptions.WAREHOUSE.key(), warehouse); + options.put("token.provider", AuthProviderEnum.BEAR.identifier()); + options.put("header.x-PaImOn-CaPaBiLiTiEs", "client-v0,client-v0"); + + SparkCatalog catalog = new SparkCatalog(); + catalog.initialize("case_variant", new CaseInsensitiveStringMap(options)); + Catalog paimonCatalog = catalog.paimonCatalog(); + String canonicalKey = RESTApi.HEADER_PREFIX + RESTApi.CAPABILITIES_HEADER; + String expectedHeader = "client-v0," + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY; + try { + assertThat( + paimonCatalog.options().keySet().stream() + .filter(key -> key.equalsIgnoreCase(canonicalKey))) + .containsExactly(canonicalKey); + assertThat(paimonCatalog.options()).containsEntry(canonicalKey, expectedHeader); + + restCatalogServer.clearReceivedHeaders(); + paimonCatalog.listDatabases(); + assertThat(restCatalogServer.getReceivedHeaders()) + .isNotEmpty() + .allSatisfy( + headers -> + assertThat(headers) + .containsEntry( + RESTApi.CAPABILITIES_HEADER.toLowerCase(), + expectedHeader)); + } finally { + paimonCatalog.close(); + } + } + @Test public void testFunction() throws Exception { List inputParams = new ArrayList<>(); diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSparkTestWithRestCatalogBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSparkTestWithRestCatalogBase.scala index 12b5c8febecc..abf8458b64a3 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSparkTestWithRestCatalogBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSparkTestWithRestCatalogBase.scala @@ -19,7 +19,7 @@ package org.apache.paimon.spark import org.apache.paimon.options.CatalogOptions -import org.apache.paimon.rest.{RESTCatalogFactory, RESTCatalogInternalOptions, RESTCatalogServer} +import org.apache.paimon.rest.{RESTApi, RESTCatalogFactory, RESTCatalogInternalOptions, RESTCatalogServer} import org.apache.paimon.rest.auth.{AuthProviderEnum, BearTokenAuthProvider} import org.apache.paimon.rest.responses.ConfigResponse import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap @@ -42,7 +42,10 @@ class PaimonSparkTestWithRestCatalogBase extends PaimonSparkTestBase { RESTCatalogInternalOptions.PREFIX.key, "paimon", CatalogOptions.WAREHOUSE.key, - warehouse), + warehouse, + RESTCatalogInternalOptions.SERVER_CAPABILITIES.key, + RESTApi.FORMAT_TABLE_PARTITION_LOCATION_CAPABILITY + ), ImmutableMap.of() ) val authProvider = new BearTokenAuthProvider(initToken) diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionDdlParityTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionDdlParityTest.scala index fb77ab834bae..08e59bd03ad5 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionDdlParityTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionDdlParityTest.scala @@ -36,6 +36,42 @@ import scala.collection.JavaConverters._ */ class CatalogManagedPartitionDdlParityTest extends PaimonSparkTestWithRestCatalogBase { + test("ADD PARTITION LOCATION reads external data without creating the default directory") { + val tableName = "ddl_add_location" + withTable(tableName) { + createTable(tableName) + withTempDir { + externalDir => + val table = formatTable(tableName) + val externalLocation = new Path(externalDir.toURI.toString).toString + table + .fileIO() + .writeFile(new Path(externalLocation, "part-00001.csv"), "1,a\n", false) + + sql( + s"ALTER TABLE ${qualified(tableName)} ADD " + + s"PARTITION (dt = '20260101', hour = '00') " + + s"LOCATION '$externalLocation'") + + val partitions = + paimonCatalog + .listPartitions(Identifier.create(dbName0, tableName)) + .asScala + assert(partitions.size == 1) + assert(partitions.head.location() == externalLocation) + assert( + !table + .fileIO() + .exists(new Path(table.location(), "dt=20260101/hour=00"))) + checkAnswer( + sql( + s"SELECT id, payload, dt, hour FROM ${qualified(tableName)} " + + s"WHERE dt = '20260101' AND hour = '00'"), + Seq(Row(1, "a", "20260101", "00"))) + } + } + } + test("ADD PARTITION IF NOT EXISTS is a repeatable no-op, a strict repeat is an error") { val tableName = "ddl_add_if_not_exists" withTable(tableName) { From acba8ac7e4584f3799cb5a784dc9d427b5f96d5d Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 2 Sep 2026 03:27:48 +0800 Subject: [PATCH 08/15] docs: document explicit format table partition locations --- docs/docs/flink/sql-ddl.md | 5 +++++ docs/docs/spark/sql-ddl.md | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/docs/docs/flink/sql-ddl.md b/docs/docs/flink/sql-ddl.md index 654e477079ef..3d4927181ad1 100644 --- a/docs/docs/flink/sql-ddl.md +++ b/docs/docs/flink/sql-ddl.md @@ -49,6 +49,11 @@ and a table whose catalog holds no partitions reads as empty. Flink has no SQL c them: use Spark's `MSCK REPAIR TABLE` or the catalog's partition API. Flink writes on a current version do register the partitions they produce. +Flink does not currently support catalog-managed Format Table partitions registered with an +explicit `LOCATION` and does not declare the corresponding REST capability. A REST service should +reject access to a table containing such partitions instead of letting Flink derive and read the +wrong directory. Use Spark for this workflow. + In a REST catalog, asking for catalog-managed partitions on a table that cannot have them — an external table, or `format-table.implementation = engine` — fails. In any other catalog the option keeps the meaning it has always had on a Format Table — none — and partitions come from the diff --git a/docs/docs/spark/sql-ddl.md b/docs/docs/spark/sql-ddl.md index 657ef33220b4..608105c16d0a 100644 --- a/docs/docs/spark/sql-ddl.md +++ b/docs/docs/spark/sql-ddl.md @@ -213,6 +213,8 @@ partitions and Spark supports the standard partition DDL: ```sql ALTER TABLE my_table ADD PARTITION (dt='2025-01-01'); +ALTER TABLE my_table ADD PARTITION (dt='2024-12-31') + LOCATION 'oss://archive-bucket/events/dt=2024-12-31'; ALTER TABLE my_table DROP PARTITION (dt='2025-01-01'); MSCK REPAIR TABLE my_table; SHOW PARTITIONS my_table; @@ -226,6 +228,28 @@ On a Format Table whose partitions are discovered from the filesystem, `ADD PART added partition before any data is written returns no rows. `DROP PARTITION` unregisters the partition and deletes its directory. +`ADD PARTITION ... LOCATION` is available when the REST catalog server advertises the +`format-table-partition-location-v1` capability. It registers the supplied directory URI as +metadata without creating, listing, moving, or deleting data there. The location must be an +absolute, non-root URI; +it is normalized before registration, and a catalog may restrict the allowed schemes. Partitions +with and without `LOCATION` can coexist in one table. + +An explicitly located partition is read with the credentials in the client's catalog context when +it is outside the table root; the table's REST data token is not reused for that location. Dropping +such a partition only unregisters it and never deletes or probes the explicit URI. The drop is +rejected if the partition's derived directory under the table root exists, because a later +`MSCK REPAIR TABLE` could otherwise register that directory as the same partition. + +Explicitly located partitions are currently read-only. `INSERT`, `INSERT OVERWRITE`, and +`TRUNCATE TABLE` fail before changing files or catalog metadata when their scope includes one. +`ANALYZE TABLE` also rejects a selected explicitly located partition. `MSCK REPAIR TABLE` leaves +these registrations alone and does not collect statistics from their locations. + +Spark declares this capability for REST catalogs. Other clients must not declare it until their +complete read and mutation paths implement the same rules; a REST service should reject reads of a +table containing explicit partition locations from a client that did not declare the capability. + A partition value that is empty or all whitespace is rejected by `ADD PARTITION`, `DROP PARTITION` and `TRUNCATE PARTITION`. Such a value is written to the partition named by `partition.default-name` (`__DEFAULT_PARTITION__` unless configured otherwise), the same partition From 5165a059284ff0dd6381af69a464e1c2a39c66c0 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 2 Sep 2026 11:35:18 +0800 Subject: [PATCH 09/15] fix(format): validate partition registry before mutations --- .../table/format/CatalogSplitEnumerator.java | 25 +- .../table/format/FormatTableCommit.java | 249 ++++++++----- .../FormatTablePartitionPathResolver.java | 2 +- ...FormatTablePartitionRegistryValidator.java | 95 +++++ .../apache/paimon/rest/RESTCatalogServer.java | 2 +- .../CatalogManagedPartitionScanTest.java | 4 +- .../table/format/FormatTableCommitTest.java | 337 ++++++++++++++++++ .../FormatTablePartitionPathResolverTest.java | 16 +- .../PaimonFormatTablePartitionDdlExec.scala | 17 +- .../spark/format/PaimonFormatTable.scala | 114 +++--- .../FormatTablePartitionDdlPlanningTest.scala | 91 ++++- .../FormatTablePartitionManagementTest.scala | 90 ++++- ...CatalogManagedPartitionDdlParityTest.scala | 53 +++ 13 files changed, 916 insertions(+), 179 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionRegistryValidator.java diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java b/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java index 2c4c001cf1f5..5c91f2955632 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java @@ -175,7 +175,8 @@ private void validatePathOwnership( && (!explicitLocationCount.isPresent() || explicitLocationCount.getAsLong() != 0)) { authoritative = findCatalogPartitions(null); } - validateExplicitLocationCount(authoritative, explicitLocationCount); + FormatTablePartitionRegistryValidator.validateExplicitLocationCount( + authoritative, explicitLocationCount, table.fullName()); if (authoritative.stream().noneMatch(partition -> partition.location() != null)) { return; } @@ -185,26 +186,6 @@ private void validatePathOwnership( toSpecsAndPaths(authoritative, coreOptions.formatTablePartitionOnlyValueInPath()); } - private void validateExplicitLocationCount( - List authoritative, OptionalLong explicitLocationCount) { - if (!explicitLocationCount.isPresent()) { - return; - } - long observed = - authoritative.stream() - .filter( - partition -> - partition.location() != null - && !partition.location().isEmpty()) - .count(); - if (observed != explicitLocationCount.getAsLong()) { - throw new IllegalStateException( - String.format( - "Catalog reported %d explicit partition locations for format table %s, but its complete listing contained %d.", - explicitLocationCount.getAsLong(), table.fullName(), observed)); - } - } - private List findCatalogPartitions(@Nullable PartitionPredicate partitionFilter) { Optional extracted = FormatTableScan.extractPartitionPredicate(partitionFilter); Map prefix = leadingEqualityPrefix(extracted); @@ -283,7 +264,7 @@ private List, Path>> toSpecsAndPaths( for (Partition partition : partitions) { LinkedHashMap spec = normalizeSpec(partition.spec(), onlyValueInPath); Path partitionPath = pathResolver.resolve(spec, partition.location()); - if (pathResolver.remember(spec, partitionPath)) { + if (pathResolver.validateAndRecord(spec, partitionPath)) { result.add(Pair.of(spec, partitionPath)); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java index 78f268833f66..625f0fa50e65 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java @@ -63,6 +63,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -70,6 +71,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; import static org.apache.paimon.table.format.FormatBatchWriteBuilder.validateStaticPartition; @@ -204,7 +207,8 @@ public void commit(List commitMessages) { } } - rejectWritesToExplicitLocationPartitions(messages); + List validatedPartitions = + rejectWritesToExplicitLocationPartitions(messages); Set> partitionSpecs = new HashSet<>(); Set clearedPartitionPaths = new HashSet<>(); @@ -246,7 +250,10 @@ public void commit(List commitMessages) { // is everything the table holds rather than the files this commit happens to // write: a statement whose query returns nothing still empties the table. clearedPartitionPaths.addAll( - deletePreviousDataFiles(tableDataDirectories(), 0, cleanupThreadNum)); + deletePreviousDataFiles( + tableDataDirectories(validatedPartitions), + 0, + cleanupThreadNum)); } } if (overwrite) { @@ -392,13 +399,14 @@ private void publishMessages(List messages) throws IOExce } /** Rejects writes whose files would belong to a catalog partition outside the table root. */ - private void rejectWritesToExplicitLocationPartitions(List messages) { + private List rejectWritesToExplicitLocationPartitions( + List messages) { if (partitionManager == null || partitionKeys == null || partitionKeys.isEmpty()) { - return; + return Collections.emptyList(); } try { - rejectWritesToExplicitLocationPartitionsBeforeMutation(messages); + return rejectWritesToExplicitLocationPartitionsBeforeMutation(messages); } catch (RuntimeException failure) { // Nothing has been published yet. Abort should clean staging only: the target may be // a pre-existing file in a directory owned by another partition. @@ -407,21 +415,27 @@ private void rejectWritesToExplicitLocationPartitions(List rejectWritesToExplicitLocationPartitionsBeforeMutation( List messages) { - List affectedPartitions; - Set> affectedDefaultPaths = new LinkedHashSet<>(); - LinkedHashMap overwritePrefix = null; + Predicate affectsPartition; + Supplier> zeroCountSelection; + Set> affectedDefaultSpecs = new LinkedHashSet<>(); + LinkedHashMap affectedDefaultPrefix = null; if (overwrite && staticPartitions != null && !staticPartitions.isEmpty()) { - affectedPartitions = registeredPartitionMetadata(staticPartitions); LinkedHashMap staticSpec = orderedPartitionPrefix(staticPartitions); if (staticSpec.size() == partitionKeys.size()) { - affectedDefaultPaths.add(staticSpec); + affectedDefaultSpecs.add(staticSpec); + affectsPartition = partition -> partition.spec().equals(staticPartitions); } else { - overwritePrefix = staticSpec; + affectedDefaultPrefix = staticSpec; + affectsPartition = + partition -> partitionSpecMatchesPrefix(partition.spec(), staticSpec); } + zeroCountSelection = () -> registeredPartitionMetadata(staticPartitions); } else if (overwrite && !replacesOnlyWrittenPartitions()) { - affectedPartitions = partitionManager.listPartitions(Collections.emptyMap(), null); + affectsPartition = ignored -> true; + zeroCountSelection = + () -> partitionManager.listPartitions(Collections.emptyMap(), null); } else { Set> affectedSpecs = new LinkedHashSet<>(); for (TwoPhaseCommitMessage message : messages) { @@ -434,38 +448,55 @@ private void rejectWritesToExplicitLocationPartitionsBeforeMutation( affectedSpecs.add( extractPartitionSpecFromPath(targetPath.getParent(), partitionKeys)); } - if (!overwrite - && affectedSpecs.isEmpty() - && staticPartitions != null - && staticPartitions.size() == partitionKeys.size()) { - affectedSpecs.add(staticPartitions); + if (!overwrite && staticPartitions != null && !staticPartitions.isEmpty()) { + LinkedHashMap staticSpec = orderedPartitionPrefix(staticPartitions); + if (staticSpec.size() == partitionKeys.size()) { + if (affectedSpecs.isEmpty()) { + affectedSpecs.add(staticPartitions); + } + } else { + affectedDefaultPrefix = staticSpec; + } } - if (affectedSpecs.isEmpty()) { - return; + if (affectedSpecs.isEmpty() && affectedDefaultPrefix == null) { + return Collections.emptyList(); } - affectedPartitions = - partitionManager.listPartitionsByNames(new ArrayList<>(affectedSpecs)); for (Map affectedSpec : affectedSpecs) { - affectedDefaultPaths.add(orderedPartitionPrefix(affectedSpec)); + affectedDefaultSpecs.add(orderedPartitionPrefix(affectedSpec)); } - } + affectsPartition = + affectedSpecs.isEmpty() + ? ignored -> false + : partition -> affectedSpecs.contains(partition.spec()); + zeroCountSelection = + affectedSpecs.isEmpty() + ? () -> registeredPartitionMetadata(staticPartitions) + : () -> + partitionManager.listPartitionsByNames( + new ArrayList<>(affectedSpecs)); + } + + PartitionRegistryView registry = loadPartitionRegistryView(zeroCountSelection); + List affectedPartitions = + registry.partitions.stream().filter(affectsPartition).collect(Collectors.toList()); for (Partition partition : affectedPartitions) { if (partition.location() != null) { throw unsupportedExplicitLocation(overwrite ? "Overwriting" : "Writing", partition); } } - rejectOverlappingDefaultPaths(affectedDefaultPaths, overwritePrefix); + rejectOverlappingDefaultPaths(affectedDefaultSpecs, affectedDefaultPrefix, registry); + return registry.partitions; } private void rejectOverlappingDefaultPaths( - Set> affectedDefaultPaths, - @Nullable LinkedHashMap overwritePrefix) { - if (affectedDefaultPaths.isEmpty() && overwritePrefix == null) { + Set> affectedDefaultSpecs, + @Nullable LinkedHashMap affectedDefaultPrefix, + PartitionRegistryView registry) { + if (affectedDefaultSpecs.isEmpty() && affectedDefaultPrefix == null) { return; } - OptionalLong explicitLocationCount = partitionManager.explicitPartitionLocationCount(); - if (explicitLocationCount.isPresent() && explicitLocationCount.getAsLong() == 0) { + if (registry.explicitLocationsReportedAbsent) { return; } @@ -475,27 +506,28 @@ private void rejectOverlappingDefaultPaths( tableIdentifier.getFullName(), formatTablePartitionOnlyValueInPath); FormatTablePartitionPathResolver explicitOwnership = - overwritePrefix == null + affectedDefaultPrefix == null ? null : new FormatTablePartitionPathResolver( new Path(location), tableIdentifier.getFullName(), formatTablePartitionOnlyValueInPath); - for (Partition partition : partitionManager.listPartitions(Collections.emptyMap(), null)) { + for (Partition partition : registry.partitions) { LinkedHashMap spec = orderedRegisteredPartitionSpec(partition.spec()); Path resolved = ownership.resolve(spec, partition.location()); - ownership.remember(spec, resolved); + ownership.validateAndRecord(spec, resolved); if (explicitOwnership != null && partition.location() != null) { - explicitOwnership.remember(spec, resolved); + explicitOwnership.validateAndRecord(spec, resolved); } } - for (LinkedHashMap affectedDefaultPath : affectedDefaultPaths) { - ownership.remember(affectedDefaultPath, ownership.resolve(affectedDefaultPath, null)); + for (LinkedHashMap affectedDefaultSpec : affectedDefaultSpecs) { + ownership.validateAndRecord( + affectedDefaultSpec, ownership.resolve(affectedDefaultSpec, null)); } - if (overwritePrefix != null) { - explicitOwnership.remember( - overwritePrefix, explicitOwnership.resolve(overwritePrefix, null)); + if (affectedDefaultPrefix != null) { + explicitOwnership.validateAndRecord( + affectedDefaultPrefix, explicitOwnership.resolve(affectedDefaultPrefix, null)); } } @@ -533,6 +565,54 @@ private List registeredPartitionMetadata(Map partitio return partitionManager.listPartitions(partitionSpec, null); } + /** + * Loads the requested selection after an authoritative zero explicit-location count; otherwise + * loads the complete registry required for count and location-ownership validation. The + * returned view has had both its count and every returned partition validated. + */ + private PartitionRegistryView loadPartitionRegistryView( + Supplier> zeroCountSelection) { + OptionalLong explicitLocationCount = partitionManager.explicitPartitionLocationCount(); + boolean explicitLocationsReportedAbsent = + explicitLocationCount.isPresent() && explicitLocationCount.getAsLong() == 0; + List partitions = + explicitLocationsReportedAbsent + ? zeroCountSelection.get() + : partitionManager.listPartitions(Collections.emptyMap(), null); + FormatTablePartitionRegistryValidator.validateExplicitLocationCount( + partitions, explicitLocationCount, tableIdentifier.getFullName()); + FormatTablePartitionRegistryValidator.validatePartitionLocations( + partitions, + partitionKeys, + new Path(location), + tableIdentifier.getFullName(), + formatTablePartitionOnlyValueInPath); + return new PartitionRegistryView(partitions, explicitLocationsReportedAbsent); + } + + private static boolean partitionSpecMatchesPrefix( + Map partitionSpec, Map prefix) { + for (Map.Entry entry : prefix.entrySet()) { + if (!partitionSpec.containsKey(entry.getKey()) + || !Objects.equals(entry.getValue(), partitionSpec.get(entry.getKey()))) { + return false; + } + } + return true; + } + + private static final class PartitionRegistryView { + + private final List partitions; + private final boolean explicitLocationsReportedAbsent; + + private PartitionRegistryView( + List partitions, boolean explicitLocationsReportedAbsent) { + this.partitions = partitions; + this.explicitLocationsReportedAbsent = explicitLocationsReportedAbsent; + } + } + private UnsupportedOperationException unsupportedExplicitLocation( String operation, Partition partition) { return new UnsupportedOperationException( @@ -828,17 +908,17 @@ private boolean replacesOnlyWrittenPartitions() { * has not registered, or one whose name does not parse into the partition keys - and replacing * what the table holds leaves it alone, the way {@link #truncateTable()} does. */ - private List tableDataDirectories() { + private List tableDataDirectories(List validatedPartitions) { if (partitionKeys == null || partitionKeys.isEmpty()) { return Collections.singletonList(new Path(location)); } List directories = new ArrayList<>(); if (partitionManager != null) { - for (Map spec : registeredPartitions(Collections.emptyMap())) { + for (Partition partition : validatedPartitions) { directories.add( buildPartitionPath( location, - spec, + partition.spec(), formatTablePartitionOnlyValueInPath, partitionKeys)); } @@ -1225,7 +1305,11 @@ public void truncateTable() { // by whatever the table reads its partitions from. if (partitionManager != null) { List partitions = - partitionManager.listPartitions(Collections.emptyMap(), null); + loadPartitionRegistryView( + () -> + partitionManager.listPartitions( + Collections.emptyMap(), null)) + .partitions; for (Partition partition : partitions) { if (partition.location() != null) { throw unsupportedExplicitLocation("Truncating", partition); @@ -1252,40 +1336,17 @@ public void truncateTable() { @Override public void truncatePartitions(List> partitionSpecs) { + if (partitionSpecs.isEmpty()) { + return; + } if (partitionManager == null) { truncate(partitionSpecs); return; } - // Complete specs are asked for in one request; only a prefix has to be listed on its own. - List> complete = new ArrayList<>(); - for (Map partitionSpec : partitionSpecs) { - if (partitionSpec.size() == partitionKeys.size()) { - complete.add(partitionSpec); - } - } - Map, Partition> registered = - complete.isEmpty() - ? Collections.emptyMap() - : partitionManager.listPartitionsByNames(complete).stream() - .collect( - Collectors.toMap( - Partition::spec, - Function.identity(), - (left, right) -> left, - LinkedHashMap::new)); - Map, Partition> partitions = new LinkedHashMap<>(); - for (Map partitionSpec : partitionSpecs) { - if (partitionSpec.size() == partitionKeys.size()) { - Partition partition = registered.get(partitionSpec); - if (partition != null) { - partitions.put(partition.spec(), partition); - } - } else { - for (Partition partition : partitionManager.listPartitions(partitionSpec, null)) { - partitions.putIfAbsent(partition.spec(), partition); - } - } - } + PartitionRegistryView registry = + loadPartitionRegistryView(() -> listRequestedPartitions(partitionSpecs)); + Map, Partition> partitions = + selectRequestedPartitions(registry.partitions, partitionSpecs); for (Partition partition : partitions.values()) { if (partition.location() != null) { throw unsupportedExplicitLocation("Truncating", partition); @@ -1294,15 +1355,39 @@ public void truncatePartitions(List> partitionSpecs) { truncate(partitions.values().stream().map(Partition::spec).collect(Collectors.toList())); } - /** - * The registered partitions named by {@code prefix}, which names only the leading partition - * keys, or none of them. The catalog says which partitions a catalog-managed table has, so - * truncating neither empties nor registers a directory still waiting for MSCK REPAIR TABLE. - */ - private List> registeredPartitions(Map prefix) { - return partitionManager.listPartitions(prefix, null).stream() - .map(Partition::spec) - .collect(Collectors.toList()); + private List listRequestedPartitions(List> partitionSpecs) { + List> complete = + partitionSpecs.stream() + .filter(partitionSpec -> partitionSpec.size() == partitionKeys.size()) + .collect(Collectors.toList()); + List selected = new ArrayList<>(); + if (!complete.isEmpty()) { + selected.addAll(partitionManager.listPartitionsByNames(complete)); + } + for (Map partitionSpec : partitionSpecs) { + if (partitionSpec.size() < partitionKeys.size()) { + selected.addAll(partitionManager.listPartitions(partitionSpec, null)); + } + } + return selected; + } + + private Map, Partition> selectRequestedPartitions( + List registry, List> partitionSpecs) { + Map, Partition> selected = new LinkedHashMap<>(); + for (Map partitionSpec : partitionSpecs) { + boolean complete = partitionSpec.size() == partitionKeys.size(); + for (Partition partition : registry) { + boolean matches = + complete + ? partition.spec().equals(partitionSpec) + : partitionSpecMatchesPrefix(partition.spec(), partitionSpec); + if (matches) { + selected.putIfAbsent(partition.spec(), partition); + } + } + } + return selected; } private void truncate(List> partitionSpecs) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionPathResolver.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionPathResolver.java index 389407ecd6dd..c443e6a767a6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionPathResolver.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionPathResolver.java @@ -93,7 +93,7 @@ public static Path resolveExplicitLocation( * Records a resolved path. Returns false only for an identical duplicate entry for the same * spec, which must not duplicate every row in that partition. */ - boolean remember(LinkedHashMap spec, Path path) { + boolean validateAndRecord(LinkedHashMap spec, Path path) { ResolvedPath resolved = ResolvedPath.of(path); ResolvedPath previousForSpec = pathsBySpec.get(spec); if (previousForSpec != null) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionRegistryValidator.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionRegistryValidator.java new file mode 100644 index 000000000000..f5116130c128 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionRegistryValidator.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.format; + +import org.apache.paimon.fs.Path; +import org.apache.paimon.partition.Partition; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; + +/** Validates consistency properties of a catalog-managed Format Table partition registry. */ +public final class FormatTablePartitionRegistryValidator { + + private FormatTablePartitionRegistryValidator() {} + + /** + * Validates that a registry listing contains the reported number of non-empty explicit + * partition locations. + * + *

Callers must pass the complete authoritative registry unless the authoritative count is + * zero. In that case a requested selection is sufficient because any explicit location in the + * selection disproves the reported zero. + */ + public static void validateExplicitLocationCount( + List partitions, OptionalLong explicitLocationCount, String tableName) { + if (!explicitLocationCount.isPresent()) { + return; + } + long observed = + partitions.stream() + .filter( + partition -> + partition.location() != null + && !partition.location().isEmpty()) + .count(); + if (observed != explicitLocationCount.getAsLong()) { + throw new IllegalStateException( + String.format( + "Catalog reported %d explicit partition locations for format table %s, but its validated listing contained %d.", + explicitLocationCount.getAsLong(), tableName, observed)); + } + } + + /** + * Validates complete partition specs and rejects equal or nested resolved locations. + * + *

When the explicit-location count is non-zero or unavailable, callers must pass the + * complete authoritative registry. A pruned selection is safe only after an authoritative zero + * count proves that no explicit location can own a selected default path. + */ + public static void validatePartitionLocations( + List partitions, + List partitionKeys, + Path tablePath, + String tableName, + boolean onlyValueInPath) { + FormatTablePartitionPathResolver resolver = + new FormatTablePartitionPathResolver(tablePath, tableName, onlyValueInPath); + for (Partition partition : partitions) { + Map spec = partition.spec(); + if (spec == null + || spec.size() != partitionKeys.size() + || !spec.keySet().containsAll(partitionKeys)) { + throw new IllegalStateException( + String.format( + "Catalog returned incomplete partition spec %s for Format Table %s.", + spec, tableName)); + } + LinkedHashMap orderedSpec = new LinkedHashMap<>(); + for (String partitionKey : partitionKeys) { + orderedSpec.put(partitionKey, spec.get(partitionKey)); + } + Path resolved = resolver.resolve(orderedSpec, partition.location()); + resolver.validateAndRecord(orderedSpec, resolved); + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index 142d8fb14e18..5a620c6e623e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -354,7 +354,7 @@ public void setExplicitPartitionLocationCountReported(boolean reported) { this.explicitPartitionLocationCountReported = reported; } - public void setExplicitPartitionLocationCount(long explicitPartitionLocationCount) { + public void setExplicitPartitionLocationCount(@Nullable Long explicitPartitionLocationCount) { this.explicitPartitionLocationCountOverride = explicitPartitionLocationCount; } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java index 14f177c57f19..0f43b13d28e6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java @@ -462,7 +462,7 @@ void testFullScanFailsWhenExplicitLocationCountExceedsObservedLocations() { assertThatThrownBy(() -> new FormatTableScan(table, null, null).plan().splits()) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("reported 1 explicit partition locations") - .hasMessageContaining("complete listing contained 0"); + .hasMessageContaining("validated listing contained 0"); assertThat(fileIO.listedPaths).isEmpty(); } @@ -485,7 +485,7 @@ void testFilteredScanDoesNotCountEmptyLocationAsObserved() { assertThatThrownBy(() -> new FormatTableScan(table, filter, null).plan().splits()) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("reported 1 explicit partition locations") - .hasMessageContaining("complete listing contained 0"); + .hasMessageContaining("validated listing contained 0"); assertThat(fileIO.listedPaths).isEmpty(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java index f0251bb713e5..a57fee136385 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java @@ -91,6 +91,7 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doAnswer; @@ -105,6 +106,111 @@ class FormatTableCommitTest { @TempDir java.nio.file.Path tempDir; + @Test + void testAppendRejectsExplicitLocationCountMismatchBeforePublishingOrRegistering() + throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "append-location-count-mismatch"); + Map targetSpec = Collections.singletonMap("part", "target"); + Map explicitSpec = Collections.singletonMap("part", "explicit"); + Map emptyLocationSpec = Collections.singletonMap("part", "empty"); + Path targetPath = new Path(tablePath, "part=target/data-new.csv"); + AtomicInteger publishCalls = new AtomicInteger(); + TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class); + when(committer.targetPath()).thenReturn(targetPath); + doAnswer( + invocation -> { + publishCalls.incrementAndGet(); + return null; + }) + .when(committer) + .commit(fileIO); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(2)); + when(partitionManager.listPartitionsByNames(Collections.singletonList(targetSpec))) + .thenReturn(Collections.singletonList(partitionAt(targetSpec, null))); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Arrays.asList( + partitionAt(targetSpec, null), + partitionAt(explicitSpec, "file:/external/part=explicit"), + partitionAt(emptyLocationSpec, ""))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Collections.singletonList("part"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + false, + Identifier.create("location_db", "location_table"), + null, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ true); + fileIO.startTrackingMutations(); + + Throwable failure = + catchThrowable( + () -> + commit.commit( + Collections.singletonList( + new TwoPhaseCommitMessage(committer)))); + + assertThat(publishCalls.get()).isZero(); + assertThat(fileIO.exists(targetPath)).isFalse(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(committer, never()).commit(fileIO); + verify(committer, never()).clean(fileIO); + verify(partitionManager, never()).createPartitions(anyList(), anyBoolean()); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + assertExplicitLocationCountMismatch( + failure, 2, 1, Identifier.create("location_db", "location_table")); + } + + @Test + void testWholeTableOverwriteRejectsExplicitLocationCountMismatchBeforeDeletingOldData() + throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = + new Path(new Path(tempDir.toUri()), "whole-overwrite-location-count-mismatch"); + Map spec = Collections.singletonMap("part", "target"); + Path oldData = new Path(tablePath, "part=target/data-old.csv"); + fileIO.writeFile(oldData, "old", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(1)); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn(Collections.singletonList(partitionAt(spec, null))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Collections.singletonList("part"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + true, + Identifier.create("location_db", "location_table"), + null, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ false); + fileIO.startTrackingMutations(); + + Throwable failure = catchThrowable(() -> commit.commit(Collections.emptyList())); + + assertThat(fileIO.exists(oldData)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + assertExplicitLocationCountMismatch( + failure, 1, 0, Identifier.create("location_db", "location_table")); + } + @Test void testAppendRejectsRegisteredExplicitLocationBeforeAnyTableMutation() throws Exception { MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); @@ -119,6 +225,10 @@ void testAppendRejectsRegisteredExplicitLocationBeforeAnyTableMutation() throws .thenReturn( Collections.singletonList( partitionAt(spec, "file:/external/part=external"))); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Collections.singletonList( + partitionAt(spec, "file:/external/part=external"))); FormatTableCommit commit = new FormatTableCommit( tablePath.toString(), @@ -168,6 +278,10 @@ void testStaticOverwriteRejectsExplicitLocationBeforeDeletingOldData() throws Ex .thenReturn( Collections.singletonList( partitionAt(spec, "file:/external/part=external"))); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Collections.singletonList( + partitionAt(spec, "file:/external/part=external"))); FormatTableCommit commit = new FormatTableCommit( tablePath.toString(), @@ -301,6 +415,55 @@ void testAppendRejectsFutureDefaultPathOwnedByExplicitPartition() throws Excepti .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); } + @Test + void testAppendWithEmptyMessagesRejectsPartialStaticPrefixOwnedByExplicitPartition() + throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "append-partial-static-owned-prefix"); + Map staticPrefix = Collections.singletonMap("year", "2025"); + Map explicitSpec = partitionSpec("2024", "11"); + Path staticPrefixPath = new Path(tablePath, "year=2025"); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(1)); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Collections.singletonList( + partitionAt(explicitSpec, staticPrefixPath.toString()))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Arrays.asList("year", "month"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + false, + Identifier.create("location_db", "location_table"), + staticPrefix, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ true); + fileIO.startTrackingMutations(); + + assertThatThrownBy(() -> commit.commit(Collections.emptyList())) + .isInstanceOf(RuntimeException.class) + .hasRootCauseInstanceOf(IllegalStateException.class) + .hasRootCauseMessage( + "Catalog returned overlapping locations for different partitions of " + + "Format Table location_db.location_table."); + + assertThat(fileIO.exists(staticPrefixPath)).isFalse(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager).explicitPartitionLocationCount(); + verify(partitionManager).listPartitions(Collections.emptyMap(), null); + verify(partitionManager, never()).listPartitions(staticPrefix, null); + verify(partitionManager, never()).listPartitionsByNames(anyList()); + verify(partitionManager, never()).createPartitions(anyList(), anyBoolean()); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + @Test void testDynamicOverwriteRejectsAffectedExplicitLocationBeforeReplacingData() throws Exception { MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); @@ -315,6 +478,10 @@ void testDynamicOverwriteRejectsAffectedExplicitLocationBeforeReplacingData() th .thenReturn( Collections.singletonList( partitionAt(spec, "file:/external/part=external"))); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Collections.singletonList( + partitionAt(spec, "file:/external/part=external"))); FormatTableCommit commit = new FormatTableCommit( tablePath.toString(), @@ -427,6 +594,11 @@ void testStaticPrefixOverwriteRejectsExplicitDescendantBeforeDeletingAnyPartitio Arrays.asList( partitionAt(defaultSpec, null), partitionAt(explicitSpec, "file:/external/year=2025/month=11"))); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Arrays.asList( + partitionAt(defaultSpec, null), + partitionAt(explicitSpec, "file:/external/year=2025/month=11"))); FormatTableCommit commit = new FormatTableCommit( tablePath.toString(), @@ -1372,6 +1544,34 @@ void testTruncateTableRejectsExplicitLocationBeforeDeletingOrReporting() throws .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); } + @Test + void testTruncateTableRejectsExplicitLocationCountMismatchBeforeDeletingOrReporting() + throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = + new Path(new Path(tempDir.toUri()), "truncate-table-location-count-mismatch"); + Map spec = Collections.singletonMap("part", "target"); + Path oldData = new Path(tablePath, "part=target/data-old.csv"); + fileIO.writeFile(oldData, "old", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(1)); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn(Collections.singletonList(partitionAt(spec, null))); + FormatTableCommit commit = + truncatingCommit(tablePath, fileIO, false, partitionManager, "part"); + fileIO.startTrackingMutations(); + + Throwable failure = catchThrowable(commit::truncateTable); + + assertThat(fileIO.exists(oldData)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + assertExplicitLocationCountMismatch( + failure, 1, 0, Identifier.create("truncate_db", "truncate_table")); + } + @Test void testTruncateTableOnlyEmptiesTheDirectoriesThatAreItsPartitions() throws Exception { LocalFileIO fileIO = LocalFileIO.create(); @@ -1470,6 +1670,10 @@ void testTruncateNamedPartitionRejectsExplicitLocationBeforeDeletingOrReporting( .thenReturn( Collections.singletonList( partitionAt(spec, "file:/external/year=2025/month=10"))); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Collections.singletonList( + partitionAt(spec, "file:/external/year=2025/month=10"))); FormatTableCommit commit = truncatingCommit(tablePath, fileIO, false, partitionManager, "year", "month"); fileIO.startTrackingMutations(); @@ -1488,6 +1692,113 @@ void testTruncateNamedPartitionRejectsExplicitLocationBeforeDeletingOrReporting( .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); } + @Test + void testTruncateNamedPartitionValidatesCompleteRegistryBeforeDeletingOnCountMismatch() + throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = + new Path(new Path(tempDir.toUri()), "truncate-named-location-count-mismatch"); + Map targetSpec = partitionSpec("2025", "10"); + Map unrelatedSpec = partitionSpec("2024", "11"); + Path oldData = new Path(tablePath, "year=2025/month=10/data-old.csv"); + fileIO.writeFile(oldData, "old", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(2)); + when(partitionManager.listPartitionsByNames(Collections.singletonList(targetSpec))) + .thenReturn(Collections.singletonList(partitionAt(targetSpec, null))); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Arrays.asList( + partitionAt(targetSpec, null), + partitionAt(unrelatedSpec, "file:/external/year=2024/month=11"))); + FormatTableCommit commit = + truncatingCommit(tablePath, fileIO, false, partitionManager, "year", "month"); + fileIO.startTrackingMutations(); + + Throwable failure = + catchThrowable( + () -> commit.truncatePartitions(Collections.singletonList(targetSpec))); + + assertThat(fileIO.exists(oldData)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager).listPartitions(Collections.emptyMap(), null); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + assertExplicitLocationCountMismatch( + failure, 2, 1, Identifier.create("truncate_db", "truncate_table")); + } + + @Test + void testTruncateNamedPartitionRejectsIncompleteCatalogSpecAfterZeroCountBeforeMutation() + throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "truncate-named-incomplete-spec"); + Map targetSpec = partitionSpec("2025", "10"); + Map incompleteSpec = Collections.singletonMap("year", "2025"); + Path oldData = new Path(tablePath, "year=2025/month=10/data-old.csv"); + fileIO.writeFile(oldData, "old", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(0)); + when(partitionManager.listPartitionsByNames(Collections.singletonList(targetSpec))) + .thenReturn(Collections.singletonList(partitionAt(incompleteSpec, null))); + FormatTableCommit commit = + truncatingCommit(tablePath, fileIO, false, partitionManager, "year", "month"); + fileIO.startTrackingMutations(); + + assertThatThrownBy(() -> commit.truncatePartitions(Collections.singletonList(targetSpec))) + .isInstanceOf(IllegalStateException.class) + .hasMessage( + "Catalog returned incomplete partition spec {year=2025} for Format Table " + + "truncate_db.truncate_table."); + + assertThat(fileIO.exists(oldData)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager).explicitPartitionLocationCount(); + verify(partitionManager).listPartitionsByNames(Collections.singletonList(targetSpec)); + verify(partitionManager, never()).listPartitions(anyMap(), isNull()); + verify(partitionManager, never()).createPartitions(anyList(), anyBoolean()); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + + @Test + void testTruncateNamedPartitionRejectsDefaultPathOwnedByExplicitPartitionBeforeMutation() + throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "truncate-named-overlapping-location"); + Map targetSpec = partitionSpec("2025", "10"); + Map explicitSpec = partitionSpec("2024", "11"); + Path targetPartitionPath = new Path(tablePath, "year=2025/month=10"); + Path explicitData = new Path(targetPartitionPath, "data-explicit.csv"); + fileIO.writeFile(explicitData, "explicit", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(1)); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Arrays.asList( + partitionAt(targetSpec, null), + partitionAt(explicitSpec, targetPartitionPath.toString()))); + FormatTableCommit commit = + truncatingCommit(tablePath, fileIO, false, partitionManager, "year", "month"); + fileIO.startTrackingMutations(); + + assertThatThrownBy(() -> commit.truncatePartitions(Collections.singletonList(targetSpec))) + .isInstanceOf(IllegalStateException.class) + .hasMessage( + "Catalog returned overlapping locations for different partitions of " + + "Format Table truncate_db.truncate_table."); + + assertThat(fileIO.exists(explicitData)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager).listPartitions(Collections.emptyMap(), null); + verify(partitionManager, never()).listPartitionsByNames(anyList()); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + } + @Test void testTruncatePrefixRejectsExplicitDescendantBeforeMutatingDefaultDescendant() throws Exception { @@ -1506,6 +1817,11 @@ void testTruncatePrefixRejectsExplicitDescendantBeforeMutatingDefaultDescendant( Arrays.asList( partitionAt(defaultSpec, null), partitionAt(explicitSpec, "file:/external/year=2025/month=11"))); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Arrays.asList( + partitionAt(defaultSpec, null), + partitionAt(explicitSpec, "file:/external/year=2025/month=11"))); FormatTableCommit commit = truncatingCommit(tablePath, fileIO, false, partitionManager, "year", "month"); fileIO.startTrackingMutations(); @@ -2802,6 +3118,17 @@ void testBuilderCleanupConcurrencyDoesNotApplyToTruncateOperations() throws Exce 0, -1, false))); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Collections.singletonList( + new Partition( + Collections.singletonMap("part", "p"), + 0, + 0, + 0, + 0, + -1, + false))); builderTruncateCommit(partitionsPath, partitionFileIO, partitionManager) .truncatePartitions( Collections.singletonList(Collections.singletonMap("part", "p"))); @@ -3671,6 +3998,16 @@ private static Partition partitionAt(Map spec, String location) location); } + private static void assertExplicitLocationCountMismatch( + Throwable failure, long reported, long observed, Identifier identifier) { + assertThat(failure).isInstanceOf(RuntimeException.class); + assertThat(getRootCause(failure)) + .isInstanceOf(IllegalStateException.class) + .hasMessage( + "Catalog reported %d explicit partition locations for format table %s, but its validated listing contained %d.", + reported, identifier.getFullName(), observed); + } + private static Map partitionSpec(String year, String month) { LinkedHashMap spec = new LinkedHashMap<>(); spec.put("year", year); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTablePartitionPathResolverTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTablePartitionPathResolverTest.java index 7f0efaf931c8..c8d453d987bb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTablePartitionPathResolverTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTablePartitionPathResolverTest.java @@ -133,7 +133,7 @@ void testOwnershipValidationScalesToLargePartitionRegistries() { Path path = resolver.resolve( spec, "oss://bucket/archive/partition-" + partition); - assertThat(resolver.remember(spec, path)).isTrue(); + assertThat(resolver.validateAndRecord(spec, path)).isTrue(); } }); } @@ -145,13 +145,17 @@ void testPathSegmentBoundariesDistinguishPrefixesFromAncestors() { LinkedHashMap second = spec("second"); LinkedHashMap child = spec("child"); - assertThat(resolver.remember(first, resolver.resolve(first, "oss://bucket/archive/a-b"))) + assertThat( + resolver.validateAndRecord( + first, resolver.resolve(first, "oss://bucket/archive/a-b"))) .isTrue(); - assertThat(resolver.remember(second, resolver.resolve(second, "oss://bucket/archive/a"))) + assertThat( + resolver.validateAndRecord( + second, resolver.resolve(second, "oss://bucket/archive/a"))) .isTrue(); assertThatThrownBy( () -> - resolver.remember( + resolver.validateAndRecord( child, resolver.resolve(child, "oss://bucket/archive/a/child"))) .isInstanceOf(IllegalStateException.class) @@ -161,14 +165,14 @@ void testPathSegmentBoundariesDistinguishPrefixesFromAncestors() { LinkedHashMap descendant = spec("descendant"); LinkedHashMap ancestor = spec("ancestor"); assertThat( - reverseOrder.remember( + reverseOrder.validateAndRecord( descendant, reverseOrder.resolve( descendant, "oss://bucket/archive/root/child"))) .isTrue(); assertThatThrownBy( () -> - reverseOrder.remember( + reverseOrder.validateAndRecord( ancestor, reverseOrder.resolve( ancestor, "oss://bucket/archive/root"))) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala index 84cd80d12d27..767fb1ff1f9a 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala @@ -145,11 +145,10 @@ case class PaimonDropFormatTablePartitionsExec( val partitionKeyCount = table.table.partitionKeys().size() val (completeSpecs, partialSpecs) = partSpecs.partition(_.ident.numFields == partitionKeyCount) - val registration = table - .formatTablePartitionsRegistered( - completeSpecs.map(_.names.toArray).toArray, - completeSpecs.map(_.ident).toArray) - .toSeq + val requestedSpecs = completeSpecs ++ partialSpecs + val (registration, partitions) = table.resolveFormatTablePartitionsForDrop( + requestedSpecs.map(_.names.toArray).toArray, + requestedSpecs.map(_.ident).toArray) val missingSpecs = completeSpecs.zip(registration).collect { case (spec, false) => spec } if (missingSpecs.nonEmpty && !ifExists) { throw new NoSuchPartitionsException( @@ -158,17 +157,13 @@ case class PaimonDropFormatTablePartitionsExec( table.partitionSchema) } - val specsToDrop = - completeSpecs.zip(registration).collect { case (spec, true) => spec } ++ partialSpecs - if (specsToDrop.nonEmpty) { + if (partitions.nonEmpty) { // Catalog-managed semantics (PaimonPartitionManagement#dropFormatTablePartitions): resolve // partial specs, unregister the exact catalog partitions, then delete their directories // with the table FileIO client-side. A directory-deletion failure stays unregistered so // partially deleted data is not exposed again. PaimonFormatTablePartitionDdlExec.refreshingCache(refreshCache) { - table.dropFormatTablePartitions( - specsToDrop.map(_.names.toArray).toArray, - specsToDrop.map(_.ident).toArray) + table.dropCatalogRegisteredPartitions(partitions) } } Seq.empty diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala index a410f22e2fbc..2882527db620 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala @@ -25,7 +25,7 @@ import org.apache.paimon.partition.{Partition, PartitionLocation} import org.apache.paimon.spark.{BaseTable, FormatTableScanBuilder} import org.apache.paimon.spark.write.{BaseV2WriteBuilder, PaimonWriteRequirement} import org.apache.paimon.table.FormatTable -import org.apache.paimon.table.format.{FormatTablePartitionManager, FormatTablePartitionPathResolver} +import org.apache.paimon.table.format.{FormatTablePartitionManager, FormatTablePartitionPathResolver, FormatTablePartitionRegistryValidator} import org.apache.paimon.table.sink.BatchTableCommit import org.apache.paimon.types.RowType import org.apache.paimon.utils.{PartitionPathUtils, StringUtils} @@ -327,14 +327,29 @@ case class PaimonFormatTable(table: FormatTable) } /** - * Drops the given partitions: complete specs are unregistered and their directories deleted - * as-is, partial specs are expanded to the registered leaf partitions they cover. Callers are - * responsible for resolving which complete specs are actually registered first (see - * [[formatTablePartitionsRegistered]]), so unregistered data directories are never deleted. + * Drops the registered partitions covered by the given specs. Complete specs that are not + * registered are ignored; partial specs are expanded to the registered leaf partitions they + * cover. */ private[spark] def dropFormatTablePartitions( partitionNames: Array[Array[String]], rows: Array[InternalRow]): Boolean = { + val (_, partitions) = resolveFormatTablePartitionsForDrop(partitionNames, rows) + dropCatalogRegisteredPartitions(partitions) + } + + /** + * Resolves DROP requests from one validated view of the catalog registry. The boolean array is + * aligned with the requests and tells callers which complete specs are registered; partial-spec + * entries are not used for existence reporting. The returned partitions are the deduplicated + * registered leaves covered by all requests. + */ + private[spark] def resolveFormatTablePartitionsForDrop( + partitionNames: Array[Array[String]], + rows: Array[InternalRow]): (Array[Boolean], Seq[Partition]) = { + if (rows.isEmpty) { + return (Array.empty[Boolean], Seq.empty) + } val partitionKeyCount = table.partitionKeys().size() rows.zip(partitionNames).foreach { case (row, names) => requireNameablePartitionValues("DROP PARTITION", row, names.toSeq) @@ -346,60 +361,55 @@ case class PaimonFormatTable(table: FormatTable) // Validate every user-supplied spec before asking the catalog for its authoritative metadata. // This preserves the path-safety boundary even when an invalid spec is not registered. requested.foreach(spec => resolvePartitionPathWithinTable(orderedSpec(spec), onlyValueInPath)) - val partitions = ArrayBuffer.empty[Partition] - val seenPartitions = mutable.LinkedHashMap.empty[Map[String, String], Partition] - - def addPartition(partition: Partition): Unit = { - val validated = validateCatalogRegisteredPartition(partition.spec()) - val key = validated.asScala.toMap - seenPartitions.get(key) match { - case Some(previous) if !Objects.equals(previous.location(), partition.location()) => - throw new IllegalStateException( - s"Catalog returned conflicting locations for partition $key of Format Table " + - s"${table.fullName()}.") - case Some(_) => - case None => - seenPartitions.put(key, partition) - partitions += partition + val manager = requirePartitionManager() + val explicitLocationCount = manager.explicitPartitionLocationCount() + val completeOnly = requested.forall(_.size() == partitionKeyCount) + val registry = + if (completeOnly && explicitLocationCount.isPresent && explicitLocationCount.getAsLong == 0) { + manager.listPartitionsByNames(requested.toSeq.asJava) + } else { + manager.listPartitions(Collections.emptyMap[String, String](), null) } - } + FormatTablePartitionRegistryValidator.validateExplicitLocationCount( + registry, + explicitLocationCount, + table.fullName()) + FormatTablePartitionRegistryValidator.validatePartitionLocations( + registry, + table.partitionKeys(), + new Path(table.location()), + table.fullName(), + onlyValueInPath) - // Resolve exact requests back to their full metadata. Location is authoritative catalog state, - // so DROP must not infer it from the request or silently treat an unknown response as default. - val completeRequests = requested.filter(_.size() == partitionKeyCount) - if (completeRequests.nonEmpty) { - val requestedKeys = completeRequests.map(_.asScala.toMap).toSet - requirePartitionManager() - .listPartitionsByNames(completeRequests.toSeq.asJava) - .asScala - .filter(partition => requestedKeys.contains(partition.spec().asScala.toMap)) - .foreach(addPartition) + val bySpec = mutable.LinkedHashMap.empty[Map[String, String], Partition] + registry.asScala.foreach { + partition => + val spec = validateCatalogRegisteredPartition(partition.spec()).asScala.toMap + bySpec.get(spec) match { + case Some(previous) if !Objects.equals(previous.location(), partition.location()) => + throw new IllegalStateException( + s"Catalog returned conflicting locations for partition $spec of Format Table " + + s"${table.fullName()}.") + case Some(_) => + case None => bySpec.put(spec, partition) + } } - val partialSpecs = requested.filter(_.size() < partitionKeyCount).toSeq.distinct - if (partialSpecs.nonEmpty) { - def matchesRequestedPartial(partition: JMap[String, String]): Boolean = { - partialSpecs.exists(_.asScala.forall { - case (key, value) => Objects.equals(value, partition.get(key)) - }) - } - // One unfiltered traversal resolves every partial spec; the requested constraints are - // enforced client-side. - requirePartitionManager() - .listPartitions(Collections.emptyMap[String, String](), null) - .asScala - .foreach { - partition => - val validated = validateCatalogRegisteredPartition(partition.spec()) - if (matchesRequestedPartial(validated)) { - addPartition(partition) - } - } + val registered = + requested.map(spec => spec.size() == partitionKeyCount && bySpec.contains(spec.asScala.toMap)) + val partitions = ArrayBuffer.empty[Partition] + val requestedMaps = requested.map(_.asScala.toMap) + bySpec.foreach { + case (registeredSpec, partition) if requestedMaps.exists(_.forall { + case (key, value) => Objects.equals(value, registeredSpec.getOrElse(key, null)) + }) => + partitions += partition + case _ => } - dropCatalogRegisteredPartitions(partitions.toSeq) + (registered, partitions.toSeq) } - private def dropCatalogRegisteredPartitions(partitions: Seq[Partition]): Boolean = { + private[spark] def dropCatalogRegisteredPartitions(partitions: Seq[Partition]): Boolean = { if (partitions.isEmpty) { return true } diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala index 8c24e43247a8..ce6d57905c97 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala @@ -42,7 +42,7 @@ import org.apache.spark.sql.types.StringType import java.io.IOException import java.lang.reflect.InvocationTargetException import java.nio.file.Files -import java.util.{Collections, List => JList, Map => JMap} +import java.util.{Collections, List => JList, Map => JMap, OptionalLong} import java.util.concurrent.{Callable, Executors, TimeUnit} import scala.collection.JavaConverters._ @@ -366,6 +366,8 @@ class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalog var dropCalls = 0 var refreshCalls = 0 val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -408,6 +410,79 @@ class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalog } } + test("mixed DROP keeps complete existence aligned when a partial spec precedes it") { + val fileIO = LocalFileIO.create() + val tablePath = new Path(Files.createTempDirectory("format-table-drop-mixed-order").toUri) + val partialMatch = Map("dt" -> "20260715", "hh" -> "10") + val completeMatch = Map("dt" -> "20260716", "hh" -> "11") + val unrelated = Map("dt" -> "20260717", "hh" -> "12") + val partialDir = new Path(tablePath, "dt=20260715/hh=10") + val completeDir = new Path(tablePath, "dt=20260716/hh=11") + val unrelatedDir = new Path(tablePath, "dt=20260717/hh=12") + var listCalls = 0 + var listByNamesCalls = 0 + var dropCalls = 0 + var dropped = Seq.empty[Map[String, String]] + var refreshCalls = 0 + val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean, + statistics: JList[PartitionStatistics], + replaceStatistics: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = { + dropCalls += 1 + dropped = partitions.asScala.map(_.asScala.toMap).toSeq + } + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[Partition] = { + listByNamesCalls += 1 + registeredPartitions(completeMatch) + } + + override def listPartitions( + prefix: JMap[String, String], + filter: Predicate): JList[Partition] = { + listCalls += 1 + registeredPartitions(partialMatch, completeMatch, unrelated) + } + } + + try { + fileIO.mkdirs(partialDir) + fileIO.mkdirs(completeDir) + fileIO.mkdirs(unrelatedDir) + val table = new PaimonFormatTable( + rawFormatTable(withCatalogManagedPartitions = true, gateway, tablePath.toString, fileIO)) + val partialFirst = + ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](10))) + + // A partial request has no existence bit, so placing it first exposes positional drift. + runCommand( + PaimonDropFormatTablePartitionsExec( + table, + Seq(partialFirst, partition(20260716, 11)), + ifExists = false, + purge = false, + () => refreshCalls += 1)) + + assert(listCalls == 1) + assert(listByNamesCalls == 0) + assert(dropCalls == 1) + assert(dropped == Seq(partialMatch, completeMatch)) + assert(refreshCalls == 1) + assert(!fileIO.exists(partialDir)) + assert(!fileIO.exists(completeDir)) + assert(fileIO.exists(unrelatedDir)) + } finally { + fileIO.delete(tablePath, true) + } + } + test( "catalog-managed DROP IF EXISTS drops only registered partitions and preserves pending data") { val fileIO = LocalFileIO.create() @@ -419,6 +494,8 @@ class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalog var dropped = Seq.empty[Map[String, String]] var refreshCalls = 0 val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -488,6 +565,8 @@ class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalog var registered = Set(failedSpec) var compensationCreates = Seq.empty[(Seq[Map[String, String]], Boolean)] def newGateway(): FormatTablePartitionManager = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -555,6 +634,8 @@ class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalog var registered = Set(droppedSpec) var compensationCreates = Seq.empty[(Seq[Map[String, String]], Boolean)] def newGateway(): FormatTablePartitionManager = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -621,6 +702,8 @@ class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalog var dropped = Seq.empty[Map[String, String]] val matching = Map("dt" -> "20260715", "hh" -> "10") val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -665,6 +748,8 @@ class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalog var listedPrefixes = Seq.empty[Map[String, String]] var dropped = Seq.empty[Map[String, String]] val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -723,6 +808,8 @@ class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalog test("catalog-managed partial DROP rejects an incomplete catalog result before mutation") { var dropCalls = 0 val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -919,6 +1006,8 @@ class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalog var dropCalls = 0 var dropped = Seq.empty[JMap[String, String]] + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala index 61baa9396a58..8f435025c54f 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala @@ -36,7 +36,7 @@ import org.apache.spark.unsafe.types.UTF8String import java.lang.reflect.{InvocationHandler, InvocationTargetException, Method, Proxy} import java.nio.file.Files -import java.util.{ArrayList, Collections, List => JList, Map => JMap} +import java.util.{ArrayList, Collections, List => JList, Map => JMap, OptionalLong} import java.util.concurrent.{CountDownLatch, TimeUnit} import scala.collection.JavaConverters._ @@ -109,6 +109,8 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { test("catalog-managed DROP rejects a partition value that would escape the table location") { var dropCalls = 0 val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -330,6 +332,8 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { val partitionDir = new Path(tablePath, "dt=20260715/hh=10") val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -372,6 +376,74 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { } } + test("catalog-managed DROP rejects an explicit location that owns the target default directory") { + val fileIO = LocalFileIO.create() + val tablePath = + new Path( + Files.createTempDirectory("catalog-partition-format-drop-overlapping-location").toUri) + val explicitSpec = partitionSpec(20260715, 10) + val targetSpec = partitionSpec(20260716, 11) + val targetDir = new Path(tablePath, "dt=20260716/hh=11") + val marker = new Path(targetDir, "marker.csv") + val explicitPartition = + new Partition( + explicitSpec.asJava, + 0L, + 0L, + 0L, + 0L, + 0, + false, + null, + null, + null, + null, + null, + targetDir.toString) + val targetPartition = + new Partition(targetSpec.asJava, 0L, 0L, 0L, 0L, 0, false) + var dropCalls = 0 + val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.empty() + + override def createPartitions( + partitions: JList[JMap[String, String]], + ignoreIfExists: Boolean, + statistics: JList[PartitionStatistics], + replaceStatistics: Boolean): Unit = {} + + override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1 + + override def listPartitionsByNames( + partitions: JList[JMap[String, String]]): JList[Partition] = + Collections.singletonList(targetPartition) + + override def listPartitions( + prefix: JMap[String, String], + filter: Predicate): JList[Partition] = + Seq(explicitPartition, targetPartition).asJava + } + + try { + fileIO.writeFile(marker, "target", false) + val sparkTable = + new PaimonFormatTable( + formatTableWithCatalogManagedPartitions(fileIO, tablePath.toString, gateway)) + + val error = intercept[IllegalStateException] { + sparkTable.dropFormatTablePartitions( + Array(Array("dt", "hh")), + Array(partitionRow(20260716, 11))) + } + + assert(error.getMessage.contains("overlapping locations")) + assert(dropCalls == 0) + assert(fileIO.exists(marker)) + } finally { + fileIO.delete(tablePath, true) + } + } + test("catalog-managed DROP of an explicit-location partition rejects default-directory residue") { val fileIO = LocalFileIO.create() val tablePath = @@ -397,6 +469,8 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { null, externalDir.toString) val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(1) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -463,6 +537,8 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { externalDir.toString) var dropped = Seq.empty[Map[String, String]] val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(1) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -514,6 +590,8 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { var listedPrefixes = Seq.empty[Map[String, String]] var dropped = Seq.empty[Map[String, String]] val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -632,6 +710,8 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { val matching = partitionSpec(20260715, 10) var dropped = Seq.empty[Map[String, String]] val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -677,6 +757,8 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { val partitionDir = new Path(tablePath, "dt=20260715/hh=10") var dropCalls = 0 val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -727,6 +809,8 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { var dropCalls = 0 var failDropResponse = true val gateway = new FormatTablePartitionManager { + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -965,6 +1049,8 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { def partitions: Seq[Map[String, String]] = synchronized(storedPartitions.toSeq) + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, @@ -1010,6 +1096,8 @@ class FormatTablePartitionManagementTest extends SparkFunSuite { private class RecordingDropCatalog extends FormatTablePartitionManager { var dropCalls = 0 + override def explicitPartitionLocationCount(): OptionalLong = OptionalLong.of(0) + override def createPartitions( partitions: JList[JMap[String, String]], ignoreIfExists: Boolean, diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionDdlParityTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionDdlParityTest.scala index 08e59bd03ad5..92f2a443c196 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionDdlParityTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionDdlParityTest.scala @@ -20,6 +20,7 @@ package org.apache.paimon.spark.sql import org.apache.paimon.catalog.Identifier import org.apache.paimon.fs.Path +import org.apache.paimon.rest.ResourcePaths import org.apache.paimon.spark.PaimonSparkTestWithRestCatalogBase import org.apache.paimon.table.FormatTable @@ -109,6 +110,58 @@ class CatalogManagedPartitionDdlParityTest extends PaimonSparkTestWithRestCatalo } } + test( + "public SQL DROP preserves registration and default data when explicit location count mismatches listing") { + val tableName = "ddl_drop_location_count_mismatch" + withTable(tableName) { + createTable(tableName) + withTempDir { + unrelatedDir => + sql(s"ALTER TABLE ${qualified(tableName)} ADD PARTITION (dt = '20260101', hour = '00')") + val unrelatedLocation = new Path(unrelatedDir.toURI.toString).toString + sql( + s"ALTER TABLE ${qualified(tableName)} ADD " + + s"PARTITION (dt = '20260102', hour = '00') LOCATION '$unrelatedLocation'") + val table = formatTable(tableName) + val marker = new Path(table.location(), "dt=20260101/hour=00/marker.csv") + table.fileIO().writeFile(marker, "1,a\n", false) + restCatalogServer.setExplicitPartitionLocationCount(2L) + val dropResource = new ResourcePaths("paimon").dropPartitions(dbName0, tableName) + restCatalogServer.clearReceivedHeaders() + try { + val failure = + try { + sql( + s"ALTER TABLE ${qualified(tableName)} DROP " + + s"PARTITION (dt = '20260101', hour = '00')") + null + } catch { + case error: Throwable => error + } + + assert(restCatalogServer.getReceivedHeaders(dropResource).isEmpty) + assert(registered(tableName) == Set("20260101/00", "20260102/00")) + assert(table.fileIO().exists(marker)) + assert(failure != null) + assert( + Iterator + .iterate(failure)(_.getCause) + .takeWhile(_ != null) + .exists(_.isInstanceOf[IllegalStateException])) + assert( + causeMessages(failure).contains( + s"Catalog reported 2 explicit partition locations for format table " + + s"$dbName0.$tableName, but its validated listing contained 1."), + causeMessages(failure) + ) + } finally { + restCatalogServer.setExplicitPartitionLocationCount(null) + restCatalogServer.clearReceivedHeaders() + } + } + } + } + test("ADD and DROP resolve partition column names the way the rest of Spark resolves them") { val tableName = "ddl_case_insensitive" withTable(tableName) { From 4ac45164ef04c29ff834d66919d5b8c5c5af5fc1 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 2 Sep 2026 11:59:30 +0800 Subject: [PATCH 10/15] test(format): split partition registry validation coverage --- ...rmatTableCommitRegistryValidationTest.java | 224 ++++++++++++++++++ .../table/format/FormatTableCommitTest.java | 105 -------- 2 files changed, 224 insertions(+), 105 deletions(-) create mode 100644 paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitRegistryValidationTest.java diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitRegistryValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitRegistryValidationTest.java new file mode 100644 index 000000000000..97d7660184bf --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitRegistryValidationTest.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.format; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.TwoPhaseOutputStream; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.partition.Partition; +import org.apache.paimon.partition.PartitionStatistics; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.paimon.CoreOptions.PARTITION_DEFAULT_NAME; +import static org.apache.paimon.shade.guava30.com.google.common.base.Throwables.getRootCause; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Partition registry validation tests for {@link FormatTableCommit}. */ +class FormatTableCommitRegistryValidationTest { + + @TempDir java.nio.file.Path tempDir; + + @Test + void testAppendRejectsExplicitLocationCountMismatchBeforePublishingOrRegistering() + throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = new Path(new Path(tempDir.toUri()), "append-location-count-mismatch"); + Map targetSpec = Collections.singletonMap("part", "target"); + Map explicitSpec = Collections.singletonMap("part", "explicit"); + Map emptyLocationSpec = Collections.singletonMap("part", "empty"); + Path targetPath = new Path(tablePath, "part=target/data-new.csv"); + AtomicInteger publishCalls = new AtomicInteger(); + TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class); + when(committer.targetPath()).thenReturn(targetPath); + doAnswer( + invocation -> { + publishCalls.incrementAndGet(); + return null; + }) + .when(committer) + .commit(fileIO); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(2)); + when(partitionManager.listPartitionsByNames(Collections.singletonList(targetSpec))) + .thenReturn(Collections.singletonList(partitionAt(targetSpec, null))); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn( + Arrays.asList( + partitionAt(targetSpec, null), + partitionAt(explicitSpec, "file:/external/part=explicit"), + partitionAt(emptyLocationSpec, ""))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Collections.singletonList("part"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + false, + Identifier.create("location_db", "location_table"), + null, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ true); + fileIO.startTrackingMutations(); + + Throwable failure = + catchThrowable( + () -> + commit.commit( + Collections.singletonList( + new TwoPhaseCommitMessage(committer)))); + + assertThat(publishCalls.get()).isZero(); + assertThat(fileIO.exists(targetPath)).isFalse(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(committer, never()).commit(fileIO); + verify(committer, never()).clean(fileIO); + verify(partitionManager, never()).createPartitions(anyList(), anyBoolean()); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + assertExplicitLocationCountMismatch( + failure, 2, 1, Identifier.create("location_db", "location_table")); + } + + @Test + void testWholeTableOverwriteRejectsExplicitLocationCountMismatchBeforeDeletingOldData() + throws Exception { + MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); + Path tablePath = + new Path(new Path(tempDir.toUri()), "whole-overwrite-location-count-mismatch"); + Map spec = Collections.singletonMap("part", "target"); + Path oldData = new Path(tablePath, "part=target/data-old.csv"); + fileIO.writeFile(oldData, "old", false); + FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(1)); + when(partitionManager.listPartitions(Collections.emptyMap(), null)) + .thenReturn(Collections.singletonList(partitionAt(spec, null))); + FormatTableCommit commit = + new FormatTableCommit( + tablePath.toString(), + Collections.singletonList("part"), + fileIO, + false, + PARTITION_DEFAULT_NAME.defaultValue(), + true, + Identifier.create("location_db", "location_table"), + null, + null, + null, + partitionManager, + /* dynamicPartitionOverwrite */ false); + fileIO.startTrackingMutations(); + + Throwable failure = catchThrowable(() -> commit.commit(Collections.emptyList())); + + assertThat(fileIO.exists(oldData)).isTrue(); + assertThat(fileIO.deleteCalls()).isZero(); + assertThat(fileIO.mkdirsCalls()).isZero(); + verify(partitionManager, never()) + .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); + assertExplicitLocationCountMismatch( + failure, 1, 0, Identifier.create("location_db", "location_table")); + } + + private static Partition partitionAt(Map spec, String location) { + return new Partition( + spec, + 0, + 0, + 0, + 0, + PartitionStatistics.UNKNOWN_TOTAL_BUCKETS, + false, + null, + null, + null, + null, + null, + location); + } + + private static void assertExplicitLocationCountMismatch( + Throwable failure, long reported, long observed, Identifier identifier) { + assertThat(failure).isInstanceOf(RuntimeException.class); + assertThat(getRootCause(failure)) + .isInstanceOf(IllegalStateException.class) + .hasMessage( + "Catalog reported %d explicit partition locations for format table %s, but its validated listing contained %d.", + reported, identifier.getFullName(), observed); + } + + private static class MutationTrackingLocalFileIO extends LocalFileIO { + + private final AtomicInteger deleteCalls = new AtomicInteger(); + private final AtomicInteger mkdirsCalls = new AtomicInteger(); + private boolean tracking; + + private void startTrackingMutations() { + deleteCalls.set(0); + mkdirsCalls.set(0); + tracking = true; + } + + @Override + public boolean delete(Path path, boolean recursive) throws IOException { + if (tracking) { + deleteCalls.incrementAndGet(); + } + return super.delete(path, recursive); + } + + @Override + public boolean mkdirs(Path path) throws IOException { + if (tracking) { + mkdirsCalls.incrementAndGet(); + } + return super.mkdirs(path); + } + + private int deleteCalls() { + return deleteCalls.get(); + } + + private int mkdirsCalls() { + return mkdirsCalls.get(); + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java index a57fee136385..1f70979743b6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java @@ -106,111 +106,6 @@ class FormatTableCommitTest { @TempDir java.nio.file.Path tempDir; - @Test - void testAppendRejectsExplicitLocationCountMismatchBeforePublishingOrRegistering() - throws Exception { - MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); - Path tablePath = new Path(new Path(tempDir.toUri()), "append-location-count-mismatch"); - Map targetSpec = Collections.singletonMap("part", "target"); - Map explicitSpec = Collections.singletonMap("part", "explicit"); - Map emptyLocationSpec = Collections.singletonMap("part", "empty"); - Path targetPath = new Path(tablePath, "part=target/data-new.csv"); - AtomicInteger publishCalls = new AtomicInteger(); - TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class); - when(committer.targetPath()).thenReturn(targetPath); - doAnswer( - invocation -> { - publishCalls.incrementAndGet(); - return null; - }) - .when(committer) - .commit(fileIO); - FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); - when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(2)); - when(partitionManager.listPartitionsByNames(Collections.singletonList(targetSpec))) - .thenReturn(Collections.singletonList(partitionAt(targetSpec, null))); - when(partitionManager.listPartitions(Collections.emptyMap(), null)) - .thenReturn( - Arrays.asList( - partitionAt(targetSpec, null), - partitionAt(explicitSpec, "file:/external/part=explicit"), - partitionAt(emptyLocationSpec, ""))); - FormatTableCommit commit = - new FormatTableCommit( - tablePath.toString(), - Collections.singletonList("part"), - fileIO, - false, - PARTITION_DEFAULT_NAME.defaultValue(), - false, - Identifier.create("location_db", "location_table"), - null, - null, - null, - partitionManager, - /* dynamicPartitionOverwrite */ true); - fileIO.startTrackingMutations(); - - Throwable failure = - catchThrowable( - () -> - commit.commit( - Collections.singletonList( - new TwoPhaseCommitMessage(committer)))); - - assertThat(publishCalls.get()).isZero(); - assertThat(fileIO.exists(targetPath)).isFalse(); - assertThat(fileIO.deleteCalls()).isZero(); - assertThat(fileIO.mkdirsCalls()).isZero(); - verify(committer, never()).commit(fileIO); - verify(committer, never()).clean(fileIO); - verify(partitionManager, never()).createPartitions(anyList(), anyBoolean()); - verify(partitionManager, never()) - .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); - assertExplicitLocationCountMismatch( - failure, 2, 1, Identifier.create("location_db", "location_table")); - } - - @Test - void testWholeTableOverwriteRejectsExplicitLocationCountMismatchBeforeDeletingOldData() - throws Exception { - MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); - Path tablePath = - new Path(new Path(tempDir.toUri()), "whole-overwrite-location-count-mismatch"); - Map spec = Collections.singletonMap("part", "target"); - Path oldData = new Path(tablePath, "part=target/data-old.csv"); - fileIO.writeFile(oldData, "old", false); - FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); - when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(1)); - when(partitionManager.listPartitions(Collections.emptyMap(), null)) - .thenReturn(Collections.singletonList(partitionAt(spec, null))); - FormatTableCommit commit = - new FormatTableCommit( - tablePath.toString(), - Collections.singletonList("part"), - fileIO, - false, - PARTITION_DEFAULT_NAME.defaultValue(), - true, - Identifier.create("location_db", "location_table"), - null, - null, - null, - partitionManager, - /* dynamicPartitionOverwrite */ false); - fileIO.startTrackingMutations(); - - Throwable failure = catchThrowable(() -> commit.commit(Collections.emptyList())); - - assertThat(fileIO.exists(oldData)).isTrue(); - assertThat(fileIO.deleteCalls()).isZero(); - assertThat(fileIO.mkdirsCalls()).isZero(); - verify(partitionManager, never()) - .createPartitions(anyList(), anyBoolean(), any(), anyBoolean()); - assertExplicitLocationCountMismatch( - failure, 1, 0, Identifier.create("location_db", "location_table")); - } - @Test void testAppendRejectsRegisteredExplicitLocationBeforeAnyTableMutation() throws Exception { MutationTrackingLocalFileIO fileIO = new MutationTrackingLocalFileIO(); From dc8e766bcdb4b02c55009d8476e4129a6e23606f Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 2 Sep 2026 12:37:07 +0800 Subject: [PATCH 11/15] test(format): declare registry state in statistics fixtures --- .../FormatTableCommitStatisticsTest.java | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitStatisticsTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitStatisticsTest.java index 5ad2c306e180..eceb80cca684 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitStatisticsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitStatisticsTest.java @@ -57,6 +57,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.OptionalLong; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; @@ -294,6 +295,7 @@ void testTruncatingPartitionsReportsAnExactZeroAsTheTotal() throws Exception { Path tablePath = new Path(tempDir.toUri()); FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); registered(partitionManager, spec("2025", "10"), spec("2025", "11")); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(0)); writeDataFile(fileIO, tablePath, "year=2025/month=10", "data.csv", 4096); writeDataFile(fileIO, tablePath, "year=2025/month=11", "data.csv", 2048); @@ -384,6 +386,7 @@ void testTruncatingAPrefixReportsThePartitionsUnderneathIt() throws Exception { Path tablePath = new Path(tempDir.toUri()); FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class); Map prefix = Collections.singletonMap("year", "2025"); + when(partitionManager.explicitPartitionLocationCount()).thenReturn(OptionalLong.of(0)); when(partitionManager.listPartitions(prefix, null)) .thenReturn( Arrays.asList( @@ -909,6 +912,11 @@ public void createPartitions(List> partitions, boolean ignor registered.addAll(partitions); } + @Override + public OptionalLong explicitPartitionLocationCount() { + return OptionalLong.of(0); + } + @Override public List listPartitions( Map prefix, @Nullable Predicate filter) { @@ -917,7 +925,13 @@ public List listPartitions( @Override public List listPartitionsByNames(List> partitions) { - throw new UnsupportedOperationException(); + List found = new ArrayList<>(); + for (Map partitionSpec : partitions) { + if (registered.contains(partitionSpec)) { + found.add(partition(partitionSpec)); + } + } + return found; } @Override @@ -1027,6 +1041,9 @@ void testAppendRegistrationBatchFailureDeletesAllTargetsWithoutReportingStatisti AtomicInteger requests = new AtomicInteger(); List> registered = new ArrayList<>(); List appliedStatistics = new ArrayList<>(); + when(catalog.getExplicitPartitionLocationCount(TABLE)).thenReturn(OptionalLong.of(0)); + when(catalog.listPartitionsByNames(eq(TABLE), anyList())) + .thenReturn(Collections.emptyList()); doAnswer( invocation -> { @SuppressWarnings("unchecked") @@ -1137,6 +1154,11 @@ private static class ApplyingThenFailingStatisticsManager private int statisticsAttempts; private long appliedRecordCount; + @Override + public OptionalLong explicitPartitionLocationCount() { + return OptionalLong.of(0); + } + @Override public void createPartitions( List> partitions, @@ -1164,7 +1186,7 @@ public List listPartitions( @Override public List listPartitionsByNames(List> partitions) { - throw new UnsupportedOperationException(); + return Collections.emptyList(); } @Override From 95f4894077cc186f0f2af35208d956c917eb5b5e Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Thu, 3 Sep 2026 03:12:28 +0800 Subject: [PATCH 12/15] refactor(format): simplify explicit partition location protocol --- docs/docs/concepts/rest/rest-api.md | 8 + docs/docs/flink/sql-ddl.md | 5 +- docs/docs/spark/sql-ddl.md | 41 +- docs/scripts/validate-rest-openapi.js | 59 ++ docs/static/rest-catalog-open-api.yaml | 115 +-- .../apache/paimon/partition/Partition.java | 3 +- .../paimon/partition/PartitionLocation.java | 87 -- .../java/org/apache/paimon/rest/RESTApi.java | 113 +-- .../rest/RESTCatalogInternalOptions.java | 6 - .../org/apache/paimon/rest/ResourcePaths.java | 13 - .../requests/CreatePartitionsRequest.java | 21 +- .../responses/CreatePartitionsResponse.java | 13 +- .../rest/responses/GetTableResponse.java | 47 -- .../apache/paimon/catalog/CachingCatalog.java | 3 +- .../org/apache/paimon/catalog/Catalog.java | 60 +- .../paimon/catalog/DelegateCatalog.java | 10 +- .../org/apache/paimon/rest/RESTCatalog.java | 142 +--- .../CatalogFormatTablePartitionManager.java | 91 +-- .../table/format/CatalogSplitEnumerator.java | 103 ++- .../table/format/FormatReadBuilder.java | 18 +- .../table/format/FormatTableCommit.java | 135 +-- .../format/FormatTableFileIOResolver.java | 7 +- .../format/FormatTablePartitionManager.java | 26 +- .../FormatTablePartitionPathResolver.java | 308 ++++--- ...FormatTablePartitionRegistryValidator.java | 53 +- .../paimon/catalog/CachingCatalogTest.java | 7 +- .../paimon/catalog/DelegateCatalogTest.java | 4 +- .../paimon/rest/MockRESTCatalogTest.java | 542 ++++++------ .../apache/paimon/rest/MockRESTMessage.java | 1 - .../apache/paimon/rest/RESTApiJsonTest.java | 48 +- .../rest/RESTCatalogPartitionSupport.java | 151 +++- .../apache/paimon/rest/RESTCatalogServer.java | 773 ++++++++---------- .../apache/paimon/rest/ResourcePathsTest.java | 5 +- ...atalogFormatTablePartitionManagerTest.java | 144 +++- .../CatalogManagedPartitionScanTest.java | 188 ++--- .../table/format/FormatReadBuilderTest.java | 1 + ...rmatTableCommitRegistryValidationTest.java | 116 +-- .../FormatTableCommitStatisticsTest.java | 39 +- .../table/format/FormatTableCommitTest.java | 131 +-- .../FormatTablePartitionPathResolverTest.java | 293 +++++++ .../paimon/flink/FlinkRestCatalogITCase.java | 15 - .../paimon/flink/RESTCatalogITCaseBase.java | 5 +- .../org/apache/paimon/spark/SparkCatalog.java | 28 - ...nAnalyzeFormatTablePartitionsCommand.scala | 22 +- .../spark/format/PaimonFormatTable.scala | 40 +- .../spark/SparkCatalogWithRestTest.java | 69 +- .../PaimonSparkTestWithRestCatalogBase.scala | 6 +- .../FormatTablePartitionDdlPlanningTest.scala | 58 +- .../FormatTablePartitionManagementTest.scala | 76 +- .../CatalogManagedPartitionAnalyzeTest.scala | 51 +- ...CatalogManagedPartitionDdlParityTest.scala | 53 -- 51 files changed, 2099 insertions(+), 2254 deletions(-) delete mode 100644 paimon-api/src/main/java/org/apache/paimon/partition/PartitionLocation.java diff --git a/docs/docs/concepts/rest/rest-api.md b/docs/docs/concepts/rest/rest-api.md index 8a2f2e69e0a5..34a424b79799 100644 --- a/docs/docs/concepts/rest/rest-api.md +++ b/docs/docs/concepts/rest/rest-api.md @@ -25,6 +25,14 @@ under the License. The OpenAPI 3.1 document below defines the language-neutral wire contract for REST Catalog servers and clients. It can also be used to generate or validate SDK models in other languages. +Explicit partition locations extend the existing `POST .../partitions` request. When present, +`partitionLocations` has the same length and order as `partitionSpecs`; a null entry selects the +derived default. An upgraded server echoes the canonical stored locations in the same order, and a +client must verify every entry. A missing echo means the server did not confirm the extension, so +deploy the upgraded server before enabling clients that create explicit locations. +Upgrade all readers before registering such locations because the server cannot identify and +selectively reject older clients. +