From 480370111dce7fe8226546264ea19741a18e3248 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 4 Aug 2026 18:57:27 +0800 Subject: [PATCH 01/10] [refactor](fe) Add scoped metadata cache prototype ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Metadata cache invalidation needs catalog, database, table, partition, and exact-key isolation without stale publication races or unbounded secondary-index retention. Add an isolated catalog-local scoped cache prototype with generation-aware single-flight loading, exact-key and hierarchical publication fencing, atomic bulk commit, conditional physical cleanup, and bottom-up scope/key-node reclamation. The prototype is not wired into existing metadata cache callers yet. Add deterministic tests for the complete scope matrix, refresh/load/bulk/close races, delayed removal callbacks, generation overflow, eviction, lifecycle closure, high-cardinality churn, and a seeded reference-model state machine. ### Release note None ### Check List (For Author) - Test: Unit Test and FE build - Unit Test: fe-connector-cache Maven reactor, 86 tests passed - Unit Test: focused JDK 17 run-fe-ut review suite passed - FE build: JDK 17 ./build.sh --fe with a fresh output directory passed - Behavior changed: No. The prototype is not connected to production callers. - Does this need documentation: No --- .../doris/connector/cache/ScopePath.java | 149 ++++ .../connector/cache/ScopedMetaCache.java | 758 ++++++++++++++++ .../cache/ScopedMetaCacheRegistry.java | 615 +++++++++++++ .../doris/connector/cache/ScopePathTest.java | 77 ++ .../cache/ScopedMetaCacheConcurrencyTest.java | 842 ++++++++++++++++++ .../cache/ScopedMetaCacheHierarchyTest.java | 349 ++++++++ .../cache/ScopedMetaCacheLeakTest.java | 335 +++++++ 7 files changed, 3125 insertions(+) create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java create mode 100644 fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopePathTest.java create mode 100644 fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java create mode 100644 fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheHierarchyTest.java create mode 100644 fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheLeakTest.java diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java new file mode 100644 index 00000000000000..a3fe243cb70fbf --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.util.Objects; + +/** + * Hierarchical invalidation path within one catalog-owned scoped cache registry. + * + *

The catalog identity is implicit in the owning {@link ScopedMetaCacheRegistry}. Database and table + * identities are strings because FE and connector callers already invalidate by their respective local or remote + * names. A partition identity is deliberately opaque: Hive partition values, Iceberg transforms, and other + * connector-specific partition identities do not share one representation. Like a cache key, it must keep stable + * {@link Object#equals(Object)} and {@link Object#hashCode()} semantics while registered. + */ +public final class ScopePath { + public enum Level { + CATALOG, + DATABASE, + TABLE, + PARTITION + } + + private static final ScopePath CATALOG = new ScopePath(Level.CATALOG, null, null, null); + + private final Level level; + private final String database; + private final String table; + private final Object partition; + + private ScopePath(Level level, String database, String table, Object partition) { + this.level = level; + this.database = database; + this.table = table; + this.partition = partition; + } + + public static ScopePath catalog() { + return CATALOG; + } + + public static ScopePath database(String database) { + return new ScopePath(Level.DATABASE, Objects.requireNonNull(database, "database can not be null"), + null, null); + } + + public static ScopePath table(String database, String table) { + return new ScopePath( + Level.TABLE, + Objects.requireNonNull(database, "database can not be null"), + Objects.requireNonNull(table, "table can not be null"), + null); + } + + public static ScopePath partition(String database, String table, Object partition) { + return new ScopePath( + Level.PARTITION, + Objects.requireNonNull(database, "database can not be null"), + Objects.requireNonNull(table, "table can not be null"), + Objects.requireNonNull(partition, "partition can not be null")); + } + + public Level level() { + return level; + } + + public String database() { + return database; + } + + public String table() { + return table; + } + + public Object partition() { + return partition; + } + + public boolean contains(ScopePath other) { + Objects.requireNonNull(other, "other can not be null"); + if (level.ordinal() > other.level.ordinal()) { + return false; + } + if (level == Level.CATALOG) { + return true; + } + if (!database.equals(other.database)) { + return false; + } + if (level == Level.DATABASE) { + return true; + } + if (!table.equals(other.table)) { + return false; + } + return level == Level.TABLE || partition.equals(other.partition); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof ScopePath)) { + return false; + } + ScopePath other = (ScopePath) obj; + return level == other.level + && Objects.equals(database, other.database) + && Objects.equals(table, other.table) + && Objects.equals(partition, other.partition); + } + + @Override + public int hashCode() { + return Objects.hash(level, database, table, partition); + } + + @Override + public String toString() { + switch (level) { + case CATALOG: + return "catalog"; + case DATABASE: + return "database[" + database + "]"; + case TABLE: + return "table[" + database + "." + table + "]"; + case PARTITION: + return "partition[" + database + "." + table + ":" + partition + "]"; + default: + throw new IllegalStateException("Unknown scope level: " + level); + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java new file mode 100644 index 00000000000000..53bc58ca50cd65 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java @@ -0,0 +1,758 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.apache.doris.connector.cache.ScopedMetaCacheRegistry.CacheAddress; +import org.apache.doris.connector.cache.ScopedMetaCacheRegistry.PublicationState; +import org.apache.doris.connector.cache.ScopedMetaCacheRegistry.ScopeLease; +import org.apache.doris.connector.cache.ScopedMetaCacheRegistry.ScopeSnapshot; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.RemovalCause; +import com.github.benmanes.caffeine.cache.Ticker; + +import java.math.BigInteger; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Objects; +import java.util.OptionalLong; +import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; +import java.util.function.Function; + +/** + * One physical Caffeine cache participating in a {@link ScopedMetaCacheRegistry}. + * + *

Every value is wrapped with both its hierarchical scope-state identities and an exact-key state. Hierarchical + * invalidation can therefore detach a whole catalog/database/table/partition subtree, while exact-key invalidation + * fences only one physical cache key. Removal listeners conditionally remove the exact wrapper from its original + * scope bucket and key node, so delayed callbacks cannot delete a replacement. + */ +public final class ScopedMetaCache implements AutoCloseable { + private static final Runnable NO_OP = () -> { + }; + + private final ScopedMetaCacheRegistry registry; + private final String name; + private final boolean effectiveEnabled; + private final Cache> data; + private final ConcurrentMap> keyNodes = new ConcurrentHashMap<>(); + private final ConcurrentMap, CompletableFuture> inFlightLoads = new ConcurrentHashMap<>(); + private final Object bulkInvalidationLock = new Object(); + private final Map exactInvalidations = new HashMap<>(); + private final NavigableMap activeBulkStarts = new TreeMap<>(); + private final AtomicBoolean closed = new AtomicBoolean(false); + private final BiConsumer beforeRemoval; + private final Runnable afterLoadElection; + private final Runnable afterBulkStage; + private BigInteger exactInvalidationSequence = BigInteger.ZERO; + + ScopedMetaCache( + ScopedMetaCacheRegistry registry, + String name, + CacheSpec cacheSpec, + Ticker ticker, + BiConsumer beforeRemoval, + Runnable afterLoadElection, + Runnable afterBulkStage) { + this.registry = Objects.requireNonNull(registry, "registry can not be null"); + this.name = Objects.requireNonNull(name, "name can not be null"); + Objects.requireNonNull(cacheSpec, "cacheSpec can not be null"); + this.beforeRemoval = beforeRemoval; + this.afterLoadElection = + Objects.requireNonNull(afterLoadElection, "afterLoadElection can not be null"); + this.afterBulkStage = Objects.requireNonNull(afterBulkStage, "afterBulkStage can not be null"); + this.effectiveEnabled = CacheSpec.isCacheEnabled( + cacheSpec.isEnable(), cacheSpec.getTtlSecond(), cacheSpec.getCapacity()); + + Caffeine builder = Caffeine.newBuilder() + .maximumSize(effectiveEnabled ? cacheSpec.getCapacity() : 0L) + .executor(Runnable::run) + .removalListener(this::onRemoval); + OptionalLong expiry = effectiveEnabled + ? CacheSpec.toExpireAfterAccess(cacheSpec.getTtlSecond()) + : OptionalLong.empty(); + if (expiry.isPresent()) { + builder.expireAfterAccess(Duration.ofSeconds(expiry.getAsLong())); + } + if (ticker != null) { + builder.ticker(ticker); + } + this.data = builder.build(); + } + + public String name() { + return name; + } + + public V get(K key, ScopePath path, Function loader) { + Objects.requireNonNull(key, "key can not be null"); + Objects.requireNonNull(path, "path can not be null"); + Function loadFunction = Objects.requireNonNull(loader, "loader can not be null"); + checkOpen(); + if (!effectiveEnabled) { + return loadFunction.apply(key); + } + + V present = getIfPresent(key, path); + if (present != null) { + return present; + } + try (PublicationLease lease = acquirePublicationLease(key, path, true)) { + LoadAddress loadAddress = new LoadAddress<>(key, path, lease); + CompletableFuture ownLoad = new CompletableFuture<>(); + CompletableFuture existingLoad = inFlightLoads.putIfAbsent(loadAddress, ownLoad); + if (existingLoad != null) { + return awaitLoad(existingLoad); + } + try { + afterLoadElection.run(); + synchronized (lease.keyNode) { + present = getIfPresent(key, path); + if (present != null) { + ownLoad.complete(present); + return present; + } + } + V loaded = loadFunction.apply(key); + if (loaded != null) { + synchronized (lease.keyNode) { + publishCommitted(lease, key, loaded); + } + } + ownLoad.complete(loaded); + return loaded; + } catch (RuntimeException | Error throwable) { + ownLoad.completeExceptionally(throwable); + throw throwable; + } finally { + inFlightLoads.remove(loadAddress, ownLoad); + } + } + } + + public V getIfPresent(K key, ScopePath path) { + Objects.requireNonNull(key, "key can not be null"); + Objects.requireNonNull(path, "path can not be null"); + checkOpen(); + if (!effectiveEnabled) { + return null; + } + VersionedValue versioned = data.getIfPresent(key); + if (versioned == null) { + return null; + } + if (!versioned.scopeSnapshot.path().equals(path)) { + return null; + } + if (!versioned.committed.get()) { + return null; + } + if (!versioned.isCurrent(registry, keyNodes)) { + data.asMap().remove(key, versioned); + return null; + } + return versioned.value; + } + + public void put(K key, ScopePath path, V value) { + Objects.requireNonNull(key, "key can not be null"); + Objects.requireNonNull(path, "path can not be null"); + Objects.requireNonNull(value, "value can not be null"); + checkOpen(); + if (!effectiveEnabled) { + return; + } + try (PublicationLease lease = acquirePublicationLease(key, path, false)) { + synchronized (lease.keyNode) { + lease.keyNode.loadPublicationState.set(new Object()); + publishCommitted(lease, key, value); + } + } + } + + public void invalidateKey(K key) { + invalidateKey(key, NO_OP, NO_OP); + } + + void invalidateKey(K key, Runnable afterStateReplacement) { + invalidateKey(key, NO_OP, afterStateReplacement); + } + + void invalidateKey( + K key, Runnable beforeInvalidationLock, Runnable afterStateReplacement) { + Objects.requireNonNull(key, "key can not be null"); + Objects.requireNonNull(beforeInvalidationLock, "beforeInvalidationLock can not be null"); + Objects.requireNonNull(afterStateReplacement, "afterStateReplacement can not be null"); + checkOpen(); + beforeInvalidationLock.run(); + KeyNode node; + KeyState invalidatedState = null; + synchronized (bulkInvalidationLock) { + if (closed.get()) { + return; + } + exactInvalidationSequence = exactInvalidationSequence.add(BigInteger.ONE); + if (!activeBulkStarts.isEmpty()) { + exactInvalidations.put(key, exactInvalidationSequence); + } + node = keyNodes.get(key); + if (node != null) { + invalidatedState = replaceKeyState(node); + } + } + if (node == null) { + return; + } + afterStateReplacement.run(); + VersionedValue registered = node.registration.get(); + if (registered != null && registered.keyState == invalidatedState) { + data.asMap().remove(key, registered); + node.registration.compareAndSet(registered, null); + } + tryPruneKey(key, node); + } + + public BulkLoadHandle beginBulkLoad(ScopePath parentScope) { + Objects.requireNonNull(parentScope, "parentScope can not be null"); + checkOpen(); + if (!effectiveEnabled) { + return BulkLoadHandle.disabled(this, parentScope); + } + ScopeLease scopeLease = registry.acquire(parentScope); + BigInteger exactSequence; + synchronized (bulkInvalidationLock) { + if (closed.get()) { + scopeLease.close(); + throw new IllegalStateException("Scoped meta cache '" + name + "' is closed"); + } + exactSequence = exactInvalidationSequence; + activeBulkStarts.merge(exactSequence, 1, Integer::sum); + } + return new BulkLoadHandle( + this, + parentScope, + scopeLease, + scopeLease.publicationState(), + exactSequence); + } + + public boolean publish( + BulkLoadHandle handle, K key, ScopePath actualScope, V value) { + Objects.requireNonNull(handle, "handle can not be null"); + Objects.requireNonNull(key, "key can not be null"); + Objects.requireNonNull(actualScope, "actualScope can not be null"); + Objects.requireNonNull(value, "value can not be null"); + checkOpen(); + handle.checkOwner(this); + if (!handle.parentScope.contains(actualScope)) { + throw new IllegalArgumentException( + "Actual scope " + actualScope + " is outside bulk-load parent " + handle.parentScope); + } + if (!effectiveEnabled) { + return false; + } + try (PublicationLease lease = acquirePublicationLease(key, actualScope, false)) { + synchronized (lease.keyNode) { + if (!handle.canStage(key)) { + return false; + } + lease.keyNode.loadPublicationState.set(new Object()); + VersionedValue staged = newVersionedValue(lease, key, value); + afterBulkStage.run(); + if (handle.tryCommit(key, lease, staged)) { + return true; + } + return false; + } + } + } + + public CacheMetrics metrics() { + synchronized (bulkInvalidationLock) { + return new CacheMetrics( + data.estimatedSize(), + keyNodes.size(), + inFlightLoads.size(), + activeBulkStarts.values().stream().mapToInt(Integer::intValue).sum(), + exactInvalidations.size()); + } + } + + public void cleanUp() { + data.cleanUp(); + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + registry.removeCache(this); + closePhysicalState(); + } + + void closeFromRegistry() { + if (closed.compareAndSet(false, true)) { + closePhysicalState(); + } + } + + void removeExpectedRaw(Object rawKey, Object expectedValue) { + @SuppressWarnings("unchecked") + K key = (K) rawKey; + @SuppressWarnings("unchecked") + VersionedValue versionedValue = (VersionedValue) expectedValue; + data.asMap().remove(key, versionedValue); + } + + private PublicationLease acquirePublicationLease( + K key, ScopePath path, boolean fenceAgainstDirectPublication) { + while (true) { + ScopeLease scopeLease = registry.acquire(path); + KeyNode keyNode = keyNodes.computeIfAbsent(key, ignored -> new KeyNode<>()); + keyNode.activeLoads.incrementAndGet(); + KeyState keyState = keyNode.current.get(); + Object loadPublicationState = + fenceAgainstDirectPublication ? keyNode.loadPublicationState.get() : null; + if (keyNodes.get(key) == keyNode && scopeLease.isCurrent()) { + return new PublicationLease<>( + this, key, scopeLease, keyNode, keyState, loadPublicationState); + } + releaseKey(key, keyNode); + scopeLease.close(); + } + } + + private VersionedValue publish(PublicationLease lease, K key, V value) { + if (!lease.isCurrent()) { + return null; + } + VersionedValue versioned = newVersionedValue(lease, key, value); + versioned.committed.set(true); + install(versioned, lease); + if (!lease.isCurrent()) { + data.asMap().remove(key, versioned); + return null; + } + return versioned; + } + + private VersionedValue publishCommitted(PublicationLease lease, K key, V value) { + return publish(lease, key, value); + } + + private VersionedValue newVersionedValue( + PublicationLease lease, K key, V value) { + CacheAddress address = new CacheAddress(this, key); + return new VersionedValue<>( + key, value, address, lease.scopeLease.snapshot(), lease.keyNode, lease.keyState); + } + + private void install( + VersionedValue versioned, PublicationLease lease) { + registry.register(versioned.address, versioned, versioned.scopeSnapshot); + lease.keyNode.registration.set(versioned); + data.asMap().put(versioned.key, versioned); + } + + private boolean canStageBulk(BulkLoadHandle handle, Object rawKey) { + synchronized (bulkInvalidationLock) { + return isBulkKeyCurrent(handle, rawKey); + } + } + + private boolean tryCommitBulk( + BulkLoadHandle handle, + Object rawKey, + PublicationLease rawLease, + VersionedValue rawStaged) { + @SuppressWarnings("unchecked") + K key = (K) rawKey; + @SuppressWarnings("unchecked") + PublicationLease lease = (PublicationLease) rawLease; + @SuppressWarnings("unchecked") + VersionedValue staged = (VersionedValue) rawStaged; + synchronized (bulkInvalidationLock) { + if (!isBulkKeyCurrent(handle, key)) { + return false; + } + return handle.scopeLease.commitIfPublicationCurrent( + handle.scopePublicationState, () -> { + if (!lease.isCurrent()) { + return false; + } + staged.committed.set(true); + install(staged, lease); + return true; + }); + } + } + + private boolean isBulkKeyCurrent(BulkLoadHandle handle, Object rawKey) { + BigInteger invalidation = exactInvalidations.get(rawKey); + return !closed.get() + && !handle.closed.get() + && handle.scopeLease.isPublicationCurrent(handle.scopePublicationState) + && (invalidation == null || invalidation.compareTo(handle.exactInvalidationSequence) <= 0); + } + + private void closeBulkHandle(BulkLoadHandle handle) { + ScopeLease leaseToClose = null; + synchronized (bulkInvalidationLock) { + if (!handle.closed.compareAndSet(false, true)) { + return; + } + if (handle.scopeLease != null) { + Integer count = activeBulkStarts.get(handle.exactInvalidationSequence); + if (count == null) { + throw new IllegalStateException("Bulk-load handle start sequence is not registered"); + } + if (count == 1) { + activeBulkStarts.remove(handle.exactInvalidationSequence); + } else { + activeBulkStarts.put(handle.exactInvalidationSequence, count - 1); + } + pruneExactInvalidations(); + leaseToClose = handle.scopeLease; + } + } + if (leaseToClose != null) { + leaseToClose.close(); + } + } + + private void pruneExactInvalidations() { + if (activeBulkStarts.isEmpty()) { + exactInvalidations.clear(); + return; + } + BigInteger oldestStart = activeBulkStarts.firstKey(); + exactInvalidations.entrySet().removeIf(entry -> entry.getValue().compareTo(oldestStart) <= 0); + } + + private V awaitLoad(CompletableFuture load) { + try { + return load.join(); + } catch (CompletionException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + throw new IllegalStateException("Unexpected checked exception from metadata loader", cause); + } + } + + private void closePhysicalState() { + synchronized (bulkInvalidationLock) { + exactInvalidations.clear(); + } + keyNodes.forEach((key, node) -> { + replaceKeyState(node); + VersionedValue versioned = node.registration.get(); + if (versioned != null) { + data.asMap().remove(key, versioned); + node.registration.compareAndSet(versioned, null); + } + tryPruneKey(key, node); + }); + data.invalidateAll(); + data.cleanUp(); + } + + private void onRemoval( + Object rawKey, Object rawValue, RemovalCause cause) { + Objects.requireNonNull(rawKey, "removed cache key can not be null"); + Objects.requireNonNull(rawValue, "removed cache value can not be null"); + @SuppressWarnings("unchecked") + K key = (K) rawKey; + @SuppressWarnings("unchecked") + VersionedValue versioned = (VersionedValue) rawValue; + if (beforeRemoval != null) { + beforeRemoval.accept(key, versioned.value); + } + registry.unregister(versioned.address, versioned, versioned.scopeSnapshot); + versioned.keyNode.registration.compareAndSet(versioned, null); + tryPruneKey(key, versioned.keyNode); + } + + private KeyState replaceKeyState(KeyNode node) { + while (true) { + KeyState oldState = node.current.get(); + if (node.current.compareAndSet(oldState, new KeyState())) { + return oldState; + } + } + } + + private void releaseKey(K key, KeyNode node) { + int remaining = node.activeLoads.decrementAndGet(); + if (remaining < 0) { + throw new IllegalStateException("Cache key active-load count became negative"); + } + tryPruneKey(key, node); + } + + private void tryPruneKey(K key, KeyNode node) { + if (node.activeLoads.get() == 0 && node.registration.get() == null) { + keyNodes.remove(key, node); + } + } + + private void checkOpen() { + if (closed.get()) { + throw new IllegalStateException("Scoped meta cache '" + name + "' is closed"); + } + registry.checkOpen(); + } + + public static final class BulkLoadHandle implements AutoCloseable { + private final ScopedMetaCache owner; + private final ScopePath parentScope; + private final ScopeLease scopeLease; + private final PublicationState scopePublicationState; + private final BigInteger exactInvalidationSequence; + private final AtomicBoolean closed = new AtomicBoolean(false); + + private BulkLoadHandle( + ScopedMetaCache owner, + ScopePath parentScope, + ScopeLease scopeLease, + PublicationState scopePublicationState, + BigInteger exactInvalidationSequence) { + this.owner = owner; + this.parentScope = parentScope; + this.scopeLease = scopeLease; + this.scopePublicationState = scopePublicationState; + this.exactInvalidationSequence = exactInvalidationSequence; + } + + private static BulkLoadHandle disabled( + ScopedMetaCache owner, ScopePath parentScope) { + return new BulkLoadHandle(owner, parentScope, null, null, BigInteger.ZERO); + } + + private void checkOwner(ScopedMetaCache expectedOwner) { + if (owner != expectedOwner) { + throw new IllegalArgumentException("Bulk-load handle belongs to another cache"); + } + if (closed.get()) { + throw new IllegalStateException("Bulk-load handle is closed"); + } + } + + private boolean canStage(Object key) { + return owner.canStageBulk(this, key); + } + + private boolean tryCommit( + Object key, PublicationLease lease, VersionedValue staged) { + return owner.tryCommitBulk(this, key, lease, staged); + } + + @Override + public void close() { + owner.closeBulkHandle(this); + } + } + + public static final class CacheMetrics { + private final long physicalEntryCount; + private final int keyNodeCount; + private final int inFlightLoadCount; + private final int activeBulkHandleCount; + private final int exactInvalidationTombstoneCount; + + private CacheMetrics( + long physicalEntryCount, + int keyNodeCount, + int inFlightLoadCount, + int activeBulkHandleCount, + int exactInvalidationTombstoneCount) { + this.physicalEntryCount = physicalEntryCount; + this.keyNodeCount = keyNodeCount; + this.inFlightLoadCount = inFlightLoadCount; + this.activeBulkHandleCount = activeBulkHandleCount; + this.exactInvalidationTombstoneCount = exactInvalidationTombstoneCount; + } + + public long getPhysicalEntryCount() { + return physicalEntryCount; + } + + public int getKeyNodeCount() { + return keyNodeCount; + } + + public int getInFlightLoadCount() { + return inFlightLoadCount; + } + + public int getActiveBulkHandleCount() { + return activeBulkHandleCount; + } + + public int getExactInvalidationTombstoneCount() { + return exactInvalidationTombstoneCount; + } + } + + private static final class PublicationLease implements AutoCloseable { + private final ScopedMetaCache owner; + private final K key; + private final ScopeLease scopeLease; + private final KeyNode keyNode; + private final KeyState keyState; + private final Object loadPublicationState; + private final AtomicBoolean released = new AtomicBoolean(false); + + private PublicationLease( + ScopedMetaCache owner, + K key, + ScopeLease scopeLease, + KeyNode keyNode, + KeyState keyState, + Object loadPublicationState) { + this.owner = owner; + this.key = key; + this.scopeLease = scopeLease; + this.keyNode = keyNode; + this.keyState = keyState; + this.loadPublicationState = loadPublicationState; + } + + private boolean isCurrent() { + return !owner.closed.get() + && scopeLease.isCurrent() + && owner.keyNodes.get(key) == keyNode + && keyNode.current.get() == keyState + && (loadPublicationState == null + || keyNode.loadPublicationState.get() == loadPublicationState); + } + + @Override + public void close() { + if (released.compareAndSet(false, true)) { + owner.releaseKey(key, keyNode); + scopeLease.close(); + } + } + } + + private static final class VersionedValue { + private final K key; + private final V value; + private final CacheAddress address; + private final ScopeSnapshot scopeSnapshot; + private final KeyNode keyNode; + private final KeyState keyState; + private final AtomicBoolean committed = new AtomicBoolean(false); + + private VersionedValue( + K key, + V value, + CacheAddress address, + ScopeSnapshot scopeSnapshot, + KeyNode keyNode, + KeyState keyState) { + this.key = key; + this.value = value; + this.address = address; + this.scopeSnapshot = scopeSnapshot; + this.keyNode = keyNode; + this.keyState = keyState; + } + + private boolean isCurrent( + ScopedMetaCacheRegistry registry, + Map> currentKeyNodes) { + return scopeSnapshot.isCurrent(registry) + && currentKeyNodes.get(key) == keyNode + && keyNode.current.get() == keyState + && keyNode.registration.get() == this + && committed.get(); + } + } + + private static final class KeyNode { + private final AtomicReference current = new AtomicReference<>(new KeyState()); + private final AtomicReference loadPublicationState = new AtomicReference<>(new Object()); + private final AtomicReference> registration = new AtomicReference<>(); + private final AtomicInteger activeLoads = new AtomicInteger(); + } + + private static final class KeyState { + } + + private static final class LoadAddress { + private final K key; + private final ScopePath path; + private final ScopeSnapshot scopeSnapshot; + private final KeyNode keyNode; + private final KeyState keyState; + private final Object loadPublicationState; + + private LoadAddress(K key, ScopePath path, PublicationLease lease) { + this.key = key; + this.path = path; + this.scopeSnapshot = lease.scopeLease.snapshot(); + this.keyNode = lease.keyNode; + this.keyState = lease.keyState; + this.loadPublicationState = lease.loadPublicationState; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof LoadAddress)) { + return false; + } + LoadAddress other = (LoadAddress) obj; + return key.equals(other.key) + && path.equals(other.path) + && scopeSnapshot.sameGeneration(other.scopeSnapshot) + && keyNode == other.keyNode + && keyState == other.keyState + && loadPublicationState == other.loadPublicationState; + } + + @Override + public int hashCode() { + int result = 31 * key.hashCode() + path.hashCode(); + result = 31 * result + scopeSnapshot.generationHashCode(); + result = 31 * result + System.identityHashCode(keyNode); + result = 31 * result + System.identityHashCode(keyState); + return 31 * result + System.identityHashCode(loadPublicationState); + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java new file mode 100644 index 00000000000000..cdd222564ad95d --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java @@ -0,0 +1,615 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import com.github.benmanes.caffeine.cache.Ticker; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; +import java.util.function.BooleanSupplier; + +/** + * Catalog-local hierarchy shared by independent physical metadata caches. + * + *

Logical invalidation replaces one state object. Values capture state identities and are rejected after a + * replacement even when physical Caffeine cleanup is delayed. Each state also owns the exact registrations that + * must be physically removed. Empty child nodes are pruned bottom-up so scope-directory memory remains bounded by + * live values and in-flight loads rather than by every name ever observed. + */ +public final class ScopedMetaCacheRegistry implements AutoCloseable { + private static final Runnable NO_OP = () -> { + }; + + private final ScopeNode root = new ScopeNode(null, null, ScopePath.Level.CATALOG); + private final Set> caches = ConcurrentHashMap.newKeySet(); + private final Object publicationLock = new Object(); + private final AtomicBoolean closed = new AtomicBoolean(false); + private final AtomicIntegerArray retainedActiveLoads = + new AtomicIntegerArray(ScopePath.Level.values().length); + + public ScopedMetaCache createCache(String name, CacheSpec cacheSpec) { + return createCache(name, cacheSpec, null, null, NO_OP, NO_OP); + } + + ScopedMetaCache createCache( + String name, + CacheSpec cacheSpec, + Ticker ticker, + BiConsumer beforeRemoval) { + return createCache(name, cacheSpec, ticker, beforeRemoval, NO_OP, NO_OP); + } + + ScopedMetaCache createCache( + String name, + CacheSpec cacheSpec, + Ticker ticker, + BiConsumer beforeRemoval, + Runnable afterBulkStage) { + return createCache(name, cacheSpec, ticker, beforeRemoval, NO_OP, afterBulkStage); + } + + ScopedMetaCache createCache( + String name, + CacheSpec cacheSpec, + Ticker ticker, + BiConsumer beforeRemoval, + Runnable afterLoadElection, + Runnable afterBulkStage) { + checkOpen(); + ScopedMetaCache cache = new ScopedMetaCache<>( + this, + name, + cacheSpec, + ticker, + beforeRemoval, + afterLoadElection, + afterBulkStage); + caches.add(cache); + if (closed.get()) { + caches.remove(cache); + cache.closeFromRegistry(); + throw new IllegalStateException("Scoped meta cache registry is closed"); + } + return cache; + } + + public void invalidate(ScopePath path) { + invalidate(path, NO_OP); + } + + void invalidate(ScopePath path, Runnable afterStateReplacement) { + Objects.requireNonNull(path, "path can not be null"); + Objects.requireNonNull(afterStateReplacement, "afterStateReplacement can not be null"); + checkOpen(); + List existing; + ScopeState oldState; + synchronized (publicationLock) { + existing = resolveExisting(path); + bumpPublicationStates(existing); + if (existing.size() != path.level().ordinal() + 1) { + return; + } + ScopeNode target = existing.get(existing.size() - 1); + oldState = replaceState(target); + } + afterStateReplacement.run(); + cleanDetachedState(oldState); + tryPrune(existing); + } + + public ScopeMetrics metrics() { + ScopeMetricsAccumulator accumulator = new ScopeMetricsAccumulator(); + collectMetrics(root, accumulator); + for (ScopePath.Level level : ScopePath.Level.values()) { + accumulator.retainedActiveLoads[level.ordinal()] = retainedActiveLoads.get(level.ordinal()); + } + return accumulator.snapshot(); + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + ScopeState oldState; + synchronized (publicationLock) { + bumpPublicationStates(Collections.singletonList(root)); + oldState = replaceState(root); + } + cleanDetachedState(oldState); + List> snapshot = new ArrayList<>(caches); + caches.clear(); + snapshot.forEach(ScopedMetaCache::closeFromRegistry); + } + + ScopeLease acquire(ScopePath path) { + Objects.requireNonNull(path, "path can not be null"); + while (true) { + checkOpen(); + List nodes = resolveOrCreate(path); + nodes.forEach(node -> { + node.activeLoads.incrementAndGet(); + retainedActiveLoads.incrementAndGet(node.level.ordinal()); + }); + ScopeSnapshot snapshot = ScopeSnapshot.capture(path, nodes); + if (snapshot.isCurrent(this)) { + return new ScopeLease(this, snapshot); + } + release(nodes); + } + } + + void checkOpen() { + if (closed.get()) { + throw new IllegalStateException("Scoped meta cache registry is closed"); + } + } + + void removeCache(ScopedMetaCache cache) { + caches.remove(cache); + } + + void register(CacheAddress address, Object versionedValue, ScopeSnapshot snapshot) { + snapshot.leafState().entries.put(address, versionedValue); + } + + void unregister(CacheAddress address, Object versionedValue, ScopeSnapshot snapshot) { + snapshot.leafState().entries.remove(address, versionedValue); + tryPrune(snapshot.nodes); + } + + private ScopeState replaceState(ScopeNode node) { + while (true) { + ScopeState oldState = node.current.get(); + ScopeState newState = new ScopeState(oldState.generation + 1L); + if (node.current.compareAndSet(oldState, newState)) { + return oldState; + } + } + } + + private List resolveOrCreate(ScopePath path) { + List nodes = new ArrayList<>(path.level().ordinal() + 1); + ScopeNode catalogNode = root; + nodes.add(catalogNode); + if (path.level() == ScopePath.Level.CATALOG) { + return nodes; + } + ScopeState catalogState = catalogNode.current.get(); + ScopeNode dbNode = child(catalogNode, catalogState, path.database(), ScopePath.Level.DATABASE); + nodes.add(dbNode); + if (path.level() == ScopePath.Level.DATABASE) { + return nodes; + } + ScopeState dbState = dbNode.current.get(); + ScopeNode tableNode = child(dbNode, dbState, path.table(), ScopePath.Level.TABLE); + nodes.add(tableNode); + if (path.level() == ScopePath.Level.TABLE) { + return nodes; + } + ScopeState tableState = tableNode.current.get(); + nodes.add(child(tableNode, tableState, path.partition(), ScopePath.Level.PARTITION)); + return nodes; + } + + private List resolveExisting(ScopePath path) { + List nodes = new ArrayList<>(path.level().ordinal() + 1); + ScopeNode catalogNode = root; + nodes.add(catalogNode); + if (path.level() == ScopePath.Level.CATALOG) { + return nodes; + } + ScopeNode dbNode = catalogNode.current.get().children.get(path.database()); + if (dbNode == null) { + return nodes; + } + nodes.add(dbNode); + if (path.level() == ScopePath.Level.DATABASE) { + return nodes; + } + ScopeNode tableNode = dbNode.current.get().children.get(path.table()); + if (tableNode == null) { + return nodes; + } + nodes.add(tableNode); + if (path.level() == ScopePath.Level.TABLE) { + return nodes; + } + ScopeNode partitionNode = tableNode.current.get().children.get(path.partition()); + if (partitionNode != null) { + nodes.add(partitionNode); + } + return nodes; + } + + private ScopeNode child( + ScopeNode parent, ScopeState parentState, Object childKey, ScopePath.Level childLevel) { + return parentState.children.computeIfAbsent( + childKey, ignored -> new ScopeNode(parent, childKey, childLevel)); + } + + private void bumpPublicationStates(List nodes) { + nodes.forEach(node -> { + while (true) { + PublicationState oldState = node.descendantPublication.get(); + synchronized (oldState) { + if (node.descendantPublication.compareAndSet(oldState, new PublicationState())) { + break; + } + } + } + }); + } + + private void cleanDetachedState(ScopeState state) { + state.entries.forEach((address, value) -> { + address.removeExpected(value); + state.entries.remove(address, value); + }); + state.children.values().forEach(child -> cleanDetachedState(child.current.get())); + state.children.clear(); + } + + private void release(List nodes) { + nodes.forEach(node -> { + int remaining = node.activeLoads.decrementAndGet(); + int retainedRemaining = retainedActiveLoads.decrementAndGet(node.level.ordinal()); + if (remaining < 0) { + throw new IllegalStateException("Scope active-load count became negative"); + } + if (retainedRemaining < 0) { + throw new IllegalStateException("Retained scope active-load count became negative"); + } + }); + tryPrune(nodes); + } + + private void tryPrune(List nodes) { + for (int i = nodes.size() - 1; i > 0; i--) { + tryPrune(nodes.get(i)); + } + } + + private boolean tryPrune(ScopeNode node) { + if (node.parent == null) { + return false; + } + ScopeState state = node.current.get(); + if (node.activeLoads.get() != 0 || !state.entries.isEmpty() || !state.children.isEmpty()) { + return false; + } + ScopeState parentState = node.parent.current.get(); + return parentState.children.remove(node.childKey, node); + } + + private void collectMetrics(ScopeNode node, ScopeMetricsAccumulator accumulator) { + ScopeState state = node.current.get(); + accumulator.registrations += state.entries.size(); + state.children.values().forEach(child -> { + switch (child.level) { + case DATABASE: + accumulator.databaseNodes++; + break; + case TABLE: + accumulator.tableNodes++; + break; + case PARTITION: + accumulator.partitionNodes++; + break; + default: + throw new IllegalStateException("Unexpected child scope level: " + child.level); + } + collectMetrics(child, accumulator); + }); + } + + void forceGenerationForTest(ScopePath path, long generation) { + List existing = resolveExisting(path); + if (existing.size() != path.level().ordinal() + 1) { + throw new IllegalStateException("Scope does not exist: " + path); + } + ScopeNode node = existing.get(existing.size() - 1); + ScopeState oldState = node.current.get(); + if (!oldState.entries.isEmpty()) { + throw new IllegalStateException("Test generation can only be changed on an empty scope"); + } + ScopeState replacement = new ScopeState(generation); + replacement.children.putAll(oldState.children); + if (!node.current.compareAndSet(oldState, replacement)) { + throw new IllegalStateException("Concurrent scope mutation while forcing test generation"); + } + } + + long generationForTest(ScopePath path) { + List existing = resolveExisting(path); + if (existing.size() != path.level().ordinal() + 1) { + throw new IllegalStateException("Scope does not exist: " + path); + } + return existing.get(existing.size() - 1).current.get().generation; + } + + static final class ScopeLease implements AutoCloseable { + private final ScopedMetaCacheRegistry registry; + private final ScopeSnapshot snapshot; + private final AtomicBoolean released = new AtomicBoolean(false); + + private ScopeLease(ScopedMetaCacheRegistry registry, ScopeSnapshot snapshot) { + this.registry = registry; + this.snapshot = snapshot; + } + + ScopeSnapshot snapshot() { + return snapshot; + } + + PublicationState publicationState() { + return snapshot.leafNode().descendantPublication.get(); + } + + boolean isCurrent() { + return snapshot.isCurrent(registry); + } + + boolean isPublicationCurrent(PublicationState publicationState) { + return isCurrent() && snapshot.leafNode().descendantPublication.get() == publicationState; + } + + boolean commitIfPublicationCurrent( + PublicationState publicationState, BooleanSupplier commitAction) { + synchronized (registry.publicationLock) { + return isPublicationCurrent(publicationState) && commitAction.getAsBoolean(); + } + } + + @Override + public void close() { + if (released.compareAndSet(false, true)) { + registry.release(snapshot.nodes); + } + } + } + + static final class ScopeSnapshot { + private final ScopePath path; + private final List nodes; + private final List states; + + private ScopeSnapshot(ScopePath path, List nodes, List states) { + this.path = path; + this.nodes = Collections.unmodifiableList(new ArrayList<>(nodes)); + this.states = Collections.unmodifiableList(states); + } + + static ScopeSnapshot capture(ScopePath path, List nodes) { + List states = new ArrayList<>(nodes.size()); + nodes.forEach(node -> states.add(node.current.get())); + return new ScopeSnapshot(path, nodes, states); + } + + boolean isCurrent(ScopedMetaCacheRegistry registry) { + if (registry.closed.get() || registry.root != nodes.get(0) + || registry.root.current.get() != states.get(0)) { + return false; + } + for (int i = 1; i < nodes.size(); i++) { + ScopeNode parent = nodes.get(i - 1); + ScopeNode node = nodes.get(i); + ScopeState parentState = states.get(i - 1); + if (parent.current.get() != parentState + || parentState.children.get(node.childKey) != node + || node.current.get() != states.get(i)) { + return false; + } + } + return true; + } + + ScopePath path() { + return path; + } + + ScopeNode leafNode() { + return nodes.get(nodes.size() - 1); + } + + ScopeState leafState() { + return states.get(states.size() - 1); + } + + boolean sameGeneration(ScopeSnapshot other) { + if (nodes.size() != other.nodes.size()) { + return false; + } + for (int i = 0; i < nodes.size(); i++) { + if (nodes.get(i) != other.nodes.get(i) || states.get(i) != other.states.get(i)) { + return false; + } + } + return true; + } + + int generationHashCode() { + int result = 1; + for (int i = 0; i < nodes.size(); i++) { + result = 31 * result + System.identityHashCode(nodes.get(i)); + result = 31 * result + System.identityHashCode(states.get(i)); + } + return result; + } + } + + static final class PublicationState { + } + + static final class CacheAddress { + private final ScopedMetaCache owner; + private final Object key; + + CacheAddress(ScopedMetaCache owner, Object key) { + this.owner = owner; + this.key = key; + } + + void removeExpected(Object expectedValue) { + owner.removeExpectedRaw(key, expectedValue); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof CacheAddress)) { + return false; + } + CacheAddress other = (CacheAddress) obj; + return owner == other.owner && key.equals(other.key); + } + + @Override + public int hashCode() { + return 31 * System.identityHashCode(owner) + key.hashCode(); + } + } + + private static final class ScopeNode { + private final ScopeNode parent; + private final Object childKey; + private final ScopePath.Level level; + private final AtomicReference current = new AtomicReference<>(new ScopeState(0L)); + private final AtomicReference descendantPublication = + new AtomicReference<>(new PublicationState()); + private final AtomicInteger activeLoads = new AtomicInteger(); + + private ScopeNode( + ScopeNode parent, Object childKey, ScopePath.Level level) { + this.parent = parent; + this.childKey = childKey; + this.level = level; + } + } + + private static final class ScopeState { + private final long generation; + private final ConcurrentMap children = new ConcurrentHashMap<>(); + private final ConcurrentMap entries = new ConcurrentHashMap<>(); + + private ScopeState(long generation) { + this.generation = generation; + } + } + + public static final class ScopeMetrics { + private final int databaseNodeCount; + private final int tableNodeCount; + private final int partitionNodeCount; + private final int registrationCount; + private final int activeCatalogLoadCount; + private final int activeDatabaseLoadCount; + private final int activeTableLoadCount; + private final int activePartitionLoadCount; + + private ScopeMetrics( + int databaseNodeCount, + int tableNodeCount, + int partitionNodeCount, + int registrationCount, + int activeCatalogLoadCount, + int activeDatabaseLoadCount, + int activeTableLoadCount, + int activePartitionLoadCount) { + this.databaseNodeCount = databaseNodeCount; + this.tableNodeCount = tableNodeCount; + this.partitionNodeCount = partitionNodeCount; + this.registrationCount = registrationCount; + this.activeCatalogLoadCount = activeCatalogLoadCount; + this.activeDatabaseLoadCount = activeDatabaseLoadCount; + this.activeTableLoadCount = activeTableLoadCount; + this.activePartitionLoadCount = activePartitionLoadCount; + } + + public int getDatabaseNodeCount() { + return databaseNodeCount; + } + + public int getTableNodeCount() { + return tableNodeCount; + } + + public int getPartitionNodeCount() { + return partitionNodeCount; + } + + public int getRegistrationCount() { + return registrationCount; + } + + public int getActiveLoadCount() { + return activeCatalogLoadCount + + activeDatabaseLoadCount + + activeTableLoadCount + + activePartitionLoadCount; + } + + public int getActiveCatalogLoadCount() { + return activeCatalogLoadCount; + } + + public int getActiveDatabaseLoadCount() { + return activeDatabaseLoadCount; + } + + public int getActiveTableLoadCount() { + return activeTableLoadCount; + } + + public int getActivePartitionLoadCount() { + return activePartitionLoadCount; + } + } + + private static final class ScopeMetricsAccumulator { + private int databaseNodes; + private int tableNodes; + private int partitionNodes; + private int registrations; + private final int[] retainedActiveLoads = new int[ScopePath.Level.values().length]; + + private ScopeMetrics snapshot() { + return new ScopeMetrics( + databaseNodes, + tableNodes, + partitionNodes, + registrations, + retainedActiveLoads[ScopePath.Level.CATALOG.ordinal()], + retainedActiveLoads[ScopePath.Level.DATABASE.ordinal()], + retainedActiveLoads[ScopePath.Level.TABLE.ordinal()], + retainedActiveLoads[ScopePath.Level.PARTITION.ordinal()]); + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopePathTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopePathTest.java new file mode 100644 index 00000000000000..2233d7431258bf --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopePathTest.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ScopePathTest { + @Test + public void containsExactlyMatchesHierarchyPrefixes() { + ScopePath catalog = ScopePath.catalog(); + ScopePath database = ScopePath.database("db"); + ScopePath table = ScopePath.table("db", "tbl"); + ScopePath partition = ScopePath.partition("db", "tbl", "p=1"); + + Assertions.assertTrue(catalog.contains(catalog)); + Assertions.assertTrue(catalog.contains(database)); + Assertions.assertTrue(catalog.contains(table)); + Assertions.assertTrue(catalog.contains(partition)); + Assertions.assertTrue(database.contains(database)); + Assertions.assertTrue(database.contains(table)); + Assertions.assertTrue(database.contains(partition)); + Assertions.assertTrue(table.contains(table)); + Assertions.assertTrue(table.contains(partition)); + Assertions.assertTrue(partition.contains(partition)); + + Assertions.assertFalse(database.contains(catalog)); + Assertions.assertFalse(table.contains(database)); + Assertions.assertFalse(partition.contains(table)); + Assertions.assertFalse(database.contains(ScopePath.table("other", "tbl"))); + Assertions.assertFalse(table.contains(ScopePath.partition("db", "other", "p=1"))); + Assertions.assertFalse(partition.contains(ScopePath.partition("db", "tbl", "p=2"))); + } + + @Test + public void valueSemanticsIncludeEveryScopeComponent() { + Assertions.assertEquals(ScopePath.catalog(), ScopePath.catalog()); + Assertions.assertEquals(ScopePath.database("db"), ScopePath.database("db")); + Assertions.assertEquals(ScopePath.table("db", "tbl"), ScopePath.table("db", "tbl")); + Assertions.assertEquals( + ScopePath.partition("db", "tbl", "p=1"), + ScopePath.partition("db", "tbl", "p=1")); + Assertions.assertEquals( + ScopePath.partition("db", "tbl", "p=1").hashCode(), + ScopePath.partition("db", "tbl", "p=1").hashCode()); + Assertions.assertNotEquals(ScopePath.database("db"), ScopePath.database("other")); + Assertions.assertNotEquals(ScopePath.table("db", "tbl"), ScopePath.table("db", "other")); + Assertions.assertNotEquals( + ScopePath.partition("db", "tbl", "p=1"), + ScopePath.partition("db", "tbl", "p=2")); + } + + @Test + public void rejectsIncompletePaths() { + Assertions.assertThrows(NullPointerException.class, () -> ScopePath.database(null)); + Assertions.assertThrows(NullPointerException.class, () -> ScopePath.table(null, "tbl")); + Assertions.assertThrows(NullPointerException.class, () -> ScopePath.table("db", null)); + Assertions.assertThrows( + NullPointerException.class, () -> ScopePath.partition("db", "tbl", null)); + Assertions.assertThrows(NullPointerException.class, () -> ScopePath.catalog().contains(null)); + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java new file mode 100644 index 00000000000000..3c168f662f891c --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java @@ -0,0 +1,842 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.apache.doris.connector.cache.ScopedMetaCache.BulkLoadHandle; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class ScopedMetaCacheConcurrencyTest { + private static final long TIMEOUT_SECONDS = 10L; + private static final CacheSpec ENABLED = + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 1_000L); + private static final ScopePath TABLE = ScopePath.table("db", "tbl"); + private static final ScopePath PARTITION = ScopePath.partition("db", "tbl", "p=1"); + + @Test + public void concurrentMissesForTheSameKeyRunOneLoader() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(2); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + AtomicInteger loads = new AtomicInteger(); + Future first = executor.submit(() -> cache.get("key", TABLE, ignored -> { + loads.incrementAndGet(); + loaderStarted.countDown(); + await(releaseLoader); + return 7; + })); + await(loaderStarted); + Future second = executor.submit( + () -> cache.get("key", TABLE, ignored -> { + loads.incrementAndGet(); + return 8; + })); + + releaseLoader.countDown(); + Assertions.assertEquals(7, first.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Assertions.assertEquals(7, second.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Assertions.assertEquals(1, loads.get()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void everyContainingInvalidationFencesAnInFlightSingleKeyLoad() throws Exception { + List invalidations = Arrays.asList( + ScopePath.catalog(), + ScopePath.database("db"), + TABLE, + PARTITION); + for (ScopePath invalidation : invalidations) { + assertInFlightLoadPublication(invalidation, false); + } + } + + @Test + public void descendantAndSiblingInvalidationsDoNotFenceUnrelatedSingleKeyLoad() throws Exception { + List invalidations = Arrays.asList( + ScopePath.partition("db", "tbl", "other"), + ScopePath.table("db", "other"), + ScopePath.database("other")); + for (ScopePath invalidation : invalidations) { + assertInFlightLoadPublication(invalidation, true); + } + } + + @Test + public void exactKeyInvalidationFencesInFlightLoadButNotOtherKeys() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + cache.put("other", TABLE, "other"); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + Future result = executor.submit(() -> cache.get("key", TABLE, ignored -> { + loaderStarted.countDown(); + await(releaseLoader); + return "stale"; + })); + await(loaderStarted); + + cache.invalidateKey("key"); + releaseLoader.countDown(); + + Assertions.assertEquals("stale", result.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Assertions.assertNull(cache.getIfPresent("key", TABLE)); + Assertions.assertEquals("other", cache.getIfPresent("other", TABLE)); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void everyRelevantScopeInvalidationFencesUnknownKeyBulkPublication() { + List invalidations = Arrays.asList( + ScopePath.catalog(), + ScopePath.database("db"), + TABLE, + PARTITION, + ScopePath.partition("db", "tbl", "sibling")); + for (ScopePath invalidation : invalidations) { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + try (BulkLoadHandle handle = cache.beginBulkLoad(TABLE)) { + registry.invalidate(invalidation); + Assertions.assertFalse( + cache.publish(handle, "key", PARTITION, "stale"), + invalidation + " must fence a table bulk load"); + Assertions.assertNull(cache.getIfPresent("key", PARTITION)); + } + } + } + } + + @Test + public void siblingScopesDoNotFenceUnknownKeyBulkPublication() { + List invalidations = Arrays.asList( + ScopePath.table("db", "sibling"), + ScopePath.database("sibling")); + for (ScopePath invalidation : invalidations) { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + try (BulkLoadHandle handle = cache.beginBulkLoad(TABLE)) { + registry.invalidate(invalidation); + Assertions.assertTrue( + cache.publish(handle, "key", PARTITION, "current"), + invalidation + " must not fence an unrelated table bulk load"); + } + Assertions.assertEquals("current", cache.getIfPresent("key", PARTITION)); + } + } + } + + @Test + public void exactKeyInvalidationFencesUnknownKeyBulkPublication() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + try (BulkLoadHandle handle = cache.beginBulkLoad(TABLE)) { + cache.invalidateKey("key"); + Assertions.assertFalse(cache.publish(handle, "key", PARTITION, "stale")); + } + Assertions.assertNull(cache.getIfPresent("key", PARTITION)); + } + } + + @Test + public void exactKeyInvalidationDoesNotFenceUnrelatedBulkPublication() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + try (BulkLoadHandle handle = cache.beginBulkLoad(TABLE)) { + cache.invalidateKey("unrelated"); + Assertions.assertTrue(cache.publish(handle, "key", PARTITION, "current")); + Assertions.assertEquals(1, cache.metrics().getExactInvalidationTombstoneCount()); + } + Assertions.assertEquals("current", cache.getIfPresent("key", PARTITION)); + Assertions.assertEquals(0, cache.metrics().getActiveBulkHandleCount()); + Assertions.assertEquals(0, cache.metrics().getExactInvalidationTombstoneCount()); + } + } + + @Test + public void stagedBulkValueIsNeverVisibleAfterHandleInvalidation() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch valueStaged = new CountDownLatch(1); + CountDownLatch allowCommit = new CountDownLatch(1); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache( + "test", + ENABLED, + null, + null, + () -> { + valueStaged.countDown(); + await(allowCommit); + }); + cache.put("key", PARTITION, "current"); + try (BulkLoadHandle handle = cache.beginBulkLoad(TABLE)) { + Future publication = executor.submit( + () -> cache.publish(handle, "key", PARTITION, "stale")); + await(valueStaged); + + registry.invalidate(ScopePath.partition("db", "tbl", "sibling")); + Assertions.assertEquals("current", cache.getIfPresent("key", PARTITION)); + + allowCommit.countDown(); + Assertions.assertFalse(publication.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } + Assertions.assertEquals("current", cache.getIfPresent("key", PARTITION)); + registry.invalidate(ScopePath.catalog()); + assertEmpty(registry, cache); + } finally { + allowCommit.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void invalidationAfterPartialBulkPublicationRemovesPublishedPartAndFencesTheRest() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + ScopePath firstPartition = ScopePath.partition("db", "tbl", "p=1"); + ScopePath secondPartition = ScopePath.partition("db", "tbl", "p=2"); + try (BulkLoadHandle handle = cache.beginBulkLoad(TABLE)) { + Assertions.assertTrue(cache.publish(handle, "first", firstPartition, "first")); + registry.invalidate(firstPartition); + Assertions.assertFalse(cache.publish(handle, "second", secondPartition, "second")); + } + + Assertions.assertNull(cache.getIfPresent("first", firstPartition)); + Assertions.assertNull(cache.getIfPresent("second", secondPartition)); + assertEmpty(registry, cache); + } + } + + @Test + public void delayedOldRemovalCallbackCannotDeleteReplacementRegistration() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch oldRemovalStarted = new CountDownLatch(1); + CountDownLatch releaseOldRemoval = new CountDownLatch(1); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache( + "test", + ENABLED, + null, + (key, value) -> { + if ("old".equals(value)) { + oldRemovalStarted.countDown(); + await(releaseOldRemoval); + } + }); + cache.put("key", TABLE, "old"); + Future replacement = executor.submit(() -> cache.put("key", TABLE, "new")); + await(oldRemovalStarted); + + Assertions.assertEquals(1, registry.metrics().getRegistrationCount()); + releaseOldRemoval.countDown(); + replacement.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + + Assertions.assertEquals("new", cache.getIfPresent("key", TABLE)); + Assertions.assertEquals(1, registry.metrics().getRegistrationCount()); + Assertions.assertEquals(1, cache.metrics().getKeyNodeCount()); + } finally { + releaseOldRemoval.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void collidingKeyDoesNotWaitForUnrelatedRemoteLoader() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + CollisionKey firstKey = new CollisionKey("first"); + CollisionKey secondKey = new CollisionKey("second"); + Future first = executor.submit(() -> cache.get(firstKey, TABLE, ignored -> { + loaderStarted.countDown(); + await(releaseLoader); + return "first"; + })); + await(loaderStarted); + + Future second = executor.submit(() -> cache.put(secondKey, TABLE, "second")); + second.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + Assertions.assertEquals("second", cache.getIfPresent(secondKey, TABLE)); + + releaseLoader.countDown(); + Assertions.assertEquals("first", first.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } finally { + releaseLoader.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void directPutWinsOverAnEarlierInFlightLoader() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + Future loaded = executor.submit(() -> cache.get("key", TABLE, ignored -> { + loaderStarted.countDown(); + await(releaseLoader); + return "stale-load-result"; + })); + await(loaderStarted); + + cache.put("key", TABLE, "direct-put"); + releaseLoader.countDown(); + + Assertions.assertEquals( + "stale-load-result", loaded.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Assertions.assertEquals("direct-put", cache.getIfPresent("key", TABLE)); + } finally { + releaseLoader.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void postInvalidationQueryDoesNotJoinStaleSingleFlight() throws Exception { + List invalidations = Arrays.asList( + ScopePath.catalog(), + ScopePath.database("db"), + TABLE, + PARTITION); + for (ScopePath invalidation : invalidations) { + assertPostInvalidationQueryStartsFreshLoad(invalidation, false); + } + assertPostInvalidationQueryStartsFreshLoad(TABLE, true); + } + + @Test + public void siblingInvalidationStillSharesCurrentSingleFlight() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch oldLoaderStarted = new CountDownLatch(1); + CountDownLatch releaseOldLoader = new CountDownLatch(1); + AtomicInteger secondLoads = new AtomicInteger(); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + Future first = executor.submit(() -> cache.get("key", PARTITION, ignored -> { + oldLoaderStarted.countDown(); + await(releaseOldLoader); + return "shared"; + })); + await(oldLoaderStarted); + + registry.invalidate(ScopePath.table("other-db", "other-table")); + Future second = executor.submit(() -> cache.get("key", PARTITION, ignored -> { + secondLoads.incrementAndGet(); + return "unexpected"; + })); + releaseOldLoader.countDown(); + + Assertions.assertEquals("shared", first.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Assertions.assertEquals("shared", second.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Assertions.assertEquals(0, secondLoads.get()); + } finally { + releaseOldLoader.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void leaderRechecksCacheAfterDirectPutWinsElectionWindow() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch leaderElected = new CountDownLatch(1); + CountDownLatch allowLeader = new CountDownLatch(1); + AtomicInteger loads = new AtomicInteger(); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache( + "test", + ENABLED, + null, + null, + () -> { + leaderElected.countDown(); + await(allowLeader); + }, + () -> { + }); + Future result = executor.submit(() -> cache.get("key", TABLE, ignored -> { + loads.incrementAndGet(); + return "stale"; + })); + await(leaderElected); + + cache.put("key", TABLE, "direct"); + allowLeader.countDown(); + + Assertions.assertEquals("direct", result.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Assertions.assertEquals(0, loads.get()); + Assertions.assertEquals("direct", cache.getIfPresent("key", TABLE)); + } finally { + allowLeader.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void leaderRechecksCacheAfterBulkPublishWinsElectionWindow() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch leaderElected = new CountDownLatch(1); + CountDownLatch allowLeader = new CountDownLatch(1); + AtomicInteger loads = new AtomicInteger(); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache( + "test", + ENABLED, + null, + null, + () -> { + leaderElected.countDown(); + await(allowLeader); + }, + () -> { + }); + Future result = executor.submit(() -> cache.get("key", PARTITION, ignored -> { + loads.incrementAndGet(); + return "stale"; + })); + await(leaderElected); + + try (BulkLoadHandle handle = cache.beginBulkLoad(TABLE)) { + Assertions.assertTrue(cache.publish(handle, "key", PARTITION, "bulk")); + } + allowLeader.countDown(); + + Assertions.assertEquals("bulk", result.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Assertions.assertEquals(0, loads.get()); + Assertions.assertEquals("bulk", cache.getIfPresent("key", PARTITION)); + } finally { + allowLeader.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void failedBulkCandidatePreservesExistingValueAndCapacity() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch candidateReady = new CountDownLatch(1); + CountDownLatch allowCommit = new CountDownLatch(1); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache( + "test", + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 1L), + null, + null, + () -> { + candidateReady.countDown(); + await(allowCommit); + }); + cache.put("key", PARTITION, "current"); + try (BulkLoadHandle handle = cache.beginBulkLoad(TABLE)) { + Future publication = executor.submit( + () -> cache.publish(handle, "candidate", PARTITION, "stale")); + await(candidateReady); + + registry.invalidate(ScopePath.partition("db", "tbl", "sibling")); + Assertions.assertEquals("current", cache.getIfPresent("key", PARTITION)); + Assertions.assertEquals(1L, cache.metrics().getPhysicalEntryCount()); + + allowCommit.countDown(); + Assertions.assertFalse(publication.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } + Assertions.assertEquals("current", cache.getIfPresent("key", PARTITION)); + Assertions.assertNull(cache.getIfPresent("candidate", PARTITION)); + } finally { + allowCommit.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void keyPublicationAfterInvalidationLinearizationSurvivesCleanup() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch stateReplaced = new CountDownLatch(1); + CountDownLatch continueCleanup = new CountDownLatch(1); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + cache.put("key", TABLE, "old"); + Future invalidation = executor.submit(() -> cache.invalidateKey("key", () -> { + stateReplaced.countDown(); + await(continueCleanup); + })); + await(stateReplaced); + + Assertions.assertNull(cache.getIfPresent("key", TABLE)); + cache.put("key", TABLE, "new"); + continueCleanup.countDown(); + invalidation.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + + Assertions.assertEquals("new", cache.getIfPresent("key", TABLE)); + Assertions.assertEquals(1, registry.metrics().getRegistrationCount()); + Assertions.assertEquals(1, cache.metrics().getKeyNodeCount()); + } finally { + continueCleanup.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void staleScopeValueIsUnreadableBeforePhysicalCleanupStarts() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch stateReplaced = new CountDownLatch(1); + CountDownLatch startCleanup = new CountDownLatch(1); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + cache.put("key", TABLE, "old"); + Future invalidation = executor.submit(() -> registry.invalidate(TABLE, () -> { + stateReplaced.countDown(); + await(startCleanup); + })); + await(stateReplaced); + + Assertions.assertNull(cache.getIfPresent("key", TABLE)); + startCleanup.countDown(); + invalidation.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEmpty(registry, cache); + } finally { + startCleanup.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void scopePublicationAfterInvalidationLinearizationSurvivesCleanup() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch oldRemovalStarted = new CountDownLatch(1); + CountDownLatch continueCleanup = new CountDownLatch(1); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache( + "test", + ENABLED, + null, + (key, value) -> { + if ("old".equals(value)) { + oldRemovalStarted.countDown(); + await(continueCleanup); + } + }); + cache.put("old", TABLE, "old"); + Future invalidation = executor.submit(() -> registry.invalidate(TABLE)); + await(oldRemovalStarted); + + cache.put("new", TABLE, "new"); + continueCleanup.countDown(); + invalidation.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + + Assertions.assertEquals("new", cache.getIfPresent("new", TABLE)); + Assertions.assertEquals(1, registry.metrics().getRegistrationCount()); + Assertions.assertEquals(1, cache.metrics().getKeyNodeCount()); + } finally { + continueCleanup.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void cacheAndRegistryCloseFenceInFlightPublications() throws Exception { + assertCloseFencesPublication(false); + assertCloseFencesPublication(true); + } + + @Test + public void closeCannotRaceWithLateExactInvalidationTombstone() throws Exception { + assertCloseFencesLateExactInvalidation(false); + assertCloseFencesLateExactInvalidation(true); + } + + @Test + public void concurrentPutEvictAndInvalidationFinishWithoutDeadlock() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(4); + ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry(); + ScopedMetaCache cache = registry.createCache( + "test", CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 16L)); + CountDownLatch start = new CountDownLatch(1); + try { + Future writer = executor.submit(() -> { + await(start); + for (int i = 0; i < 2_000; i++) { + cache.put(i % 64, ScopePath.partition("db", "tbl", i % 8), i); + } + }); + Future keyInvalidator = executor.submit(() -> { + await(start); + for (int i = 0; i < 2_000; i++) { + cache.invalidateKey(i % 64); + } + }); + Future scopeInvalidator = executor.submit(() -> { + await(start); + for (int i = 0; i < 500; i++) { + registry.invalidate(ScopePath.partition("db", "tbl", i % 8)); + } + }); + Future cleaner = executor.submit(() -> { + await(start); + for (int i = 0; i < 2_000; i++) { + cache.cleanUp(); + } + }); + start.countDown(); + writer.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + keyInvalidator.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + scopeInvalidator.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + cleaner.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + + registry.invalidate(ScopePath.catalog()); + cache.cleanUp(); + assertEmpty(registry, cache); + } finally { + registry.close(); + executor.shutdownNow(); + } + } + + @Test + public void concurrentSameKeyPublicationsLeaveOneExactRegistration() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(4); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + CountDownLatch start = new CountDownLatch(1); + Future first = submitSameKeyWriter(executor, start, cache, "db1"); + Future second = submitSameKeyWriter(executor, start, cache, "db2"); + Future third = submitSameKeyWriter(executor, start, cache, "db3"); + Future fourth = submitSameKeyWriter(executor, start, cache, "db4"); + start.countDown(); + first.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + second.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + third.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + fourth.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + ScopePath finalScope = ScopePath.table("final", "tbl"); + cache.put("key", finalScope, 100); + cache.cleanUp(); + + Assertions.assertEquals(100, cache.getIfPresent("key", finalScope)); + Assertions.assertEquals(1L, cache.metrics().getPhysicalEntryCount()); + Assertions.assertEquals(1, cache.metrics().getKeyNodeCount()); + Assertions.assertEquals(1, registry.metrics().getRegistrationCount()); + Assertions.assertEquals(1, registry.metrics().getDatabaseNodeCount()); + Assertions.assertEquals(1, registry.metrics().getTableNodeCount()); + Assertions.assertEquals(0, registry.metrics().getPartitionNodeCount()); + } finally { + executor.shutdownNow(); + } + } + + private static void assertInFlightLoadPublication( + ScopePath invalidation, boolean expectedCached) throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + Future result = executor.submit(() -> cache.get("key", PARTITION, ignored -> { + loaderStarted.countDown(); + await(releaseLoader); + return "loaded"; + })); + await(loaderStarted); + registry.invalidate(invalidation); + releaseLoader.countDown(); + + Assertions.assertEquals("loaded", result.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + if (expectedCached) { + Assertions.assertEquals("loaded", cache.getIfPresent("key", PARTITION)); + } else { + Assertions.assertNull(cache.getIfPresent("key", PARTITION)); + } + } finally { + executor.shutdownNow(); + } + } + + private static void assertCloseFencesPublication(boolean closeRegistry) throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry(); + ScopedMetaCache cache = registry.createCache("test", ENABLED); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + try { + Future result = executor.submit(() -> cache.get("key", PARTITION, ignored -> { + loaderStarted.countDown(); + await(releaseLoader); + return "loaded"; + })); + await(loaderStarted); + if (closeRegistry) { + registry.close(); + } else { + cache.close(); + } + releaseLoader.countDown(); + Assertions.assertEquals("loaded", result.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + assertEmpty(registry, cache); + } finally { + releaseLoader.countDown(); + registry.close(); + executor.shutdownNow(); + } + } + + private static void assertCloseFencesLateExactInvalidation(boolean closeRegistry) throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry(); + ScopedMetaCache cache = registry.createCache("test", ENABLED); + BulkLoadHandle handle = cache.beginBulkLoad(TABLE); + CountDownLatch passedOpenCheck = new CountDownLatch(1); + CountDownLatch enterInvalidationLock = new CountDownLatch(1); + try { + Future invalidation = executor.submit(() -> cache.invalidateKey( + "key", + () -> { + passedOpenCheck.countDown(); + await(enterInvalidationLock); + }, + () -> { + })); + await(passedOpenCheck); + + if (closeRegistry) { + registry.close(); + } else { + cache.close(); + } + enterInvalidationLock.countDown(); + invalidation.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + + Assertions.assertEquals(0, cache.metrics().getExactInvalidationTombstoneCount()); + handle.close(); + Assertions.assertEquals(0, cache.metrics().getActiveBulkHandleCount()); + Assertions.assertEquals(0, registry.metrics().getActiveLoadCount()); + } finally { + enterInvalidationLock.countDown(); + handle.close(); + registry.close(); + executor.shutdownNow(); + } + } + + private static void assertPostInvalidationQueryStartsFreshLoad( + ScopePath invalidation, boolean exactKey) throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch oldLoaderStarted = new CountDownLatch(1); + CountDownLatch releaseOldLoader = new CountDownLatch(1); + AtomicInteger freshLoads = new AtomicInteger(); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + Future oldResult = executor.submit(() -> cache.get("key", PARTITION, ignored -> { + oldLoaderStarted.countDown(); + await(releaseOldLoader); + return "stale"; + })); + await(oldLoaderStarted); + if (exactKey) { + cache.invalidateKey("key"); + } else { + registry.invalidate(invalidation); + } + + Future freshResult = executor.submit(() -> cache.get("key", PARTITION, ignored -> { + freshLoads.incrementAndGet(); + return "fresh"; + })); + Assertions.assertEquals("fresh", freshResult.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Assertions.assertEquals(1, freshLoads.get()); + + releaseOldLoader.countDown(); + Assertions.assertEquals("stale", oldResult.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Assertions.assertEquals("fresh", cache.getIfPresent("key", PARTITION)); + } finally { + releaseOldLoader.countDown(); + executor.shutdownNow(); + } + } + + private static Future submitSameKeyWriter( + ExecutorService executor, + CountDownLatch start, + ScopedMetaCache cache, + String database) { + return executor.submit(() -> { + await(start); + ScopePath path = ScopePath.table(database, "tbl"); + for (int i = 0; i < 1_000; i++) { + cache.put("key", path, i); + } + }); + } + + private static void assertEmpty( + ScopedMetaCacheRegistry registry, ScopedMetaCache cache) { + ScopedMetaCacheRegistry.ScopeMetrics scopeMetrics = registry.metrics(); + Assertions.assertEquals(0, scopeMetrics.getDatabaseNodeCount()); + Assertions.assertEquals(0, scopeMetrics.getTableNodeCount()); + Assertions.assertEquals(0, scopeMetrics.getPartitionNodeCount()); + Assertions.assertEquals(0, scopeMetrics.getRegistrationCount()); + Assertions.assertEquals(0, scopeMetrics.getActiveLoadCount()); + Assertions.assertEquals(0L, cache.metrics().getPhysicalEntryCount()); + Assertions.assertEquals(0, cache.metrics().getKeyNodeCount()); + Assertions.assertEquals(0, cache.metrics().getInFlightLoadCount()); + Assertions.assertEquals(0, cache.metrics().getActiveBulkHandleCount()); + Assertions.assertEquals(0, cache.metrics().getExactInvalidationTombstoneCount()); + } + + private static void await(CountDownLatch latch) { + try { + Assertions.assertTrue(latch.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for test latch", e); + } + } + + private static final class CollisionKey { + private final String value; + + private CollisionKey(String value) { + this.value = value; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof CollisionKey && value.equals(((CollisionKey) obj).value); + } + + @Override + public int hashCode() { + return 1; + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheHierarchyTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheHierarchyTest.java new file mode 100644 index 00000000000000..92f48045f10c06 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheHierarchyTest.java @@ -0,0 +1,349 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.apache.doris.connector.cache.ScopedMetaCache.BulkLoadHandle; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +public class ScopedMetaCacheHierarchyTest { + private static final CacheSpec ENABLED = + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 1_000L); + private static final ScopePath CATALOG = ScopePath.catalog(); + private static final ScopePath DB = ScopePath.database("db"); + private static final ScopePath TABLE = ScopePath.table("db", "tbl"); + private static final ScopePath PARTITION = ScopePath.partition("db", "tbl", "p=1"); + + @Test + public void everyInvalidationLevelMatchesOnlyItsContainedEntries() { + List entryScopes = Arrays.asList(CATALOG, DB, TABLE, PARTITION); + List invalidationScopes = Arrays.asList(CATALOG, DB, TABLE, PARTITION); + + for (ScopePath entryScope : entryScopes) { + for (ScopePath invalidationScope : invalidationScopes) { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + cache.put("key", entryScope, "value"); + registry.invalidate(invalidationScope); + if (invalidationScope.contains(entryScope)) { + Assertions.assertNull( + cache.getIfPresent("key", entryScope), + invalidationScope + " must invalidate " + entryScope); + } else { + Assertions.assertEquals( + "value", + cache.getIfPresent("key", entryScope), + invalidationScope + " must not invalidate " + entryScope); + } + } + } + } + } + + @Test + public void siblingDatabaseTableAndPartitionRemainIsolated() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + ScopePath db1Table1Part1 = ScopePath.partition("db1", "tbl1", "p=1"); + ScopePath db1Table1Part2 = ScopePath.partition("db1", "tbl1", "p=2"); + ScopePath db1Table2Part1 = ScopePath.partition("db1", "tbl2", "p=1"); + ScopePath db2Table1Part1 = ScopePath.partition("db2", "tbl1", "p=1"); + cache.put("p11", db1Table1Part1, "p11"); + cache.put("p12", db1Table1Part2, "p12"); + cache.put("p21", db1Table2Part1, "p21"); + cache.put("db2", db2Table1Part1, "db2"); + + registry.invalidate(db1Table1Part1); + Assertions.assertNull(cache.getIfPresent("p11", db1Table1Part1)); + Assertions.assertEquals("p12", cache.getIfPresent("p12", db1Table1Part2)); + Assertions.assertEquals("p21", cache.getIfPresent("p21", db1Table2Part1)); + Assertions.assertEquals("db2", cache.getIfPresent("db2", db2Table1Part1)); + + registry.invalidate(ScopePath.table("db1", "tbl1")); + Assertions.assertNull(cache.getIfPresent("p12", db1Table1Part2)); + Assertions.assertEquals("p21", cache.getIfPresent("p21", db1Table2Part1)); + Assertions.assertEquals("db2", cache.getIfPresent("db2", db2Table1Part1)); + + registry.invalidate(ScopePath.database("db1")); + Assertions.assertNull(cache.getIfPresent("p21", db1Table2Part1)); + Assertions.assertEquals("db2", cache.getIfPresent("db2", db2Table1Part1)); + } + } + + @Test + public void sharedRegistryInvalidatesAllPhysicalCachesAtTheSameScope() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache tableCache = registry.createCache("tables", ENABLED); + ScopedMetaCache fileCache = registry.createCache("files", ENABLED); + ScopePath sibling = ScopePath.table("db", "sibling"); + tableCache.put("tbl", TABLE, "table"); + tableCache.put("sibling", sibling, "sibling"); + fileCache.put(1, PARTITION, "file"); + + registry.invalidate(TABLE); + + Assertions.assertNull(tableCache.getIfPresent("tbl", TABLE)); + Assertions.assertNull(fileCache.getIfPresent(1, PARTITION)); + Assertions.assertEquals("sibling", tableCache.getIfPresent("sibling", sibling)); + } + } + + @Test + public void exactKeyInvalidationAffectsOnlyOneKeyInOnePhysicalCache() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache first = registry.createCache("first", ENABLED); + ScopedMetaCache second = registry.createCache("second", ENABLED); + first.put("a", TABLE, "first-a"); + first.put("b", TABLE, "first-b"); + second.put("a", TABLE, "second-a"); + + first.invalidateKey("a"); + + Assertions.assertNull(first.getIfPresent("a", TABLE)); + Assertions.assertEquals("first-b", first.getIfPresent("b", TABLE)); + Assertions.assertEquals("second-a", second.getIfPresent("a", TABLE)); + } + } + + @Test + public void samePhysicalKeyCanMoveToANewScopeWithoutOldScopeOwningIt() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + ScopePath oldScope = ScopePath.table("old_db", "tbl"); + ScopePath newScope = ScopePath.table("new_db", "tbl"); + cache.put("key", oldScope, "old"); + cache.put("key", newScope, "new"); + + Assertions.assertNull(cache.getIfPresent("key", oldScope)); + Assertions.assertEquals("new", cache.getIfPresent("key", newScope)); + registry.invalidate(oldScope); + Assertions.assertEquals("new", cache.getIfPresent("key", newScope)); + Assertions.assertEquals(1, registry.metrics().getRegistrationCount()); + } + } + + @Test + public void disabledCacheNeverPublishesOrAllocatesIndexes() { + List disabledSpecs = Arrays.asList( + CacheSpec.of(false, CacheSpec.CACHE_NO_TTL, 100L), + CacheSpec.of(true, CacheSpec.CACHE_TTL_DISABLE_CACHE, 100L), + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 0L)); + for (CacheSpec disabled : disabledSpecs) { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("disabled", disabled); + AtomicInteger loads = new AtomicInteger(); + + Assertions.assertEquals(1, cache.get("key", PARTITION, key -> loads.incrementAndGet())); + Assertions.assertEquals(2, cache.get("key", PARTITION, key -> loads.incrementAndGet())); + cache.put("key", PARTITION, 3); + try (BulkLoadHandle handle = cache.beginBulkLoad(TABLE)) { + Assertions.assertFalse(cache.publish(handle, "bulk", PARTITION, 4)); + } + + Assertions.assertNull(cache.getIfPresent("key", PARTITION)); + assertEmpty(registry, cache); + } + } + } + + @Test + public void nullAndFailedLoadsDoNotLeaveState() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + Assertions.assertNull(cache.get("null", PARTITION, ignored -> null)); + Assertions.assertThrows( + IllegalStateException.class, + () -> cache.get("failure", PARTITION, ignored -> { + throw new IllegalStateException("expected"); + })); + assertEmpty(registry, cache); + } + } + + @Test + public void generationWrapUsesStateIdentityInsteadOfOrdering() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + try (BulkLoadHandle ignored = cache.beginBulkLoad(TABLE)) { + registry.forceGenerationForTest(TABLE, Long.MAX_VALUE); + Assertions.assertEquals(Long.MAX_VALUE, registry.generationForTest(TABLE)); + registry.invalidate(TABLE); + Assertions.assertEquals(Long.MIN_VALUE, registry.generationForTest(TABLE)); + } + cache.put("key", TABLE, "new"); + Assertions.assertEquals("new", cache.getIfPresent("key", TABLE)); + } + } + + @Test + public void bulkLoadRequiresMatchingOwnerOpenHandleAndContainedScope() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache first = registry.createCache("first", ENABLED); + ScopedMetaCache second = registry.createCache("second", ENABLED); + BulkLoadHandle handle = first.beginBulkLoad(TABLE); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> second.publish(handle, "key", TABLE, "value")); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> first.publish( + handle, "key", ScopePath.table("other", "tbl"), "value")); + handle.close(); + Assertions.assertThrows( + IllegalStateException.class, + () -> first.publish(handle, "key", TABLE, "value")); + } + } + + @Test + public void closedCacheAndRegistryRejectOperations() { + ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry(); + ScopedMetaCache cache = registry.createCache("test", ENABLED); + cache.close(); + Assertions.assertThrows( + IllegalStateException.class, () -> cache.getIfPresent("key", TABLE)); + Assertions.assertThrows( + IllegalStateException.class, () -> cache.put("key", TABLE, "value")); + + registry.close(); + Assertions.assertThrows( + IllegalStateException.class, () -> registry.createCache("late", ENABLED)); + Assertions.assertThrows(IllegalStateException.class, () -> registry.invalidate(TABLE)); + registry.close(); + cache.close(); + } + + @Test + public void deterministicStateMachineMatchesReferenceModelAcrossMixedOperations() { + Random random = new Random(0x5C0FE5L); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache first = registry.createCache("first", ENABLED); + ScopedMetaCache second = registry.createCache("second", ENABLED); + Map firstReference = new HashMap<>(); + Map secondReference = new HashMap<>(); + + for (int operation = 0; operation < 20_000; operation++) { + ScopedMetaCache cache = random.nextBoolean() ? first : second; + Map reference = + cache == first ? firstReference : secondReference; + int key = random.nextInt(32); + int action = random.nextInt(5); + if (action <= 1) { + ScopePath scope = randomScope(random); + cache.put(key, scope, operation); + reference.put(key, new ReferenceEntry(scope, operation)); + } else if (action == 2) { + ScopePath invalidation = randomScope(random); + registry.invalidate(invalidation); + firstReference.entrySet().removeIf( + entry -> invalidation.contains(entry.getValue().scope)); + secondReference.entrySet().removeIf( + entry -> invalidation.contains(entry.getValue().scope)); + } else if (action == 3) { + cache.invalidateKey(key); + reference.remove(key); + } else { + ReferenceEntry expected = reference.get(key); + if (expected == null) { + Assertions.assertNull(cache.getIfPresent(key, randomScope(random))); + } else { + Assertions.assertEquals( + expected.value, cache.getIfPresent(key, expected.scope)); + } + } + + if ((operation & 255) == 0) { + assertMatchesReference(first, firstReference); + assertMatchesReference(second, secondReference); + Assertions.assertEquals( + firstReference.size() + secondReference.size(), + registry.metrics().getRegistrationCount()); + } + } + + assertMatchesReference(first, firstReference); + assertMatchesReference(second, secondReference); + registry.invalidate(CATALOG); + first.cleanUp(); + second.cleanUp(); + Assertions.assertEquals(0L, first.metrics().getPhysicalEntryCount()); + Assertions.assertEquals(0L, second.metrics().getPhysicalEntryCount()); + Assertions.assertEquals(0, registry.metrics().getRegistrationCount()); + Assertions.assertEquals(0, registry.metrics().getDatabaseNodeCount()); + Assertions.assertEquals(0, registry.metrics().getTableNodeCount()); + Assertions.assertEquals(0, registry.metrics().getPartitionNodeCount()); + } + } + + private static ScopePath randomScope(Random random) { + int level = random.nextInt(4); + String database = "db" + random.nextInt(3); + String table = "tbl" + random.nextInt(3); + if (level == 0) { + return ScopePath.catalog(); + } + if (level == 1) { + return ScopePath.database(database); + } + if (level == 2) { + return ScopePath.table(database, table); + } + return ScopePath.partition(database, table, "p=" + random.nextInt(3)); + } + + private static void assertMatchesReference( + ScopedMetaCache cache, + Map reference) { + cache.cleanUp(); + reference.forEach((key, entry) -> + Assertions.assertEquals(entry.value, cache.getIfPresent(key, entry.scope))); + Assertions.assertEquals(reference.size(), cache.metrics().getPhysicalEntryCount()); + Assertions.assertEquals(reference.size(), cache.metrics().getKeyNodeCount()); + } + + private static void assertEmpty( + ScopedMetaCacheRegistry registry, ScopedMetaCache cache) { + ScopedMetaCacheRegistry.ScopeMetrics scopeMetrics = registry.metrics(); + Assertions.assertEquals(0, scopeMetrics.getDatabaseNodeCount()); + Assertions.assertEquals(0, scopeMetrics.getTableNodeCount()); + Assertions.assertEquals(0, scopeMetrics.getPartitionNodeCount()); + Assertions.assertEquals(0, scopeMetrics.getRegistrationCount()); + Assertions.assertEquals(0, scopeMetrics.getActiveLoadCount()); + Assertions.assertEquals(0L, cache.metrics().getPhysicalEntryCount()); + Assertions.assertEquals(0, cache.metrics().getKeyNodeCount()); + } + + private static final class ReferenceEntry { + private final ScopePath scope; + private final int value; + + private ReferenceEntry(ScopePath scope, int value) { + this.scope = scope; + this.value = value; + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheLeakTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheLeakTest.java new file mode 100644 index 00000000000000..a8f295301c1061 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheLeakTest.java @@ -0,0 +1,335 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.apache.doris.connector.cache.ScopedMetaCache.BulkLoadHandle; + +import com.github.benmanes.caffeine.cache.Ticker; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +public class ScopedMetaCacheLeakTest { + private static final CacheSpec ENABLED = + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10_000L); + private static final ScopePath TABLE = ScopePath.table("db", "tbl"); + private static final ScopePath PARTITION = ScopePath.partition("db", "tbl", "p=1"); + + @Test + public void explicitInvalidationReclaimsEveryPossibleLeafLevel() { + List scopes = Arrays.asList( + ScopePath.catalog(), + ScopePath.database("db"), + TABLE, + PARTITION); + for (ScopePath scope : scopes) { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + cache.put("key", scope, "value"); + Assertions.assertEquals(1, registry.metrics().getRegistrationCount()); + + registry.invalidate(scope); + cache.cleanUp(); + + assertEmpty(registry, cache); + } + } + } + + @Test + public void subtreeInvalidationReclaimsOnlyItsDatabaseTableAndPartitionNodes() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + for (int table = 0; table < 4; table++) { + for (int partition = 0; partition < 5; partition++) { + String key = "db1-t" + table + "-p" + partition; + cache.put( + key, + ScopePath.partition("db1", "t" + table, "p=" + partition), + key); + } + } + cache.put("db2", ScopePath.partition("db2", "t", "p=1"), "retained"); + + registry.invalidate(ScopePath.database("db1")); + cache.cleanUp(); + + ScopedMetaCacheRegistry.ScopeMetrics metrics = registry.metrics(); + Assertions.assertEquals(1, metrics.getDatabaseNodeCount()); + Assertions.assertEquals(1, metrics.getTableNodeCount()); + Assertions.assertEquals(1, metrics.getPartitionNodeCount()); + Assertions.assertEquals(1, metrics.getRegistrationCount()); + Assertions.assertEquals("retained", cache.getIfPresent( + "db2", ScopePath.partition("db2", "t", "p=1"))); + Assertions.assertEquals(1L, cache.metrics().getPhysicalEntryCount()); + Assertions.assertEquals(1, cache.metrics().getKeyNodeCount()); + } + } + + @Test + public void exactKeyInvalidationPrunesLastScopePathAndKeyNode() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + cache.put("key", PARTITION, "value"); + + cache.invalidateKey("key"); + cache.cleanUp(); + + assertEmpty(registry, cache); + } + } + + @Test + public void capacityEvictionReclaimsRegistrationsKeyNodesAndScopeNodes() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache( + "test", CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 3L)); + for (int i = 0; i < 100; i++) { + cache.put(i, ScopePath.partition("db-" + i, "tbl-" + i, "p=" + i), i); + } + cache.cleanUp(); + + ScopedMetaCacheRegistry.ScopeMetrics metrics = registry.metrics(); + long physical = cache.metrics().getPhysicalEntryCount(); + Assertions.assertTrue(physical <= 3L); + Assertions.assertEquals(physical, metrics.getRegistrationCount()); + Assertions.assertEquals(physical, cache.metrics().getKeyNodeCount()); + Assertions.assertEquals(physical, metrics.getDatabaseNodeCount()); + Assertions.assertEquals(physical, metrics.getTableNodeCount()); + Assertions.assertEquals(physical, metrics.getPartitionNodeCount()); + + registry.invalidate(ScopePath.catalog()); + cache.cleanUp(); + assertEmpty(registry, cache); + } + } + + @Test + public void expirationReclaimsRegistrationsKeyNodesAndScopeNodes() { + AtomicLong nowNanos = new AtomicLong(); + Ticker ticker = nowNanos::get; + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache( + "test", CacheSpec.of(true, 1L, 10L), ticker, null); + cache.put("key", PARTITION, "value"); + nowNanos.set(java.util.concurrent.TimeUnit.SECONDS.toNanos(2L)); + cache.cleanUp(); + + assertEmpty(registry, cache); + } + } + + @Test + public void repeatedReplacementNeverAccumulatesRegistrations() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + for (int i = 0; i < 10_000; i++) { + cache.put("key", PARTITION, i); + } + cache.cleanUp(); + + Assertions.assertEquals(1L, cache.metrics().getPhysicalEntryCount()); + Assertions.assertEquals(1, cache.metrics().getKeyNodeCount()); + Assertions.assertEquals(1, registry.metrics().getRegistrationCount()); + Assertions.assertEquals(1, registry.metrics().getDatabaseNodeCount()); + Assertions.assertEquals(1, registry.metrics().getTableNodeCount()); + Assertions.assertEquals(1, registry.metrics().getPartitionNodeCount()); + } + } + + @Test + public void highCardinalityChurnRemainsBoundedByLiveEntries() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache( + "test", CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 32L)); + for (int i = 0; i < 20_000; i++) { + cache.put(i, ScopePath.partition("db-" + i, "tbl-" + i, "p=" + i), i); + if ((i & 255) == 0) { + cache.cleanUp(); + } + } + cache.cleanUp(); + + long physical = cache.metrics().getPhysicalEntryCount(); + ScopedMetaCacheRegistry.ScopeMetrics metrics = registry.metrics(); + Assertions.assertTrue(physical <= 32L); + Assertions.assertEquals(physical, cache.metrics().getKeyNodeCount()); + Assertions.assertEquals(physical, metrics.getRegistrationCount()); + Assertions.assertEquals(physical, metrics.getDatabaseNodeCount()); + Assertions.assertEquals(physical, metrics.getTableNodeCount()); + Assertions.assertEquals(physical, metrics.getPartitionNodeCount()); + } + } + + @Test + public void bulkHandleWithoutPublicationReleasesAllAllocatedNodes() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + BulkLoadHandle handle = cache.beginBulkLoad(TABLE); + Assertions.assertEquals(1, registry.metrics().getDatabaseNodeCount()); + Assertions.assertEquals(1, registry.metrics().getTableNodeCount()); + Assertions.assertEquals(3, registry.metrics().getActiveLoadCount()); + + handle.close(); + + assertEmpty(registry, cache); + } + } + + @Test + public void invalidationDuringActiveLeaseReclaimsAfterLeaseRelease() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + BulkLoadHandle handle = cache.beginBulkLoad(TABLE); + registry.invalidate(TABLE); + Assertions.assertEquals(3, registry.metrics().getActiveLoadCount()); + + Assertions.assertFalse(cache.publish(handle, "key", PARTITION, "stale")); + handle.close(); + + assertEmpty(registry, cache); + } + } + + @Test + public void detachedScopeLeasesRemainObservableUntilHandleRelease() { + List invalidations = Arrays.asList( + ScopePath.catalog(), + ScopePath.database("db"), + TABLE, + PARTITION); + for (ScopePath invalidation : invalidations) { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + BulkLoadHandle handle = cache.beginBulkLoad(PARTITION); + + registry.invalidate(invalidation); + + ScopedMetaCacheRegistry.ScopeMetrics metrics = registry.metrics(); + Assertions.assertEquals(1, metrics.getActiveCatalogLoadCount()); + Assertions.assertEquals(1, metrics.getActiveDatabaseLoadCount()); + Assertions.assertEquals(1, metrics.getActiveTableLoadCount()); + Assertions.assertEquals(1, metrics.getActivePartitionLoadCount()); + Assertions.assertEquals(4, metrics.getActiveLoadCount()); + Assertions.assertEquals(1, cache.metrics().getActiveBulkHandleCount()); + + handle.close(); + assertEmpty(registry, cache); + } + } + } + + @Test + public void invalidatingNeverSeenScopesDoesNotCreateDirectoryNodes() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + for (int i = 0; i < 10_000; i++) { + registry.invalidate(ScopePath.partition("db-" + i, "tbl-" + i, "p=" + i)); + } + + assertEmpty(registry, cache); + } + } + + @Test + public void closingOnePhysicalCachePreservesSharedScopeForOtherCache() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache first = registry.createCache("first", ENABLED); + ScopedMetaCache second = registry.createCache("second", ENABLED); + first.put("first", PARTITION, "first"); + second.put("second", PARTITION, "second"); + + first.close(); + + Assertions.assertEquals(1, registry.metrics().getRegistrationCount()); + Assertions.assertEquals(1, registry.metrics().getDatabaseNodeCount()); + Assertions.assertEquals(1, registry.metrics().getTableNodeCount()); + Assertions.assertEquals(1, registry.metrics().getPartitionNodeCount()); + Assertions.assertEquals("second", second.getIfPresent("second", PARTITION)); + second.close(); + assertEmpty(registry, second); + } + } + + @Test + public void registryCloseReclaimsAllPhysicalAndIndexStateAndIsIdempotent() { + ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry(); + ScopedMetaCache first = registry.createCache("first", ENABLED); + ScopedMetaCache second = registry.createCache("second", ENABLED); + for (int i = 0; i < 100; i++) { + ScopePath path = ScopePath.partition("db", "tbl-" + (i % 5), "p=" + i); + first.put("first-" + i, path, "value"); + second.put("second-" + i, path, "value"); + } + + registry.close(); + registry.close(); + + assertEmpty(registry, first); + Assertions.assertEquals(0L, second.metrics().getPhysicalEntryCount()); + Assertions.assertEquals(0, second.metrics().getKeyNodeCount()); + } + + @Test + public void closeClearsTombstonesWithOutstandingBulkHandle() { + assertCloseClearsTombstones(false); + assertCloseClearsTombstones(true); + } + + private static void assertCloseClearsTombstones(boolean closeRegistry) { + ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry(); + ScopedMetaCache cache = registry.createCache("test", ENABLED); + BulkLoadHandle handle = cache.beginBulkLoad(TABLE); + for (int i = 0; i < 1_000; i++) { + cache.invalidateKey("key-" + i); + } + Assertions.assertEquals(1_000, cache.metrics().getExactInvalidationTombstoneCount()); + + if (closeRegistry) { + registry.close(); + } else { + cache.close(); + } + Assertions.assertEquals(0, cache.metrics().getExactInvalidationTombstoneCount()); + Assertions.assertEquals(1, cache.metrics().getActiveBulkHandleCount()); + + handle.close(); + handle.close(); + Assertions.assertEquals(0, cache.metrics().getActiveBulkHandleCount()); + Assertions.assertEquals(0, registry.metrics().getActiveLoadCount()); + registry.close(); + } + + private static void assertEmpty( + ScopedMetaCacheRegistry registry, ScopedMetaCache cache) { + ScopedMetaCacheRegistry.ScopeMetrics scopeMetrics = registry.metrics(); + Assertions.assertEquals(0, scopeMetrics.getDatabaseNodeCount()); + Assertions.assertEquals(0, scopeMetrics.getTableNodeCount()); + Assertions.assertEquals(0, scopeMetrics.getPartitionNodeCount()); + Assertions.assertEquals(0, scopeMetrics.getRegistrationCount()); + Assertions.assertEquals(0, scopeMetrics.getActiveLoadCount()); + Assertions.assertEquals(0L, cache.metrics().getPhysicalEntryCount()); + Assertions.assertEquals(0, cache.metrics().getKeyNodeCount()); + Assertions.assertEquals(0, cache.metrics().getInFlightLoadCount()); + Assertions.assertEquals(0, cache.metrics().getActiveBulkHandleCount()); + Assertions.assertEquals(0, cache.metrics().getExactInvalidationTombstoneCount()); + } +} From 899e7bf3bda414d861b0aed8b349dd964ad3716d Mon Sep 17 00:00:00 2001 From: 924060929 Date: Wed, 5 Aug 2026 10:00:56 +0800 Subject: [PATCH 02/10] [improvement](fe) Reduce scoped metadata cache overhead ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: The scoped metadata cache correctness prototype allocated generic lists and snapshots on scope acquisition, recomputed single-flight address hashes, serialized bulk candidates through a redundant preliminary lock, and performed duplicate detached-entry cleanup. Share an immutable fixed-layout generation snapshot per leaf state, cache address hashes, publish bulk values only after the final fence succeeds, and make invalidation own each detached registration before physical removal. In local microbenchmarks this reduces cache-hit latency from 84-95 ns to about 61 ns, reduces same-table retained overhead from about 569 bytes to 369 bytes per entry, and raises eight-thread bulk throughput from about 1.33 million to 1.56 million operations per second while preserving the state-identity invalidation protocol. ### Release note None ### Check List (For Author) - Test: Unit Test and FE build - Unit Test: fe-connector-cache Maven reactor, 87 tests passed - Unit Test: focused JDK 17 run-fe-ut review suite, 54 tests passed - FE build: JDK 17 ./build.sh --fe with a fresh output directory passed - Behavior changed: No. The prototype is not connected to production callers. - Does this need documentation: No --- .../connector/cache/ScopedMetaCache.java | 36 +- .../cache/ScopedMetaCacheRegistry.java | 309 +++++++++++++----- .../cache/ScopedMetaCacheConcurrencyTest.java | 44 +++ 3 files changed, 288 insertions(+), 101 deletions(-) diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java index 53bc58ca50cd65..dfe658c7416183 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java @@ -170,9 +170,6 @@ public V getIfPresent(K key, ScopePath path) { if (!versioned.scopeSnapshot.path().equals(path)) { return null; } - if (!versioned.committed.get()) { - return null; - } if (!versioned.isCurrent(registry, keyNodes)) { data.asMap().remove(key, versioned); return null; @@ -279,10 +276,6 @@ public boolean publish( } try (PublicationLease lease = acquirePublicationLease(key, actualScope, false)) { synchronized (lease.keyNode) { - if (!handle.canStage(key)) { - return false; - } - lease.keyNode.loadPublicationState.set(new Object()); VersionedValue staged = newVersionedValue(lease, key, value); afterBulkStage.run(); if (handle.tryCommit(key, lease, staged)) { @@ -354,7 +347,6 @@ private VersionedValue publish(PublicationLease lease, K key, V valu return null; } VersionedValue versioned = newVersionedValue(lease, key, value); - versioned.committed.set(true); install(versioned, lease); if (!lease.isCurrent()) { data.asMap().remove(key, versioned); @@ -381,12 +373,6 @@ private void install( data.asMap().put(versioned.key, versioned); } - private boolean canStageBulk(BulkLoadHandle handle, Object rawKey) { - synchronized (bulkInvalidationLock) { - return isBulkKeyCurrent(handle, rawKey); - } - } - private boolean tryCommitBulk( BulkLoadHandle handle, Object rawKey, @@ -407,7 +393,7 @@ private boolean tryCommitBulk( if (!lease.isCurrent()) { return false; } - staged.committed.set(true); + lease.keyNode.loadPublicationState.set(new Object()); install(staged, lease); return true; }); @@ -569,10 +555,6 @@ private void checkOwner(ScopedMetaCache expectedOwner) { } } - private boolean canStage(Object key) { - return owner.canStageBulk(this, key); - } - private boolean tryCommit( Object key, PublicationLease lease, VersionedValue staged) { return owner.tryCommitBulk(this, key, lease, staged); @@ -674,7 +656,6 @@ private static final class VersionedValue { private final ScopeSnapshot scopeSnapshot; private final KeyNode keyNode; private final KeyState keyState; - private final AtomicBoolean committed = new AtomicBoolean(false); private VersionedValue( K key, @@ -697,8 +678,7 @@ private boolean isCurrent( return scopeSnapshot.isCurrent(registry) && currentKeyNodes.get(key) == keyNode && keyNode.current.get() == keyState - && keyNode.registration.get() == this - && committed.get(); + && keyNode.registration.get() == this; } } @@ -719,6 +699,7 @@ private static final class LoadAddress { private final KeyNode keyNode; private final KeyState keyState; private final Object loadPublicationState; + private final int hashCode; private LoadAddress(K key, ScopePath path, PublicationLease lease) { this.key = key; @@ -727,6 +708,11 @@ private LoadAddress(K key, ScopePath path, PublicationLease lease) { this.keyNode = lease.keyNode; this.keyState = lease.keyState; this.loadPublicationState = lease.loadPublicationState; + int addressHashCode = 31 * key.hashCode() + path.hashCode(); + addressHashCode = 31 * addressHashCode + scopeSnapshot.generationHashCode(); + addressHashCode = 31 * addressHashCode + System.identityHashCode(keyNode); + addressHashCode = 31 * addressHashCode + System.identityHashCode(keyState); + this.hashCode = 31 * addressHashCode + System.identityHashCode(loadPublicationState); } @Override @@ -748,11 +734,7 @@ public boolean equals(Object obj) { @Override public int hashCode() { - int result = 31 * key.hashCode() + path.hashCode(); - result = 31 * result + scopeSnapshot.generationHashCode(); - result = 31 * result + System.identityHashCode(keyNode); - result = 31 * result + System.identityHashCode(keyState); - return 31 * result + System.identityHashCode(loadPublicationState); + return hashCode; } } } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java index cdd222564ad95d..bf85d1ccc6ad9e 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java @@ -151,16 +151,12 @@ ScopeLease acquire(ScopePath path) { Objects.requireNonNull(path, "path can not be null"); while (true) { checkOpen(); - List nodes = resolveOrCreate(path); - nodes.forEach(node -> { - node.activeLoads.incrementAndGet(); - retainedActiveLoads.incrementAndGet(node.level.ordinal()); - }); - ScopeSnapshot snapshot = ScopeSnapshot.capture(path, nodes); + ScopeSnapshot snapshot = resolveOrCreateSnapshot(path); + retain(snapshot); if (snapshot.isCurrent(this)) { return new ScopeLease(this, snapshot); } - release(nodes); + release(snapshot); } } @@ -180,7 +176,7 @@ void register(CacheAddress address, Object versionedValue, ScopeSnapshot snapsho void unregister(CacheAddress address, Object versionedValue, ScopeSnapshot snapshot) { snapshot.leafState().entries.remove(address, versionedValue); - tryPrune(snapshot.nodes); + tryPrune(snapshot); } private ScopeState replaceState(ScopeNode node) { @@ -193,28 +189,25 @@ private ScopeState replaceState(ScopeNode node) { } } - private List resolveOrCreate(ScopePath path) { - List nodes = new ArrayList<>(path.level().ordinal() + 1); + private ScopeSnapshot resolveOrCreateSnapshot(ScopePath path) { ScopeNode catalogNode = root; - nodes.add(catalogNode); if (path.level() == ScopePath.Level.CATALOG) { - return nodes; + return ScopeSnapshot.capture(path, catalogNode, null, null, null); } ScopeState catalogState = catalogNode.current.get(); ScopeNode dbNode = child(catalogNode, catalogState, path.database(), ScopePath.Level.DATABASE); - nodes.add(dbNode); if (path.level() == ScopePath.Level.DATABASE) { - return nodes; + return ScopeSnapshot.capture(path, catalogNode, dbNode, null, null); } ScopeState dbState = dbNode.current.get(); ScopeNode tableNode = child(dbNode, dbState, path.table(), ScopePath.Level.TABLE); - nodes.add(tableNode); if (path.level() == ScopePath.Level.TABLE) { - return nodes; + return ScopeSnapshot.capture(path, catalogNode, dbNode, tableNode, null); } ScopeState tableState = tableNode.current.get(); - nodes.add(child(tableNode, tableState, path.partition(), ScopePath.Level.PARTITION)); - return nodes; + ScopeNode partitionNode = child( + tableNode, tableState, path.partition(), ScopePath.Level.PARTITION); + return ScopeSnapshot.capture(path, catalogNode, dbNode, tableNode, partitionNode); } private List resolveExisting(ScopePath path) { @@ -268,25 +261,55 @@ private void bumpPublicationStates(List nodes) { private void cleanDetachedState(ScopeState state) { state.entries.forEach((address, value) -> { - address.removeExpected(value); - state.entries.remove(address, value); + if (state.entries.remove(address, value)) { + address.removeExpected(value); + } }); state.children.values().forEach(child -> cleanDetachedState(child.current.get())); state.children.clear(); } - private void release(List nodes) { - nodes.forEach(node -> { - int remaining = node.activeLoads.decrementAndGet(); - int retainedRemaining = retainedActiveLoads.decrementAndGet(node.level.ordinal()); - if (remaining < 0) { - throw new IllegalStateException("Scope active-load count became negative"); - } - if (retainedRemaining < 0) { - throw new IllegalStateException("Retained scope active-load count became negative"); - } - }); - tryPrune(nodes); + private void retain(ScopeSnapshot snapshot) { + retain(snapshot.catalogNode); + if (snapshot.databaseNode != null) { + retain(snapshot.databaseNode); + } + if (snapshot.tableNode != null) { + retain(snapshot.tableNode); + } + if (snapshot.partitionNode != null) { + retain(snapshot.partitionNode); + } + } + + private void retain(ScopeNode node) { + node.activeLoads.incrementAndGet(); + retainedActiveLoads.incrementAndGet(node.level.ordinal()); + } + + private void release(ScopeSnapshot snapshot) { + release(snapshot.catalogNode); + if (snapshot.databaseNode != null) { + release(snapshot.databaseNode); + } + if (snapshot.tableNode != null) { + release(snapshot.tableNode); + } + if (snapshot.partitionNode != null) { + release(snapshot.partitionNode); + } + tryPrune(snapshot); + } + + private void release(ScopeNode node) { + int remaining = node.activeLoads.decrementAndGet(); + int retainedRemaining = retainedActiveLoads.decrementAndGet(node.level.ordinal()); + if (remaining < 0) { + throw new IllegalStateException("Scope active-load count became negative"); + } + if (retainedRemaining < 0) { + throw new IllegalStateException("Retained scope active-load count became negative"); + } } private void tryPrune(List nodes) { @@ -295,6 +318,18 @@ private void tryPrune(List nodes) { } } + private void tryPrune(ScopeSnapshot snapshot) { + if (snapshot.partitionNode != null) { + tryPrune(snapshot.partitionNode); + } + if (snapshot.tableNode != null) { + tryPrune(snapshot.tableNode); + } + if (snapshot.databaseNode != null) { + tryPrune(snapshot.databaseNode); + } + } + private boolean tryPrune(ScopeNode node) { if (node.parent == null) { return false; @@ -389,44 +424,131 @@ boolean commitIfPublicationCurrent( @Override public void close() { if (released.compareAndSet(false, true)) { - registry.release(snapshot.nodes); + registry.release(snapshot); } } } static final class ScopeSnapshot { private final ScopePath path; - private final List nodes; - private final List states; - - private ScopeSnapshot(ScopePath path, List nodes, List states) { + private final ScopeNode catalogNode; + private final ScopeState catalogState; + private final ScopeNode databaseNode; + private final ScopeState databaseState; + private final ScopeNode tableNode; + private final ScopeState tableState; + private final ScopeNode partitionNode; + private final ScopeState partitionState; + private final int generationHashCode; + + private ScopeSnapshot( + ScopePath path, + ScopeNode catalogNode, + ScopeState catalogState, + ScopeNode databaseNode, + ScopeState databaseState, + ScopeNode tableNode, + ScopeState tableState, + ScopeNode partitionNode, + ScopeState partitionState) { this.path = path; - this.nodes = Collections.unmodifiableList(new ArrayList<>(nodes)); - this.states = Collections.unmodifiableList(states); - } - - static ScopeSnapshot capture(ScopePath path, List nodes) { - List states = new ArrayList<>(nodes.size()); - nodes.forEach(node -> states.add(node.current.get())); - return new ScopeSnapshot(path, nodes, states); + this.catalogNode = catalogNode; + this.catalogState = catalogState; + this.databaseNode = databaseNode; + this.databaseState = databaseState; + this.tableNode = tableNode; + this.tableState = tableState; + this.partitionNode = partitionNode; + this.partitionState = partitionState; + int hashCode = generationHashCode(1, catalogNode, catalogState); + if (databaseNode != null) { + hashCode = generationHashCode(hashCode, databaseNode, databaseState); + } + if (tableNode != null) { + hashCode = generationHashCode(hashCode, tableNode, tableState); + } + if (partitionNode != null) { + hashCode = generationHashCode(hashCode, partitionNode, partitionState); + } + this.generationHashCode = hashCode; + } + + static ScopeSnapshot capture( + ScopePath path, + ScopeNode catalogNode, + ScopeNode databaseNode, + ScopeNode tableNode, + ScopeNode partitionNode) { + ScopeState catalogState = catalogNode.current.get(); + ScopeState databaseState = databaseNode == null ? null : databaseNode.current.get(); + ScopeState tableState = tableNode == null ? null : tableNode.current.get(); + ScopeState partitionState = partitionNode == null ? null : partitionNode.current.get(); + ScopeState leafState = partitionState != null + ? partitionState + : tableState != null ? tableState : databaseState != null ? databaseState : catalogState; + ScopeSnapshot currentSnapshot = leafState.scopeSnapshot; + if (currentSnapshot != null && currentSnapshot.matches( + catalogNode, + catalogState, + databaseNode, + databaseState, + tableNode, + tableState, + partitionNode, + partitionState)) { + return currentSnapshot; + } + synchronized (leafState) { + currentSnapshot = leafState.scopeSnapshot; + if (currentSnapshot == null || !currentSnapshot.matches( + catalogNode, + catalogState, + databaseNode, + databaseState, + tableNode, + tableState, + partitionNode, + partitionState)) { + currentSnapshot = new ScopeSnapshot( + path, + catalogNode, + catalogState, + databaseNode, + databaseState, + tableNode, + tableState, + partitionNode, + partitionState); + leafState.scopeSnapshot = currentSnapshot; + } + return currentSnapshot; + } } boolean isCurrent(ScopedMetaCacheRegistry registry) { - if (registry.closed.get() || registry.root != nodes.get(0) - || registry.root.current.get() != states.get(0)) { + if (registry.closed.get() || registry.root != catalogNode + || catalogNode.current.get() != catalogState) { return false; } - for (int i = 1; i < nodes.size(); i++) { - ScopeNode parent = nodes.get(i - 1); - ScopeNode node = nodes.get(i); - ScopeState parentState = states.get(i - 1); - if (parent.current.get() != parentState - || parentState.children.get(node.childKey) != node - || node.current.get() != states.get(i)) { - return false; - } + if (databaseNode == null) { + return true; + } + if (catalogState.children.get(databaseNode.childKey) != databaseNode + || databaseNode.current.get() != databaseState) { + return false; + } + if (tableNode == null) { + return true; + } + if (databaseState.children.get(tableNode.childKey) != tableNode + || tableNode.current.get() != tableState) { + return false; } - return true; + if (partitionNode == null) { + return true; + } + return tableState.children.get(partitionNode.childKey) == partitionNode + && partitionNode.current.get() == partitionState; } ScopePath path() { @@ -434,32 +556,70 @@ ScopePath path() { } ScopeNode leafNode() { - return nodes.get(nodes.size() - 1); + if (partitionNode != null) { + return partitionNode; + } + if (tableNode != null) { + return tableNode; + } + if (databaseNode != null) { + return databaseNode; + } + return catalogNode; } ScopeState leafState() { - return states.get(states.size() - 1); + if (partitionState != null) { + return partitionState; + } + if (tableState != null) { + return tableState; + } + if (databaseState != null) { + return databaseState; + } + return catalogState; } boolean sameGeneration(ScopeSnapshot other) { - if (nodes.size() != other.nodes.size()) { - return false; - } - for (int i = 0; i < nodes.size(); i++) { - if (nodes.get(i) != other.nodes.get(i) || states.get(i) != other.states.get(i)) { - return false; - } - } - return true; + return matches( + other.catalogNode, + other.catalogState, + other.databaseNode, + other.databaseState, + other.tableNode, + other.tableState, + other.partitionNode, + other.partitionState); } int generationHashCode() { - int result = 1; - for (int i = 0; i < nodes.size(); i++) { - result = 31 * result + System.identityHashCode(nodes.get(i)); - result = 31 * result + System.identityHashCode(states.get(i)); - } - return result; + return generationHashCode; + } + + private static int generationHashCode( + int currentHashCode, ScopeNode node, ScopeState state) { + int hashCode = 31 * currentHashCode + System.identityHashCode(node); + return 31 * hashCode + System.identityHashCode(state); + } + + private boolean matches( + ScopeNode expectedCatalogNode, + ScopeState expectedCatalogState, + ScopeNode expectedDatabaseNode, + ScopeState expectedDatabaseState, + ScopeNode expectedTableNode, + ScopeState expectedTableState, + ScopeNode expectedPartitionNode, + ScopeState expectedPartitionState) { + return catalogNode == expectedCatalogNode + && catalogState == expectedCatalogState + && databaseNode == expectedDatabaseNode + && databaseState == expectedDatabaseState + && tableNode == expectedTableNode + && tableState == expectedTableState + && partitionNode == expectedPartitionNode + && partitionState == expectedPartitionState; } } @@ -518,6 +678,7 @@ private static final class ScopeState { private final long generation; private final ConcurrentMap children = new ConcurrentHashMap<>(); private final ConcurrentMap entries = new ConcurrentHashMap<>(); + private volatile ScopeSnapshot scopeSnapshot; private ScopeState(long generation) { this.generation = generation; diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java index 3c168f662f891c..3ed99fbbb75ce0 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java @@ -221,6 +221,50 @@ public void stagedBulkValueIsNeverVisibleAfterHandleInvalidation() throws Except } } + @Test + public void failedBulkPublicationDoesNotFenceIndependentDirectLoader() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch allowLoaderReturn = new CountDownLatch(1); + CountDownLatch valueStaged = new CountDownLatch(1); + CountDownLatch allowBulkCommit = new CountDownLatch(1); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache( + "test", + ENABLED, + null, + null, + () -> { + valueStaged.countDown(); + await(allowBulkCommit); + }); + Future directLoad = executor.submit(() -> cache.get("key", PARTITION, ignored -> { + loaderStarted.countDown(); + await(allowLoaderReturn); + return "direct"; + })); + await(loaderStarted); + + try (BulkLoadHandle handle = cache.beginBulkLoad(TABLE)) { + Future bulkPublication = executor.submit( + () -> cache.publish(handle, "key", PARTITION, "stale")); + await(valueStaged); + registry.invalidate(ScopePath.partition("db", "tbl", "sibling")); + + allowBulkCommit.countDown(); + Assertions.assertFalse(bulkPublication.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } + allowLoaderReturn.countDown(); + + Assertions.assertEquals("direct", directLoad.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Assertions.assertEquals("direct", cache.getIfPresent("key", PARTITION)); + } finally { + allowBulkCommit.countDown(); + allowLoaderReturn.countDown(); + executor.shutdownNow(); + } + } + @Test public void invalidationAfterPartialBulkPublicationRemovesPublishedPartAndFencesTheRest() { try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { From dc81bc25f11cd04634798880e96ce0a8e396dd04 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Mon, 10 Aug 2026 12:03:25 +0800 Subject: [PATCH 03/10] [improvement](fe) Reduce scoped metadata load overhead ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Cold scoped metadata loads repeatedly allocated varargs arrays and capturing lambdas and computed identity hashes for state tokens even though single-flight hashing only needs stable key and path distribution. Use allocation-free path hashing cached per shared scope snapshot, use a get/putIfAbsent scope-child fast path, cache cache-address hashes, keep full generation identity comparisons in LoadAddress.equals, and remove a redundant bulk scope check before the final publication fence. Matched local microbenchmarks reduced cold load from 652.5-768.5 ns/op to 442.6 ns/op while independent cache-hit latency remained effectively unchanged. Current-head profiling no longer shows the identityHashCode, ScopePath Object array, or child lambda hotspots. ### Release note None ### Check List (For Author) - Test: Unit Test and FE build - Unit Test: 87 fe-connector-cache tests and 54 focused scoped-cache tests - FE build: ./build.sh --fe - Behavior changed: No - Does this need documentation: No --- .../doris/connector/cache/ScopePath.java | 6 ++- .../connector/cache/ScopedMetaCache.java | 7 +--- .../cache/ScopedMetaCacheRegistry.java | 37 +++++++------------ 3 files changed, 20 insertions(+), 30 deletions(-) diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java index a3fe243cb70fbf..82a031be8c327a 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java @@ -128,7 +128,11 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash(level, database, table, partition); + int pathHashCode = 1; + pathHashCode = 31 * pathHashCode + Objects.hashCode(level); + pathHashCode = 31 * pathHashCode + Objects.hashCode(database); + pathHashCode = 31 * pathHashCode + Objects.hashCode(table); + return 31 * pathHashCode + Objects.hashCode(partition); } @Override diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java index dfe658c7416183..da4f549cd8b372 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java @@ -404,7 +404,6 @@ private boolean isBulkKeyCurrent(BulkLoadHandle handle, Object rawKey) { BigInteger invalidation = exactInvalidations.get(rawKey); return !closed.get() && !handle.closed.get() - && handle.scopeLease.isPublicationCurrent(handle.scopePublicationState) && (invalidation == null || invalidation.compareTo(handle.exactInvalidationSequence) <= 0); } @@ -708,11 +707,7 @@ private LoadAddress(K key, ScopePath path, PublicationLease lease) { this.keyNode = lease.keyNode; this.keyState = lease.keyState; this.loadPublicationState = lease.loadPublicationState; - int addressHashCode = 31 * key.hashCode() + path.hashCode(); - addressHashCode = 31 * addressHashCode + scopeSnapshot.generationHashCode(); - addressHashCode = 31 * addressHashCode + System.identityHashCode(keyNode); - addressHashCode = 31 * addressHashCode + System.identityHashCode(keyState); - this.hashCode = 31 * addressHashCode + System.identityHashCode(loadPublicationState); + this.hashCode = 31 * key.hashCode() + scopeSnapshot.pathHashCode(); } @Override diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java index bf85d1ccc6ad9e..d1de6a49102e46 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java @@ -242,8 +242,13 @@ private List resolveExisting(ScopePath path) { private ScopeNode child( ScopeNode parent, ScopeState parentState, Object childKey, ScopePath.Level childLevel) { - return parentState.children.computeIfAbsent( - childKey, ignored -> new ScopeNode(parent, childKey, childLevel)); + ScopeNode existing = parentState.children.get(childKey); + if (existing != null) { + return existing; + } + ScopeNode created = new ScopeNode(parent, childKey, childLevel); + ScopeNode raced = parentState.children.putIfAbsent(childKey, created); + return raced == null ? created : raced; } private void bumpPublicationStates(List nodes) { @@ -439,7 +444,7 @@ static final class ScopeSnapshot { private final ScopeState tableState; private final ScopeNode partitionNode; private final ScopeState partitionState; - private final int generationHashCode; + private final int pathHashCode; private ScopeSnapshot( ScopePath path, @@ -460,17 +465,7 @@ private ScopeSnapshot( this.tableState = tableState; this.partitionNode = partitionNode; this.partitionState = partitionState; - int hashCode = generationHashCode(1, catalogNode, catalogState); - if (databaseNode != null) { - hashCode = generationHashCode(hashCode, databaseNode, databaseState); - } - if (tableNode != null) { - hashCode = generationHashCode(hashCode, tableNode, tableState); - } - if (partitionNode != null) { - hashCode = generationHashCode(hashCode, partitionNode, partitionState); - } - this.generationHashCode = hashCode; + this.pathHashCode = path.hashCode(); } static ScopeSnapshot capture( @@ -593,14 +588,8 @@ boolean sameGeneration(ScopeSnapshot other) { other.partitionState); } - int generationHashCode() { - return generationHashCode; - } - - private static int generationHashCode( - int currentHashCode, ScopeNode node, ScopeState state) { - int hashCode = 31 * currentHashCode + System.identityHashCode(node); - return 31 * hashCode + System.identityHashCode(state); + int pathHashCode() { + return pathHashCode; } private boolean matches( @@ -629,10 +618,12 @@ static final class PublicationState { static final class CacheAddress { private final ScopedMetaCache owner; private final Object key; + private final int hashCode; CacheAddress(ScopedMetaCache owner, Object key) { this.owner = owner; this.key = key; + this.hashCode = 31 * System.identityHashCode(owner) + key.hashCode(); } void removeExpected(Object expectedValue) { @@ -653,7 +644,7 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return 31 * System.identityHashCode(owner) + key.hashCode(); + return hashCode; } } From fe8dae508ed02d7bff1f6e4de5850ae74b43adee Mon Sep 17 00:00:00 2001 From: 924060929 Date: Mon, 10 Aug 2026 16:23:22 +0800 Subject: [PATCH 04/10] [improvement](fe) Scale scoped metadata cache publication ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Scoped metadata bulk publication used exclusive per-cache and registry monitors, serializing independent keys and cache owners. Replace those publication barriers with writer-preferred striped phase gates while keeping exact-key and hierarchical invalidation exclusive. Defer synchronous Caffeine removal callbacks until publication phases are released, preserve expected-value cleanup under callback reentrancy and failures, and coordinate scope-node retain, child creation, invalidation, and pruning with an exact lifecycle marker. In matched eight-thread benchmarks, the correctness-safe implementation improves same-cache publication from about 1.3-1.4 M ops/s to 2.8-3.2 M ops/s and same-registry publication from about 1.8 M ops/s to 3.0-3.3 M ops/s. CPU and lock profiles show the remaining same-cache lock bottleneck is Caffeine maintenance rather than the framework publication barriers. ### Release note None ### Check List (For Author) - Test: Unit Test and manual performance profiling - Unit Test: `./run-fe-ut.sh --run org.apache.doris.connector.cache.*Test` (99 tests; full FE reactor BUILD SUCCESS) - Manual test: matched eight-thread topology benchmarks and async-profiler CPU/lock profiles - Behavior changed: Yes (independent bulk publications can overlap while invalidation and close remain exclusive) - Does this need documentation: No --- .../connector/cache/ScopedMetaCache.java | 155 ++++++++--- .../cache/ScopedMetaCacheRegistry.java | 208 ++++++++++---- .../connector/cache/StripedPhaseGate.java | 111 ++++++++ .../cache/ScopedMetaCacheConcurrencyTest.java | 260 ++++++++++++++++++ .../connector/cache/StripedPhaseGateTest.java | 160 +++++++++++ 5 files changed, 796 insertions(+), 98 deletions(-) create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/StripedPhaseGate.java create mode 100644 fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/StripedPhaseGateTest.java diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java index da4f549cd8b372..c2b891ce8fa8c8 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java @@ -29,6 +29,8 @@ import java.math.BigInteger; import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.HashMap; import java.util.Map; import java.util.NavigableMap; @@ -54,6 +56,7 @@ * scope bucket and key node, so delayed callbacks cannot delete a replacement. */ public final class ScopedMetaCache implements AutoCloseable { + private static final System.Logger LOG = System.getLogger(ScopedMetaCache.class.getName()); private static final Runnable NO_OP = () -> { }; @@ -63,13 +66,15 @@ public final class ScopedMetaCache implements AutoCloseable { private final Cache> data; private final ConcurrentMap> keyNodes = new ConcurrentHashMap<>(); private final ConcurrentMap, CompletableFuture> inFlightLoads = new ConcurrentHashMap<>(); - private final Object bulkInvalidationLock = new Object(); + private final StripedPhaseGate bulkInvalidationGate = new StripedPhaseGate(); private final Map exactInvalidations = new HashMap<>(); private final NavigableMap activeBulkStarts = new TreeMap<>(); private final AtomicBoolean closed = new AtomicBoolean(false); private final BiConsumer beforeRemoval; private final Runnable afterLoadElection; private final Runnable afterBulkStage; + private final ThreadLocal> removalDeferrals = + ThreadLocal.withInitial(RemovalDeferral::new); private BigInteger exactInvalidationSequence = BigInteger.ZERO; ScopedMetaCache( @@ -208,31 +213,31 @@ void invalidateKey( Objects.requireNonNull(afterStateReplacement, "afterStateReplacement can not be null"); checkOpen(); beforeInvalidationLock.run(); - KeyNode node; - KeyState invalidatedState = null; - synchronized (bulkInvalidationLock) { + InvalidatedKey invalidated = bulkInvalidationGate.write(() -> { if (closed.get()) { - return; + return null; } exactInvalidationSequence = exactInvalidationSequence.add(BigInteger.ONE); if (!activeBulkStarts.isEmpty()) { exactInvalidations.put(key, exactInvalidationSequence); } - node = keyNodes.get(key); + KeyNode node = keyNodes.get(key); + KeyState invalidatedState = null; if (node != null) { invalidatedState = replaceKeyState(node); } - } - if (node == null) { + return new InvalidatedKey<>(node, invalidatedState); + }); + if (invalidated == null || invalidated.node == null) { return; } afterStateReplacement.run(); - VersionedValue registered = node.registration.get(); - if (registered != null && registered.keyState == invalidatedState) { + VersionedValue registered = invalidated.node.registration.get(); + if (registered != null && registered.keyState == invalidated.keyState) { data.asMap().remove(key, registered); - node.registration.compareAndSet(registered, null); + invalidated.node.registration.compareAndSet(registered, null); } - tryPruneKey(key, node); + tryPruneKey(key, invalidated.node); } public BulkLoadHandle beginBulkLoad(ScopePath parentScope) { @@ -242,15 +247,15 @@ public BulkLoadHandle beginBulkLoad(ScopePath parentScope) { return BulkLoadHandle.disabled(this, parentScope); } ScopeLease scopeLease = registry.acquire(parentScope); - BigInteger exactSequence; - synchronized (bulkInvalidationLock) { + BigInteger exactSequence = bulkInvalidationGate.write(() -> { if (closed.get()) { scopeLease.close(); throw new IllegalStateException("Scoped meta cache '" + name + "' is closed"); } - exactSequence = exactInvalidationSequence; - activeBulkStarts.merge(exactSequence, 1, Integer::sum); - } + BigInteger sequence = exactInvalidationSequence; + activeBulkStarts.merge(sequence, 1, Integer::sum); + return sequence; + }); return new BulkLoadHandle( this, parentScope, @@ -287,14 +292,12 @@ public boolean publish( } public CacheMetrics metrics() { - synchronized (bulkInvalidationLock) { - return new CacheMetrics( + return bulkInvalidationGate.read(() -> new CacheMetrics( data.estimatedSize(), keyNodes.size(), inFlightLoads.size(), activeBulkStarts.values().stream().mapToInt(Integer::intValue).sum(), - exactInvalidations.size()); - } + exactInvalidations.size())); } public void cleanUp() { @@ -384,19 +387,28 @@ private boolean tryCommitBulk( PublicationLease lease = (PublicationLease) rawLease; @SuppressWarnings("unchecked") VersionedValue staged = (VersionedValue) rawStaged; - synchronized (bulkInvalidationLock) { - if (!isBulkKeyCurrent(handle, key)) { - return false; + RemovalDeferral removalDeferral = removalDeferrals.get(); + removalDeferral.depth++; + try { + return bulkInvalidationGate.readBoolean(() -> { + if (!isBulkKeyCurrent(handle, key)) { + return false; + } + return handle.scopeLease.commitIfPublicationCurrent( + handle.scopePublicationState, () -> { + if (!lease.isCurrent()) { + return false; + } + lease.keyNode.loadPublicationState.set(new Object()); + install(staged, lease); + return true; + }); + }); + } finally { + removalDeferral.depth--; + if (removalDeferral.depth == 0 && !removalDeferral.draining) { + drainDeferredRemovals(removalDeferral); } - return handle.scopeLease.commitIfPublicationCurrent( - handle.scopePublicationState, () -> { - if (!lease.isCurrent()) { - return false; - } - lease.keyNode.loadPublicationState.set(new Object()); - install(staged, lease); - return true; - }); } } @@ -408,10 +420,9 @@ private boolean isBulkKeyCurrent(BulkLoadHandle handle, Object rawKey) { } private void closeBulkHandle(BulkLoadHandle handle) { - ScopeLease leaseToClose = null; - synchronized (bulkInvalidationLock) { + ScopeLease leaseToClose = bulkInvalidationGate.write(() -> { if (!handle.closed.compareAndSet(false, true)) { - return; + return null; } if (handle.scopeLease != null) { Integer count = activeBulkStarts.get(handle.exactInvalidationSequence); @@ -424,9 +435,10 @@ private void closeBulkHandle(BulkLoadHandle handle) { activeBulkStarts.put(handle.exactInvalidationSequence, count - 1); } pruneExactInvalidations(); - leaseToClose = handle.scopeLease; + return handle.scopeLease; } - } + return null; + }); if (leaseToClose != null) { leaseToClose.close(); } @@ -457,9 +469,7 @@ private V awaitLoad(CompletableFuture load) { } private void closePhysicalState() { - synchronized (bulkInvalidationLock) { - exactInvalidations.clear(); - } + bulkInvalidationGate.write(exactInvalidations::clear); keyNodes.forEach((key, node) -> { replaceKeyState(node); VersionedValue versioned = node.registration.get(); @@ -478,17 +488,74 @@ private void onRemoval( Objects.requireNonNull(rawKey, "removed cache key can not be null"); Objects.requireNonNull(rawValue, "removed cache value can not be null"); @SuppressWarnings("unchecked") - K key = (K) rawKey; - @SuppressWarnings("unchecked") VersionedValue versioned = (VersionedValue) rawValue; + RemovalDeferral removalDeferral = removalDeferrals.get(); + removalDeferral.removals.addLast(versioned); + if (removalDeferral.depth > 0 || removalDeferral.draining) { + return; + } + drainDeferredRemovals(removalDeferral); + } + + private void drainDeferredRemovals(RemovalDeferral removalDeferral) { + removalDeferral.draining = true; + Throwable failure = null; + try { + VersionedValue versioned; + while ((versioned = removalDeferral.removals.pollFirst()) != null) { + try { + completeRemoval(versioned); + } catch (RuntimeException | Error e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + } + } finally { + removalDeferral.draining = false; + removalDeferral.removals.clear(); + removalDeferrals.remove(); + } + if (failure instanceof RuntimeException) { + throw (RuntimeException) failure; + } + if (failure != null) { + throw (Error) failure; + } + } + + private void completeRemoval(VersionedValue versioned) { + K key = versioned.key; if (beforeRemoval != null) { - beforeRemoval.accept(key, versioned.value); + try { + beforeRemoval.accept(key, versioned.value); + } catch (Throwable t) { + LOG.log(System.Logger.Level.WARNING, "Scoped metadata cache removal callback failed", t); + } } registry.unregister(versioned.address, versioned, versioned.scopeSnapshot); versioned.keyNode.registration.compareAndSet(versioned, null); tryPruneKey(key, versioned.keyNode); } + private static final class RemovalDeferral { + private final Deque> removals = new ArrayDeque<>(); + private int depth; + private boolean draining; + } + + private static final class InvalidatedKey { + private final KeyNode node; + private final KeyState keyState; + + private InvalidatedKey(KeyNode node, KeyState keyState) { + this.node = node; + this.keyState = keyState; + } + } + private KeyState replaceKeyState(KeyNode node) { while (true) { KeyState oldState = node.current.get(); diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java index d1de6a49102e46..0d3b076fe1688f 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java @@ -28,8 +28,8 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicIntegerArray; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.LongAdder; import java.util.function.BiConsumer; import java.util.function.BooleanSupplier; @@ -44,13 +44,30 @@ public final class ScopedMetaCacheRegistry implements AutoCloseable { private static final Runnable NO_OP = () -> { }; + private static final BiConsumer NO_OP_SCOPE = (level, key) -> { + }; private final ScopeNode root = new ScopeNode(null, null, ScopePath.Level.CATALOG); private final Set> caches = ConcurrentHashMap.newKeySet(); - private final Object publicationLock = new Object(); + private final StripedPhaseGate publicationGate = new StripedPhaseGate(); private final AtomicBoolean closed = new AtomicBoolean(false); - private final AtomicIntegerArray retainedActiveLoads = - new AtomicIntegerArray(ScopePath.Level.values().length); + private final LongAdder[] retainedActiveLoads = newCounters(ScopePath.Level.values().length); + private final BiConsumer afterParentPin; + private final BiConsumer beforePruneMark; + private final BiConsumer afterPruneMark; + + public ScopedMetaCacheRegistry() { + this(NO_OP_SCOPE, NO_OP_SCOPE, NO_OP_SCOPE); + } + + ScopedMetaCacheRegistry( + BiConsumer afterParentPin, + BiConsumer beforePruneMark, + BiConsumer afterPruneMark) { + this.afterParentPin = Objects.requireNonNull(afterParentPin, "afterParentPin can not be null"); + this.beforePruneMark = Objects.requireNonNull(beforePruneMark, "beforePruneMark can not be null"); + this.afterPruneMark = Objects.requireNonNull(afterPruneMark, "afterPruneMark can not be null"); + } public ScopedMetaCache createCache(String name, CacheSpec cacheSpec) { return createCache(name, cacheSpec, null, null, NO_OP, NO_OP); @@ -106,27 +123,28 @@ void invalidate(ScopePath path, Runnable afterStateReplacement) { Objects.requireNonNull(path, "path can not be null"); Objects.requireNonNull(afterStateReplacement, "afterStateReplacement can not be null"); checkOpen(); - List existing; - ScopeState oldState; - synchronized (publicationLock) { - existing = resolveExisting(path); + InvalidatedScope invalidated = publicationGate.write(() -> { + List existing = resolveExisting(path); bumpPublicationStates(existing); if (existing.size() != path.level().ordinal() + 1) { - return; + return null; } ScopeNode target = existing.get(existing.size() - 1); - oldState = replaceState(target); + return new InvalidatedScope(existing, replaceState(target)); + }); + if (invalidated == null) { + return; } afterStateReplacement.run(); - cleanDetachedState(oldState); - tryPrune(existing); + cleanDetachedState(invalidated.oldState); + tryPrune(invalidated.existing); } public ScopeMetrics metrics() { ScopeMetricsAccumulator accumulator = new ScopeMetricsAccumulator(); collectMetrics(root, accumulator); for (ScopePath.Level level : ScopePath.Level.values()) { - accumulator.retainedActiveLoads[level.ordinal()] = retainedActiveLoads.get(level.ordinal()); + accumulator.retainedActiveLoads[level.ordinal()] = retainedActiveLoads[level.ordinal()].intValue(); } return accumulator.snapshot(); } @@ -136,11 +154,10 @@ public void close() { if (!closed.compareAndSet(false, true)) { return; } - ScopeState oldState; - synchronized (publicationLock) { + ScopeState oldState = publicationGate.write(() -> { bumpPublicationStates(Collections.singletonList(root)); - oldState = replaceState(root); - } + return replaceState(root); + }); cleanDetachedState(oldState); List> snapshot = new ArrayList<>(caches); caches.clear(); @@ -152,7 +169,9 @@ ScopeLease acquire(ScopePath path) { while (true) { checkOpen(); ScopeSnapshot snapshot = resolveOrCreateSnapshot(path); - retain(snapshot); + if (snapshot == null || !retain(snapshot)) { + continue; + } if (snapshot.isCurrent(this)) { return new ScopeLease(this, snapshot); } @@ -196,17 +215,26 @@ private ScopeSnapshot resolveOrCreateSnapshot(ScopePath path) { } ScopeState catalogState = catalogNode.current.get(); ScopeNode dbNode = child(catalogNode, catalogState, path.database(), ScopePath.Level.DATABASE); + if (dbNode == null) { + return null; + } if (path.level() == ScopePath.Level.DATABASE) { return ScopeSnapshot.capture(path, catalogNode, dbNode, null, null); } ScopeState dbState = dbNode.current.get(); ScopeNode tableNode = child(dbNode, dbState, path.table(), ScopePath.Level.TABLE); + if (tableNode == null) { + return null; + } if (path.level() == ScopePath.Level.TABLE) { return ScopeSnapshot.capture(path, catalogNode, dbNode, tableNode, null); } ScopeState tableState = tableNode.current.get(); ScopeNode partitionNode = child( tableNode, tableState, path.partition(), ScopePath.Level.PARTITION); + if (partitionNode == null) { + return null; + } return ScopeSnapshot.capture(path, catalogNode, dbNode, tableNode, partitionNode); } @@ -246,9 +274,29 @@ private ScopeNode child( if (existing != null) { return existing; } - ScopeNode created = new ScopeNode(parent, childKey, childLevel); - ScopeNode raced = parentState.children.putIfAbsent(childKey, created); - return raced == null ? created : raced; + if (!parent.tryRetain()) { + return null; + } + try { + if (parent.current.get() != parentState) { + return null; + } + afterParentPin.accept(childLevel, childKey); + existing = parentState.children.get(childKey); + if (existing != null) { + return existing; + } + ScopeNode created = new ScopeNode(parent, childKey, childLevel); + ScopeNode raced = parentState.children.putIfAbsent(childKey, created); + ScopeNode child = raced == null ? created : raced; + if (parent.current.get() != parentState) { + parentState.children.remove(childKey, child); + return null; + } + return child; + } finally { + parent.releaseRetain(); + } } private void bumpPublicationStates(List nodes) { @@ -274,49 +322,38 @@ private void cleanDetachedState(ScopeState state) { state.children.clear(); } - private void retain(ScopeSnapshot snapshot) { - retain(snapshot.catalogNode); + private boolean retain(ScopeSnapshot snapshot) { + if (!snapshot.leafNode().tryRetain()) { + return false; + } + retainedActiveLoads[ScopePath.Level.CATALOG.ordinal()].increment(); if (snapshot.databaseNode != null) { - retain(snapshot.databaseNode); + retainedActiveLoads[ScopePath.Level.DATABASE.ordinal()].increment(); } if (snapshot.tableNode != null) { - retain(snapshot.tableNode); + retainedActiveLoads[ScopePath.Level.TABLE.ordinal()].increment(); } if (snapshot.partitionNode != null) { - retain(snapshot.partitionNode); + retainedActiveLoads[ScopePath.Level.PARTITION.ordinal()].increment(); } - } - - private void retain(ScopeNode node) { - node.activeLoads.incrementAndGet(); - retainedActiveLoads.incrementAndGet(node.level.ordinal()); + return true; } private void release(ScopeSnapshot snapshot) { - release(snapshot.catalogNode); + snapshot.leafNode().releaseRetain(); + retainedActiveLoads[ScopePath.Level.CATALOG.ordinal()].decrement(); if (snapshot.databaseNode != null) { - release(snapshot.databaseNode); + retainedActiveLoads[ScopePath.Level.DATABASE.ordinal()].decrement(); } if (snapshot.tableNode != null) { - release(snapshot.tableNode); + retainedActiveLoads[ScopePath.Level.TABLE.ordinal()].decrement(); } if (snapshot.partitionNode != null) { - release(snapshot.partitionNode); + retainedActiveLoads[ScopePath.Level.PARTITION.ordinal()].decrement(); } tryPrune(snapshot); } - private void release(ScopeNode node) { - int remaining = node.activeLoads.decrementAndGet(); - int retainedRemaining = retainedActiveLoads.decrementAndGet(node.level.ordinal()); - if (remaining < 0) { - throw new IllegalStateException("Scope active-load count became negative"); - } - if (retainedRemaining < 0) { - throw new IllegalStateException("Retained scope active-load count became negative"); - } - } - private void tryPrune(List nodes) { for (int i = nodes.size() - 1; i > 0; i--) { tryPrune(nodes.get(i)); @@ -339,12 +376,28 @@ private boolean tryPrune(ScopeNode node) { if (node.parent == null) { return false; } - ScopeState state = node.current.get(); - if (node.activeLoads.get() != 0 || !state.entries.isEmpty() || !state.children.isEmpty()) { - return false; + while (true) { + ScopeState state = node.current.get(); + if (!state.entries.isEmpty() || !state.children.isEmpty()) { + return false; + } + beforePruneMark.accept(node.level, node.childKey); + if (!node.tryBeginPrune()) { + return false; + } + try { + afterPruneMark.accept(node.level, node.childKey); + } catch (RuntimeException | Error e) { + node.cancelPrune(); + throw e; + } + if (node.current.get() != state || !state.entries.isEmpty() || !state.children.isEmpty()) { + node.cancelPrune(); + continue; + } + ScopeState parentState = node.parent.current.get(); + return parentState.children.remove(node.childKey, node); } - ScopeState parentState = node.parent.current.get(); - return parentState.children.remove(node.childKey, node); } private void collectMetrics(ScopeNode node, ScopeMetricsAccumulator accumulator) { @@ -368,6 +421,14 @@ private void collectMetrics(ScopeNode node, ScopeMetricsAccumulator accumulator) }); } + private static LongAdder[] newCounters(int count) { + LongAdder[] counters = new LongAdder[count]; + for (int index = 0; index < count; index++) { + counters[index] = new LongAdder(); + } + return counters; + } + void forceGenerationForTest(ScopePath path, long generation) { List existing = resolveExisting(path); if (existing.size() != path.level().ordinal() + 1) { @@ -421,9 +482,8 @@ boolean isPublicationCurrent(PublicationState publicationState) { boolean commitIfPublicationCurrent( PublicationState publicationState, BooleanSupplier commitAction) { - synchronized (registry.publicationLock) { - return isPublicationCurrent(publicationState) && commitAction.getAsBoolean(); - } + return registry.publicationGate.readBoolean( + () -> isPublicationCurrent(publicationState) && commitAction.getAsBoolean()); } @Override @@ -434,6 +494,16 @@ public void close() { } } + private static final class InvalidatedScope { + private final List existing; + private final ScopeState oldState; + + private InvalidatedScope(List existing, ScopeState oldState) { + this.existing = existing; + this.oldState = oldState; + } + } + static final class ScopeSnapshot { private final ScopePath path; private final ScopeNode catalogNode; @@ -655,6 +725,7 @@ private static final class ScopeNode { private final AtomicReference current = new AtomicReference<>(new ScopeState(0L)); private final AtomicReference descendantPublication = new AtomicReference<>(new PublicationState()); + // -1 reserves the node for pruning. A canceled prune restores zero; a detached node keeps the marker. private final AtomicInteger activeLoads = new AtomicInteger(); private ScopeNode( @@ -663,6 +734,35 @@ private ScopeNode( this.childKey = childKey; this.level = level; } + + private boolean tryRetain() { + while (true) { + int active = activeLoads.get(); + if (active < 0) { + return false; + } + if (activeLoads.compareAndSet(active, active + 1)) { + return true; + } + } + } + + private void releaseRetain() { + int active = activeLoads.decrementAndGet(); + if (active < 0) { + throw new IllegalStateException("Scope node retain count became negative"); + } + } + + private boolean tryBeginPrune() { + return activeLoads.compareAndSet(0, -1); + } + + private void cancelPrune() { + if (!activeLoads.compareAndSet(-1, 0)) { + throw new IllegalStateException("Scope node prune marker changed unexpectedly"); + } + } } private static final class ScopeState { diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/StripedPhaseGate.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/StripedPhaseGate.java new file mode 100644 index 00000000000000..b585e8f020b64f --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/StripedPhaseGate.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.function.BooleanSupplier; +import java.util.function.Supplier; + +/** + * Writer-preferred phase gate for short cache-publication critical sections. + * + *

The gate is intentionally non-reentrant: an action must not enter either phase of the same gate. In particular, + * synchronous removal callbacks must run after the read phase. Reader actions must also avoid remote I/O or other + * unbounded work because a waiting writer and newly arriving readers spin until the current reader phase drains. + */ +final class StripedPhaseGate { + // Sixteen counters spread across distinct 64-byte cache lines avoid one shared reader counter under FE workers. + private static final int READER_STRIPE_COUNT = 16; + private static final int READER_STRIDE = 16; + + private final Object writerLock = new Object(); + private final AtomicIntegerArray activeReaders = + new AtomicIntegerArray(READER_STRIPE_COUNT * READER_STRIDE); + private volatile boolean writerActive; + + boolean readBoolean(BooleanSupplier action) { + int readerIndex = enterRead(); + try { + return action.getAsBoolean(); + } finally { + activeReaders.decrementAndGet(readerIndex); + } + } + + T read(Supplier action) { + int readerIndex = enterRead(); + try { + return action.get(); + } finally { + activeReaders.decrementAndGet(readerIndex); + } + } + + void write(Runnable action) { + write(() -> { + action.run(); + return null; + }); + } + + T write(Supplier action) { + synchronized (writerLock) { + writerActive = true; + try { + awaitReaders(); + return action.get(); + } finally { + writerActive = false; + } + } + } + + private int enterRead() { + int readerIndex = readerIndex(); + while (true) { + while (writerActive) { + Thread.onSpinWait(); + } + activeReaders.incrementAndGet(readerIndex); + if (!writerActive) { + return readerIndex; + } + activeReaders.decrementAndGet(readerIndex); + } + } + + private void awaitReaders() { + while (hasReaders()) { + Thread.onSpinWait(); + } + } + + private boolean hasReaders() { + for (int stripe = 0; stripe < READER_STRIPE_COUNT; stripe++) { + if (activeReaders.get(stripe * READER_STRIDE) != 0) { + return true; + } + } + return false; + } + + private int readerIndex() { + return ((int) Thread.currentThread().getId() & (READER_STRIPE_COUNT - 1)) + * READER_STRIDE; + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java index 3ed99fbbb75ce0..c6f928fa7124a0 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java @@ -29,7 +29,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; public class ScopedMetaCacheConcurrencyTest { private static final long TIMEOUT_SECONDS = 10L; @@ -524,6 +526,264 @@ public void failedBulkCandidatePreservesExistingValueAndCapacity() throws Except } } + @Test + public void bulkReplacementRemovalCanReenterInvalidation() { + AtomicBoolean reentered = new AtomicBoolean(false); + AtomicReference> cacheReference = new AtomicReference<>(); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache( + "test", + ENABLED, + null, + (key, value) -> { + if ("old".equals(value) && reentered.compareAndSet(false, true)) { + cacheReference.get().invalidateKey(key); + registry.invalidate(TABLE); + } + }); + cacheReference.set(cache); + cache.put("key", TABLE, "old"); + + try (BulkLoadHandle handle = cache.beginBulkLoad(TABLE)) { + Assertions.assertTrue(cache.publish(handle, "key", TABLE, "new")); + } + + Assertions.assertTrue(reentered.get()); + Assertions.assertNull(cache.getIfPresent("key", TABLE)); + assertEmpty(registry, cache); + } + } + + @Test + public void bulkReplacementRemovalCanReenterBulkPublication() { + AtomicBoolean reentered = new AtomicBoolean(false); + AtomicBoolean nestedPublished = new AtomicBoolean(false); + AtomicInteger removalCallbacks = new AtomicInteger(); + AtomicReference> cacheReference = new AtomicReference<>(); + AtomicReference handleReference = new AtomicReference<>(); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache( + "test", + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 1L), + null, + (key, value) -> { + removalCallbacks.incrementAndGet(); + if ("old".equals(value) && reentered.compareAndSet(false, true)) { + nestedPublished.set(cacheReference.get().publish( + handleReference.get(), "nested", TABLE, "nested")); + throw new AssertionError("injected removal callback error"); + } + }); + cacheReference.set(cache); + cache.put("key", TABLE, "old"); + + try (BulkLoadHandle handle = cache.beginBulkLoad(TABLE)) { + handleReference.set(handle); + Assertions.assertTrue(cache.publish(handle, "key", TABLE, "new")); + } + + Assertions.assertTrue(reentered.get()); + Assertions.assertTrue(nestedPublished.get()); + Assertions.assertNull(cache.getIfPresent("key", TABLE)); + Assertions.assertEquals("nested", cache.getIfPresent("nested", TABLE)); + Assertions.assertEquals(2, removalCallbacks.get()); + Assertions.assertEquals(1, registry.metrics().getRegistrationCount()); + Assertions.assertEquals(1, cache.metrics().getKeyNodeCount()); + } + } + + @Test + public void bulkRemovalFailureDoesNotChangeCommittedPublication() { + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry()) { + ScopedMetaCache cache = registry.createCache( + "test", + ENABLED, + null, + (key, value) -> { + throw new IllegalStateException("injected removal callback failure"); + }); + cache.put("key", TABLE, "old"); + + try (BulkLoadHandle handle = cache.beginBulkLoad(TABLE)) { + Assertions.assertTrue(cache.publish(handle, "key", TABLE, "new")); + } + + Assertions.assertEquals("new", cache.getIfPresent("key", TABLE)); + Assertions.assertEquals(1, registry.metrics().getRegistrationCount()); + Assertions.assertEquals(1, cache.metrics().getKeyNodeCount()); + } + } + + @Test + public void childCreationPinPreventsParentPrune() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch parentPinned = new CountDownLatch(1); + CountDownLatch allowChildInsert = new CountDownLatch(1); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry( + (level, key) -> { + if (level == ScopePath.Level.TABLE) { + parentPinned.countDown(); + await(allowChildInsert); + } + }, + (level, key) -> { + }, + (level, key) -> { + })) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + BulkLoadHandle databaseHandle = cache.beginBulkLoad(ScopePath.database("db")); + Future tableHandleFuture = executor.submit(() -> cache.beginBulkLoad(TABLE)); + await(parentPinned); + + databaseHandle.close(); + allowChildInsert.countDown(); + try (BulkLoadHandle tableHandle = tableHandleFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + Assertions.assertTrue(cache.publish(tableHandle, "key", TABLE, "value")); + } + Assertions.assertEquals("value", cache.getIfPresent("key", TABLE)); + } finally { + allowChildInsert.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void parentPruneMarkerMakesChildCreationRetry() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch parentMarked = new CountDownLatch(1); + CountDownLatch allowParentRemoval = new CountDownLatch(1); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry( + (level, key) -> { + }, + (level, key) -> { + }, + (level, key) -> { + if (level == ScopePath.Level.DATABASE) { + parentMarked.countDown(); + await(allowParentRemoval); + } + })) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + BulkLoadHandle databaseHandle = cache.beginBulkLoad(ScopePath.database("db")); + Future prune = executor.submit(databaseHandle::close); + await(parentMarked); + + Future tableHandleFuture = executor.submit(() -> cache.beginBulkLoad(TABLE)); + Assertions.assertFalse(tableHandleFuture.isDone()); + allowParentRemoval.countDown(); + prune.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + try (BulkLoadHandle tableHandle = tableHandleFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + Assertions.assertTrue(cache.publish(tableHandle, "key", TABLE, "value")); + } + Assertions.assertEquals("value", cache.getIfPresent("key", TABLE)); + } finally { + allowParentRemoval.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void parentPruneRechecksChildInsertedAfterFastCheck() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch parentObservedEmpty = new CountDownLatch(1); + CountDownLatch allowParentMark = new CountDownLatch(1); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry( + (level, key) -> { + }, + (level, key) -> { + if (level == ScopePath.Level.DATABASE) { + parentObservedEmpty.countDown(); + await(allowParentMark); + } + }, + (level, key) -> { + })) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + BulkLoadHandle databaseHandle = cache.beginBulkLoad(ScopePath.database("db")); + Future prune = executor.submit(databaseHandle::close); + await(parentObservedEmpty); + + try (BulkLoadHandle tableHandle = cache.beginBulkLoad(TABLE)) { + allowParentMark.countDown(); + prune.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + Assertions.assertTrue(cache.publish(tableHandle, "key", TABLE, "value")); + } + Assertions.assertEquals("value", cache.getIfPresent("key", TABLE)); + } finally { + allowParentMark.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void parentPruneRetriesAfterConcurrentStateReplacement() throws Exception { + assertScopePruneRetriesAfterConcurrentStateReplacement( + ScopePath.database("db"), ScopePath.Level.DATABASE); + assertScopePruneRetriesAfterConcurrentStateReplacement(TABLE, ScopePath.Level.TABLE); + assertScopePruneRetriesAfterConcurrentStateReplacement(PARTITION, ScopePath.Level.PARTITION); + } + + @Test + public void pruneMarkerIsReleasedWhenPostMarkActionFails() { + AtomicBoolean failOnce = new AtomicBoolean(true); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry( + (level, key) -> { + }, + (level, key) -> { + }, + (level, key) -> { + if (level == ScopePath.Level.DATABASE && failOnce.compareAndSet(true, false)) { + throw new IllegalStateException("injected post-mark failure"); + } + })) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + BulkLoadHandle failedHandle = cache.beginBulkLoad(ScopePath.database("db")); + IllegalStateException failure = Assertions.assertThrows( + IllegalStateException.class, failedHandle::close); + Assertions.assertEquals("injected post-mark failure", failure.getMessage()); + Assertions.assertFalse(failOnce.get()); + + try (BulkLoadHandle recoveredHandle = cache.beginBulkLoad(ScopePath.database("db"))) { + Assertions.assertTrue(cache.publish( + recoveredHandle, "key", ScopePath.database("db"), "value")); + } + Assertions.assertEquals("value", cache.getIfPresent("key", ScopePath.database("db"))); + registry.invalidate(ScopePath.database("db")); + assertEmpty(registry, cache); + } + } + + private static void assertScopePruneRetriesAfterConcurrentStateReplacement( + ScopePath scope, ScopePath.Level level) throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch scopeMarked = new CountDownLatch(1); + CountDownLatch allowPruneRecheck = new CountDownLatch(1); + try (ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry( + (ignoredLevel, key) -> { + }, + (ignoredLevel, key) -> { + }, + (markedLevel, key) -> { + if (markedLevel == level) { + scopeMarked.countDown(); + await(allowPruneRecheck); + } + })) { + ScopedMetaCache cache = registry.createCache("test", ENABLED); + BulkLoadHandle handle = cache.beginBulkLoad(scope); + Future prune = executor.submit(handle::close); + await(scopeMarked); + + registry.invalidate(scope); + allowPruneRecheck.countDown(); + prune.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEmpty(registry, cache); + } finally { + allowPruneRecheck.countDown(); + executor.shutdownNow(); + } + } + @Test public void keyPublicationAfterInvalidationLinearizationSurvivesCleanup() throws Exception { ExecutorService executor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/StripedPhaseGateTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/StripedPhaseGateTest.java new file mode 100644 index 00000000000000..1e562ec85ec3a6 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/StripedPhaseGateTest.java @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; + +public class StripedPhaseGateTest { + private static final long TIMEOUT_SECONDS = 10L; + + @Test + public void readersCanOverlap() throws Exception { + StripedPhaseGate gate = new StripedPhaseGate(); + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch readersEntered = new CountDownLatch(2); + CountDownLatch releaseReaders = new CountDownLatch(1); + try { + Future first = executor.submit(() -> gate.readBoolean(() -> { + readersEntered.countDown(); + await(releaseReaders); + return true; + })); + Future second = executor.submit(() -> gate.readBoolean(() -> { + readersEntered.countDown(); + await(releaseReaders); + return true; + })); + + await(readersEntered); + releaseReaders.countDown(); + Assertions.assertTrue(first.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Assertions.assertTrue(second.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } finally { + releaseReaders.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void writerWaitsForCurrentReadersAndBlocksNewReaders() throws Exception { + StripedPhaseGate gate = new StripedPhaseGate(); + ExecutorService executor = Executors.newFixedThreadPool(3); + CountDownLatch firstReaderEntered = new CountDownLatch(1); + CountDownLatch releaseFirstReader = new CountDownLatch(1); + CountDownLatch writerEntered = new CountDownLatch(1); + CountDownLatch releaseWriter = new CountDownLatch(1); + try { + Future firstReader = executor.submit(() -> gate.read(() -> { + firstReaderEntered.countDown(); + await(releaseFirstReader); + return null; + })); + await(firstReaderEntered); + Future writer = executor.submit(() -> gate.write(() -> { + writerEntered.countDown(); + await(releaseWriter); + })); + + Assertions.assertThrows( + TimeoutException.class, () -> writer.get(100, TimeUnit.MILLISECONDS)); + releaseFirstReader.countDown(); + await(writerEntered); + + Future secondReader = executor.submit(() -> gate.readBoolean(() -> true)); + Assertions.assertThrows( + TimeoutException.class, () -> secondReader.get(100, TimeUnit.MILLISECONDS)); + releaseWriter.countDown(); + + firstReader.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + writer.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + Assertions.assertTrue(secondReader.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } finally { + releaseFirstReader.countDown(); + releaseWriter.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void writerCompletesUnderContinuousReaders() throws Exception { + StripedPhaseGate gate = new StripedPhaseGate(); + ExecutorService executor = Executors.newFixedThreadPool(9); + AtomicBoolean run = new AtomicBoolean(true); + CountDownLatch readersStarted = new CountDownLatch(8); + List> readers = new ArrayList<>(); + try { + for (int index = 0; index < 8; index++) { + readers.add(executor.submit(() -> { + readersStarted.countDown(); + while (run.get()) { + gate.readBoolean(() -> true); + } + })); + } + await(readersStarted); + + Future writer = executor.submit(() -> gate.write(() -> { + })); + writer.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } finally { + run.set(false); + for (Future reader : readers) { + reader.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + executor.shutdownNow(); + } + } + + @Test + public void exceptionReleasesReaderAndWriterPhases() { + StripedPhaseGate gate = new StripedPhaseGate(); + + Assertions.assertThrows(IllegalStateException.class, () -> gate.read(() -> { + throw new IllegalStateException("read"); + })); + gate.write(() -> { + }); + + Assertions.assertThrows(IllegalStateException.class, () -> gate.write(() -> { + throw new IllegalStateException("write"); + })); + Assertions.assertTrue(gate.readBoolean(() -> true)); + } + + private static void await(CountDownLatch latch) { + try { + if (!latch.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + throw new AssertionError("Timed out waiting for test latch"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("Interrupted while waiting for test latch", e); + } + } +} From 49003932ac5b2d0ec0bdda832d88eb7e119f6e05 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Mon, 10 Aug 2026 19:08:23 +0800 Subject: [PATCH 05/10] [refactor](fe) Unify external metadata cache framework Issue Number: None Related PR: None Problem Summary: FE naming caches and connector metadata caches used duplicate cache wrappers and independently maintained invalidation dependencies. This refactor makes fe-connector-cache the shared runtime for scoped generation fencing, hierarchical invalidation, load deduplication, refresh, eviction cleanup, and lifecycle management. FE keeps only its naming and ID-index publication adapter, while ADBC, Hive, HMS, Hudi, Iceberg, MaxCompute, and Paimon use catalog-scoped shared cache owners. It also removes the duplicate CacheSpec and obsolete cache implementations, preserves per-catalog isolation, and closes concurrent identity-publication and refresh-close cleanup windows. None - Test: Unit Test - fe-connector-cache: 100 tests passed - FE targeted metadata cache tests: 93 tests passed - Iceberg cache tests: 66 tests passed - ./build.sh --fe - Behavior changed: No. This is an internal cache framework refactor preserving existing metadata semantics. - Does this need documentation: No --- fe/fe-connector/fe-connector-adbc/pom.xml | 5 +- .../doris/connector/adbc/AdbcConnector.java | 5 +- .../connector/adbc/AdbcMetadataCache.java | 98 +- fe/fe-connector/fe-connector-cache/pom.xml | 15 +- .../doris/connector/cache/CacheFactory.java | 127 -- .../doris/connector/cache/CacheSpec.java | 11 +- .../connector/cache/CatalogMetaCache.java | 99 + .../cache/ConnectorMetadataCache.java | 36 +- .../doris/connector/cache/MetaCache.java | 121 ++ .../connector/cache/MetaCacheDefinition.java | 136 ++ .../doris/connector/cache/MetaCacheEntry.java | 377 ---- .../connector/cache/MetaCacheEntryStats.java | 201 -- .../cache/MetaCacheRemovalListener.java | 24 + .../cache/MetaCacheRemovalReason.java | 27 + .../doris/connector/cache/ScopePath.java | 13 + .../connector/cache/ScopedMetaCache.java | 282 ++- .../cache/ScopedMetaCacheRegistry.java | 55 +- .../doris/connector/cache/CacheSpecTest.java | 5 +- .../connector/cache/CatalogMetaCacheTest.java | 203 ++ .../connector/cache/MetaCacheEntryTest.java | 298 --- .../cache/ScopedMetaCacheConcurrencyTest.java | 34 + fe/fe-connector/fe-connector-hive/pom.xml | 11 +- .../doris/connector/hive/HiveConnector.java | 41 +- .../connector/hive/HiveFileListingCache.java | 53 +- fe/fe-connector/fe-connector-hms/pom.xml | 7 +- .../doris/connector/hms/CachingHmsClient.java | 84 +- .../doris/connector/hudi/HudiConnector.java | 11 +- fe/fe-connector/fe-connector-iceberg/pom.xml | 7 +- .../iceberg/IcebergCommentCache.java | 39 +- .../connector/iceberg/IcebergConnector.java | 92 +- .../connector/iceberg/IcebergFormatCache.java | 39 +- .../iceberg/IcebergLatestSnapshotCache.java | 38 +- .../iceberg/IcebergManifestCache.java | 53 +- .../iceberg/IcebergPartitionCache.java | 39 +- .../connector/iceberg/IcebergTableCache.java | 48 +- .../IcebergLatestSnapshotCacheTest.java | 2 +- .../iceberg/IcebergTableCacheTest.java | 2 +- .../fe-connector-maxcompute/pom.xml | 11 +- .../maxcompute/MaxComputeDorisConnector.java | 13 +- .../maxcompute/MaxComputePartitionCache.java | 44 +- fe/fe-connector/fe-connector-paimon/pom.xml | 10 +- .../connector/paimon/PaimonConnector.java | 25 +- .../paimon/PaimonLatestSnapshotCache.java | 32 +- .../connector/paimon/PaimonSchemaAtMemo.java | 56 +- .../paimon/PaimonLatestSnapshotCacheTest.java | 2 +- fe/fe-core/pom.xml | 5 + .../doris/datasource/ExternalCatalog.java | 33 +- .../doris/datasource/ExternalDatabase.java | 33 +- .../doris/DorisExternalMetaCache.java | 6 +- .../metacache/AbstractExternalMetaCache.java | 79 +- .../doris/datasource/metacache/CacheSpec.java | 287 --- .../metacache/CatalogEntryGroup.java | 51 +- .../metacache/ExternalMetaCache.java | 3 +- .../datasource/metacache/MetaCacheEntry.java | 849 +++----- .../metacache/MetaCacheEntryDef.java | 2 + .../metacache/MetaCacheEntryInvalidation.java | 13 +- .../metacache/MetaCacheEntryStats.java | 2 + .../doris/catalog/RefreshManagerTest.java | 4 +- .../doris/datasource/ExternalCatalogTest.java | 8 +- .../datasource/ExternalDatabaseTest.java | 5 +- .../doris/DorisExternalMetaCacheTest.java | 4 +- .../AbstractExternalMetaCacheTest.java | 16 +- .../datasource/metacache/CacheSpecTest.java | 10 +- .../metacache/MetaCacheDeadlockTest.java | 1 + .../metacache/MetaCacheEntryTest.java | 1757 ++--------------- 65 files changed, 2007 insertions(+), 4092 deletions(-) delete mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheFactory.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java delete mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java delete mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntryStats.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheRemovalListener.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheRemovalReason.java create mode 100644 fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java delete mode 100644 fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEntryTest.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java diff --git a/fe/fe-connector/fe-connector-adbc/pom.xml b/fe/fe-connector/fe-connector-adbc/pom.xml index 4a8af27d1c4722..d97f44c317b8f5 100644 --- a/fe/fe-connector/fe-connector-adbc/pom.xml +++ b/fe/fe-connector/fe-connector-adbc/pom.xml @@ -62,10 +62,7 @@ under the License. ${project.version} - + ${project.groupId} fe-connector-cache diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java index fe2d8d4396d59c..0dcb468c063dff 100644 --- a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java @@ -17,6 +17,7 @@ package org.apache.doris.connector.adbc; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorMetadata; @@ -46,6 +47,7 @@ public class AdbcConnector implements Connector { private final AdbcSchemaStrategy schemaStrategy = new AdbcSchemaStrategy(); private final AdbcPartitionedReadSupport partitionedRead = new AdbcPartitionedReadSupport(); private final AdbcDialectSelector dialectSelector; + private final CatalogMetaCache metaCache = new CatalogMetaCache(); private final AdbcMetadataCache metadataCache; private volatile AdbcClient client; @@ -65,7 +67,7 @@ public AdbcConnector(Map properties, ConnectorContext context) { // The raw map, because the cache knobs are the shared framework's keys rather than this // connector's: CacheSpec owns their names, and mirroring them as fields here would give the // validator and the reader two things to drift apart. - this.metadataCache = new AdbcMetadataCache(props.getRaw()); + this.metadataCache = new AdbcMetadataCache(metaCache, props.getRaw()); } @Override @@ -212,6 +214,7 @@ private AdbcClient getOrCreateClient() { @Override public synchronized void close() throws IOException { closed = true; + metaCache.close(); if (client != null) { client.close(); client = null; diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java index c083552da0ea50..743c1612c80d44 100644 --- a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java @@ -18,14 +18,16 @@ package org.apache.doris.connector.adbc; import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; import org.apache.arrow.vector.types.pojo.Schema; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.ForkJoinPool; import java.util.function.Supplier; /** @@ -85,19 +87,33 @@ public final class AdbcMetadataCache { /** The database listing is a single value, so it needs a single key. */ private static final String THE_ONLY_KEY = ""; - private final MetaCacheEntry> namespaces; - private final MetaCacheEntry> tableNames; - private final MetaCacheEntry tableSchemas; + private final CatalogMetaCache owner; + private final MetaCache> namespaces; + private final MetaCache> tableNames; + private final MetaCache tableSchemas; /** * Built in {@link AdbcConnector}'s constructor, which also runs on an FE replaying the edit log, so this * reads properties and nothing else -- no driver, no filesystem, no remote call. */ public AdbcMetadataCache(Map properties) { + this(new CatalogMetaCache(), properties); + } + + AdbcMetadataCache(CatalogMetaCache owner, Map properties) { + this.owner = Objects.requireNonNull(owner, "owner can not be null"); CacheSpec spec = cacheSpec(properties); - this.namespaces = entry("adbc-namespaces", spec); - this.tableNames = entry("adbc-table-names", spec); - this.tableSchemas = entry("adbc-table-schema", spec); + this.namespaces = owner.create(MetaCacheDefinition + .>builder("adbc-namespaces", spec, ignored -> ScopePath.catalog()) + .build()); + this.tableNames = owner.create(MetaCacheDefinition + .>builder("adbc-table-names", spec, + key -> ScopePath.database(key.dorisDbName)) + .build()); + this.tableSchemas = owner.create(MetaCacheDefinition + .builder("adbc-table-schema", spec, + key -> ScopePath.table(key.dorisDbName, key.table)) + .build()); } static CacheSpec cacheSpec(Map properties) { @@ -114,15 +130,6 @@ static CacheSpec.PropertySpec propertySpec() { CacheSpec.of(true, DEFAULT_TTL_SECOND, DEFAULT_CAPACITY)); } - /** - * Contextual-only with manual miss load, as the iceberg caches are: the remote read runs OUTSIDE - * Caffeine's compute lock, so a slow source does not stall unrelated keys, the driver's own exception - * arrives unwrapped, and a load that failed is not remembered as an answer. - */ - private static MetaCacheEntry entry(String name, CacheSpec spec) { - return new MetaCacheEntry<>(name, null, spec, ForkJoinPool.commonPool(), false, true, 0L, true); - } - // ========= reads ========= List namespaces(Supplier> loader) { @@ -136,12 +143,12 @@ List reloadNamespaces(Supplier> loader) { } List tableNames(AdbcNamespace namespace, Supplier> loader) { - return tableNames.get(namespace, ignored -> loader.get()); + return tableNames.get(TableNameKey.forNamespace(namespace), ignored -> loader.get()); } /** Re-reads one database's table listing, replacing whatever was remembered. See the class note. */ List reloadTableNames(AdbcNamespace namespace, Supplier> loader) { - tableNames.invalidateKey(namespace); + tableNames.invalidateKey(TableNameKey.forNamespace(namespace)); return tableNames(namespace, loader); } @@ -160,14 +167,16 @@ Schema tableSchema(AdbcTableHandle handle, Supplier loader) { * new name in, and the one the user tried would appear to do nothing. */ void invalidateTable(String dbName, String tableName) { - tableSchemas.invalidateIf(key -> key.is(dbName, tableName)); - tableNames.invalidateIf(namespace -> namespace.dorisDatabaseName().equals(dbName)); + owner.invalidateTable(dbName, tableName); + // A table refresh must also forget the parent name listing, so a remotely-created table can be found. + // This is an ancestor materialization, not a sibling-cache dependency, and therefore cannot be selected + // by the table's descendant scope. + tableNames.invalidateKey(TableNameKey.forDatabase(dbName)); } /** {@code REFRESH DATABASE}: that database's table listing and every schema in it. */ void invalidateDb(String dbName) { - tableSchemas.invalidateIf(key -> key.isIn(dbName)); - tableNames.invalidateIf(namespace -> namespace.dorisDatabaseName().equals(dbName)); + owner.invalidateDatabase(dbName); } /** @@ -175,9 +184,38 @@ void invalidateDb(String dbName) { * names no database to invalidate and is reached for when the catalog's own shape changed. */ void invalidateAll() { - namespaces.invalidateAll(); - tableNames.invalidateAll(); - tableSchemas.invalidateAll(); + owner.invalidateCatalog(); + } + + /** + * A database's listing is addressed by the Doris database name. The remote namespace is carried only so + * the miss loader can query it; it is deliberately not part of identity because Doris cannot address two + * remote namespaces that project to the same database name separately. + */ + private static final class TableNameKey { + private final String dorisDbName; + + private TableNameKey(String dorisDbName) { + this.dorisDbName = Objects.requireNonNull(dorisDbName, "dorisDbName can not be null"); + } + + private static TableNameKey forNamespace(AdbcNamespace namespace) { + return new TableNameKey(namespace.dorisDatabaseName()); + } + + private static TableNameKey forDatabase(String database) { + return new TableNameKey(database); + } + + @Override + public boolean equals(Object o) { + return o instanceof TableNameKey && dorisDbName.equals(((TableNameKey) o).dorisDbName); + } + + @Override + public int hashCode() { + return dorisDbName.hashCode(); + } } /** @@ -199,14 +237,6 @@ private TableKey(AdbcTableHandle handle) { this.dorisDbName = handle.getDorisDbName(); } - private boolean is(String db, String tableName) { - return dorisDbName.equals(db) && table.equals(tableName); - } - - private boolean isIn(String db) { - return dorisDbName.equals(db); - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-connector/fe-connector-cache/pom.xml b/fe/fe-connector/fe-connector-cache/pom.xml index f9ba332609f91b..6ff5e53731a875 100644 --- a/fe/fe-connector/fe-connector-cache/pom.xml +++ b/fe/fe-connector/fe-connector-cache/pom.xml @@ -33,17 +33,10 @@ under the License. jar Doris FE Connector Cache Framework - Connector-side meta-cache framework (CacheSpec + MetaCacheEntry + CacheFactory + MetaCacheEntryStats), - an INDEPENDENT copy of fe-core's `org.apache.doris.datasource.metacache` framework re-homed under the - `org.apache.doris.connector.*` prefix so the connector plugins can reuse it (they cannot import fe-core). - fe-core keeps its own copy untouched; the two live side-by-side until every connector has migrated, then - the fe-core copy is retired. fe-core does NOT depend on this module. - - This module is bundled into each connector plugin zip (child-first), so it uses the plugin's own bundled - Caffeine at runtime; Caffeine is therefore `provided` here (compiled against, never packaged by this - module). The framework's public API (MetaCacheEntry) is Caffeine-free, and fe-core and the connectors - never share a cache object across the classloader boundary, so no Caffeine type crosses and there is no - split-brain. Two knobs fe-core reads from static Config are constructor-injected here. + Shared catalog-local metadata cache framework. Cache definitions declare their metadata scope, and every + physical cache in one catalog participates in hierarchical invalidation through a shared registry. The + public API is Caffeine-free so connector plugins can use their child-first Caffeine runtime without + exposing third-party types across the FE/connector classloader boundary. diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheFactory.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheFactory.java deleted file mode 100644 index 03e4126e9911be..00000000000000 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheFactory.java +++ /dev/null @@ -1,127 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.cache; - -import com.github.benmanes.caffeine.cache.AsyncCacheLoader; -import com.github.benmanes.caffeine.cache.AsyncLoadingCache; -import com.github.benmanes.caffeine.cache.CacheLoader; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.LoadingCache; -import com.github.benmanes.caffeine.cache.RemovalListener; -import com.github.benmanes.caffeine.cache.Ticker; - -import java.time.Duration; -import java.util.OptionalLong; -import java.util.concurrent.ExecutorService; - -/** - * Factory to create Caffeine cache. - * - *

Connector-side copy of fe-core {@code org.apache.doris.common.CacheFactory} (independent-copy meta-cache - * migration): connector plugins cannot import fe-core, so the framework is duplicated under - * {@code org.apache.doris.connector.cache}. This type is framework-internal — its public methods RETURN - * Caffeine types, which must never cross to connector (child-first) code; connectors only touch the - * Caffeine-free {@link MetaCacheEntry} API. Keep behaviourally in sync with the fe-core original. - * - *

This class is used to create Caffeine cache with specified parameters. - * It is used to create both sync and async cache. - * The cache is created with the following parameters: - * - expireAfterAccessSec: The duration after which the cache entries will expire. - * - refreshAfterWriteSec: The duration after which the cache entries will be refreshed. - * - maxSize: The maximum size of the cache. - * - enableStats: Whether to enable stats for the cache. - * - ticker: The ticker to use for the cache. - */ -public class CacheFactory { - - private OptionalLong expireAfterAccessSec; - private OptionalLong refreshAfterWriteSec; - private long maxSize; - private boolean enableStats; - // Ticker is used to provide a time source for the cache. - // Only used for test, to provide a fake time source. - // If not provided, the system time is used. - private Ticker ticker; - - public CacheFactory( - OptionalLong expireAfterAccessSec, - OptionalLong refreshAfterWriteSec, - long maxSize, - boolean enableStats, - Ticker ticker) { - this.expireAfterAccessSec = expireAfterAccessSec; - this.refreshAfterWriteSec = refreshAfterWriteSec; - this.maxSize = maxSize; - this.enableStats = enableStats; - this.ticker = ticker; - } - - // Build a loading cache, without executor, it will use fork-join pool for refresh - public LoadingCache buildCache(CacheLoader cacheLoader) { - Caffeine builder = buildWithParams(); - return builder.build(cacheLoader); - } - - // Build a loading cache, with executor, it will use given executor for refresh - public LoadingCache buildCache(CacheLoader cacheLoader, - ExecutorService executor) { - Caffeine builder = buildWithParams(); - builder.executor(executor); - return builder.build(cacheLoader); - } - - // Build cache with sync removal listener to prevent deadlock when listener calls invalidateAll() - public LoadingCache buildCacheWithSyncRemovalListener(CacheLoader cacheLoader, - RemovalListener removalListener) { - Caffeine builder = buildWithParams(); - if (removalListener != null) { - builder.removalListener(removalListener); - } - builder.executor(Runnable::run); // Sync execution to avoid thread pool deadlock - return builder.build(cacheLoader); - } - - // Build an async loading cache - public AsyncLoadingCache buildAsyncCache(AsyncCacheLoader cacheLoader, - ExecutorService executor) { - Caffeine builder = buildWithParams(); - builder.executor(executor); - return builder.buildAsync(cacheLoader); - } - - private Caffeine buildWithParams() { - Caffeine builder = Caffeine.newBuilder(); - builder.maximumSize(maxSize); - - if (expireAfterAccessSec.isPresent()) { - builder.expireAfterAccess(Duration.ofSeconds(expireAfterAccessSec.getAsLong())); - } - if (refreshAfterWriteSec.isPresent()) { - builder.refreshAfterWrite(Duration.ofSeconds(refreshAfterWriteSec.getAsLong())); - } - - if (enableStats) { - builder.recordStats(); - } - - if (ticker != null) { - builder.ticker(ticker); - } - return builder; - } -} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java index 0524b31d402f48..01ee88a1fc905b 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java @@ -25,13 +25,8 @@ /** * Common cache specification for external metadata caches. * - *

Connector-side copy of the meta-cache property model (independent-copy meta-cache migration). fe-core is - * NOT changed: it keeps its own {@code org.apache.doris.datasource.metacache.CacheSpec}; this is a separate - * class under {@code org.apache.doris.connector.*} used only by the connector plugins. Although that prefix is - * parent-first, fe-core does not depend on this module, so the class resolves parent → miss → CHILD and is - * child-loaded per plugin — fe-core and the plugins do NOT share one {@code Class} identity. It carries no - * third-party dependency (JDK only) and never crosses the fe-core↔connector boundary as an object (only its - * {@code IllegalArgumentException}, a JDK type, crosses), so it is safe on both classpaths. + *

This is the single property model shared by fe-core and connector plugins. It carries no fe-core dependency, + * so cache configuration and validation remain available to independently loaded connectors. * *

The {@code check*Property} validators throw {@link IllegalArgumentException} (fe-core's * {@code PluginDrivenExternalCatalog.checkProperties} re-wraps it into a {@code DdlException} verbatim; the @@ -204,7 +199,7 @@ public static boolean isMetaCacheKeyForEngine(String key, String engine) { } /** - * Convert ttlSecond to OptionalLong for CacheFactory. + * Convert ttlSecond to the optional expiry used by the cache runtime. * ttlSecond=-1 means no expiration; ttlSecond=0 disables cache. */ public static OptionalLong toExpireAfterAccess(long ttlSecond) { diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java new file mode 100644 index 00000000000000..5ee31799f9cfa2 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Catalog-local owner of all framework metadata caches. + * + *

Connectors invalidate a semantic scope once. Every physical cache created through this owner participates + * automatically because its values are registered in the shared {@link ScopedMetaCacheRegistry}. + */ +public final class CatalogMetaCache implements AutoCloseable { + private final ScopedMetaCacheRegistry registry; + private final Set names = ConcurrentHashMap.newKeySet(); + private final AtomicBoolean closed = new AtomicBoolean(false); + + public CatalogMetaCache() { + this(new ScopedMetaCacheRegistry()); + } + + CatalogMetaCache(ScopedMetaCacheRegistry registry) { + this.registry = Objects.requireNonNull(registry, "registry can not be null"); + } + + public MetaCache create(MetaCacheDefinition definition) { + MetaCacheDefinition nonNullDefinition = + Objects.requireNonNull(definition, "definition can not be null"); + checkOpen(); + if (!names.add(nonNullDefinition.name())) { + throw new IllegalArgumentException("Duplicate meta cache name: " + nonNullDefinition.name()); + } + try { + return new MetaCache<>(nonNullDefinition, + registry.createCacheWithMetaRemovalListener(nonNullDefinition.name(), + nonNullDefinition.cacheSpec(), nonNullDefinition.removalListener(), + nonNullDefinition.refreshAfterWrite(), nonNullDefinition.refreshExecutor())); + } catch (RuntimeException | Error throwable) { + names.remove(nonNullDefinition.name()); + throw throwable; + } + } + + public void invalidateCatalog() { + registry.invalidate(ScopePath.catalog()); + } + + public void invalidateDatabase(String database) { + registry.invalidate(ScopePath.database(database)); + } + + public void invalidateTable(String database, String table) { + registry.invalidate(ScopePath.table(database, table)); + } + + public void invalidatePartition(String database, String table, Object partition) { + registry.invalidate(ScopePath.partition(database, table, partition)); + } + + public void invalidatePartitionCollection(String database, String table) { + registry.invalidate(ScopePath.partitionCollection(database, table)); + } + + public ScopedMetaCacheRegistry.ScopeMetrics metrics() { + return registry.metrics(); + } + + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + registry.close(); + names.clear(); + } + } + + private void checkOpen() { + if (closed.get()) { + throw new IllegalStateException("Catalog meta cache is closed"); + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java index fde68573e73c50..41e678465a4687 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java @@ -20,7 +20,7 @@ import java.util.Collections; import java.util.Map; import java.util.Objects; -import java.util.concurrent.ForkJoinPool; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -50,7 +50,8 @@ public final class ConnectorMetadataCache { /** Default capacity, matching {@code IcebergPartitionCache}'s {@code DEFAULT_TABLE_CACHE_CAPACITY}. */ static final long DEFAULT_CAPACITY = 1000L; - private final MetaCacheEntry entry; + private final CatalogMetaCache owner; + private final MetaCache entry; /** * @param engine engine token for the {@code meta.cache...*} property namespace, e.g. @@ -61,20 +62,31 @@ public final class ConnectorMetadataCache { * {@code null}, treated as empty (defaults apply). */ public ConnectorMetadataCache(String engine, String entryName, Map props) { + this(new CatalogMetaCache(), engine + "." + entryName, engine, entryName, props); + } + + public ConnectorMetadataCache( + CatalogMetaCache owner, String cacheName, String engine, String entryName, Map props) { + this(owner, cacheName, engine, entryName, props, + key -> ScopePath.table(key.getDb(), key.getTable())); + } + + public ConnectorMetadataCache(CatalogMetaCache owner, String cacheName, String engine, String entryName, + Map props, Function scopeResolver) { + this.owner = Objects.requireNonNull(owner, "owner can not be null"); Objects.requireNonNull(engine, "engine can not be null"); Objects.requireNonNull(entryName, "entryName can not be null"); Map properties = props == null ? Collections.emptyMap() : props; CacheSpec spec = CacheSpec.fromProperties(properties, engine, entryName, CacheSpec.of(true, DEFAULT_TTL_SECOND, DEFAULT_CAPACITY)); - // contextual-only (loader == null, supplied per-call by get()) + manual-miss-load, no auto-refresh -- - // identical shape to IcebergPartitionCache / MaxComputePartitionCache's entry construction. - this.entry = new MetaCacheEntry<>(engine + "." + entryName, null, spec, - ForkJoinPool.commonPool(), false, true, 0L, true); + this.entry = owner.create(MetaCacheDefinition + .builder(cacheName, spec, scopeResolver) + .build()); } /** Caching is on only when the resolved {@link CacheSpec} is effectively enabled (see {@link #entry}'s spec). */ public boolean isEnabled() { - return entry.stats().isEffectiveEnabled(); + return entry.isEnabled(); } /** @@ -89,23 +101,21 @@ public V get(ConnectorTableKey key, Supplier loader) { /** Evicts every cached snapshot/schema entry of one (db, table). Backs REFRESH TABLE / table-level invalidation. */ public void invalidateTable(String db, String table) { - entry.invalidateIf(key -> key.matches(db, table)); + owner.invalidateTable(db, table); } /** Evicts every cached entry of one db (all its tables). Backs REFRESH DATABASE / db-level invalidation. */ public void invalidateDb(String db) { - entry.invalidateIf(key -> key.matchesDb(db)); + owner.invalidateDatabase(db); } /** Evicts every cached entry. Backs REFRESH CATALOG. */ public void invalidateAll() { - entry.invalidateAll(); + owner.invalidateCatalog(); } /** Test-only: current number of cached entries (accurate map membership, not Caffeine's estimate). */ long size() { - int[] count = {0}; - entry.forEach((key, value) -> count[0]++); - return count[0]; + return entry.size(); } } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java new file mode 100644 index 00000000000000..6191a575ac5519 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.util.Objects; +import java.util.function.BiConsumer; +import java.util.function.Function; + +/** One typed physical cache owned by a {@link CatalogMetaCache}. */ +public final class MetaCache { + private final MetaCacheDefinition definition; + private final ScopedMetaCache delegate; + + MetaCache(MetaCacheDefinition definition, ScopedMetaCache delegate) { + this.definition = Objects.requireNonNull(definition, "definition can not be null"); + this.delegate = Objects.requireNonNull(delegate, "delegate can not be null"); + } + + public String name() { + return definition.name(); + } + + public V get(K key) { + Function loader = definition.loader(); + if (loader == null) { + throw new IllegalStateException("Meta cache '" + name() + "' has no default loader"); + } + return get(key, loader); + } + + public V get(K key, Function loader) { + K nonNullKey = Objects.requireNonNull(key, "key can not be null"); + return delegate.get(nonNullKey, definition.scope(nonNullKey), loader); + } + + public V getIfPresent(K key) { + K nonNullKey = Objects.requireNonNull(key, "key can not be null"); + return delegate.getIfPresent(nonNullKey, definition.scope(nonNullKey)); + } + + public void put(K key, V value) { + K nonNullKey = Objects.requireNonNull(key, "key can not be null"); + delegate.put(nonNullKey, definition.scope(nonNullKey), value); + } + + public void invalidateKey(K key) { + delegate.invalidateKey(Objects.requireNonNull(key, "key can not be null")); + } + + /** + * Starts a bulk remote load whose results all belong below {@code parentScope}. Publishing through the + * returned handle is fenced against concurrent scope and exact-key invalidation. + */ + public BulkLoad beginBulkLoad(ScopePath parentScope) { + return new BulkLoad<>(this, delegate.beginBulkLoad(parentScope)); + } + + public ScopedMetaCache.CacheMetrics metrics() { + return delegate.metrics(); + } + + public boolean isEnabled() { + return metrics().isEffectiveEnabled(); + } + + public long size() { + return metrics().getPhysicalEntryCount(); + } + + public long loadSuccessCount() { + return metrics().getLoadSuccessCount(); + } + + public CacheSpec cacheSpec() { + return definition.cacheSpec(); + } + + public boolean isAutoRefresh() { + return definition.refreshAfterWrite() != null; + } + + public void forEach(BiConsumer consumer) { + delegate.forEach(consumer); + } + + public static final class BulkLoad implements AutoCloseable { + private final MetaCache owner; + private final ScopedMetaCache.BulkLoadHandle delegate; + + private BulkLoad(MetaCache owner, ScopedMetaCache.BulkLoadHandle delegate) { + this.owner = owner; + this.delegate = delegate; + } + + /** Returns false when an invalidation raced the remote load and the value was intentionally not cached. */ + public boolean publish(K key, V value) { + K nonNullKey = Objects.requireNonNull(key, "key can not be null"); + return owner.delegate.publish(delegate, nonNullKey, owner.definition.scope(nonNullKey), value); + } + + @Override + public void close() { + delegate.close(); + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java new file mode 100644 index 00000000000000..728e1f4d1229c7 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.Executor; +import java.util.function.Function; + +/** + * Declarative definition of one physical metadata cache in a catalog. + * + *

The scope resolver is mandatory. It is the only invalidation dependency a framework-owned cache declares: + * every value published by the cache is registered at that scope, so catalog/database/table/partition invalidation + * automatically covers it without a connector-maintained list of sibling caches. + */ +public final class MetaCacheDefinition { + private final String name; + private final CacheSpec cacheSpec; + private final Function scopeResolver; + private final Function loader; + private final MetaCacheRemovalListener removalListener; + private final Duration refreshAfterWrite; + private final Executor refreshExecutor; + + private MetaCacheDefinition(Builder builder) { + name = requireName(builder.name); + cacheSpec = Objects.requireNonNull(builder.cacheSpec, "cacheSpec can not be null"); + scopeResolver = Objects.requireNonNull(builder.scopeResolver, "scopeResolver can not be null"); + loader = builder.loader; + removalListener = builder.removalListener; + refreshAfterWrite = builder.refreshAfterWrite; + refreshExecutor = builder.refreshExecutor; + if (refreshAfterWrite != null && loader == null) { + throw new IllegalArgumentException("refresh-after-write requires a default loader"); + } + } + + public static Builder builder( + String name, CacheSpec cacheSpec, Function scopeResolver) { + return new Builder<>(name, cacheSpec, scopeResolver); + } + + public String name() { + return name; + } + + CacheSpec cacheSpec() { + return cacheSpec; + } + + ScopePath scope(K key) { + return Objects.requireNonNull( + scopeResolver.apply(Objects.requireNonNull(key, "key can not be null")), + "scopeResolver can not return null"); + } + + Function loader() { + return loader; + } + + MetaCacheRemovalListener removalListener() { + return removalListener; + } + + Duration refreshAfterWrite() { + return refreshAfterWrite; + } + + Executor refreshExecutor() { + return refreshExecutor; + } + + private static String requireName(String name) { + String nonNullName = Objects.requireNonNull(name, "name can not be null"); + if (nonNullName.isEmpty()) { + throw new IllegalArgumentException("name can not be empty"); + } + return nonNullName; + } + + public static final class Builder { + private final String name; + private final CacheSpec cacheSpec; + private final Function scopeResolver; + private Function loader; + private MetaCacheRemovalListener removalListener; + private Duration refreshAfterWrite; + private Executor refreshExecutor; + + private Builder(String name, CacheSpec cacheSpec, Function scopeResolver) { + this.name = name; + this.cacheSpec = cacheSpec; + this.scopeResolver = scopeResolver; + } + + public Builder loader(Function loader) { + this.loader = Objects.requireNonNull(loader, "loader can not be null"); + return this; + } + + public Builder removalListener(MetaCacheRemovalListener removalListener) { + this.removalListener = Objects.requireNonNull(removalListener, "removalListener can not be null"); + return this; + } + + public Builder refreshAfterWrite(Duration duration, Executor executor) { + Duration nonNullDuration = Objects.requireNonNull(duration, "duration can not be null"); + if (nonNullDuration.isZero() || nonNullDuration.isNegative()) { + throw new IllegalArgumentException("refresh-after-write duration must be positive"); + } + this.refreshAfterWrite = nonNullDuration; + this.refreshExecutor = Objects.requireNonNull(executor, "refreshExecutor can not be null"); + return this; + } + + public MetaCacheDefinition build() { + return new MetaCacheDefinition<>(this); + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java deleted file mode 100644 index 97a64ecc159a4a..00000000000000 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntry.java +++ /dev/null @@ -1,377 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.cache; - -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.LoadingCache; -import com.github.benmanes.caffeine.cache.stats.CacheStats; - -import java.util.Objects; -import java.util.OptionalLong; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BiConsumer; -import java.util.function.Function; -import java.util.function.Predicate; - -/** - * Unified cache entry abstraction. - * It stores one logical cache dataset and provides optional lazy loading, - * key/predicate/full invalidation, and lightweight runtime stats. - * - *

Connector-side copy of fe-core {@code org.apache.doris.datasource.metacache.MetaCacheEntry} - * (independent-copy meta-cache migration): connector plugins cannot import fe-core, so the framework is - * duplicated under {@code org.apache.doris.connector.cache}. The public API is Caffeine-free (Caffeine is - * encapsulated), so instances are safe to hold in connector (child-first) code. Two knobs that fe-core reads - * from static {@code Config} are here supplied by the connector via the constructor - * ({@code refreshAfterWriteSeconds}, {@code manualMissLoadEnabled}); otherwise keep in sync with fe-core. - */ -public class MetaCacheEntry { - // Use striped locks to deduplicate slow external loads without managing per-key lock lifecycle. - private static final int LOAD_LOCK_STRIPES = 128; - - private final String name; - private final Function loader; - private final CacheSpec cacheSpec; - private final boolean effectiveEnabled; - private final boolean autoRefresh; - // fe-core reads these two from Config; the connector copy has no fe-core Config, so they are injected. - // refreshAfterWriteSeconds is already in seconds (fe-core computes Config.*_minutes * 60 at its call site). - private final long refreshAfterWriteSeconds; - private final boolean manualMissLoadEnabled; - // Connector-owned cleanup hook. Keep the public API free of Caffeine types. - private final BiConsumer retirementListener; - // Keep the loading cache for refreshAfterWrite and the legacy sync-load path when the feature is disabled. - private final LoadingCache loadingData; - // Use the plain cache view for manual miss load so slow I/O does not happen in Caffeine's sync load path. - private final Cache data; - // Protect one key stripe at a time to deduplicate concurrent miss loads with bounded lock count. - private final Object[] loadLocks = new Object[LOAD_LOCK_STRIPES]; - private final AtomicLong invalidateCount = new AtomicLong(0); - // Bump generation before invalidation so in-flight manual loads do not repopulate stale values. - private final AtomicLong invalidateGeneration = new AtomicLong(0); - // Track load statistics outside Caffeine because manual miss loads bypass the built-in load counters. - private final AtomicLong loadSuccessCount = new AtomicLong(0); - private final AtomicLong loadFailureCount = new AtomicLong(0); - private final AtomicLong totalLoadTimeNanos = new AtomicLong(0); - private final AtomicLong lastLoadSuccessTimeMs = new AtomicLong(-1L); - private final AtomicLong lastLoadFailureTimeMs = new AtomicLong(-1L); - private final AtomicReference lastError = new AtomicReference<>(""); - - /** - * Convenience constructor for the common connector case: a loader-backed entry with no auto-refresh and no - * manual miss load (Caffeine's sync load path). Use the full constructor for contextual-only entries, - * auto-refresh, or manual miss load. - */ - public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor) { - this(name, loader, cacheSpec, refreshExecutor, false, false, 0L, false); - } - - public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, - ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, - long refreshAfterWriteSeconds, boolean manualMissLoadEnabled) { - this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, - refreshAfterWriteSeconds, manualMissLoadEnabled, null); - } - - /** - * Creates an entry with a synchronous retirement callback. The callback is invoked whenever a loaded value - * stops being owned by this cache: eviction/invalidation, a disabled cache, or an invalidation racing a - * manual miss load before publication. The callback deliberately uses only JDK types; Caffeine remains an - * implementation detail of this module. - */ - public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, - ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, - long refreshAfterWriteSeconds, boolean manualMissLoadEnabled, - BiConsumer retirementListener) { - this.name = name; - if (contextualOnly) { - if (loader != null) { - throw new IllegalArgumentException("contextual-only entry loader must be null"); - } - if (autoRefresh) { - throw new IllegalArgumentException("contextual-only entry can not enable auto refresh"); - } - } else { - Objects.requireNonNull(loader, "loader can not be null"); - } - this.loader = loader; - this.cacheSpec = Objects.requireNonNull(cacheSpec, "cacheSpec can not be null"); - this.autoRefresh = autoRefresh; - this.refreshAfterWriteSeconds = refreshAfterWriteSeconds; - this.manualMissLoadEnabled = manualMissLoadEnabled; - this.retirementListener = retirementListener; - Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); - this.effectiveEnabled = CacheSpec.isCacheEnabled( - this.cacheSpec.isEnable(), this.cacheSpec.getTtlSecond(), this.cacheSpec.getCapacity()); - OptionalLong expireAfterAccessSec = - effectiveEnabled ? CacheSpec.toExpireAfterAccess(this.cacheSpec.getTtlSecond()) : OptionalLong.empty(); - OptionalLong refreshAfterWriteSec = - effectiveEnabled && autoRefresh - ? OptionalLong.of(refreshAfterWriteSeconds) - : OptionalLong.empty(); - long maxSize = effectiveEnabled ? this.cacheSpec.getCapacity() : 0L; - CacheFactory cacheFactory = new CacheFactory( - expireAfterAccessSec, - refreshAfterWriteSec, - maxSize, - true, - null); - if (retirementListener != null) { - this.loadingData = cacheFactory.buildCacheWithSyncRemovalListener( - this::loadFromDefaultLoader, - (key, value, cause) -> retirementListener.accept(key, value)); - } else { - this.loadingData = cacheFactory.buildCache(this::loadFromDefaultLoader, refreshExecutor); - } - this.data = loadingData; - // Initialize striped locks eagerly to keep the hot path allocation-free. - for (int i = 0; i < loadLocks.length; i++) { - loadLocks[i] = new Object(); - } - } - - public String name() { - return name; - } - - public V get(K key) { - if (!isManualMissLoadEnabled()) { - return loadingData.get(key); - } - return getWithManualLoad(key, this::applyDefaultLoader); - } - - public V get(K key, Function missLoader) { - Function loadFunction = Objects.requireNonNull(missLoader, "missLoader can not be null"); - if (!isManualMissLoadEnabled()) { - return loadingData.get(key, typedKey -> loadAndTrack(typedKey, loadFunction)); - } - return getWithManualLoad(key, loadFunction); - } - - public V getIfPresent(K key) { - if (!effectiveEnabled) { - return null; - } - return data.getIfPresent(key); - } - - public void put(K key, V value) { - if (!effectiveEnabled) { - return; - } - data.put(key, value); - } - - /** - * The current invalidation generation. Capture this BEFORE a slow external load, then hand it to - * {@link #putIfNotInvalidatedSince} so a {@code flush}/{@code invalidate*} that raced the load does not - * get its clear silently undone by a stale write-back. Mirrors the guard the manual-miss-load path - * ({@link #getWithManualLoad}) applies around its own put; exposed so a caller that does its OWN bulk - * external read (e.g. a decorator batching a multi-key RPC and putting each result under its own key) - * can reuse the same generation guard instead of an unguarded {@link #put}. - */ - public long invalidationGeneration() { - return invalidateGeneration.get(); - } - - /** - * Generation-guarded put: caches {@code (key, value)} only if no invalidation has happened since - * {@code generation} was captured (before the caller's external load). If a {@code flush}/{@code - * invalidate*} raced the load — bumping the generation either before the put (skip) or between the put - * and the recheck (drop only the value we wrote, via {@link #removeLoadedValue}) — the stale value is - * NOT left cached, exactly as {@link #getWithManualLoad} does for its single-key load. Additive: the - * existing {@link #put} is unchanged; a disabled entry is a no-op. - */ - public void putIfNotInvalidatedSince(long generation, K key, V value) { - if (!effectiveEnabled) { - return; - } - synchronized (loadLock(key)) { - // A racing flush already bumped the generation before we could put: skip so a stale pre-flush - // value is not re-cached (mirrors getWithManualLoad's pre-put guard). - if (generation != invalidateGeneration.get()) { - return; - } - data.put(key, value); - // A flush landing between the check and the put: drop only the value we just wrote, keeping any - // newer replacement intact (mirrors getWithManualLoad's post-put guard). - if (generation != invalidateGeneration.get()) { - removeLoadedValue(key, value); - } - } - } - - public void invalidateKey(K key) { - invalidateGeneration.incrementAndGet(); - if (data.asMap().remove(key) != null) { - invalidateCount.incrementAndGet(); - } - } - - public void invalidateIf(Predicate predicate) { - invalidateGeneration.incrementAndGet(); - data.asMap().keySet().removeIf(key -> { - if (predicate.test(key)) { - invalidateCount.incrementAndGet(); - return true; - } - return false; - }); - } - - public void invalidateAll() { - invalidateGeneration.incrementAndGet(); - long size = data.estimatedSize(); - data.invalidateAll(); - invalidateCount.addAndGet(size); - } - - public void forEach(BiConsumer consumer) { - data.asMap().forEach(consumer); - } - - public MetaCacheEntryStats stats() { - CacheStats cacheStats = loadingData.stats(); - long successCount = loadSuccessCount.get(); - long failureCount = loadFailureCount.get(); - long totalLoadTime = totalLoadTimeNanos.get(); - long totalLoadCount = successCount + failureCount; - return new MetaCacheEntryStats( - cacheSpec.isEnable(), - effectiveEnabled, - autoRefresh, - cacheSpec.getTtlSecond(), - cacheSpec.getCapacity(), - data.estimatedSize(), - cacheStats.requestCount(), - cacheStats.hitCount(), - cacheStats.missCount(), - cacheStats.hitRate(), - successCount, - failureCount, - totalLoadTime, - totalLoadCount == 0 ? 0D : (double) totalLoadTime / totalLoadCount, - cacheStats.evictionCount(), - invalidateCount.get(), - lastLoadSuccessTimeMs.get(), - lastLoadFailureTimeMs.get(), - lastError.get()); - } - - // Injected at construction (fe-core reads Config.enable_external_meta_cache_manual_miss_load dynamically). - private boolean isManualMissLoadEnabled() { - return manualMissLoadEnabled; - } - - // Execute slow miss loads outside Caffeine's sync load path and suppress stale write-back after invalidation. - private V getWithManualLoad(K key, Function loadFunction) { - if (!effectiveEnabled) { - // Bypass cache entirely when the entry is disabled so manual miss load does not relax disable semantics. - V loaded = loadAndTrack(key, loadFunction); - notifyRetirement(key, loaded); - return loaded; - } - - V value = data.getIfPresent(key); - if (value != null) { - return value; - } - - synchronized (loadLock(key)) { - value = data.asMap().get(key); - if (value != null) { - return value; - } - - long generation = invalidateGeneration.get(); - V loaded = loadAndTrack(key, loadFunction); - if (generation != invalidateGeneration.get()) { - notifyRetirement(key, loaded); - return loaded; - } - - // Keep null results uncached so manual miss load matches LoadingCache null-return behavior. - if (loaded == null) { - return null; - } - - // Leave a narrow hook for tests to pause exactly before the cache put race window. - beforeManualCachePutForTest(key, loaded); - data.put(key, loaded); - if (generation != invalidateGeneration.get()) { - removeLoadedValue(key, loaded); - } - return loaded; - } - } - - // Remove only the value loaded by the current request and keep newer replacements intact. - private void removeLoadedValue(K key, V loaded) { - data.asMap().computeIfPresent(key, (ignored, currentValue) -> currentValue == loaded ? null : currentValue); - } - - private void notifyRetirement(K key, V value) { - if (retirementListener != null && value != null) { - retirementListener.accept(key, value); - } - } - - // Map keys to a fixed lock stripe set to bound memory usage while keeping same-key deduplication. - private Object loadLock(K key) { - int hash = key == null ? 0 : key.hashCode(); - return loadLocks[(hash & Integer.MAX_VALUE) % loadLocks.length]; - } - - // Let tests pause between the first generation check and data.put without affecting production behavior. - void beforeManualCachePutForTest(K key, V loaded) { - } - - private V loadFromDefaultLoader(K key) { - return loadAndTrack(key, this::applyDefaultLoader); - } - - // Resolve the default loader separately so the manual path can share tracking without double counting. - private V applyDefaultLoader(K key) { - if (loader == null) { - throw new UnsupportedOperationException( - String.format("Entry '%s' requires a contextual miss loader.", name)); - } - return loader.apply(key); - } - - // Track load outcomes locally because manual miss loads do not contribute to Caffeine load statistics. - private V loadAndTrack(K key, Function loadFunction) { - long startNanos = System.nanoTime(); - try { - V value = loadFunction.apply(key); - loadSuccessCount.incrementAndGet(); - totalLoadTimeNanos.addAndGet(System.nanoTime() - startNanos); - lastLoadSuccessTimeMs.set(System.currentTimeMillis()); - return value; - } catch (RuntimeException | Error e) { - loadFailureCount.incrementAndGet(); - totalLoadTimeNanos.addAndGet(System.nanoTime() - startNanos); - lastLoadFailureTimeMs.set(System.currentTimeMillis()); - lastError.set(e.toString()); - throw e; - } - } -} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntryStats.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntryStats.java deleted file mode 100644 index 41c8b89192cd1b..00000000000000 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheEntryStats.java +++ /dev/null @@ -1,201 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.cache; - -import java.util.Objects; - -/** - * Immutable stats snapshot of one {@link MetaCacheEntry}. - * - *

Connector-side copy of fe-core {@code org.apache.doris.datasource.metacache.MetaCacheEntryStats} - * (independent-copy migration): the connector plugins cannot import fe-core, so the meta-cache framework is - * duplicated under {@code org.apache.doris.connector.cache}. Keep behaviourally in sync with the fe-core - * original until the fe-core copy is retired. - * - *

Time fields use the following units: - *

    - *
  • {@code totalLoadTimeNanos}/{@code averageLoadPenaltyNanos}: nanoseconds
    • - *
  • {@code lastLoadSuccessTimeMs}/{@code lastLoadFailureTimeMs}: epoch milliseconds
    • - *
- * - *

For last-load timestamps, {@code -1} means no corresponding event happened yet. - * {@code lastError} keeps the latest load failure message; empty string means no failure recorded. - */ -public final class MetaCacheEntryStats { - private final boolean configEnabled; - private final boolean effectiveEnabled; - private final boolean autoRefresh; - private final long ttlSecond; - private final long capacity; - private final long estimatedSize; - private final long requestCount; - private final long hitCount; - private final long missCount; - private final double hitRate; - private final long loadSuccessCount; - private final long loadFailureCount; - private final long totalLoadTimeNanos; - private final double averageLoadPenaltyNanos; - private final long evictionCount; - private final long invalidateCount; - private final long lastLoadSuccessTimeMs; - private final long lastLoadFailureTimeMs; - private final String lastError; - - /** - * Build an immutable stats snapshot. - */ - public MetaCacheEntryStats( - boolean configEnabled, - boolean effectiveEnabled, - boolean autoRefresh, - long ttlSecond, - long capacity, - long estimatedSize, - long requestCount, - long hitCount, - long missCount, - double hitRate, - long loadSuccessCount, - long loadFailureCount, - long totalLoadTimeNanos, - double averageLoadPenaltyNanos, - long evictionCount, - long invalidateCount, - long lastLoadSuccessTimeMs, - long lastLoadFailureTimeMs, - String lastError) { - this.configEnabled = configEnabled; - this.effectiveEnabled = effectiveEnabled; - this.autoRefresh = autoRefresh; - this.ttlSecond = ttlSecond; - this.capacity = capacity; - this.estimatedSize = estimatedSize; - this.requestCount = requestCount; - this.hitCount = hitCount; - this.missCount = missCount; - this.hitRate = hitRate; - this.loadSuccessCount = loadSuccessCount; - this.loadFailureCount = loadFailureCount; - this.totalLoadTimeNanos = totalLoadTimeNanos; - this.averageLoadPenaltyNanos = averageLoadPenaltyNanos; - this.evictionCount = evictionCount; - this.invalidateCount = invalidateCount; - this.lastLoadSuccessTimeMs = lastLoadSuccessTimeMs; - this.lastLoadFailureTimeMs = lastLoadFailureTimeMs; - this.lastError = Objects.requireNonNull(lastError, "lastError"); - } - - public boolean isConfigEnabled() { - return configEnabled; - } - - /** - * Effective cache enable state evaluated by {@link CacheSpec#isCacheEnabled(boolean, long, long)}. - */ - public boolean isEffectiveEnabled() { - return effectiveEnabled; - } - - public boolean isAutoRefresh() { - return autoRefresh; - } - - public long getTtlSecond() { - return ttlSecond; - } - - public long getCapacity() { - return capacity; - } - - public long getEstimatedSize() { - return estimatedSize; - } - - public long getRequestCount() { - return requestCount; - } - - public long getHitCount() { - return hitCount; - } - - public long getMissCount() { - return missCount; - } - - public double getHitRate() { - return hitRate; - } - - public long getLoadSuccessCount() { - return loadSuccessCount; - } - - public long getLoadFailureCount() { - return loadFailureCount; - } - - public long getTotalLoadTimeNanos() { - return totalLoadTimeNanos; - } - - /** - * Average load penalty in nanoseconds. - */ - public double getAverageLoadPenaltyNanos() { - return averageLoadPenaltyNanos; - } - - public long getEvictionCount() { - return evictionCount; - } - - public double getEvictionRate() { - if (requestCount == 0) { - return 0D; - } - return (double) evictionCount / requestCount; - } - - public long getInvalidateCount() { - return invalidateCount; - } - - /** - * Last successful load timestamp in epoch milliseconds, or {@code -1} if absent. - */ - public long getLastLoadSuccessTimeMs() { - return lastLoadSuccessTimeMs; - } - - /** - * Last failed load timestamp in epoch milliseconds, or {@code -1} if absent. - */ - public long getLastLoadFailureTimeMs() { - return lastLoadFailureTimeMs; - } - - /** - * Latest load failure message, or empty string if no failure is recorded. - */ - public String getLastError() { - return lastError; - } -} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheRemovalListener.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheRemovalListener.java new file mode 100644 index 00000000000000..603ab3f5703807 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheRemovalListener.java @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +/** Caffeine-free callback invoked after one metadata-cache value is removed. */ +@FunctionalInterface +public interface MetaCacheRemovalListener { + void onRemoval(K key, V value, MetaCacheRemovalReason reason); +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheRemovalReason.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheRemovalReason.java new file mode 100644 index 00000000000000..ac2c2769dfc5a4 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheRemovalReason.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +/** Framework-owned reason for a metadata-cache removal. */ +public enum MetaCacheRemovalReason { + EXPLICIT, + REPLACED, + COLLECTED, + EXPIRED, + SIZE +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java index 82a031be8c327a..98d9a7abdfed3f 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java @@ -29,6 +29,10 @@ * {@link Object#equals(Object)} and {@link Object#hashCode()} semantics while registered. */ public final class ScopePath { + private enum SyntheticPartition { + COLLECTION + } + public enum Level { CATALOG, DATABASE, @@ -75,6 +79,15 @@ public static ScopePath partition(String database, String table, Object partitio Objects.requireNonNull(partition, "partition can not be null")); } + /** + * Scope for a value describing the table's partition collection as a whole, such as a partition-name list. + * It is a child of the table but distinct from every connector partition identity, allowing a partition + * change to invalidate membership views without also dropping unrelated table metadata. + */ + public static ScopePath partitionCollection(String database, String table) { + return partition(database, table, SyntheticPartition.COLLECTION); + } + public Level level() { return level; } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java index c2b891ce8fa8c8..8bdd79b7cb37ef 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java @@ -25,6 +25,7 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.RemovalCause; +import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.Ticker; import java.math.BigInteger; @@ -41,9 +42,12 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.LongAdder; import java.util.function.BiConsumer; import java.util.function.Function; @@ -66,13 +70,29 @@ public final class ScopedMetaCache implements AutoCloseable { private final Cache> data; private final ConcurrentMap> keyNodes = new ConcurrentHashMap<>(); private final ConcurrentMap, CompletableFuture> inFlightLoads = new ConcurrentHashMap<>(); + private final ConcurrentMap> refreshing = new ConcurrentHashMap<>(); private final StripedPhaseGate bulkInvalidationGate = new StripedPhaseGate(); private final Map exactInvalidations = new HashMap<>(); private final NavigableMap activeBulkStarts = new TreeMap<>(); private final AtomicBoolean closed = new AtomicBoolean(false); - private final BiConsumer beforeRemoval; + private final LongAdder requestCount = new LongAdder(); + private final LongAdder hitCount = new LongAdder(); + private final LongAdder missCount = new LongAdder(); + private final LongAdder loadSuccessCount = new LongAdder(); + private final LongAdder loadFailureCount = new LongAdder(); + private final LongAdder totalLoadTimeNanos = new LongAdder(); + private final LongAdder evictionCount = new LongAdder(); + private final LongAdder invalidateCount = new LongAdder(); + private final AtomicReference lastLoadSuccessTimeMs = new AtomicReference<>(-1L); + private final AtomicReference lastLoadFailureTimeMs = new AtomicReference<>(-1L); + private final AtomicReference lastError = new AtomicReference<>(""); + private final RemovalListener beforeRemoval; + private final Ticker ticker; + private final long refreshAfterWriteNanos; + private final Executor refreshExecutor; private final Runnable afterLoadElection; private final Runnable afterBulkStage; + private final Runnable afterRefreshRegistration; private final ThreadLocal> removalDeferrals = ThreadLocal.withInitial(RemovalDeferral::new); private BigInteger exactInvalidationSequence = BigInteger.ZERO; @@ -82,16 +102,24 @@ public final class ScopedMetaCache implements AutoCloseable { String name, CacheSpec cacheSpec, Ticker ticker, - BiConsumer beforeRemoval, + RemovalListener beforeRemoval, + Duration refreshAfterWrite, + Executor refreshExecutor, Runnable afterLoadElection, - Runnable afterBulkStage) { + Runnable afterBulkStage, + Runnable afterRefreshRegistration) { this.registry = Objects.requireNonNull(registry, "registry can not be null"); this.name = Objects.requireNonNull(name, "name can not be null"); Objects.requireNonNull(cacheSpec, "cacheSpec can not be null"); this.beforeRemoval = beforeRemoval; + this.ticker = ticker == null ? Ticker.systemTicker() : ticker; + this.refreshAfterWriteNanos = refreshAfterWrite == null ? 0L : refreshAfterWrite.toNanos(); + this.refreshExecutor = refreshExecutor; this.afterLoadElection = Objects.requireNonNull(afterLoadElection, "afterLoadElection can not be null"); this.afterBulkStage = Objects.requireNonNull(afterBulkStage, "afterBulkStage can not be null"); + this.afterRefreshRegistration = Objects.requireNonNull( + afterRefreshRegistration, "afterRefreshRegistration can not be null"); this.effectiveEnabled = CacheSpec.isCacheEnabled( cacheSpec.isEnable(), cacheSpec.getTtlSecond(), cacheSpec.getCapacity()); @@ -106,7 +134,7 @@ public final class ScopedMetaCache implements AutoCloseable { builder.expireAfterAccess(Duration.ofSeconds(expiry.getAsLong())); } if (ticker != null) { - builder.ticker(ticker); + builder.ticker(this.ticker); } this.data = builder.build(); } @@ -121,13 +149,17 @@ public V get(K key, ScopePath path, Function loader) { Function loadFunction = Objects.requireNonNull(loader, "loader can not be null"); checkOpen(); if (!effectiveEnabled) { - return loadFunction.apply(key); + recordAccess(false); + return loadAndRecord(key, loadFunction); } - V present = getIfPresent(key, path); - if (present != null) { - return present; + VersionedValue presentVersioned = currentVersionedValue(key, path); + if (presentVersioned != null) { + recordAccess(true); + scheduleRefresh(key, path, loader, presentVersioned); + return presentVersioned.value; } + recordAccess(false); try (PublicationLease lease = acquirePublicationLease(key, path, true)) { LoadAddress loadAddress = new LoadAddress<>(key, path, lease); CompletableFuture ownLoad = new CompletableFuture<>(); @@ -138,13 +170,13 @@ public V get(K key, ScopePath path, Function loader) { try { afterLoadElection.run(); synchronized (lease.keyNode) { - present = getIfPresent(key, path); + VersionedValue present = currentVersionedValue(key, path); if (present != null) { - ownLoad.complete(present); - return present; + ownLoad.complete(present.value); + return present.value; } } - V loaded = loadFunction.apply(key); + V loaded = loadAndRecord(key, loadFunction); if (loaded != null) { synchronized (lease.keyNode) { publishCommitted(lease, key, loaded); @@ -166,8 +198,15 @@ public V getIfPresent(K key, ScopePath path) { Objects.requireNonNull(path, "path can not be null"); checkOpen(); if (!effectiveEnabled) { + recordAccess(false); return null; } + VersionedValue versioned = currentVersionedValue(key, path); + recordAccess(versioned != null); + return versioned == null ? null : versioned.value; + } + + private VersionedValue currentVersionedValue(K key, ScopePath path) { VersionedValue versioned = data.getIfPresent(key); if (versioned == null) { return null; @@ -179,7 +218,7 @@ public V getIfPresent(K key, ScopePath path) { data.asMap().remove(key, versioned); return null; } - return versioned.value; + return versioned; } public void put(K key, ScopePath path, V value) { @@ -297,7 +336,58 @@ public CacheMetrics metrics() { keyNodes.size(), inFlightLoads.size(), activeBulkStarts.values().stream().mapToInt(Integer::intValue).sum(), - exactInvalidations.size())); + exactInvalidations.size(), + effectiveEnabled, + requestCount.sum(), + hitCount.sum(), + missCount.sum(), + loadSuccessCount.sum(), + loadFailureCount.sum(), + totalLoadTimeNanos.sum(), + evictionCount.sum(), + invalidateCount.sum(), + lastLoadSuccessTimeMs.get(), + lastLoadFailureTimeMs.get(), + lastError.get())); + } + + int refreshingCountForTest() { + return refreshing.size(); + } + + public void forEach(BiConsumer consumer) { + Objects.requireNonNull(consumer, "consumer can not be null"); + data.asMap().forEach((key, versioned) -> { + if (versioned.isCurrent(registry, keyNodes)) { + consumer.accept(key, versioned.value); + } + }); + } + + private V loadAndRecord(K key, Function loader) { + long startNanos = System.nanoTime(); + try { + V loaded = loader.apply(key); + loadSuccessCount.increment(); + lastLoadSuccessTimeMs.set(System.currentTimeMillis()); + return loaded; + } catch (RuntimeException | Error throwable) { + loadFailureCount.increment(); + lastLoadFailureTimeMs.set(System.currentTimeMillis()); + lastError.set(throwable.toString()); + throw throwable; + } finally { + totalLoadTimeNanos.add(System.nanoTime() - startNanos); + } + } + + private void recordAccess(boolean hit) { + requestCount.increment(); + if (hit) { + hitCount.increment(); + } else { + missCount.increment(); + } } public void cleanUp() { @@ -366,7 +456,49 @@ private VersionedValue newVersionedValue( PublicationLease lease, K key, V value) { CacheAddress address = new CacheAddress(this, key); return new VersionedValue<>( - key, value, address, lease.scopeLease.snapshot(), lease.keyNode, lease.keyState); + key, value, address, lease.scopeLease.snapshot(), lease.keyNode, lease.keyState, ticker.read()); + } + + private void scheduleRefresh(K key, ScopePath path, Function loader, VersionedValue current) { + if (refreshAfterWriteNanos == 0L || ticker.read() - current.writeTimeNanos < refreshAfterWriteNanos + || refreshing.putIfAbsent(key, current) != null) { + return; + } + PublicationLease lease; + try { + afterRefreshRegistration.run(); + lease = acquirePublicationLease(key, path, true); + } catch (RuntimeException | Error throwable) { + refreshing.remove(key, current); + throw throwable; + } + if (data.getIfPresent(key) != current || !current.isCurrent(registry, keyNodes)) { + lease.close(); + refreshing.remove(key, current); + return; + } + try { + refreshExecutor.execute(() -> { + try (PublicationLease ignored = lease) { + if (closed.get()) { + return; + } + V refreshed = loadAndRecord(key, loader); + if (refreshed != null) { + synchronized (lease.keyNode) { + publishCommitted(lease, key, refreshed); + } + } + } catch (RuntimeException | Error throwable) { + LOG.log(System.Logger.Level.WARNING, "Scoped metadata cache refresh failed", throwable); + } finally { + refreshing.remove(key, current); + } + }); + } catch (RejectedExecutionException exception) { + lease.close(); + refreshing.remove(key, current); + } } private void install( @@ -490,7 +622,7 @@ private void onRemoval( @SuppressWarnings("unchecked") VersionedValue versioned = (VersionedValue) rawValue; RemovalDeferral removalDeferral = removalDeferrals.get(); - removalDeferral.removals.addLast(versioned); + removalDeferral.removals.addLast(new DeferredRemoval<>(versioned, cause)); if (removalDeferral.depth > 0 || removalDeferral.draining) { return; } @@ -501,10 +633,10 @@ private void drainDeferredRemovals(RemovalDeferral removalDeferral) { removalDeferral.draining = true; Throwable failure = null; try { - VersionedValue versioned; - while ((versioned = removalDeferral.removals.pollFirst()) != null) { + DeferredRemoval removal; + while ((removal = removalDeferral.removals.pollFirst()) != null) { try { - completeRemoval(versioned); + completeRemoval(removal.versioned, removal.cause); } catch (RuntimeException | Error e) { if (failure == null) { failure = e; @@ -526,11 +658,16 @@ private void drainDeferredRemovals(RemovalDeferral removalDeferral) { } } - private void completeRemoval(VersionedValue versioned) { + private void completeRemoval(VersionedValue versioned, RemovalCause cause) { + if (cause.wasEvicted()) { + evictionCount.increment(); + } else if (cause == RemovalCause.EXPLICIT) { + invalidateCount.increment(); + } K key = versioned.key; if (beforeRemoval != null) { try { - beforeRemoval.accept(key, versioned.value); + beforeRemoval.onRemoval(key, versioned.value, cause); } catch (Throwable t) { LOG.log(System.Logger.Level.WARNING, "Scoped metadata cache removal callback failed", t); } @@ -541,11 +678,21 @@ private void completeRemoval(VersionedValue versioned) { } private static final class RemovalDeferral { - private final Deque> removals = new ArrayDeque<>(); + private final Deque> removals = new ArrayDeque<>(); private int depth; private boolean draining; } + private static final class DeferredRemoval { + private final VersionedValue versioned; + private final RemovalCause cause; + + private DeferredRemoval(VersionedValue versioned, RemovalCause cause) { + this.versioned = versioned; + this.cause = cause; + } + } + private static final class InvalidatedKey { private final KeyNode node; private final KeyState keyState; @@ -638,18 +785,54 @@ public static final class CacheMetrics { private final int inFlightLoadCount; private final int activeBulkHandleCount; private final int exactInvalidationTombstoneCount; + private final boolean effectiveEnabled; + private final long requestCount; + private final long hitCount; + private final long missCount; + private final long loadSuccessCount; + private final long loadFailureCount; + private final long totalLoadTimeNanos; + private final long evictionCount; + private final long invalidateCount; + private final long lastLoadSuccessTimeMs; + private final long lastLoadFailureTimeMs; + private final String lastError; private CacheMetrics( long physicalEntryCount, int keyNodeCount, int inFlightLoadCount, int activeBulkHandleCount, - int exactInvalidationTombstoneCount) { + int exactInvalidationTombstoneCount, + boolean effectiveEnabled, + long requestCount, + long hitCount, + long missCount, + long loadSuccessCount, + long loadFailureCount, + long totalLoadTimeNanos, + long evictionCount, + long invalidateCount, + long lastLoadSuccessTimeMs, + long lastLoadFailureTimeMs, + String lastError) { this.physicalEntryCount = physicalEntryCount; this.keyNodeCount = keyNodeCount; this.inFlightLoadCount = inFlightLoadCount; this.activeBulkHandleCount = activeBulkHandleCount; this.exactInvalidationTombstoneCount = exactInvalidationTombstoneCount; + this.effectiveEnabled = effectiveEnabled; + this.requestCount = requestCount; + this.hitCount = hitCount; + this.missCount = missCount; + this.loadSuccessCount = loadSuccessCount; + this.loadFailureCount = loadFailureCount; + this.totalLoadTimeNanos = totalLoadTimeNanos; + this.evictionCount = evictionCount; + this.invalidateCount = invalidateCount; + this.lastLoadSuccessTimeMs = lastLoadSuccessTimeMs; + this.lastLoadFailureTimeMs = lastLoadFailureTimeMs; + this.lastError = lastError; } public long getPhysicalEntryCount() { @@ -671,6 +854,54 @@ public int getActiveBulkHandleCount() { public int getExactInvalidationTombstoneCount() { return exactInvalidationTombstoneCount; } + + public boolean isEffectiveEnabled() { + return effectiveEnabled; + } + + public long getLoadSuccessCount() { + return loadSuccessCount; + } + + public long getRequestCount() { + return requestCount; + } + + public long getHitCount() { + return hitCount; + } + + public long getMissCount() { + return missCount; + } + + public long getLoadFailureCount() { + return loadFailureCount; + } + + public long getTotalLoadTimeNanos() { + return totalLoadTimeNanos; + } + + public long getEvictionCount() { + return evictionCount; + } + + public long getInvalidateCount() { + return invalidateCount; + } + + public long getLastLoadSuccessTimeMs() { + return lastLoadSuccessTimeMs; + } + + public long getLastLoadFailureTimeMs() { + return lastLoadFailureTimeMs; + } + + public String getLastError() { + return lastError; + } } private static final class PublicationLease implements AutoCloseable { @@ -722,6 +953,7 @@ private static final class VersionedValue { private final ScopeSnapshot scopeSnapshot; private final KeyNode keyNode; private final KeyState keyState; + private final long writeTimeNanos; private VersionedValue( K key, @@ -729,13 +961,15 @@ private VersionedValue( CacheAddress address, ScopeSnapshot scopeSnapshot, KeyNode keyNode, - KeyState keyState) { + KeyState keyState, + long writeTimeNanos) { this.key = key; this.value = value; this.address = address; this.scopeSnapshot = scopeSnapshot; this.keyNode = keyNode; this.keyState = keyState; + this.writeTimeNanos = writeTimeNanos; } private boolean isCurrent( diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java index 0d3b076fe1688f..d949f589ae7b0a 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java @@ -17,8 +17,10 @@ package org.apache.doris.connector.cache; +import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.Ticker; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -26,6 +28,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -73,6 +76,23 @@ public ScopedMetaCache createCache(String name, CacheSpec cacheSpec return createCache(name, cacheSpec, null, null, NO_OP, NO_OP); } + ScopedMetaCache createCacheWithRemovalListener( + String name, CacheSpec cacheSpec, RemovalListener removalListener, + Duration refreshAfterWrite, Executor refreshExecutor) { + return createCacheWithRemovalListener(name, cacheSpec, null, removalListener, + refreshAfterWrite, refreshExecutor, NO_OP, NO_OP, NO_OP); + } + + ScopedMetaCache createCacheWithMetaRemovalListener( + String name, CacheSpec cacheSpec, MetaCacheRemovalListener removalListener, + Duration refreshAfterWrite, Executor refreshExecutor) { + RemovalListener caffeineListener = removalListener == null ? null + : (key, value, cause) -> removalListener.onRemoval( + key, value, MetaCacheRemovalReason.valueOf(cause.name())); + return createCacheWithRemovalListener( + name, cacheSpec, caffeineListener, refreshAfterWrite, refreshExecutor); + } + ScopedMetaCache createCache( String name, CacheSpec cacheSpec, @@ -97,15 +117,46 @@ ScopedMetaCache createCache( BiConsumer beforeRemoval, Runnable afterLoadElection, Runnable afterBulkStage) { + RemovalListener listener = beforeRemoval == null + ? null + : (key, value, cause) -> beforeRemoval.accept(key, value); + return createCacheWithRemovalListener( + name, cacheSpec, ticker, listener, null, null, afterLoadElection, afterBulkStage, NO_OP); + } + + ScopedMetaCache createCacheWithRefresh( + String name, + CacheSpec cacheSpec, + Duration refreshAfterWrite, + Executor refreshExecutor, + Runnable afterRefreshRegistration) { + return createCacheWithRemovalListener( + name, cacheSpec, null, null, refreshAfterWrite, refreshExecutor, + NO_OP, NO_OP, afterRefreshRegistration); + } + + private ScopedMetaCache createCacheWithRemovalListener( + String name, + CacheSpec cacheSpec, + Ticker ticker, + RemovalListener removalListener, + Duration refreshAfterWrite, + Executor refreshExecutor, + Runnable afterLoadElection, + Runnable afterBulkStage, + Runnable afterRefreshRegistration) { checkOpen(); ScopedMetaCache cache = new ScopedMetaCache<>( this, name, cacheSpec, ticker, - beforeRemoval, + removalListener, + refreshAfterWrite, + refreshExecutor, afterLoadElection, - afterBulkStage); + afterBulkStage, + afterRefreshRegistration); caches.add(cache); if (closed.get()) { caches.remove(cache); diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java index 276735b40c2f9b..c0cde92bd92229 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java @@ -25,9 +25,8 @@ import java.util.OptionalLong; /** - * Pins the shared {@link CacheSpec} validators and parsing (the single source of truth after the - * three prior copies — fe-core {@code datasource.metacache.CacheSpec}, the {@code connector.api.cache} - * mirror, and the connectors' hand-rolled checks — were collapsed here). + * Pins the shared {@link CacheSpec} validators and parsing after the prior copies and hand-rolled checks were + * collapsed here. * *

WHY this matters: this class restores the legacy CREATE/ALTER CATALOG meta-cache property * validation that was dropped at the SPI cutover. The validators MUST throw {@link IllegalArgumentException} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java new file mode 100644 index 00000000000000..0c4acd656195f8 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Queue; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicInteger; + +class CatalogMetaCacheTest { + private static final CacheSpec SPEC = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 100); + + @Test + void tableInvalidationAutomaticallyCoversEveryPhysicalCache() { + try (CatalogMetaCache catalog = new CatalogMetaCache()) { + MetaCache tableCache = catalog.create(tableDefinition("table")); + MetaCache snapshotCache = catalog.create(tableDefinition("snapshot")); + MetaCache futureCache = catalog.create(tableDefinition("future-derived-cache")); + TableKey target = new TableKey("db", "table"); + TableKey sibling = new TableKey("db", "sibling"); + + tableCache.put(target, "table-v1"); + snapshotCache.put(target, "snapshot-v1"); + futureCache.put(target, "future-v1"); + tableCache.put(sibling, "sibling-v1"); + + catalog.invalidateTable("db", "table"); + + Assertions.assertNull(tableCache.getIfPresent(target)); + Assertions.assertNull(snapshotCache.getIfPresent(target)); + Assertions.assertNull(futureCache.getIfPresent(target)); + Assertions.assertEquals("sibling-v1", tableCache.getIfPresent(sibling)); + } + } + + @Test + void databaseInvalidationCoversDescendantsButNotSiblingDatabase() { + try (CatalogMetaCache catalog = new CatalogMetaCache()) { + MetaCache cache = catalog.create(tableDefinition("table")); + TableKey first = new TableKey("db1", "table"); + TableKey second = new TableKey("db2", "table"); + cache.put(first, "first"); + cache.put(second, "second"); + + catalog.invalidateDatabase("db1"); + + Assertions.assertNull(cache.getIfPresent(first)); + Assertions.assertEquals("second", cache.getIfPresent(second)); + } + } + + @Test + void defaultAndContextualLoadersUseDefinitionScope() { + AtomicInteger loads = new AtomicInteger(); + MetaCacheDefinition definition = MetaCacheDefinition + .builder("table", SPEC, TableKey::scope) + .loader(key -> "default-" + loads.incrementAndGet()) + .build(); + try (CatalogMetaCache catalog = new CatalogMetaCache()) { + MetaCache cache = catalog.create(definition); + TableKey first = new TableKey("db", "first"); + TableKey second = new TableKey("db", "second"); + + Assertions.assertEquals("default-1", cache.get(first)); + Assertions.assertEquals("default-1", cache.get(first)); + Assertions.assertEquals("contextual", cache.get(second, key -> "contextual")); + Assertions.assertEquals(1, loads.get()); + } + } + + @Test + void rejectsMissingScopeDuplicateNamesAndUseAfterClose() { + Assertions.assertThrows(NullPointerException.class, + () -> MetaCacheDefinition.builder("missing-scope", SPEC, null).build()); + + CatalogMetaCache catalog = new CatalogMetaCache(); + catalog.create(tableDefinition("duplicate")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> catalog.create(tableDefinition("duplicate"))); + catalog.close(); + Assertions.assertThrows(IllegalStateException.class, + () -> catalog.create(tableDefinition("after-close"))); + } + + @Test + void refreshReturnsCurrentValueAndPublishesReloadedValue() { + AtomicInteger loads = new AtomicInteger(); + MetaCacheDefinition definition = MetaCacheDefinition + .builder("refresh", SPEC, TableKey::scope) + .loader(key -> "v" + loads.incrementAndGet()) + .refreshAfterWrite(Duration.ofNanos(1), Runnable::run) + .build(); + try (CatalogMetaCache catalog = new CatalogMetaCache()) { + MetaCache cache = catalog.create(definition); + TableKey key = new TableKey("db", "table"); + + Assertions.assertEquals("v1", cache.get(key)); + Assertions.assertEquals("v1", cache.get(key)); + Assertions.assertEquals("v2", cache.getIfPresent(key)); + } + } + + @Test + void invalidationRejectsRefreshAdmittedBeforeGenerationChange() { + AtomicInteger loads = new AtomicInteger(); + QueuedExecutor executor = new QueuedExecutor(); + MetaCacheDefinition definition = MetaCacheDefinition + .builder("refresh", SPEC, TableKey::scope) + .loader(key -> "v" + loads.incrementAndGet()) + .refreshAfterWrite(Duration.ofNanos(1), executor) + .build(); + try (CatalogMetaCache catalog = new CatalogMetaCache()) { + MetaCache cache = catalog.create(definition); + TableKey key = new TableKey("db", "table"); + + Assertions.assertEquals("v1", cache.get(key)); + Assertions.assertEquals("v1", cache.get(key)); + Assertions.assertEquals(1, executor.size()); + catalog.invalidateTable("db", "table"); + executor.runNext(); + + Assertions.assertNull(cache.getIfPresent(key)); + Assertions.assertEquals(2, loads.get()); + } + } + + @Test + void closeSkipsRefreshLoaderAlreadyQueued() { + AtomicInteger loads = new AtomicInteger(); + QueuedExecutor executor = new QueuedExecutor(); + MetaCacheDefinition definition = MetaCacheDefinition + .builder("refresh", SPEC, TableKey::scope) + .loader(key -> "v" + loads.incrementAndGet()) + .refreshAfterWrite(Duration.ofNanos(1), executor) + .build(); + CatalogMetaCache catalog = new CatalogMetaCache(); + MetaCache cache = catalog.create(definition); + TableKey key = new TableKey("db", "table"); + + Assertions.assertEquals("v1", cache.get(key)); + Assertions.assertEquals("v1", cache.get(key)); + Assertions.assertEquals(1, executor.size()); + catalog.close(); + executor.runNext(); + + Assertions.assertEquals(1, loads.get()); + Assertions.assertEquals(0, catalog.metrics().getRegistrationCount()); + } + + private static MetaCacheDefinition tableDefinition(String name) { + return MetaCacheDefinition.builder(name, SPEC, TableKey::scope).build(); + } + + private static final class TableKey { + private final String database; + private final String table; + + private TableKey(String database, String table) { + this.database = database; + this.table = table; + } + + private ScopePath scope() { + return ScopePath.table(database, table); + } + } + + private static final class QueuedExecutor implements Executor { + private final Queue tasks = new ArrayDeque<>(); + + @Override + public void execute(Runnable command) { + tasks.add(command); + } + + private int size() { + return tasks.size(); + } + + private void runNext() { + tasks.remove().run(); + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEntryTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEntryTest.java deleted file mode 100644 index cdc49438f20f82..00000000000000 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEntryTest.java +++ /dev/null @@ -1,298 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.cache; - -import com.github.benmanes.caffeine.cache.LoadingCache; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.lang.reflect.Field; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Behaviour parity tests for the connector-side {@link MetaCacheEntry} copy. Where fe-core's - * {@code MetaCacheEntryTest} toggles {@code Config.enable_external_meta_cache_manual_miss_load}, this copy - * passes the flag through the constructor instead (the connector copy has no fe-core Config). - */ -public class MetaCacheEntryTest { - - @Test - public void loaderGetTracksHitMissAndLastError() { - ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); - try { - CacheSpec cacheSpec = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L); - MetaCacheEntry entry = new MetaCacheEntry<>( - "test", - key -> { - if ("fail".equals(key)) { - throw new IllegalStateException("mock failure"); - } - return 1; - }, - cacheSpec, - refreshExecutor); - - Assertions.assertEquals(Integer.valueOf(1), entry.get("ok")); - Assertions.assertEquals(Integer.valueOf(1), entry.get("ok")); - Assertions.assertThrows(IllegalStateException.class, () -> entry.get("fail")); - - MetaCacheEntryStats stats = entry.stats(); - Assertions.assertEquals(3L, stats.getRequestCount()); - Assertions.assertEquals(1L, stats.getHitCount()); - Assertions.assertEquals(2L, stats.getMissCount()); - Assertions.assertEquals(1L, stats.getLoadSuccessCount()); - Assertions.assertEquals(1L, stats.getLoadFailureCount()); - Assertions.assertTrue(stats.getLastLoadSuccessTimeMs() > 0); - Assertions.assertTrue(stats.getLastLoadFailureTimeMs() > 0); - Assertions.assertTrue(stats.getLastError().contains("mock failure")); - - // A per-call miss loader overrides the default loader. - Assertions.assertEquals(Integer.valueOf(101), entry.get("miss-loader", key -> 101)); - } finally { - refreshExecutor.shutdownNow(); - } - } - - @Test - public void disabledSpecMakesEntryIneffective() { - ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); - try { - // ttl == 0 disables the cache (CacheSpec.isCacheEnabled). - CacheSpec cacheSpec = CacheSpec.of(true, 0L, 10L); - AtomicInteger loadCounter = new AtomicInteger(); - MetaCacheEntry entry = new MetaCacheEntry<>( - "test", - key -> loadCounter.incrementAndGet(), - cacheSpec, - refreshExecutor); - - // getIfPresent short-circuits to null for a disabled entry, even right after a get() populated it. - Assertions.assertNull(entry.getIfPresent("k")); - Assertions.assertEquals(Integer.valueOf(1), entry.get("k")); - Assertions.assertNull(entry.getIfPresent("k")); - - MetaCacheEntryStats stats = entry.stats(); - Assertions.assertTrue(stats.isConfigEnabled()); - Assertions.assertFalse(stats.isEffectiveEnabled()); - - // The manual miss-load path bypasses the cache entirely when disabled, so every get reloads. - AtomicInteger manualCounter = new AtomicInteger(); - MetaCacheEntry manualEntry = new MetaCacheEntry<>( - "test-manual", key -> manualCounter.incrementAndGet(), cacheSpec, refreshExecutor, - false, false, 0L, true); - manualEntry.get("k"); - manualEntry.get("k"); - Assertions.assertEquals(2, manualCounter.get()); - } finally { - refreshExecutor.shutdownNow(); - } - } - - @Test - public void capacityEvictsAndStatsCountEviction() throws Exception { - ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); - try { - CacheSpec cacheSpec = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 1L); - MetaCacheEntry entry = new MetaCacheEntry<>( - "test", - String::length, - cacheSpec, - refreshExecutor); - - Assertions.assertEquals(0D, entry.stats().getEvictionRate(), 0D); - Assertions.assertEquals(Integer.valueOf(1), entry.get("a")); - Assertions.assertEquals(Integer.valueOf(2), entry.get("bb")); - // Force Caffeine to run pending maintenance so the eviction is observable. - extractLoadingCache(entry).cleanUp(); - Assertions.assertEquals(1L, entry.stats().getEvictionCount()); - Assertions.assertEquals(0.5D, entry.stats().getEvictionRate(), 0D); - } finally { - refreshExecutor.shutdownNow(); - } - } - - @Test - public void contextualOnlyEntryRejectsDefaultGetButServesMissLoader() { - ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); - try { - CacheSpec cacheSpec = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L); - MetaCacheEntry entry = new MetaCacheEntry<>( - "contextual", null, cacheSpec, refreshExecutor, - false, true, 0L, false); - - UnsupportedOperationException ex = Assertions.assertThrows( - UnsupportedOperationException.class, () -> entry.get("k")); - Assertions.assertTrue(ex.getMessage().contains("contextual miss loader")); - Assertions.assertEquals(Integer.valueOf(7), entry.get("k", key -> 7)); - // Second call is a cache hit; the miss loader is not consulted again. - Assertions.assertEquals(Integer.valueOf(7), entry.get("k", key -> 999)); - } finally { - refreshExecutor.shutdownNow(); - } - } - - @Test - public void invalidationDropsEntriesAndCounts() { - ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); - try { - CacheSpec cacheSpec = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L); - MetaCacheEntry entry = new MetaCacheEntry<>( - "test", String::length, cacheSpec, refreshExecutor); - - entry.get("a"); - entry.get("bb"); - entry.get("ccc"); - entry.invalidateKey("a"); - Assertions.assertNull(entry.getIfPresent("a")); - Assertions.assertEquals(Integer.valueOf(2), entry.getIfPresent("bb")); - - entry.invalidateIf(key -> key.length() == 2); - Assertions.assertNull(entry.getIfPresent("bb")); - Assertions.assertEquals(Integer.valueOf(3), entry.getIfPresent("ccc")); - - entry.invalidateAll(); - Assertions.assertNull(entry.getIfPresent("ccc")); - Assertions.assertTrue(entry.stats().getInvalidateCount() >= 3L); - } finally { - refreshExecutor.shutdownNow(); - } - } - - @Test - public void manualMissLoadDeduplicatesConcurrentSameKey() throws Exception { - ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); - ExecutorService queryExecutor = Executors.newFixedThreadPool(2); - try { - CacheSpec cacheSpec = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L); - CountDownLatch loaderStarted = new CountDownLatch(1); - CountDownLatch releaseLoader = new CountDownLatch(1); - AtomicInteger loadCounter = new AtomicInteger(); - // manualMissLoadEnabled = true (last ctor arg) exercises the striped-lock manual load path. - MetaCacheEntry entry = new MetaCacheEntry<>( - "test", - key -> { - loaderStarted.countDown(); - awaitLatch(releaseLoader); - return loadCounter.incrementAndGet(); - }, - cacheSpec, refreshExecutor, - false, false, 0L, true); - - Future first = queryExecutor.submit(() -> entry.get("k")); - Assertions.assertTrue(loaderStarted.await(3L, TimeUnit.SECONDS)); - Future second = queryExecutor.submit(() -> entry.get("k")); - releaseLoader.countDown(); - - Assertions.assertEquals(Integer.valueOf(1), first.get(3L, TimeUnit.SECONDS)); - Assertions.assertEquals(Integer.valueOf(1), second.get(3L, TimeUnit.SECONDS)); - Assertions.assertEquals(1, loadCounter.get()); - } finally { - queryExecutor.shutdownNow(); - refreshExecutor.shutdownNow(); - } - } - - @Test - public void putIfNotInvalidatedSinceHonorsGenerationGuard() { - ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); - try { - // Contextual + manual-miss entry, mirroring the hive partition-object cache that uses the guarded put. - CacheSpec cacheSpec = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L); - MetaCacheEntry entry = new MetaCacheEntry<>( - "test", null, cacheSpec, refreshExecutor, false, true, 0L, true); - - // No invalidation since the captured generation -> the guarded put caches normally. - long g1 = entry.invalidationGeneration(); - entry.putIfNotInvalidatedSince(g1, "a", 1); - Assertions.assertEquals(Integer.valueOf(1), entry.getIfPresent("a")); - - // An invalidation between the capture and the put (the flush-races-an-in-flight-load case) must make - // the put a no-op, so a stale pre-invalidation value is NOT re-cached to the TTL. - long g2 = entry.invalidationGeneration(); - entry.invalidateAll(); // bumps the generation, as a racing flush / REFRESH would - entry.putIfNotInvalidatedSince(g2, "b", 2); - // MUTATION: an unguarded put (the pre-R3 raw put) leaves "b" cached here -> getIfPresent == 2 -> red. - Assertions.assertNull(entry.getIfPresent("b"), "a stale-generation put must not re-cache the value"); - - // A generation captured AFTER the invalidation puts normally again (the guard is not a one-shot latch). - long g3 = entry.invalidationGeneration(); - entry.putIfNotInvalidatedSince(g3, "c", 3); - Assertions.assertEquals(Integer.valueOf(3), entry.getIfPresent("c")); - } finally { - refreshExecutor.shutdownNow(); - } - } - - @Test - public void removalCallbackReceivesDisabledAndSuppressedLoads() throws Exception { - ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); - ExecutorService queryExecutor = Executors.newSingleThreadExecutor(); - try { - AtomicInteger removals = new AtomicInteger(); - MetaCacheEntry disabled = new MetaCacheEntry<>( - "disabled", null, CacheSpec.of(true, 0L, 10L), refreshExecutor, - false, true, 0L, true, (key, value) -> removals.incrementAndGet()); - Assertions.assertEquals(Integer.valueOf(1), disabled.get("k", key -> 1)); - Assertions.assertEquals(1, removals.get(), "a disabled cache must release the unretained value"); - - CountDownLatch loaderStarted = new CountDownLatch(1); - CountDownLatch releaseLoader = new CountDownLatch(1); - MetaCacheEntry entry = new MetaCacheEntry<>( - "racing", null, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), refreshExecutor, - false, true, 0L, true, (key, value) -> removals.incrementAndGet()); - Future load = queryExecutor.submit(() -> entry.get("k", key -> { - loaderStarted.countDown(); - awaitLatch(releaseLoader); - return 2; - })); - Assertions.assertTrue(loaderStarted.await(3L, TimeUnit.SECONDS)); - entry.invalidateKey("k"); - releaseLoader.countDown(); - Assertions.assertEquals(Integer.valueOf(2), load.get(3L, TimeUnit.SECONDS)); - Assertions.assertNull(entry.getIfPresent("k")); - Assertions.assertEquals(2, removals.get(), - "an invalidation-before-publication must release the suppressed value"); - } finally { - queryExecutor.shutdownNow(); - refreshExecutor.shutdownNow(); - } - } - - @SuppressWarnings("unchecked") - private static LoadingCache extractLoadingCache(MetaCacheEntry entry) throws Exception { - Field field = MetaCacheEntry.class.getDeclaredField("loadingData"); - field.setAccessible(true); - return (LoadingCache) field.get(entry); - } - - private static void awaitLatch(CountDownLatch latch) { - try { - if (!latch.await(3L, TimeUnit.SECONDS)) { - throw new IllegalStateException("latch timed out"); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IllegalStateException(e); - } - } -} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java index c6f928fa7124a0..bb1f1009f909b9 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.time.Duration; import java.util.Arrays; import java.util.List; import java.util.concurrent.CountDownLatch; @@ -70,6 +71,39 @@ public void concurrentMissesForTheSameKeyRunOneLoader() throws Exception { } } + @Test + public void closeDuringRefreshAdmissionDoesNotRetainRefreshMarker() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch refreshRegistered = new CountDownLatch(1); + CountDownLatch continueAdmission = new CountDownLatch(1); + ScopedMetaCacheRegistry registry = new ScopedMetaCacheRegistry(); + try { + ScopedMetaCache cache = registry.createCacheWithRefresh( + "test", ENABLED, Duration.ofNanos(1L), Runnable::run, () -> { + refreshRegistered.countDown(); + await(continueAdmission); + }); + Assertions.assertEquals("v1", cache.get("key", TABLE, ignored -> "v1")); + + Future refresh = executor.submit( + () -> cache.get("key", TABLE, ignored -> "v2")); + await(refreshRegistered); + registry.close(); + continueAdmission.countDown(); + + Exception failure = Assertions.assertThrows( + Exception.class, () -> refresh.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Assertions.assertTrue(failure.getCause() instanceof IllegalStateException); + Assertions.assertEquals(0, cache.refreshingCountForTest()); + Assertions.assertEquals(0, registry.metrics().getRegistrationCount()); + Assertions.assertEquals(0, registry.metrics().getActiveLoadCount()); + } finally { + continueAdmission.countDown(); + registry.close(); + executor.shutdownNow(); + } + } + @Test public void everyContainingInvalidationFencesAnInFlightSingleKeyLoad() throws Exception { List invalidations = Arrays.asList( diff --git a/fe/fe-connector/fe-connector-hive/pom.xml b/fe/fe-connector/fe-connector-hive/pom.xml index 2d21c24abc181b..2152c990f53162 100644 --- a/fe/fe-connector/fe-connector-hive/pom.xml +++ b/fe/fe-connector/fe-connector-hive/pom.xml @@ -54,20 +54,15 @@ under the License. ${project.version} - + ${project.groupId} fe-connector-cache ${project.version} - + dep is what puts the cache API on -hms's own compile classpath. --> ${project.groupId} fe-connector-cache diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java index b9244157131de4..98a1587af52571 100644 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java @@ -18,7 +18,10 @@ package org.apache.doris.connector.hms; import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; import org.apache.hadoop.hive.common.FileUtils; @@ -30,7 +33,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.concurrent.ForkJoinPool; import java.util.function.Function; /** @@ -46,7 +48,7 @@ * legacy after the cutover. Because the {@code HmsClient} is also held by the hudi/iceberg siblings from * this same module, the decorator is reusable by them later.

* - *

What it caches (4 methods), each on its own {@link MetaCacheEntry} configured from catalog + *

What it caches (4 methods), each on its own framework cache configured from catalog * properties {@code meta.cache.hive..(enable|ttl-second|capacity)} (defaults mirror the legacy * fe-core {@code Config} values — the connector is {@code Config}-free):

*
    @@ -111,32 +113,40 @@ public class CachingHmsClient implements HmsClient { static final long DEFAULT_PARTITION_NAMES_CAPACITY = 10000L; static final long DEFAULT_PARTITION_CAPACITY = 100000L; static final long DEFAULT_COLUMN_STATS_CAPACITY = 10000L; - private final HmsClient delegate; - private final MetaCacheEntry tableCache; - private final MetaCacheEntry> partitionNamesCache; - private final MetaCacheEntry partitionsCache; - private final MetaCacheEntry> columnStatsCache; + private final CatalogMetaCache owner; + private final MetaCache tableCache; + private final MetaCache> partitionNamesCache; + private final MetaCache partitionsCache; + private final MetaCache> columnStatsCache; public CachingHmsClient(HmsClient delegate, Map properties) { + this(new CatalogMetaCache(), delegate, properties); + } + + public CachingHmsClient(CatalogMetaCache owner, HmsClient delegate, Map properties) { + this.owner = Objects.requireNonNull(owner, "owner can not be null"); this.delegate = Objects.requireNonNull(delegate, "delegate can not be null"); Map props = applyLegacyTtlCompatibility( properties == null ? Collections.emptyMap() : properties); - this.tableCache = newEntry("hive.table", props, ENTRY_TABLE, DEFAULT_TABLE_CAPACITY); - this.partitionNamesCache = - newEntry("hive.partition_names", props, ENTRY_PARTITION_NAMES, DEFAULT_PARTITION_NAMES_CAPACITY); - this.partitionsCache = newEntry("hive.partition", props, ENTRY_PARTITION, DEFAULT_PARTITION_CAPACITY); - this.columnStatsCache = - newEntry("hive.column_stats", props, ENTRY_COLUMN_STATS, DEFAULT_COLUMN_STATS_CAPACITY); - } - - private static MetaCacheEntry newEntry(String name, Map props, - String entry, long defaultCapacity) { + this.tableCache = newEntry(owner, "hive-table", props, ENTRY_TABLE, DEFAULT_TABLE_CAPACITY, + key -> ScopePath.table(key.dbName, key.tableName)); + this.partitionNamesCache = newEntry(owner, "hive-partition-names", props, ENTRY_PARTITION_NAMES, + DEFAULT_PARTITION_NAMES_CAPACITY, + key -> ScopePath.partitionCollection(key.dbName, key.tableName)); + this.partitionsCache = newEntry(owner, "hive-partition", props, ENTRY_PARTITION, + DEFAULT_PARTITION_CAPACITY, + key -> ScopePath.partition(key.dbName, key.tableName, key.values)); + this.columnStatsCache = newEntry(owner, "hive-column-stats", props, ENTRY_COLUMN_STATS, + DEFAULT_COLUMN_STATS_CAPACITY, + key -> ScopePath.table(key.dbName, key.tableName)); + } + + private static MetaCache newEntry(CatalogMetaCache owner, String name, + Map props, String entry, long defaultCapacity, Function scopeResolver) { CacheSpec spec = CacheSpec.fromProperties(props, ENGINE, entry, CacheSpec.of(true, DEFAULT_TTL_SECOND, defaultCapacity)); - // Contextual-only + manual-miss load so a slow HMS RPC runs outside Caffeine's sync compute lock - // (deduplicated by a striped lock instead), mirroring PaimonLatestSnapshotCache / IcebergLatestSnapshotCache. - return new MetaCacheEntry<>(name, null, spec, ForkJoinPool.commonPool(), false, true, 0L, true); + return owner.create(MetaCacheDefinition.builder(name, spec, scopeResolver).build()); } /** Legacy fe-core catalog knob ({@code ExternalCatalog.SCHEMA_CACHE_TTL_SECOND}) for the table/schema cache. */ @@ -232,11 +242,12 @@ public List getPartitions(String dbName, String tableName, Lis // guard; the per-partition put must restore it (getTable/listPartitionNames/getTableColumnStatistics // still use the guarded get path). The delegate results still populate the RESULT list directly, // preserving the misparse->never-drop safety (only the CACHE put is generation-guarded). - long generation = partitionsCache.invalidationGeneration(); - for (HmsPartitionInfo info : delegate.getPartitions(dbName, tableName, missNames)) { - partitionsCache.putIfNotInvalidatedSince( - generation, new PartitionKey(dbName, tableName, info.getValues()), info); - result.add(info); + try (MetaCache.BulkLoad load = + partitionsCache.beginBulkLoad(ScopePath.table(dbName, tableName))) { + for (HmsPartitionInfo info : delegate.getPartitions(dbName, tableName, missNames)) { + load.publish(new PartitionKey(dbName, tableName, info.getValues()), info); + result.add(info); + } } } return result; @@ -282,10 +293,7 @@ public List getTableColumnStatistics(String dbName, String /** Drop every cached entry for one table. Backs {@code REFRESH TABLE}. */ public void flush(String dbName, String tableName) { - tableCache.invalidateKey(new TableKey(dbName, tableName)); - partitionNamesCache.invalidateIf(key -> key.matches(dbName, tableName)); - partitionsCache.invalidateIf(key -> key.matches(dbName, tableName)); - columnStatsCache.invalidateIf(key -> key.matches(dbName, tableName)); + owner.invalidateTable(dbName, tableName); } /** @@ -297,26 +305,20 @@ public void flush(String dbName, String tableName) { * partition-level refresh. */ public void invalidatePartitions(String dbName, String tableName, Set> partitionValues) { - partitionNamesCache.invalidateIf(key -> key.matches(dbName, tableName)); - if (!partitionValues.isEmpty()) { - partitionsCache.invalidateIf(key -> key.matchesPartitions(dbName, tableName, partitionValues)); + owner.invalidatePartitionCollection(dbName, tableName); + for (List values : partitionValues) { + owner.invalidatePartition(dbName, tableName, values); } } /** Drop every cached entry for one database (all its tables). Backs {@code REFRESH DATABASE}. */ public void flushDb(String dbName) { - tableCache.invalidateIf(key -> key.matchesDb(dbName)); - partitionNamesCache.invalidateIf(key -> key.matchesDb(dbName)); - partitionsCache.invalidateIf(key -> key.matchesDb(dbName)); - columnStatsCache.invalidateIf(key -> key.matchesDb(dbName)); + owner.invalidateDatabase(dbName); } /** Drop the whole cache. Backs {@code REFRESH CATALOG}. */ public void flushAll() { - tableCache.invalidateAll(); - partitionNamesCache.invalidateAll(); - partitionsCache.invalidateAll(); - columnStatsCache.invalidateAll(); + owner.invalidateCatalog(); } // ========== Pass-through: everything else is delegated verbatim ========== diff --git a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java index c07ecaa0bdf499..491cdbc2831634 100644 --- a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java +++ b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java @@ -17,6 +17,7 @@ package org.apache.doris.connector.hudi; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.hms.CachingHmsClient; import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsClientConfig; @@ -67,6 +68,7 @@ public class HudiConnector implements Connector { private final HudiCatalogProperties props; private final Map properties; private final ConnectorContext context; + private final CatalogMetaCache metaCache = new CatalogMetaCache(); private volatile HmsClient hmsClient; // HMS and storage deliberately have separate authenticators: hive.metastore.username must affect set_ugi @@ -157,6 +159,8 @@ public void invalidateTable(String dbName, String tableName) { void invalidateTable(HmsClient client, String dbName, String tableName) { if (client instanceof CachingHmsClient) { ((CachingHmsClient) client).flush(dbName, tableName); + } else { + metaCache.invalidateTable(dbName, tableName); } } @@ -173,6 +177,8 @@ public void invalidateDb(String dbName) { void invalidateDb(HmsClient client, String dbName) { if (client instanceof CachingHmsClient) { ((CachingHmsClient) client).flushDb(dbName); + } else { + metaCache.invalidateDatabase(dbName); } } @@ -189,6 +195,8 @@ public void invalidateAll() { void invalidateAll(HmsClient client) { if (client instanceof CachingHmsClient) { ((CachingHmsClient) client).flushAll(); + } else { + metaCache.invalidateCatalog(); } } @@ -251,7 +259,7 @@ HmsClientConfig buildHmsClientConfig() { * assert the cache decoration. */ HmsClient wrapWithCache(HmsClient raw) { - return new CachingHmsClient(raw, properties); + return new CachingHmsClient(metaCache, raw, properties); } /** @@ -338,6 +346,7 @@ private static Configuration buildHmsConf(AbstractHmsMetaStoreProperties hms) { @Override public void close() throws IOException { + metaCache.close(); HmsClient c = hmsClient; if (c != null) { c.close(); diff --git a/fe/fe-connector/fe-connector-iceberg/pom.xml b/fe/fe-connector/fe-connector-iceberg/pom.xml index 92adbcf47b3771..856594ff29f2a5 100644 --- a/fe/fe-connector/fe-connector-iceberg/pom.xml +++ b/fe/fe-connector/fe-connector-iceberg/pom.xml @@ -47,13 +47,10 @@ under the License. ${project.version} - + IcebergLatestSnapshotCache / IcebergManifestCache are backed by MetaCache. --> ${project.groupId} fe-connector-cache diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java index 6caf556f5780f6..cc76a99833b950 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java @@ -18,12 +18,13 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; -import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; -import java.util.concurrent.ForkJoinPool; import java.util.function.Supplier; /** @@ -52,20 +53,27 @@ */ final class IcebergCommentCache { - private final MetaCacheEntry entry; + private final CatalogMetaCache owner; + private final MetaCache entry; IcebergCommentCache(long ttlSeconds, int maxSize) { + this(new CatalogMetaCache(), ttlSeconds, maxSize); + } + + IcebergCommentCache(CatalogMetaCache owner, long ttlSeconds, int maxSize) { + this.owner = owner; // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). // Load-bearing here: a vended no-cache catalog builds this object but must NOT cache comments // (operator "no meta cache" intent). CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); - this.entry = new MetaCacheEntry<>("iceberg-comment", null, spec, - ForkJoinPool.commonPool(), false, true, 0L, true); + this.entry = owner.create(MetaCacheDefinition + .builder("iceberg-comment", spec, IcebergCommentCache::scope) + .build()); } /** Caching is on only when the TTL is positive; ttl-second <= 0 means "always read the comment live". */ boolean isEnabled() { - return entry.stats().isEffectiveEnabled(); + return entry.isEnabled(); } /** @@ -80,29 +88,30 @@ String getOrLoad(TableIdentifier identifier, Supplier loader) { /** Drops the cached comment for one table so the next read goes live (REFRESH TABLE). */ void invalidate(TableIdentifier identifier) { - entry.invalidateKey(identifier); + owner.invalidateTable(identifier.namespace().toString(), identifier.name()); } /** Drops every cached comment for one database (REFRESH DATABASE / DROP DATABASE); match = namespace equality. */ void invalidateDb(String dbName) { - Namespace ns = Namespace.of(dbName); - entry.invalidateIf(id -> id.namespace().equals(ns)); + owner.invalidateDatabase(dbName); } /** Drops all cached comments (REFRESH CATALOG). */ void invalidateAll() { - entry.invalidateAll(); + owner.invalidateCatalog(); } /** Test-only: current number of cached entries (accurate map membership, not Caffeine's estimate). */ int size() { - int[] count = {0}; - entry.forEach((key, value) -> count[0]++); - return count[0]; + return Math.toIntExact(entry.size()); } /** Test-only: how many times the live loader (the remote comment load) actually ran — the metric gate. */ long loadCountForTest() { - return entry.stats().getLoadSuccessCount(); + return entry.loadSuccessCount(); + } + + private static ScopePath scope(TableIdentifier identifier) { + return ScopePath.table(identifier.namespace().toString(), identifier.name()); } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java index 1dea43f325a9fa..530ac702c06057 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java @@ -17,6 +17,7 @@ package org.apache.doris.connector.iceberg; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.doris.connector.metastore.iceberg.jdbc.IcebergJdbcMetaStoreProperties; import org.apache.doris.connector.metastore.iceberg.rest.IcebergRestMetaStoreProperties; @@ -55,7 +56,6 @@ import org.apache.iceberg.catalog.BaseViewSessionCatalog; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.SessionCatalog; -import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.hive.HiveCatalog; import org.apache.iceberg.hive.HiveHadoopUtil; @@ -235,9 +235,10 @@ public class IcebergConnector implements Connector { mvccPartitionViewCache; private final ConnectorMetadataCache> // null under session=user listPartitionsViewCache; + private final CatalogMetaCache metaCache = new CatalogMetaCache(); // Manifest content cache — pure metadata, default-off (meta.cache.iceberg.manifest.enable), and consumed // ONLY after a per-user resolveTable(ForRead) -- exempt: no read path without a per-user load. - private final IcebergManifestCache manifestCache = new IcebergManifestCache(); + private final IcebergManifestCache manifestCache = new IcebergManifestCache(metaCache); // Lazily-built plugin-side Kerberos authenticator (single-owner auth; see TcclPinningConnectorContext). // null for a non-Kerberos catalog. Its doAs acts on the PLUGIN's UserGroupInformation copy — the one the @@ -271,7 +272,7 @@ public IcebergConnector(Map properties, ConnectorContext context this.latestSnapshotCache = isUserSessionEnabled() ? null : new IcebergLatestSnapshotCache( - resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY); + metaCache, resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY); // PERF-01 cross-query RAW-table cache. Disabled (null) when the catalog's credentials are // query-dependent, because a cached raw Table carries its FileIO's credentials: // - iceberg.rest.session=user: per-user delegated FileIO -> sharing across users leaks credentials. @@ -284,7 +285,7 @@ public IcebergConnector(Map properties, ConnectorContext context || IcebergScanPlanProvider.restVendedCredentialsEnabled(this.properties)) ? null : new IcebergTableCache( - resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY, + metaCache, resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY, this::cachedTableCleanup, catalogResourceTracker); // PERF-02: partition-view cache. Authorization-sensitive projection: a shared (table+snapshot-keyed, no // user dimension) hit would disclose one user's partition list. Its readers are all downstream of a @@ -295,13 +296,13 @@ public IcebergConnector(Map properties, ConnectorContext context this.partitionCache = isUserSessionEnabled() ? null : new IcebergPartitionCache( - resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY); + metaCache, resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY); // PERF-03: inferred-file-format cache. Same authorization-sensitive treatment as partitionCache (disabled // under session=user, kept otherwise); readers already tolerate a null cache (resolveFileFormatName). this.formatCache = isUserSessionEnabled() ? null : new IcebergFormatCache( - resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY); + metaCache, resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY); // PERF-05: table-comment cache, built ONLY for a REST vended-credentials catalog that is NOT session=user. // Plain catalogs (tableCache on) already serve the comment path from tableCache; session=user is excluded // because a shared comment cache would bypass the per-user loadTable authorization (a metadata disclosure). @@ -310,7 +311,7 @@ public IcebergConnector(Map properties, ConnectorContext context this.commentCache = (IcebergScanPlanProvider.restVendedCredentialsEnabled(this.properties) && !isUserSessionEnabled()) ? new IcebergCommentCache( - resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY) + metaCache, resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY) : null; // PERF-06: derived partition-view cache A (generic ConnectorMetadataCache). Same // authorization-sensitive treatment as partitionCache -- disabled (null) under iceberg.rest.session=user so @@ -319,10 +320,12 @@ public IcebergConnector(Map properties, ConnectorContext context // framework's CacheSpec (default ON / 24h / 1000). Two typed instances (MVCC view + partition-info list). this.mvccPartitionViewCache = isUserSessionEnabled() ? null - : new ConnectorMetadataCache<>("iceberg", "partition_view", this.properties); + : new ConnectorMetadataCache<>(metaCache, "iceberg.mvcc-partition-view", + "iceberg", "partition_view", this.properties); this.listPartitionsViewCache = isUserSessionEnabled() ? null - : new ConnectorMetadataCache<>("iceberg", "partition_view", this.properties); + : new ConnectorMetadataCache<>(metaCache, "iceberg.list-partitions-view", + "iceberg", "partition_view", this.properties); } /** @@ -654,27 +657,7 @@ private IcebergCatalogOps newCatalogBackedOps(ConnectorSession session) { */ @Override public void invalidateTable(String dbName, String tableName) { - if (latestSnapshotCache != null) { - latestSnapshotCache.invalidate(TableIdentifier.of(dbName, tableName)); - } - if (tableCache != null) { - tableCache.invalidate(TableIdentifier.of(dbName, tableName)); - } - if (partitionCache != null) { - partitionCache.invalidate(TableIdentifier.of(dbName, tableName)); - } - if (formatCache != null) { - formatCache.invalidate(TableIdentifier.of(dbName, tableName)); - } - if (commentCache != null) { - commentCache.invalidate(TableIdentifier.of(dbName, tableName)); - } - if (mvccPartitionViewCache != null) { - mvccPartitionViewCache.invalidateTable(dbName, tableName); - } - if (listPartitionsViewCache != null) { - listPartitionsViewCache.invalidateTable(dbName, tableName); - } + metaCache.invalidateTable(dbName, tableName); } /** @@ -690,27 +673,7 @@ public void invalidateTable(String dbName, String tableName) { */ @Override public void invalidateDb(String dbName) { - if (latestSnapshotCache != null) { - latestSnapshotCache.invalidateDb(dbName); - } - if (tableCache != null) { - tableCache.invalidateDb(dbName); - } - if (partitionCache != null) { - partitionCache.invalidateDb(dbName); - } - if (formatCache != null) { - formatCache.invalidateDb(dbName); - } - if (commentCache != null) { - commentCache.invalidateDb(dbName); - } - if (mvccPartitionViewCache != null) { - mvccPartitionViewCache.invalidateDb(dbName); - } - if (listPartitionsViewCache != null) { - listPartitionsViewCache.invalidateDb(dbName); - } + metaCache.invalidateDatabase(dbName); } /** @@ -722,28 +685,8 @@ public void invalidateDb(String dbName) { */ @Override public void invalidateAll() { - if (latestSnapshotCache != null) { - latestSnapshotCache.invalidateAll(); - } - if (tableCache != null) { - tableCache.invalidateAll(); - } - if (partitionCache != null) { - partitionCache.invalidateAll(); - } - if (formatCache != null) { - formatCache.invalidateAll(); - } - if (commentCache != null) { - commentCache.invalidateAll(); - } - if (mvccPartitionViewCache != null) { - mvccPartitionViewCache.invalidateAll(); - } - if (listPartitionsViewCache != null) { - listPartitionsViewCache.invalidateAll(); - } - manifestCache.invalidateAll(); + metaCache.invalidateCatalog(); + manifestCache.clearStats(); } /** @@ -1689,7 +1632,8 @@ public synchronized void close() throws IOException { if (tableCache != null) { tableCache.close(); } - invalidateAll(); + manifestCache.clearStats(); + metaCache.close(); Catalog c = icebergCatalog; BaseViewSessionCatalog sc = restSessionCatalog; icebergCatalog = null; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java index cb07b3e57da124..7b7c4b1cf4f2a8 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java @@ -18,13 +18,14 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; -import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import java.util.Objects; -import java.util.concurrent.ForkJoinPool; import java.util.function.Supplier; /** @@ -88,18 +89,25 @@ public int hashCode() { } } - private final MetaCacheEntry entry; + private final CatalogMetaCache owner; + private final MetaCache entry; IcebergFormatCache(long ttlSeconds, int maxSize) { + this(new CatalogMetaCache(), ttlSeconds, maxSize); + } + + IcebergFormatCache(CatalogMetaCache owner, long ttlSeconds, int maxSize) { + this.owner = owner; // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); - this.entry = new MetaCacheEntry<>("iceberg-format", null, spec, - ForkJoinPool.commonPool(), false, true, 0L, true); + this.entry = owner.create(MetaCacheDefinition + .builder("iceberg-format", spec, IcebergFormatCache::scope) + .build()); } /** Caching is on only when the TTL is positive; ttl-second <= 0 means "always infer live". */ boolean isEnabled() { - return entry.stats().isEffectiveEnabled(); + return entry.isEnabled(); } /** @@ -114,29 +122,30 @@ String getOrLoad(Key key, Supplier loader) { /** Drops every cached snapshot entry for one table so the next read infers live (REFRESH TABLE). */ void invalidate(TableIdentifier id) { - entry.invalidateIf(key -> key.id.equals(id)); + owner.invalidateTable(id.namespace().toString(), id.name()); } /** Drops every cached entry for one database (REFRESH DATABASE / DROP DATABASE); db match = namespace equality. */ void invalidateDb(String dbName) { - Namespace ns = Namespace.of(dbName); - entry.invalidateIf(key -> key.id.namespace().equals(ns)); + owner.invalidateDatabase(dbName); } /** Drops all cached entries (REFRESH CATALOG). */ void invalidateAll() { - entry.invalidateAll(); + owner.invalidateCatalog(); } /** Test-only: current number of cached entries (accurate map membership, not Caffeine's estimate). */ int size() { - int[] count = {0}; - entry.forEach((key, value) -> count[0]++); - return count[0]; + return Math.toIntExact(entry.size()); } /** Test-only: how many times the live loader (the whole-table format inference) actually ran — the metric gate. */ long loadCountForTest() { - return entry.stats().getLoadSuccessCount(); + return entry.loadSuccessCount(); + } + + private static ScopePath scope(Key key) { + return ScopePath.table(key.id.namespace().toString(), key.id.name()); } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java index 8c98d57a46e7a7..49afd5111daca0 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java @@ -18,13 +18,14 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartitionView; -import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; -import java.util.concurrent.ForkJoinPool; import java.util.function.Supplier; /** @@ -72,18 +73,26 @@ static final class CachedSnapshot { } } - private final MetaCacheEntry entry; + private final CatalogMetaCache owner; + private final MetaCache entry; IcebergLatestSnapshotCache(long ttlSeconds, int maxSize) { + this(new CatalogMetaCache(), ttlSeconds, maxSize); + } + + IcebergLatestSnapshotCache(CatalogMetaCache owner, long ttlSeconds, int maxSize) { + this.owner = owner; // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); - this.entry = new MetaCacheEntry<>("iceberg-latest-snapshot", null, spec, - ForkJoinPool.commonPool(), false, true, 0L, true); + this.entry = owner.create(MetaCacheDefinition + .builder( + "iceberg-latest-snapshot", spec, IcebergLatestSnapshotCache::scope) + .build()); } /** Caching is on only when the TTL is positive; ttl-second <= 0 means "always read live". */ boolean isEnabled() { - return entry.stats().isEffectiveEnabled(); + return entry.isEnabled(); } /** @@ -99,7 +108,7 @@ CachedSnapshot getOrLoad(TableIdentifier identifier, Supplier lo /** Drops the cached entry for one table so the next read goes live (REFRESH TABLE). */ void invalidate(TableIdentifier identifier) { - entry.invalidateKey(identifier); + owner.invalidateTable(identifier.namespace().toString(), identifier.name()); } /** @@ -109,19 +118,20 @@ void invalidate(TableIdentifier identifier) { * {@code IcebergConnectorMetadata.beginQuerySnapshot}), so a db match is namespace equality. */ void invalidateDb(String dbName) { - Namespace ns = Namespace.of(dbName); - entry.invalidateIf(id -> id.namespace().equals(ns)); + owner.invalidateDatabase(dbName); } /** Drops all cached entries. */ void invalidateAll() { - entry.invalidateAll(); + owner.invalidateCatalog(); } /** Test-only: current number of cached entries (accurate map membership, not Caffeine's estimate). */ int size() { - int[] count = {0}; - entry.forEach((key, value) -> count[0]++); - return count[0]; + return Math.toIntExact(entry.size()); + } + + private static ScopePath scope(TableIdentifier identifier) { + return ScopePath.table(identifier.namespace().toString(), identifier.name()); } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java index 7790d43062b5aa..71fea169d5d23b 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java @@ -18,7 +18,10 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; @@ -37,7 +40,6 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; import java.util.function.LongSupplier; import java.util.function.Supplier; @@ -73,8 +75,9 @@ final class IcebergManifestCache { /** Leak backstop for the per-scan stats stash (see {@link #statsByQuery}); far above any live entry's life. */ private static final long DEFAULT_STATS_TTL_SECONDS = 300L; - private final MetaCacheEntry entry; - private final MetaCacheEntry> equalityDeleteFieldIds; + private final CatalogMetaCache owner; + private final MetaCache entry; + private final MetaCache> equalityDeleteFieldIds; /** Immutable snapshot key for the compact equality-delete field-id projection. */ private static final class SnapshotKey { @@ -103,7 +106,6 @@ public int hashCode() { return Objects.hash(tableLocation, snapshotId); } } - // Per-scan manifest-cache access tally, keyed by the statement's stable queryId // (ConnectorSession.getQueryId()), so VERBOSE EXPLAIN can report THIS scan's hits/misses/failures (the // "manifest cache:" line). The provider that PLANS the scan and the (transient, fresh-per-call) provider that @@ -128,21 +130,39 @@ private static final class ScanStats { } IcebergManifestCache() { - this(DEFAULT_MANIFEST_CACHE_CAPACITY); + this(new CatalogMetaCache(), DEFAULT_MANIFEST_CACHE_CAPACITY); + } + + IcebergManifestCache(CatalogMetaCache owner) { + this(owner, DEFAULT_MANIFEST_CACHE_CAPACITY); } IcebergManifestCache(int maxSize) { - this(maxSize, DEFAULT_STATS_TTL_SECONDS, System::nanoTime); + this(new CatalogMetaCache(), maxSize); + } + + private IcebergManifestCache(CatalogMetaCache owner, int maxSize) { + this(owner, maxSize, DEFAULT_STATS_TTL_SECONDS, System::nanoTime); } /** Visible for testing: injectable stats TTL + clock so the leak sweep is deterministic without sleeping. */ IcebergManifestCache(int maxSize, long statsTtlSeconds, LongSupplier nanoClock) { + this(new CatalogMetaCache(), maxSize, statsTtlSeconds, nanoClock); + } + + private IcebergManifestCache( + CatalogMetaCache owner, int maxSize, long statsTtlSeconds, LongSupplier nanoClock) { + this.owner = owner; // Always enabled, no expiry, capacity-bounded (CACHE_NO_TTL == -1 means "no expiration", enabled). CacheSpec spec = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, Math.max(1, maxSize)); - this.entry = new MetaCacheEntry<>("iceberg-manifest", null, spec, - ForkJoinPool.commonPool(), false, true, 0L, true); - this.equalityDeleteFieldIds = new MetaCacheEntry<>("iceberg-equality-delete-field-ids", null, spec, - ForkJoinPool.commonPool(), false, true, 0L, true); + this.entry = owner.create(MetaCacheDefinition + .builder( + "iceberg-manifest", spec, ignored -> ScopePath.catalog()) + .build()); + this.equalityDeleteFieldIds = owner.create(MetaCacheDefinition + .>builder( + "iceberg-equality-delete-field-ids", spec, ignored -> ScopePath.catalog()) + .build()); this.statsTtlNanos = TimeUnit.SECONDS.toNanos(Math.max(1L, statsTtlSeconds)); this.nanoClock = nanoClock; } @@ -271,15 +291,16 @@ private static List loadDeleteFiles(ManifestFile manifest, Table tab * (catalog-wide invalidation); table-level invalidation (REFRESH TABLE) intentionally does not. */ void invalidateAll() { - entry.invalidateAll(); - equalityDeleteFieldIds.invalidateAll(); + owner.invalidateCatalog(); + statsByQuery.clear(); + } + + void clearStats() { statsByQuery.clear(); } /** Test-only: current number of cached entries (accurate map membership, not Caffeine's estimate). */ int size() { - int[] count = {0}; - entry.forEach((key, value) -> count[0]++); - return count[0]; + return Math.toIntExact(entry.size()); } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java index a85e2187043e7c..679758091fd25d 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java @@ -18,15 +18,16 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.iceberg.IcebergPartitionUtils.IcebergRawPartition; -import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import java.util.List; import java.util.Objects; -import java.util.concurrent.ForkJoinPool; import java.util.function.Supplier; /** @@ -85,18 +86,25 @@ public int hashCode() { } } - private final MetaCacheEntry> entry; + private final CatalogMetaCache owner; + private final MetaCache> entry; IcebergPartitionCache(long ttlSeconds, int maxSize) { + this(new CatalogMetaCache(), ttlSeconds, maxSize); + } + + IcebergPartitionCache(CatalogMetaCache owner, long ttlSeconds, int maxSize) { + this.owner = owner; // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); - this.entry = new MetaCacheEntry<>("iceberg-partition", null, spec, - ForkJoinPool.commonPool(), false, true, 0L, true); + this.entry = owner.create(MetaCacheDefinition + .>builder("iceberg-partition", spec, IcebergPartitionCache::scope) + .build()); } /** Caching is on only when the TTL is positive; ttl-second <= 0 means "always scan live". */ boolean isEnabled() { - return entry.stats().isEffectiveEnabled(); + return entry.isEnabled(); } /** @@ -110,29 +118,30 @@ List getOrLoad(Key key, Supplier> /** Drops every cached snapshot entry for one table so the next read scans live (REFRESH TABLE). */ void invalidate(TableIdentifier id) { - entry.invalidateIf(key -> key.id.equals(id)); + owner.invalidateTable(id.namespace().toString(), id.name()); } /** Drops every cached entry for one database (REFRESH DATABASE / DROP DATABASE); db match = namespace equality. */ void invalidateDb(String dbName) { - Namespace ns = Namespace.of(dbName); - entry.invalidateIf(key -> key.id.namespace().equals(ns)); + owner.invalidateDatabase(dbName); } /** Drops all cached entries (REFRESH CATALOG). */ void invalidateAll() { - entry.invalidateAll(); + owner.invalidateCatalog(); } /** Test-only: current number of cached entries (accurate map membership, not Caffeine's estimate). */ int size() { - int[] count = {0}; - entry.forEach((key, value) -> count[0]++); - return count[0]; + return Math.toIntExact(entry.size()); } /** Test-only: how many times the live loader (the PARTITIONS scan) actually ran — the metric gate. */ long loadCountForTest() { - return entry.stats().getLoadSuccessCount(); + return entry.loadSuccessCount(); + } + + private static ScopePath scope(Key key) { + return ScopePath.table(key.id.namespace().toString(), key.id.name()); } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java index f13c24435252f2..6e9ad1e80c555c 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java @@ -18,13 +18,14 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; import org.apache.iceberg.Table; -import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; -import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; @@ -39,7 +40,7 @@ * lets consecutive queries — and the analysis/planning phases of one query, whose handles have distinct memo * lineages — reuse a single loaded table, exactly as the legacy with-cache catalog did. * - *

    Backing. Reuses the shared {@link MetaCacheEntry} framework identically to + *

    Backing. Reuses the shared {@link MetaCache} framework identically to * {@link IcebergLatestSnapshotCache}: a contextual, access-TTL entry whose per-key loader is supplied at * {@link #getOrLoad}, with manual miss-load on so the loader runs OUTSIDE Caffeine's compute lock * (single-flight per key) and propagates its exception verbatim (a concurrent-drop @@ -61,33 +62,41 @@ */ final class IcebergTableCache { - private final MetaCacheEntry entry; + private final CatalogMetaCache owner; + private final MetaCache entry; private final Function cleanupFactory; private final IcebergCatalogResourceTracker resourceTracker; private final AtomicBoolean closed = new AtomicBoolean(); IcebergTableCache(long ttlSeconds, int maxSize) { - this(ttlSeconds, maxSize, table -> () -> { }, null); + this(new CatalogMetaCache(), ttlSeconds, maxSize, table -> () -> { }, null); } IcebergTableCache(long ttlSeconds, int maxSize, Function cleanupFactory) { - this(ttlSeconds, maxSize, cleanupFactory, null); + this(new CatalogMetaCache(), ttlSeconds, maxSize, cleanupFactory, null); } IcebergTableCache(long ttlSeconds, int maxSize, Function cleanupFactory, IcebergCatalogResourceTracker resourceTracker) { + this(new CatalogMetaCache(), ttlSeconds, maxSize, cleanupFactory, resourceTracker); + } + + IcebergTableCache(CatalogMetaCache owner, long ttlSeconds, int maxSize, + Function cleanupFactory, IcebergCatalogResourceTracker resourceTracker) { + this.owner = owner; this.cleanupFactory = cleanupFactory; this.resourceTracker = resourceTracker; // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); - this.entry = new MetaCacheEntry<>("iceberg-table", null, spec, - ForkJoinPool.commonPool(), false, true, 0L, true, - (identifier, owner) -> owner.release()); + this.entry = owner.create(MetaCacheDefinition + .builder("iceberg-table", spec, IcebergTableCache::scope) + .removalListener((identifier, tableOwner, reason) -> tableOwner.release()) + .build()); } /** Caching is on only when the TTL is positive; ttl-second <= 0 means "always read live". */ boolean isEnabled() { - return entry.stats().isEffectiveEnabled(); + return entry.isEnabled(); } /** @@ -158,7 +167,7 @@ Table getOrLoad(TableIdentifier identifier, Supplier loader) { /** Drops the cached entry for one table so the next read goes live (REFRESH TABLE). */ void invalidate(TableIdentifier identifier) { - entry.invalidateKey(identifier); + owner.invalidateTable(identifier.namespace().toString(), identifier.name()); } /** @@ -168,27 +177,28 @@ void invalidate(TableIdentifier identifier) { * namespace equality — mirroring {@link IcebergLatestSnapshotCache#invalidateDb}. */ void invalidateDb(String dbName) { - Namespace ns = Namespace.of(dbName); - entry.invalidateIf(id -> id.namespace().equals(ns)); + owner.invalidateDatabase(dbName); } /** Drops all cached entries. */ void invalidateAll() { - entry.invalidateAll(); + owner.invalidateCatalog(); } /** Seals new borrows and retires every cache owner, including loaders that publish after invalidation. */ void close() { if (closed.compareAndSet(false, true)) { - entry.invalidateAll(); + owner.invalidateCatalog(); } } /** Test-only: current number of cached entries (accurate map membership, not Caffeine's estimate). */ int size() { - int[] count = {0}; - entry.forEach((key, value) -> count[0]++); - return count[0]; + return Math.toIntExact(entry.size()); + } + + private static ScopePath scope(TableIdentifier identifier) { + return ScopePath.table(identifier.namespace().toString(), identifier.name()); } static final class TableLease implements AutoCloseable { diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCacheTest.java index c592472ffffb08..5815bd94716cfa 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCacheTest.java @@ -25,7 +25,7 @@ /** * Unit tests for {@link IcebergLatestSnapshotCache} (mirrors PaimonLatestSnapshotCacheTest). The cache is now - * backed by the shared {@link org.apache.doris.connector.cache.MetaCacheEntry} framework; these tests cover the + * backed by the shared {@link org.apache.doris.connector.cache.MetaCache} framework; these tests cover the * adapter's contract — within-TTL stability, the {@code ttl <= 0} disable, and invalidation. Timed-expiry * mechanics are the framework's responsibility (the ttl→duration mapping is unit-tested in the framework * module's {@code CacheSpecTest}; Caffeine {@code expireAfterAccess} itself is the library's behavior), so they diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java index 31f0131d0bbf1a..ac265c04e517a3 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java @@ -34,7 +34,7 @@ /** * Unit tests for {@link IcebergTableCache} (PERF-01). The cross-query RAW-table cache mirrors - * {@link IcebergLatestSnapshotCache} exactly (same {@link org.apache.doris.connector.cache.MetaCacheEntry} + * {@link IcebergLatestSnapshotCache} exactly (same {@link org.apache.doris.connector.cache.MetaCache} * backing) but stores the whole {@link Table} instead of the {@code (snapshotId, schemaId)} pin, restoring the * table-caching half of the legacy {@code IcebergExternalMetaCache}. These tests cover the adapter's contract — * within-TTL stability, the {@code ttl <= 0} disable, invalidation, and the exception-propagation guarantee the diff --git a/fe/fe-connector/fe-connector-maxcompute/pom.xml b/fe/fe-connector/fe-connector-maxcompute/pom.xml index 8ff6cbd0e580be..58f4bbe5479978 100644 --- a/fe/fe-connector/fe-connector-maxcompute/pom.xml +++ b/fe/fe-connector/fe-connector-maxcompute/pom.xml @@ -52,20 +52,15 @@ under the License. ${project.version} - + ${project.groupId} fe-connector-cache ${project.version} - + ${project.groupId} fe-connector-cache ${project.version} -