From 8d34adbda1da5807d73714fc1458c6af100e6bcf Mon Sep 17 00:00:00 2001 From: Vladimir Rodionov Date: Tue, 25 Aug 2026 20:21:13 -0700 Subject: [PATCH] HBASE-30329 Add topology-backed cache diagnostics --- .../hadoop/hbase/io/hfile/CacheConfig.java | 52 ++- .../io/hfile/cache/CacheAccessServices.java | 48 +-- .../io/hfile/cache/CacheTopologyType.java | 2 +- ...ultHBaseCachePlacementAdmissionPolicy.java | 5 +- .../io/hfile/cache/SingleEngineTopology.java | 2 +- .../io/hfile/cache/SingleTierTopology.java | 171 ++++++++ .../TopologyBackedCacheAccessService.java | 132 +++++- .../TopologyBackedCacheAccessServices.java | 150 +++++++ .../hbase/io/hfile/TestCacheConfig.java | 7 +- .../hbase/io/hfile/TestCacheOnWrite.java | 10 +- .../cache/CacheAccessServiceTestFactory.java | 6 +- ...estBlockCacheBackedCacheAccessService.java | 246 ----------- .../hfile/cache/TestCacheAccessServices.java | 2 +- ...tibleTopologyBackedCacheAccessService.java | 356 ++++++++++++++++ ...eTierTopologyBackedCacheAccessService.java | 400 ++++++++++++++++++ 15 files changed, 1276 insertions(+), 313 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleTierTopology.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestBlockCacheBackedCacheAccessService.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestInclusiveCombinedBlockCacheCompatibleTopologyBackedCacheAccessService.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestSingleTierTopologyBackedCacheAccessService.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index fa69b77f5ecb..589a97bb3549 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.io.hfile; +import java.util.Objects; import java.util.Optional; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -24,9 +25,12 @@ import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver; import org.apache.hadoop.hbase.io.ByteBuffAllocator; import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; -import org.apache.hadoop.hbase.io.hfile.cache.BlockCacheBackedCacheAccessService; +import org.apache.hadoop.hbase.io.hfile.cache.BlockCacheBackedCacheEngine; import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService; import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServices; +import org.apache.hadoop.hbase.io.hfile.cache.CacheEngine; +import org.apache.hadoop.hbase.io.hfile.cache.CacheTier; +import org.apache.hadoop.hbase.io.hfile.cache.CacheTopology; import org.apache.hadoop.hbase.io.hfile.cache.CacheTopologyType; import org.apache.hadoop.hbase.io.hfile.cache.TopologyBackedCacheAccessService; import org.apache.yetus.audience.InterfaceAudience; @@ -221,12 +225,52 @@ public CacheConfig(Configuration conf, ColumnFamilyDescriptor family, CacheAcces initFromConf(conf, family); this.byteBuffAllocator = byteBuffAllocator; this.cacheAccessService = service != null ? service : CacheAccessServices.disabled(); - this.blockCache = service instanceof BlockCacheBackedCacheAccessService - ? ((BlockCacheBackedCacheAccessService) service).getBlockCache() - : null; + /* + * Preserve the legacy BlockCache reference only when the supplied service is a direct + * single-tier BlockCache adapter. Multi-tier combined caches are represented by + * TopologyBackedCacheAccessService and intentionally do not expose a single legacy BlockCache + * through this field. Callers that need cache behavior or diagnostics should use + * cacheAccessService-level capabilities instead of relying on this legacy field. + */ + this.blockCache = unwrapSingleLegacyBlockCache(this.cacheAccessService); } + /** + * Extracts the legacy {@link BlockCache} from a cache access service when that service is backed + * by a single {@link BlockCacheBackedCacheEngine}. + *

+ * This method exists only to preserve compatibility with legacy {@link CacheConfig} callers that + * still use the {@link BlockCache} field. The active cache access path remains + * {@link CacheAccessService}. Code that needs cache behavior or diagnostics should use + * {@link CacheAccessService} capabilities instead of depending on the legacy block cache field. + *

+ * @param cacheAccessService cache access service to inspect + * @return wrapped legacy block cache when available; otherwise {@code null} + * @throws NullPointerException if {@code cacheAccessService} is {@code null} + */ + private static BlockCache unwrapSingleLegacyBlockCache(CacheAccessService cacheAccessService) { + Objects.requireNonNull(cacheAccessService, "cacheAccessService must not be null"); + + if (!(cacheAccessService instanceof TopologyBackedCacheAccessService)) { + return null; + } + + TopologyBackedCacheAccessService topologyBackedService = + (TopologyBackedCacheAccessService) cacheAccessService; + CacheTopology topology = topologyBackedService.getTopology(); + + if (topology.getType() != CacheTopologyType.SINGLE_TIER) { + return null; + } + + Optional engine = topology.getEngine(CacheTier.L1); + if (!engine.isPresent() || !(engine.get() instanceof BlockCacheBackedCacheEngine)) { + return null; + } + return ((BlockCacheBackedCacheEngine) engine.get()).getBlockCache(); + } + /** * Create a cache configuration using the specified configuration object and family descriptor. * @param conf hbase configuration diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServices.java index a020323737a4..344cfc19b501 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServices.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory; import org.apache.hadoop.hbase.io.hfile.CachedBlock; import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache; +import org.apache.hadoop.hbase.io.hfile.InclusiveCombinedBlockCache; import org.apache.yetus.audience.InterfaceAudience; /** @@ -47,26 +48,37 @@ private CacheAccessServices() { } /** - * Creates a cache access service backed by an existing block cache. + * Creates a {@link CacheAccessService} for the supplied legacy {@link BlockCache}. *

- * For regular {@link BlockCache} implementations, this returns a legacy - * {@link BlockCacheBackedCacheAccessService}. For {@link CombinedBlockCache}, this returns a - * topology-backed service using {@link TieredExclusiveTopology}. This moves combined L1/L2 - * orchestration to the new topology layer while keeping the existing combined block cache object - * available for legacy {@link BlockCache}-facing APIs. + * All legacy block caches are adapted through {@link TopologyBackedCacheAccessService}. Plain + * single-tier block caches are represented by {@link SingleTierTopology}. Exclusive combined + * caches are represented by {@link TieredExclusiveTopology}. Inclusive combined caches are + * represented by {@link TieredInclusiveTopology}. *

- * @param blockCache block cache to expose through {@link CacheAccessService} - * @return cache access service + *

+ * {@link InclusiveCombinedBlockCache} is checked before {@link CombinedBlockCache} because the + * inclusive variant has different residency, promotion, and eviction semantics. Routing it + * through the exclusive topology would be incorrect. + *

+ * @param blockCache legacy block cache to adapt + * @return topology-backed cache access service for the supplied block cache + * @throws NullPointerException if {@code blockCache} is {@code null} */ - public static CacheAccessService fromBlockCache(BlockCache blockCache) { Objects.requireNonNull(blockCache, "blockCache must not be null"); + + if (blockCache instanceof InclusiveCombinedBlockCache) { + return TopologyBackedCacheAccessServices + .fromInclusiveCombinedBlockCache((InclusiveCombinedBlockCache) blockCache); + } + if (blockCache instanceof CombinedBlockCache) { return TopologyBackedCacheAccessServices .fromCombinedBlockCache((CombinedBlockCache) blockCache); } - return new BlockCacheBackedCacheAccessService(blockCache); + return TopologyBackedCacheAccessServices.fromSingleBlockCache("single", blockCache, + DefaultHBaseCachePlacementAdmissionPolicy.INSTANCE); } /** @@ -80,7 +92,7 @@ public static CacheAccessService fromBlockCache(BlockCache blockCache) { * The method delegates block-cache construction to * {@link BlockCacheFactory#createBlockCache(Configuration)}. If the legacy factory creates a * {@link BlockCache}, the returned service is backed by that cache through - * {@link BlockCacheBackedCacheAccessService}. If the legacy factory does not create a cache, this + * {@link TopologyBackedCacheAccessService}. If the legacy factory does not create a cache, this * method returns the disabled/no-op cache access service. *

*

@@ -140,15 +152,6 @@ public static CacheAccessService disabled() { * @return optional iterable cached-block view * @throws NullPointerException if {@code cacheAccessService} is {@code null} */ - @SuppressWarnings("unchecked") - // public static Optional> - // asCachedBlockIterable(CacheAccessService cacheAccessService) { - // Objects.requireNonNull(cacheAccessService, "cacheAccessService must not be null"); - // if (cacheAccessService instanceof Iterable) { - // return Optional.of((Iterable) cacheAccessService); - // } - // return Optional.empty(); - // } public static Optional> asCachedBlockIterable(CacheAccessService service) { Objects.requireNonNull(service, "service must not be null"); @@ -156,11 +159,6 @@ public static Optional> asCachedBlockIterable(CacheAccessS if (service instanceof TopologyBackedCacheAccessService) { return ((TopologyBackedCacheAccessService) service).asCachedBlockIterable(); } - - if (service instanceof BlockCacheBackedCacheAccessService) { - return Optional.of(((BlockCacheBackedCacheAccessService) service).getBlockCache()); - } - return Optional.empty(); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheTopologyType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheTopologyType.java index 50ff46d5bde7..6a8cac5032e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheTopologyType.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheTopologyType.java @@ -28,7 +28,7 @@ public enum CacheTopologyType { /** * A topology with a single cache engine. */ - SINGLE, + SINGLE_TIER, /** * A tiered topology where a block normally resides in only one tier at a time. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/DefaultHBaseCachePlacementAdmissionPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/DefaultHBaseCachePlacementAdmissionPolicy.java index 039c620a33d3..91218aa187bd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/DefaultHBaseCachePlacementAdmissionPolicy.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/DefaultHBaseCachePlacementAdmissionPolicy.java @@ -36,6 +36,9 @@ @InterfaceAudience.Private public class DefaultHBaseCachePlacementAdmissionPolicy implements CachePlacementAdmissionPolicy { + public static final DefaultHBaseCachePlacementAdmissionPolicy INSTANCE = + new DefaultHBaseCachePlacementAdmissionPolicy(); + @Override public AdmissionDecision shouldAdmit(BlockCacheKey cacheKey, Cacheable block, CacheWriteContext context, AdmissionPriority priority, CacheTopologyView topologyView) { @@ -60,7 +63,7 @@ public TierDecision selectTier(BlockCacheKey cacheKey, Cacheable block, CacheWri * L2 when both tiers are available, but falls back to any available tier rather than rejecting * placement. */ - if (topologyView.getType() == CacheTopologyType.SINGLE) { + if (topologyView.getType() == CacheTopologyType.SINGLE_TIER) { return TierDecision.single(CacheTier.SINGLE); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleEngineTopology.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleEngineTopology.java index c94deabdae5a..1ebcf77afbb3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleEngineTopology.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleEngineTopology.java @@ -52,7 +52,7 @@ public String getName() { @Override public CacheTopologyType getType() { - return CacheTopologyType.SINGLE; + return CacheTopologyType.SINGLE_TIER; } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleTierTopology.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleTierTopology.java new file mode 100644 index 000000000000..57e910e994ea --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleTierTopology.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile.cache; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; +import org.apache.hadoop.hbase.io.hfile.CacheStats; +import org.apache.hadoop.hbase.io.hfile.Cacheable; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Single-tier cache topology. + *

+ * A single-tier topology contains exactly one cache engine. It is used to represent legacy + * single-tier {@code BlockCache} implementations inside the topology-backed cache access framework. + * Unlike tiered topologies, this topology does not perform tier orchestration, promotion, or + * demotion. All cache operations are directed to the single L1 engine. + *

+ *

+ * This topology allows plain block caches to use the same {@link TopologyBackedCacheAccessService} + * path as combined caches while preserving the existing cache implementation underneath. + *

+ */ +@InterfaceAudience.Private +public class SingleTierTopology implements CacheTopology { + + private final String name; + private final CacheEngine engine; + private final CacheTopologyView view; + + /** + * Creates a single-tier cache topology. + * @param name topology name used for diagnostics + * @param engine cache engine backing the single tier + */ + public SingleTierTopology(String name, CacheEngine engine) { + this.name = name; + this.engine = engine; + this.view = new CacheTopologyView(this); + } + + /** + * Returns the diagnostic name of this topology. + * @return topology name + */ + @Override + public String getName() { + return name; + } + + /** + * Returns the topology type. + * @return {@link CacheTopologyType#SINGLE_TIER} + */ + @Override + public CacheTopologyType getType() { + return CacheTopologyType.SINGLE_TIER; + } + + /** + * Returns the cache engines that participate in this topology. + * @return singleton list containing the single cache engine + */ + @Override + public List getEngines() { + return Collections.singletonList(engine); + } + + /** + * Returns the tiers available in this topology. + * @return singleton list containing {@link CacheTier#L1} + */ + @Override + public List getTiers() { + return Collections.singletonList(CacheTier.L1); + } + + /** + * Returns the cache engine associated with the requested tier. + * @param tier cache tier to resolve + * @return the single cache engine for {@link CacheTier#L1}; otherwise {@link Optional#empty()} + */ + @Override + public Optional getEngine(CacheTier tier) { + switch (tier) { + case L1: + return Optional.of(engine); + default: + return Optional.empty(); + } + } + + /** + * Returns the topology view associated with this topology. + * @return topology view + */ + @Override + public CacheTopologyView getView() { + return view; + } + + /** + * Returns cache statistics for the single cache engine. + * @return cache statistics exposed by the backing engine + */ + @Override + public CacheStats getStats() { + return engine.getStats(); + } + + /** + * Attempts to promote a block within this topology. + *

+ * Single-tier topology has no higher or lower tier, so promotion is not supported. The method + * returns {@code false} without modifying the cache. + *

+ * @param cacheKey cache key identifying the block + * @param block cached block + * @param sourceEngine source cache engine + * @param targetEngine target cache engine + * @return {@code false} because single-tier topology does not support promotion + */ + @Override + public boolean promote(BlockCacheKey cacheKey, Cacheable block, CacheEngine sourceEngine, + CacheEngine targetEngine) { + return false; + } + + /** + * Attempts to demote a block within this topology. + *

+ * Single-tier topology has no lower tier, so demotion is not supported. The method returns + * {@code false} without modifying the cache. + *

+ * @param cacheKey cache key identifying the block + * @param block cached block + * @param sourceEngine source cache engine + * @param targetEngine target cache engine + * @return {@code false} because single-tier topology does not support demotion + */ + @Override + public boolean demote(BlockCacheKey cacheKey, Cacheable block, CacheEngine sourceEngine, + CacheEngine targetEngine) { + return false; + } + + /** + * Shuts down the single cache engine. + */ + @Override + public void shutdown() { + engine.shutdown(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessService.java index 828368274e43..7aa540cdf060 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessService.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessService.java @@ -18,12 +18,14 @@ package org.apache.hadoop.hbase.io.hfile.cache; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; -import java.util.stream.StreamSupport; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; @@ -48,19 +50,13 @@ * operations. *

*

- * This class is the topology-backed counterpart to {@link BlockCacheBackedCacheAccessService}. The - * block-cache-backed implementation is useful for incremental migration with no behavior change. - * This implementation is useful once callers are ready to exercise the new topology and engine - * abstractions directly through {@link CacheAccessService}. - *

- *

* Representation selection is intentionally not invoked by this initial implementation. Until the * service can actually apply representation decisions safely, especially around HFileBlock * lifecycle and packed/unpacked storage, representation policy is left to a later integration step. *

*/ @InterfaceAudience.Private -public class TopologyBackedCacheAccessService implements CacheAccessService { +public class TopologyBackedCacheAccessService implements CacheAccessService, Iterable { private final CacheTopology topology; private final CachePlacementAdmissionPolicy policy; @@ -130,7 +126,6 @@ public String getName() { * @return cached block, or {@code null} if not present in any tier */ @Override - public Cacheable getBlock(BlockCacheKey cacheKey, CacheRequestContext context) { Objects.requireNonNull(cacheKey, "cacheKey must not be null"); Objects.requireNonNull(context, "context must not be null"); @@ -214,23 +209,79 @@ private void updateBlockMetrics(Cacheable block, BlockCacheKey key, CacheEngine } /** - * Adds a block to the cache using policy-selected target tiers. + * Caches a block using the topology-backed cache access service. *

- * This method first asks the configured policy whether the block should be admitted. If admitted, - * the policy selects the target tier or tiers. The block is then inserted into each selected - * engine using {@link CacheEngine#cacheBlock(BlockCacheKey, Cacheable, boolean, boolean)}. + * Single-tier topology preserves the legacy direct block-cache behavior by writing admitted + * blocks to the only backing engine. Tiered topologies use the placement policy to select one or + * more target tiers. *

+ * @param cacheKey cache key identifying the block + * @param block block to cache + * @param context cache write context + * @throws NullPointerException if {@code cacheKey}, {@code block}, or {@code context} is + * {@code null} + */ + @Override + public void cacheBlock(BlockCacheKey cacheKey, Cacheable block, CacheWriteContext context) { + Objects.requireNonNull(cacheKey, "cacheKey must not be null"); + Objects.requireNonNull(block, "block must not be null"); + Objects.requireNonNull(context, "context must not be null"); + + if (topology.getType() == CacheTopologyType.SINGLE_TIER) { + cacheBlockToSingleTier(cacheKey, block, context); + return; + } + + cacheBlockToSelectedTiers(cacheKey, block, context); + } + + /** + * Caches a block into the only engine in a single-tier topology. *

- * The policy's representation decision is intentionally not applied in this initial - * implementation. The current block object is passed through unchanged. + * This method preserves the behavior of the legacy {@link BlockCacheBackedCacheAccessService} + * path. A single-tier topology has no placement decision to make: admitted blocks are written to + * the only available engine, which is exposed as {@link CacheTier#L1}. *

- * @param cacheKey block cache key - * @param block block contents + * @param cacheKey cache key identifying the block + * @param block block to cache * @param context cache write context + * @throws NullPointerException if {@code cacheKey}, {@code block}, or {@code context} is + * {@code null} */ + private void cacheBlockToSingleTier(BlockCacheKey cacheKey, Cacheable block, + CacheWriteContext context) { + Objects.requireNonNull(cacheKey, "cacheKey must not be null"); + Objects.requireNonNull(block, "block must not be null"); + Objects.requireNonNull(context, "context must not be null"); - @Override - public void cacheBlock(BlockCacheKey cacheKey, Cacheable block, CacheWriteContext context) { + AdmissionDecision admission = + policy.shouldAdmit(cacheKey, block, context, AdmissionPriority.NORMAL, topologyView); + if (!admission.isAdmitted()) { + return; + } + + Optional engine = topology.getEngine(CacheTier.L1); + if (!engine.isPresent()) { + return; + } + + engine.get().cacheBlock(cacheKey, block, context.isInMemory(), context.isWaitWhenCache()); + } + + /** + * Caches a block into the tiers selected by the placement policy. + *

+ * This method is used for tiered topologies where the placement policy decides whether the block + * belongs in L1, L2, or multiple tiers. + *

+ * @param cacheKey cache key identifying the block + * @param block block to cache + * @param context cache write context + * @throws NullPointerException if {@code cacheKey}, {@code block}, or {@code context} is + * {@code null} + */ + private void cacheBlockToSelectedTiers(BlockCacheKey cacheKey, Cacheable block, + CacheWriteContext context) { Objects.requireNonNull(cacheKey, "cacheKey must not be null"); Objects.requireNonNull(block, "block must not be null"); Objects.requireNonNull(context, "context must not be null"); @@ -626,6 +677,22 @@ public void notifyFileCachingCompleted(Path path, int blockCount, int dataBlockC } } + /** + * Returns an iterable view over cached blocks exposed by the cache engines in this topology. + *

+ * The returned iterable aggregates cached-block iterables from all engines that support this + * diagnostic capability. Engines that do not expose cached-block iteration are skipped. If no + * engine supports cached-block iteration, this method returns {@link Optional#empty()}. + *

+ *

+ * The aggregate iterable uses each underlying iterable's {@link Iterable#iterator()} method + * instead of {@link Iterable#spliterator()}. This is intentional because some legacy cache + * implementations and Mockito-based test doubles expose iteration through {@code iterator()} but + * may not provide a usable {@code spliterator()}. + *

+ * @return an aggregated cached-block iterable when at least one engine supports this capability; + * otherwise {@link Optional#empty()} + */ public Optional> asCachedBlockIterable() { List> iterables = new ArrayList<>(); @@ -637,9 +704,32 @@ public Optional> asCachedBlockIterable() { return Optional.empty(); } - Iterable cachedBlocks = () -> iterables.stream() - .flatMap(iterable -> StreamSupport.stream(iterable.spliterator(), false)).iterator(); + Iterable cachedBlocks = () -> new Iterator() { + private final Iterator> iterableIterator = iterables.iterator(); + private Iterator currentIterator = Collections.emptyIterator(); + + @Override + public boolean hasNext() { + while (!currentIterator.hasNext() && iterableIterator.hasNext()) { + currentIterator = iterableIterator.next().iterator(); + } + return currentIterator.hasNext(); + } + + @Override + public CachedBlock next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return currentIterator.next(); + } + }; return Optional.of(cachedBlocks); } + + @Override + public Iterator iterator() { + return asCachedBlockIterable().orElse(Collections.emptyList()).iterator(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessServices.java index 3ce586c22a78..8df55427ecf2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessServices.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache; import org.apache.hadoop.hbase.io.hfile.FirstLevelBlockCache; +import org.apache.hadoop.hbase.io.hfile.InclusiveCombinedBlockCache; import org.apache.yetus.audience.InterfaceAudience; /** @@ -110,6 +111,155 @@ public static TopologyBackedCacheAccessService fromTieredExclusiveBlockCaches(St return new TopologyBackedCacheAccessService(topology, policy); } + /** + * Creates a topology-backed cache access service for an {@link InclusiveCombinedBlockCache}. + *

+ * {@link InclusiveCombinedBlockCache} represents a two-tier inclusive cache layout. Unlike the + * exclusive {@link CombinedBlockCache} path, a block may be present in more than one tier. The + * resulting service therefore uses {@link TieredInclusiveTopology}, not + * {@link TieredExclusiveTopology}. + *

+ *

+ * The supplied legacy combined cache is used only as a source of the existing first-level and + * second-level block caches. Each tier is wrapped in a {@link BlockCacheBackedCacheEngine}, and + * the new {@link TopologyBackedCacheAccessService} performs access through the topology + * abstraction. + *

+ * @param combinedBlockCache inclusive combined block cache to adapt + * @return topology-backed cache access service using a tiered inclusive topology + * @throws NullPointerException if {@code combinedBlockCache} is {@code null} + * @throws IllegalArgumentException if the combined cache does not expose exactly two tiers + */ + public static TopologyBackedCacheAccessService + fromInclusiveCombinedBlockCache(InclusiveCombinedBlockCache combinedBlockCache) { + Objects.requireNonNull(combinedBlockCache, "combinedBlockCache must not be null"); + + BlockCache[] blockCaches = combinedBlockCache.getBlockCaches(); + if (blockCaches.length != 2) { + throw new IllegalArgumentException( + "InclusiveCombinedBlockCache must expose exactly two block caches"); + } + + return fromTieredInclusiveBlockCaches("inclusive-combined", blockCaches[0], blockCaches[1], + DefaultHBaseCachePlacementAdmissionPolicy.INSTANCE); + } + + /** + * Creates a topology-backed cache access service from two legacy block caches using an inclusive + * tiered topology. + *

+ * The first supplied block cache is treated as the L1 tier and the second supplied block cache is + * treated as the L2 tier. Both legacy caches are adapted to {@link CacheEngine} instances using + * {@link CacheEngines#fromBlockCache(BlockCache)} and then assembled into a + * {@link TieredInclusiveTopology}. + *

+ *

+ * This helper is intended for compatibility with legacy inclusive combined-cache configurations + * while moving cache access and diagnostics to the {@link CacheAccessService} abstraction. + * Inclusive topology semantics differ from exclusive topology semantics: a block may exist in + * both tiers, and eviction from one tier does not necessarily imply eviction from the other tier. + *

+ * @param name topology name used for diagnostics + * @param l1 first-level block cache + * @param l2 second-level block cache + * @param policy cache placement and admission policy to use with the topology-backed service + * @return topology-backed cache access service backed by a tiered inclusive topology + * @throws NullPointerException if {@code name}, {@code l1}, {@code l2}, or {@code policy} is + * {@code null} + */ + public static TopologyBackedCacheAccessService fromTieredInclusiveBlockCaches(String name, + BlockCache l1, BlockCache l2, CachePlacementAdmissionPolicy policy) { + Objects.requireNonNull(name, "name must not be null"); + Objects.requireNonNull(l1, "l1 must not be null"); + Objects.requireNonNull(l2, "l2 must not be null"); + Objects.requireNonNull(policy, "policy must not be null"); + + CacheEngine l1Engine = CacheEngines.fromBlockCache(l1); + CacheEngine l2Engine = CacheEngines.fromBlockCache(l2); + CacheTopology topology = new TieredInclusiveTopology(name, l1Engine, l2Engine); + + return new TopologyBackedCacheAccessService(topology, policy); + } + + /** + * Creates a topology-backed cache access service for a single legacy {@link BlockCache}. + *

+ * The supplied block cache is adapted to a {@link CacheEngine} and placed behind a + * {@link SingleTierTopology}. This makes single-tier caches use the same + * {@link TopologyBackedCacheAccessService} path as combined caches while preserving the existing + * block cache implementation underneath. + *

+ * @param name topology name used for diagnostics + * @param blockCache legacy block cache to adapt + * @param policy cache placement and admission policy + * @return topology-backed cache access service backed by a single-tier topology + * @throws NullPointerException if {@code name}, {@code blockCache}, or {@code policy} is + * {@code null} + */ + public static TopologyBackedCacheAccessService fromSingleBlockCache(String name, + BlockCache blockCache, CachePlacementAdmissionPolicy policy) { + Objects.requireNonNull(name, "name must not be null"); + Objects.requireNonNull(blockCache, "blockCache must not be null"); + Objects.requireNonNull(policy, "policy must not be null"); + + CacheEngine engine = CacheEngines.fromBlockCache(blockCache); + CacheTopology topology = new SingleTierTopology(name, engine); + return new TopologyBackedCacheAccessService(topology, policy); + } + + /** + * Returns the legacy {@link BlockCache} wrapped by the cache engine for the requested tier. + *

+ * This helper is intended for tests that need to verify compatibility with legacy block cache + * implementations during the migration to topology-backed cache access. Production code should + * prefer {@link CacheAccessService} capability methods instead of unwrapping the underlying + * {@link BlockCache}. + *

+ *

+ * The supplied service must be a {@link TopologyBackedCacheAccessService}. The requested tier + * must resolve to a {@link BlockCacheBackedCacheEngine}. If either condition is not true, this + * method fails fast with an {@link IllegalArgumentException}. + *

+ * @param cacheAccessService cache access service to inspect + * @param tier cache tier to unwrap + * @return legacy block cache wrapped by the cache engine for the requested tier + * @throws NullPointerException if {@code cacheAccessService} or {@code tier} is {@code null} + * @throws IllegalArgumentException if the service is not topology-backed, if the requested tier + * is not present, or if the tier is not backed by a + * {@link BlockCacheBackedCacheEngine} + */ + public static BlockCache getBlockCache(CacheAccessService cacheAccessService, CacheTier tier) { + Objects.requireNonNull(cacheAccessService, "cacheAccessService must not be null"); + Objects.requireNonNull(tier, "tier must not be null"); + + if (!(cacheAccessService instanceof TopologyBackedCacheAccessService)) { + throw new IllegalArgumentException( + "cacheAccessService must be a TopologyBackedCacheAccessService"); + } + + TopologyBackedCacheAccessService topologyBackedService = + (TopologyBackedCacheAccessService) cacheAccessService; + CacheTopology topology = topologyBackedService.getTopology(); + + CacheEngine engine = topology.getEngine(tier) + .orElseThrow(() -> new IllegalArgumentException("No cache engine found for tier " + tier)); + + if (!(engine instanceof BlockCacheBackedCacheEngine)) { + throw new IllegalArgumentException( + "Cache engine for tier " + tier + " must be a BlockCacheBackedCacheEngine"); + } + + return ((BlockCacheBackedCacheEngine) engine).getBlockCache(); + } + + /** + * Returns the legacy {@link BlockCache} wrapped by the cache engine for the L1 tier. + * @return legacy block cache wrapped by the cache engine for the L1 tier + */ + public static BlockCache getBlockCache(CacheAccessService cacheAccessService) { + return getBlockCache(cacheAccessService, CacheTier.L1); + } + /** * Configures the legacy L1 to L2 victim-cache relationship used by CombinedBlockCache. *

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java index 1d7286dbe6c6..377331d00888 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java @@ -43,10 +43,11 @@ import org.apache.hadoop.hbase.io.ByteBuffAllocator; import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; -import org.apache.hadoop.hbase.io.hfile.cache.BlockCacheBackedCacheAccessService; import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService; import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServiceTestFactory; import org.apache.hadoop.hbase.io.hfile.cache.NoOpCacheAccessService; +import org.apache.hadoop.hbase.io.hfile.cache.TopologyBackedCacheAccessService; +import org.apache.hadoop.hbase.io.hfile.cache.TopologyBackedCacheAccessServices; import org.apache.hadoop.hbase.io.util.MemorySizeUtil; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.testclassification.IOTests; @@ -491,8 +492,8 @@ void testCacheAccessServiceBackedByBlockCacheWhenBlockCacheIsConfigured() { CacheAccessService service = cacheConfig.getCacheAccessService(); - assertInstanceOf(BlockCacheBackedCacheAccessService.class, service); - assertSame(blockCache, ((BlockCacheBackedCacheAccessService) service).getBlockCache()); + assertInstanceOf(TopologyBackedCacheAccessService.class, service); + assertSame(blockCache, TopologyBackedCacheAccessServices.getBlockCache(service)); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java index a016cbef03ab..75a03d425042 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java @@ -50,10 +50,10 @@ import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; -import org.apache.hadoop.hbase.io.hfile.cache.BlockCacheBackedCacheAccessService; import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService; import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServiceTestFactory; import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServices; +import org.apache.hadoop.hbase.io.hfile.cache.TopologyBackedCacheAccessServices; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; @@ -202,7 +202,7 @@ public static Stream parameters() throws IOException { private void clearBlockCache(CacheAccessService cache) throws InterruptedException { // TODO: HBASE-30018 refactor later - BlockCache blockCache = ((BlockCacheBackedCacheAccessService) cache).getBlockCache(); + BlockCache blockCache = TopologyBackedCacheAccessServices.getBlockCache(cache); if (blockCache instanceof LruBlockCache) { ((LruBlockCache) blockCache).clearCache(); } else { @@ -240,7 +240,7 @@ public void setUp() throws IOException { cowType.shouldBeCached(BlockType.LEAF_INDEX)); conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, cowType.shouldBeCached(BlockType.BLOOM_CHUNK)); - cacheConf = new CacheConfig(conf, ((BlockCacheBackedCacheAccessService) cache).getBlockCache()); + cacheConf = new CacheConfig(conf, TopologyBackedCacheAccessServices.getBlockCache(cache)); fs = HFileSystem.get(conf); } @@ -442,7 +442,7 @@ private void testCachingDataBlocksDuringCompactionInternals(boolean useTags, .setCompressionType(compress).setBloomFilterType(BLOOM_TYPE).setMaxVersions(maxVersions) .setDataBlockEncoding(NoOpDataBlockEncoder.INSTANCE.getDataBlockEncoding()).build(); HRegion region = TEST_UTIL.createTestRegion(table, cfd, - ((BlockCacheBackedCacheAccessService) cache).getBlockCache()); + TopologyBackedCacheAccessServices.getBlockCache(cache)); int rowIdx = 0; long ts = EnvironmentEdgeManager.currentTime(); for (int iFile = 0; iFile < 5; ++iFile) { @@ -500,7 +500,7 @@ private void testCachingDataBlocksDuringCompactionInternals(boolean useTags, // of testing // BucketCache, we cannot verify block type as it is not stored in the cache. boolean cacheOnCompactAndNonBucketCache = cacheBlocksOnCompaction - && !(((BlockCacheBackedCacheAccessService) cache).getBlockCache() instanceof BucketCache); + && !(TopologyBackedCacheAccessServices.getBlockCache(cache) instanceof BucketCache); String assertErrorMessage = "\nTest description: " + testDescription + "\ncacheBlocksOnCompaction: " + cacheBlocksOnCompaction + "\n"; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java index 37dbc0a92278..368c72617bb1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java @@ -663,11 +663,7 @@ public static CacheAccessServiceTestInstance bucketInstance(String */ public static BlockCache blockCache(CacheAccessService cacheAccessService) { Objects.requireNonNull(cacheAccessService, "cacheAccessService must not be null"); - if (cacheAccessService instanceof BlockCacheBackedCacheAccessService) { - return ((BlockCacheBackedCacheAccessService) cacheAccessService).getBlockCache(); - } - throw new IllegalArgumentException("CacheAccessService is not backed by a legacy BlockCache: " - + cacheAccessService.getClass().getName()); + return TopologyBackedCacheAccessServices.getBlockCache(cacheAccessService); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestBlockCacheBackedCacheAccessService.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestBlockCacheBackedCacheAccessService.java deleted file mode 100644 index 8a7378c7b837..000000000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestBlockCacheBackedCacheAccessService.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.io.hfile.cache; - -import static org.junit.Assert.assertThrows; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.util.Optional; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.io.hfile.BlockCache; -import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; -import org.apache.hadoop.hbase.io.hfile.BlockType; -import org.apache.hadoop.hbase.io.hfile.CacheStats; -import org.apache.hadoop.hbase.io.hfile.Cacheable; -import org.apache.hadoop.hbase.io.hfile.HFileBlock; -import org.apache.hadoop.hbase.testclassification.IOTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -/** - * Tests for {@link BlockCacheBackedCacheAccessService} and related service helpers. - */ -@Tag(IOTests.TAG) -@Tag(SmallTests.TAG) -public class TestBlockCacheBackedCacheAccessService { - - private static final String HFILE_NAME = "file"; - - private static final long BLOCK_OFFSET = 1L; - - private static final long RANGE_START_OFFSET = 1L; - - private static final long RANGE_END_OFFSET = 10L; - - /** - * Verifies that context-based lookup delegates to the block-type aware legacy lookup method. - */ - @Test - void testGetBlockWithBlockTypeDelegatesToBlockCache() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = new BlockCacheBackedCacheAccessService(blockCache); - BlockCacheKey key = new BlockCacheKey(HFILE_NAME, BLOCK_OFFSET); - Cacheable block = mock(Cacheable.class); - - when(blockCache.getBlock(key, true, true, false, BlockType.DATA)).thenReturn(block); - - CacheRequestContext context = CacheRequestContext.newBuilder().withCaching(true) - .withRepeat(true).withUpdateCacheMetrics(false).withBlockType(BlockType.DATA).build(); - - assertSame(block, service.getBlock(key, context)); - verify(blockCache).getBlock(key, true, true, false, BlockType.DATA); - } - - /** - * Verifies that context-based lookup delegates to the legacy lookup method without block type. - */ - @Test - void testGetBlockWithoutBlockTypeDelegatesToBlockCache() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = new BlockCacheBackedCacheAccessService(blockCache); - BlockCacheKey key = new BlockCacheKey(HFILE_NAME, BLOCK_OFFSET); - Cacheable block = mock(Cacheable.class); - - when(blockCache.getBlock(key, true, false, true)).thenReturn(block); - - CacheRequestContext context = CacheRequestContext.newBuilder().withCaching(true) - .withRepeat(false).withUpdateCacheMetrics(true).build(); - - assertSame(block, service.getBlock(key, context)); - verify(blockCache).getBlock(key, true, false, true); - } - - /** - * Verifies that context-based insertion delegates in-memory and wait flags correctly. - */ - @Test - void testCacheBlockDelegatesToBlockCache() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = new BlockCacheBackedCacheAccessService(blockCache); - BlockCacheKey key = new BlockCacheKey(HFILE_NAME, BLOCK_OFFSET); - Cacheable block = mock(Cacheable.class); - - CacheWriteContext context = CacheWriteContext.newBuilder().withInMemory(true) - .withWaitWhenCache(true).withSource(CacheWriteSource.READ_MISS).build(); - - service.cacheBlock(key, block, context); - - verify(blockCache).cacheBlock(key, block, true, true); - } - - /** - * Verifies that invalidation methods delegate to the wrapped block cache. - */ - @Test - void testEvictionDelegatesToBlockCache() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = new BlockCacheBackedCacheAccessService(blockCache); - BlockCacheKey key = new BlockCacheKey(HFILE_NAME, BLOCK_OFFSET); - - when(blockCache.evictBlock(key)).thenReturn(true); - when(blockCache.evictBlocksByHfileName(HFILE_NAME)).thenReturn(3); - when(blockCache.evictBlocksRangeByHfileName(HFILE_NAME, RANGE_START_OFFSET, RANGE_END_OFFSET)) - .thenReturn(2); - - assertTrue(service.evictBlock(key)); - assertEquals(3, service.evictBlocksByHfileName(HFILE_NAME)); - assertEquals(2, - service.evictBlocksRangeByHfileName(HFILE_NAME, RANGE_START_OFFSET, RANGE_END_OFFSET)); - - verify(blockCache).evictBlock(key); - verify(blockCache).evictBlocksByHfileName(HFILE_NAME); - verify(blockCache).evictBlocksRangeByHfileName(HFILE_NAME, RANGE_START_OFFSET, - RANGE_END_OFFSET); - } - - /** - * Verifies that stats, sizing, lifecycle, and optional helpers delegate to the wrapped cache. - */ - @Test - void testStatsSizingLifecycleAndHelpersDelegateToBlockCache() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = new BlockCacheBackedCacheAccessService(blockCache); - CacheStats stats = new CacheStats("test"); - HFileBlock hfileBlock = mock(HFileBlock.class); - BlockCacheKey key = new BlockCacheKey(HFILE_NAME, BLOCK_OFFSET); - Configuration conf = new Configuration(false); - - when(blockCache.getStats()).thenReturn(stats); - when(blockCache.getMaxSize()).thenReturn(100L); - when(blockCache.getFreeSize()).thenReturn(40L); - when(blockCache.size()).thenReturn(60L); - when(blockCache.getCurrentDataSize()).thenReturn(50L); - when(blockCache.getBlockCount()).thenReturn(10L); - when(blockCache.getDataBlockCount()).thenReturn(8L); - when(blockCache.blockFitsIntoTheCache(hfileBlock)).thenReturn(Optional.of(true)); - when(blockCache.isAlreadyCached(key)).thenReturn(Optional.of(false)); - when(blockCache.getBlockSize(key)).thenReturn(Optional.of(123)); - when(blockCache.isCacheEnabled()).thenReturn(true); - when(blockCache.waitForCacheInitialization(500L)).thenReturn(true); - - assertSame(stats, service.getStats()); - assertEquals(100L, service.getMaxSize()); - assertEquals(40L, service.getFreeSize()); - assertEquals(60L, service.size()); - assertEquals(50L, service.getCurrentDataSize()); - assertEquals(10L, service.getBlockCount()); - assertEquals(8L, service.getDataBlockCount()); - assertEquals(Optional.of(true), service.blockFitsIntoTheCache(hfileBlock)); - assertEquals(Optional.of(false), service.isAlreadyCached(key)); - assertEquals(Optional.of(123), service.getBlockSize(key)); - assertTrue(service.isCacheEnabled()); - assertTrue(service.waitForCacheInitialization(500L)); - - service.onConfigurationChange(conf); - service.shutdown(); - - verify(blockCache).onConfigurationChange(conf); - verify(blockCache).shutdown(); - } - - /** - * Verifies factory helper methods. - */ - @Test - void testCacheAccessServicesFactoryMethods() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = CacheAccessServices.fromBlockCache(blockCache); - - assertInstanceOf(BlockCacheBackedCacheAccessService.class, service); - assertSame(blockCache, ((BlockCacheBackedCacheAccessService) service).getBlockCache()); - assertInstanceOf(NoOpCacheAccessService.class, CacheAccessServices.disabled()); - } - - /** - * Verifies disabled-cache behavior. - */ - @Test - void testNoOpCacheAccessService() { - CacheAccessService service = new NoOpCacheAccessService(new CacheStats("noop")); - - assertEquals("NoOpCacheAccessService", service.getName()); - assertNull(service.getBlock(mock(BlockCacheKey.class), mock(CacheRequestContext.class))); - service.cacheBlock(mock(BlockCacheKey.class), mock(Cacheable.class), - mock(CacheWriteContext.class)); - assertFalse(service.evictBlock(mock(BlockCacheKey.class))); - assertEquals(0, service.evictBlocksByHfileName(HFILE_NAME)); - assertEquals(0L, service.getMaxSize()); - assertEquals(0L, service.getFreeSize()); - assertEquals(0L, service.size()); - assertEquals(0L, service.getCurrentDataSize()); - assertEquals(0L, service.getBlockCount()); - assertEquals(0L, service.getDataBlockCount()); - assertFalse(service.isCacheEnabled()); - assertFalse(service.waitForCacheInitialization(1L)); - service.onConfigurationChange(new Configuration(false)); - service.shutdown(); - } - - @Test - void testNotifyFileCachingCompletedDelegatesToBlockCache() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = new BlockCacheBackedCacheAccessService(blockCache); - Path fileName = new Path("/hbase/table/region/family/file"); - int totalBlockCount = 10; - int dataBlockCount = 8; - long size = 1024L; - - service.notifyFileCachingCompleted(fileName, totalBlockCount, dataBlockCount, size); - - verify(blockCache).notifyFileCachingCompleted(fileName, totalBlockCount, dataBlockCount, size); - } - - @Test - void testNotifyFileCachingCompletedRejectsNullPath() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = new BlockCacheBackedCacheAccessService(blockCache); - - assertThrows(NullPointerException.class, - () -> service.notifyFileCachingCompleted(null, 10, 8, 1024L)); - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCacheAccessServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCacheAccessServices.java index 1c548b9f9304..65dff577bf3e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCacheAccessServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCacheAccessServices.java @@ -39,7 +39,7 @@ public class TestCacheAccessServices { void testFromBlockCacheCreatesBlockCacheBackedServiceForRegularBlockCache() { BlockCache blockCache = mock(BlockCache.class); CacheAccessService service = CacheAccessServices.fromBlockCache(blockCache); - assertTrue(service instanceof BlockCacheBackedCacheAccessService); + assertTrue(service instanceof TopologyBackedCacheAccessService); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestInclusiveCombinedBlockCacheCompatibleTopologyBackedCacheAccessService.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestInclusiveCombinedBlockCacheCompatibleTopologyBackedCacheAccessService.java new file mode 100644 index 000000000000..42c468800d60 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestInclusiveCombinedBlockCacheCompatibleTopologyBackedCacheAccessService.java @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile.cache; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; +import org.apache.hadoop.hbase.io.hfile.Cacheable; +import org.apache.hadoop.hbase.io.hfile.CachedBlock; +import org.apache.hadoop.hbase.io.hfile.InclusiveCombinedBlockCache; +import org.apache.hadoop.hbase.testclassification.IOTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(IOTests.TAG) +@Tag(SmallTests.TAG) +public class TestInclusiveCombinedBlockCacheCompatibleTopologyBackedCacheAccessService { + + /** + * Verifies that {@link InclusiveCombinedBlockCache} is routed to a topology-backed cache access + * service using {@link CacheTopologyType#TIERED_INCLUSIVE}. + *

+ * This protects against accidentally routing {@link InclusiveCombinedBlockCache} through the + * exclusive combined-cache topology path. Inclusive and exclusive combined caches have different + * residency, promotion, and eviction semantics, so they must be represented by different topology + * types. + *

+ */ + @Test + void testInclusiveCombinedBlockCacheUsesTieredInclusiveTopology() { + InclusiveCombinedBlockCache combinedBlockCache = mock(InclusiveCombinedBlockCache.class); + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + + when(combinedBlockCache.getBlockCaches()).thenReturn(new BlockCache[] { l1, l2 }); + + CacheAccessService service = CacheAccessServices.fromBlockCache(combinedBlockCache); + + assertTrue(service instanceof TopologyBackedCacheAccessService); + TopologyBackedCacheAccessService topologyBackedService = + (TopologyBackedCacheAccessService) service; + assertEquals(CacheTopologyType.TIERED_INCLUSIVE, topologyBackedService.getTopology().getType()); + } + + /** + * Verifies that inclusive topology lookup checks L1 first and returns the L1 block when it is + * present. + *

+ * Inclusive caches prefer the first-level cache for reads. If a block is found in L1, the + * topology-backed service should return it without consulting L2. + *

+ */ + @Test + void testInclusiveL1HitReturnsBlockWithoutCheckingL2() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + when(l1.getBlock(key, true, false, true)).thenReturn(block); + + TopologyBackedCacheAccessService service = service(l1, l2, noPromotionPolicy()); + + assertSame(block, service.getBlock(key, requestContext())); + + verify(l1).getBlock(key, true, false, true); + verify(l2, never()).getBlock(any(), anyBoolean(), anyBoolean(), anyBoolean()); + } + + /** + * Verifies that inclusive topology lookup checks L2 after an L1 miss. + *

+ * Unlike the exclusive combined-cache compatibility path, inclusive lookup does not need the L1 + * membership shortcut. A normal ordered tier scan is appropriate: L1 is checked first, and L2 is + * checked only if L1 does not contain the block. + *

+ */ + @Test + void testInclusiveL2HitReturnsBlockAfterL1Miss() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + when(l1.getBlock(key, true, false, true)).thenReturn(null); + when(l2.getBlock(key, true, false, true)).thenReturn(block); + + TopologyBackedCacheAccessService service = service(l1, l2, noPromotionPolicy()); + + assertSame(block, service.getBlock(key, requestContext())); + + verify(l1).getBlock(key, true, false, true); + verify(l2).getBlock(key, true, false, true); + } + + /** + * Verifies that promotion in an inclusive topology copies a block to L1 without evicting it from + * L2. + *

+ * This is the key semantic difference from the exclusive topology. In an exclusive topology, + * promotion moves the block from L2 to L1 and removes the L2 copy. In an inclusive topology, the + * block may remain resident in both tiers. + *

+ */ + @Test + void testInclusivePromotionCopiesBlockToL1WithoutEvictingL2() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + when(l1.getBlock(key, true, false, true)).thenReturn(null); + when(l2.getBlock(key, true, false, true)).thenReturn(block); + + TopologyBackedCacheAccessService service = service(l1, l2, promoteL2HitToL1Policy()); + + assertSame(block, service.getBlock(key, requestContext())); + + verify(l1).getBlock(key, true, false, true); + verify(l2).getBlock(key, true, false, true); + verify(l1).cacheBlock(key, block); + verify(l2, never()).evictBlock(key); + } + + /** + * Verifies that service-level eviction for an inclusive topology evicts from all tiers. + *

+ * An inclusive cache may contain the same block in both L1 and L2. Evicting only the first + * matching tier could leave another resident copy behind, so the topology-backed service must ask + * both tiers to evict the key. + *

+ */ + @Test + void testInclusiveEvictBlockEvictsFromBothTiers() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + + when(l1.evictBlock(key)).thenReturn(true); + when(l2.evictBlock(key)).thenReturn(true); + + TopologyBackedCacheAccessService service = service(l1, l2, noPromotionPolicy()); + + assertTrue(service.evictBlock(key)); + + verify(l1).evictBlock(key); + verify(l2).evictBlock(key); + } + + /** + * Verifies that an inclusive topology can cache a block into both tiers when the placement policy + * selects both L1 and L2. + *

+ * This covers the inclusive cache residency model where the same block can be intentionally + * present in multiple tiers. + *

+ */ + @Test + void testInclusiveCacheBlockToBothTiers() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + TopologyBackedCacheAccessService service = + service(l1, l2, admitToTiersPolicy(CacheTier.L1, CacheTier.L2)); + + service.cacheBlock(key, block, writeContext()); + + verify(l1).cacheBlock(key, block, false, false); + verify(l2).cacheBlock(key, block, false, false); + } + + /** + * Verifies that cached-block iteration is aggregated across both tiers for inclusive topology. + *

+ * Diagnostic code and compatibility tests use + * {@link CacheAccessServices#asCachedBlockIterable(CacheAccessService)} to enumerate cached + * blocks through the active cache access service. Since an inclusive topology has multiple + * backing engines, the service must expose cached blocks from both L1 and L2. + *

+ */ + @Test + void testInclusiveCachedBlockIterableAggregatesBothTiers() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + CachedBlock l1Block = mock(CachedBlock.class); + CachedBlock l2Block = mock(CachedBlock.class); + + when(l1.iterator()).thenReturn(Arrays.asList(l1Block).iterator()); + when(l2.iterator()).thenReturn(Arrays.asList(l2Block).iterator()); + + TopologyBackedCacheAccessService service = service(l1, l2, noPromotionPolicy()); + + Optional> iterable = CacheAccessServices.asCachedBlockIterable(service); + + assertTrue(iterable.isPresent()); + assertEquals(Arrays.asList(l1Block, l2Block), toList(iterable.get())); + } + + /** + * Verifies that shutting down an inclusive topology-backed service shuts down both cache tiers. + *

+ * The topology-backed service owns the topology-level lifecycle. For a two-tier inclusive + * topology, shutdown should be delegated to both L1 and L2 engines. + *

+ */ + @Test + void testInclusiveShutdownShutsDownBothTiers() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + + TopologyBackedCacheAccessService service = service(l1, l2, noPromotionPolicy()); + + service.shutdown(); + + verify(l1).shutdown(); + verify(l2).shutdown(); + } + + /** + * Creates a topology-backed cache access service using a tiered inclusive topology. + * @param l1 first-level block cache + * @param l2 second-level block cache + * @param policy cache placement and admission policy + * @return topology-backed cache access service using the inclusive topology path + */ + private static TopologyBackedCacheAccessService service(BlockCache l1, BlockCache l2, + CachePlacementAdmissionPolicy policy) { + return TopologyBackedCacheAccessServices.fromTieredInclusiveBlockCaches("inclusive-combined", + l1, l2, policy); + } + + /** + * Creates a cache request context used by read-path tests. + *

+ * The returned context enables caching, marks the request as non-repeat, and asks the cache to + * update cache metrics. These values match the read-path behavior covered by the topology-backed + * cache access service tests. + *

+ * @return cache request context for read-path tests + */ + private static CacheRequestContext requestContext() { + return CacheRequestContext.newBuilder().withCaching(true).withRepeat(false) + .withUpdateCacheMetrics(true).build(); + } + + /** + * Creates a cache write context used by cache population tests. + *

+ * The returned context uses the default non-in-memory and non-blocking write behavior expected by + * the existing compatibility tests. + *

+ * @return cache write context for cache population tests + */ + private static CacheWriteContext writeContext() { + return CacheWriteContext.newBuilder().withInMemory(false).withWaitWhenCache(false).build(); + } + + /** + * Creates a placement policy that never promotes a block after a cache hit. + *

+ * This policy is useful for lookup tests that need to verify only the lookup order and returned + * block without introducing promotion side effects. + *

+ * @return cache placement and admission policy that disables promotion + */ + private static CachePlacementAdmissionPolicy noPromotionPolicy() { + CachePlacementAdmissionPolicy policy = mock(CachePlacementAdmissionPolicy.class); + when(policy.shouldPromote(any(), any(), any(), any(), any())) + .thenReturn(PromotionDecision.none()); + return policy; + } + + /** + * Creates a placement policy that promotes an L2 hit to L1. + *

+ * The policy returns no promotion for L1 hits and requests promotion to L1 for L2 hits. In an + * inclusive topology, this should copy the block into L1 without evicting it from L2. + *

+ * @return cache placement and admission policy that promotes L2 hits to L1 + */ + private static CachePlacementAdmissionPolicy promoteL2HitToL1Policy() { + CachePlacementAdmissionPolicy policy = mock(CachePlacementAdmissionPolicy.class); + when(policy.shouldPromote(any(), any(), eq(CacheTier.L1), any(), any())) + .thenReturn(PromotionDecision.none()); + when(policy.shouldPromote(any(), any(), eq(CacheTier.L2), any(), any())) + .thenReturn(PromotionDecision.promoteTo(CacheTier.L1, false)); + return policy; + } + + /** + * Creates a placement policy that admits writes to the supplied tiers. + *

+ * The returned policy admits every block and returns a multi-tier placement decision containing + * the tiers supplied by the caller. + *

+ * @param tiers cache tiers selected by the policy + * @return cache placement and admission policy that writes to the supplied tiers + */ + private static CachePlacementAdmissionPolicy admitToTiersPolicy(CacheTier... tiers) { + CachePlacementAdmissionPolicy policy = mock(CachePlacementAdmissionPolicy.class); + when(policy.shouldAdmit(any(), any(), any(), any(), any())) + .thenReturn(AdmissionDecision.admit()); + when(policy.selectTier(any(), any(), any(), any())) + .thenReturn(TierDecision.multiple(Arrays.asList(tiers))); + return policy; + } + + /** + * Copies an iterable of cached blocks into a list. + *

+ * The helper makes cached-block iterable assertions deterministic and easy to compare with the + * expected tier order. + *

+ * @param iterable cached-block iterable to copy + * @return list containing all cached blocks produced by the iterable + */ + private static List toList(Iterable iterable) { + List blocks = new ArrayList<>(); + for (CachedBlock block : iterable) { + blocks.add(block); + } + return blocks; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestSingleTierTopologyBackedCacheAccessService.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestSingleTierTopologyBackedCacheAccessService.java new file mode 100644 index 000000000000..e9117b3b3f47 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestSingleTierTopologyBackedCacheAccessService.java @@ -0,0 +1,400 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile.cache; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; +import org.apache.hadoop.hbase.io.hfile.CacheStats; +import org.apache.hadoop.hbase.io.hfile.Cacheable; +import org.apache.hadoop.hbase.io.hfile.CachedBlock; +import org.apache.hadoop.hbase.testclassification.IOTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(IOTests.TAG) +@Tag(SmallTests.TAG) +public class TestSingleTierTopologyBackedCacheAccessService { + + /** + * Verifies that a plain single-tier {@link BlockCache} is adapted to a topology-backed cache + * access service. + *

+ * HBASE-30329 routes legacy block caches through {@link TopologyBackedCacheAccessService}. A + * plain non-combined block cache should be represented by {@link SingleTierTopology}, not by the + * old {@link BlockCacheBackedCacheAccessService} runtime path. + *

+ */ + @Test + void testSingleTierBlockCacheUsesTopologyBackedAccessService() { + BlockCache blockCache = mock(BlockCache.class); + + CacheAccessService service = CacheAccessServices.fromBlockCache(blockCache); + + assertTrue(service instanceof TopologyBackedCacheAccessService); + TopologyBackedCacheAccessService topologyBackedService = + (TopologyBackedCacheAccessService) service; + assertEquals(CacheTopologyType.SINGLE_TIER, topologyBackedService.getTopology().getType()); + } + + /** + * Verifies that a single-tier topology exposes the expected topology metadata. + *

+ * The topology should contain exactly one engine, expose only the L1 tier, and return the same + * engine for {@link CacheTier#L1}. Other tiers should not resolve to an engine. + *

+ */ + @Test + void testSingleTierTopologyMetadata() { + CacheEngine engine = mock(CacheEngine.class); + SingleTierTopology topology = new SingleTierTopology("single", engine); + + assertEquals("single", topology.getName()); + assertEquals(CacheTopologyType.SINGLE_TIER, topology.getType()); + assertEquals(Arrays.asList(engine), topology.getEngines()); + assertEquals(Arrays.asList(CacheTier.L1), topology.getTiers()); + assertSame(engine, topology.getEngine(CacheTier.L1).orElseThrow()); + assertFalse(topology.getEngine(CacheTier.L2).isPresent()); + } + + /** + * Verifies that a single-tier topology-backed service reads blocks from the single backing cache. + *

+ * Since there is only one tier, the access service should delegate the read to the L1 engine and + * return the block supplied by the wrapped block cache. + *

+ */ + @Test + void testGetBlockDelegatesToSingleTier() { + BlockCache blockCache = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + when(blockCache.getBlock(key, true, false, true)).thenReturn(block); + + TopologyBackedCacheAccessService service = service(blockCache, noPromotionPolicy()); + + assertSame(block, service.getBlock(key, requestContext())); + + verify(blockCache).getBlock(key, true, false, true); + } + + /** + * Verifies that a single-tier topology-backed service returns {@code null} when the backing cache + * misses. + *

+ * The service should not attempt any tier fallback because the topology contains only one cache + * engine. + *

+ */ + @Test + void testGetBlockReturnsNullOnMiss() { + BlockCache blockCache = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + + when(blockCache.getBlock(key, true, false, true)).thenReturn(null); + + TopologyBackedCacheAccessService service = service(blockCache, noPromotionPolicy()); + + assertSame(null, service.getBlock(key, requestContext())); + + verify(blockCache).getBlock(key, true, false, true); + } + + /** + * Verifies that cache population for a single-tier topology writes to the single backing cache. + *

+ * The placement policy selects L1. Since single-tier topology exposes only L1, the service should + * delegate the cache write to the wrapped block cache. + *

+ */ + @Test + void testCacheBlockDelegatesToSingleTier() { + BlockCache blockCache = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + TopologyBackedCacheAccessService service = + service(blockCache, admitToTiersPolicy(CacheTier.L1)); + + service.cacheBlock(key, block, writeContext()); + + verify(blockCache).cacheBlock(key, block, false, false); + } + + /** + * Verifies that a rejected block is not written to the single backing cache. + *

+ * When the placement and admission policy rejects the write, the topology-backed service should + * not call any {@code cacheBlock} overload on the wrapped block cache. + *

+ */ + @Test + void testRejectedBlockIsNotCached() { + BlockCache blockCache = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + TopologyBackedCacheAccessService service = service(blockCache, rejectPolicy()); + + service.cacheBlock(key, block, writeContext()); + + verify(blockCache, never()).cacheBlock(any(), any()); + verify(blockCache, never()).cacheBlock(any(), any(), anyBoolean(), anyBoolean()); + } + + /** + * Verifies that service-level eviction for a single-tier topology delegates to the backing cache. + *

+ * A single-tier topology has only one possible resident tier, so eviction should be a direct + * delegation to that tier. + *

+ */ + @Test + void testEvictBlockDelegatesToSingleTier() { + BlockCache blockCache = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + + when(blockCache.evictBlock(key)).thenReturn(true); + + TopologyBackedCacheAccessService service = service(blockCache, noPromotionPolicy()); + + assertTrue(service.evictBlock(key)); + + verify(blockCache).evictBlock(key); + } + + /** + * Verifies that single-tier cached-block iteration is exposed through the topology-backed access + * service. + *

+ * Diagnostic callers use {@link CacheAccessServices#asCachedBlockIterable(CacheAccessService)} + * rather than unwrapping the legacy block cache. The single-tier topology-backed path should + * expose the same cached blocks as the wrapped block cache. + *

+ */ + @Test + void testCachedBlockIterableDelegatesToSingleTier() { + BlockCache blockCache = mock(BlockCache.class); + CachedBlock firstBlock = mock(CachedBlock.class); + CachedBlock secondBlock = mock(CachedBlock.class); + + when(blockCache.iterator()).thenReturn(Arrays.asList(firstBlock, secondBlock).iterator()); + + TopologyBackedCacheAccessService service = service(blockCache, noPromotionPolicy()); + + Optional> iterable = CacheAccessServices.asCachedBlockIterable(service); + + assertTrue(iterable.isPresent()); + assertEquals(Arrays.asList(firstBlock, secondBlock), toList(iterable.get())); + } + + /** + * Verifies that single-tier topology statistics are delegated to the backing engine. + *

+ * The topology itself does not aggregate multiple tiers, so its statistics should be exactly the + * statistics exposed by the single cache engine. + *

+ */ + @Test + void testTopologyStatsDelegatesToSingleEngine() { + CacheEngine engine = mock(CacheEngine.class); + CacheStats stats = mock(CacheStats.class); + + when(engine.getStats()).thenReturn(stats); + + SingleTierTopology topology = new SingleTierTopology("single", engine); + + assertSame(stats, topology.getStats()); + } + + /** + * Verifies that promotion is not supported by a single-tier topology. + *

+ * Promotion requires a source tier and a different target tier. Since this topology has only one + * tier, promotion is a no-op and should return {@code false}. + *

+ */ + @Test + void testSingleTierTopologyDoesNotPromote() { + CacheEngine engine = mock(CacheEngine.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + SingleTierTopology topology = new SingleTierTopology("single", engine); + + assertFalse(topology.promote(key, block, engine, engine)); + + verify(engine, never()).cacheBlock(any(), any()); + verify(engine, never()).evictBlock(any()); + } + + /** + * Verifies that demotion is not supported by a single-tier topology. + *

+ * Demotion requires a source tier and a different lower target tier. Since this topology has only + * one tier, demotion is a no-op and should return {@code false}. + *

+ */ + @Test + void testSingleTierTopologyDoesNotDemote() { + CacheEngine engine = mock(CacheEngine.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + SingleTierTopology topology = new SingleTierTopology("single", engine); + + assertFalse(topology.demote(key, block, engine, engine)); + + verify(engine, never()).cacheBlock(any(), any()); + verify(engine, never()).evictBlock(any()); + } + + /** + * Verifies that shutting down a single-tier topology-backed service shuts down the backing cache. + *

+ * The topology-backed service owns the topology lifecycle. For a single-tier topology, shutdown + * should be delegated to the only backing engine. + *

+ */ + @Test + void testShutdownDelegatesToSingleTier() { + BlockCache blockCache = mock(BlockCache.class); + + TopologyBackedCacheAccessService service = service(blockCache, noPromotionPolicy()); + + service.shutdown(); + + verify(blockCache).shutdown(); + } + + /** + * Creates a topology-backed cache access service using a single-tier topology. + * @param blockCache legacy block cache backing the single tier + * @param policy cache placement and admission policy + * @return topology-backed cache access service using a single-tier topology + */ + private static TopologyBackedCacheAccessService service(BlockCache blockCache, + CachePlacementAdmissionPolicy policy) { + return TopologyBackedCacheAccessServices.fromSingleBlockCache("single", blockCache, policy); + } + + /** + * Creates a cache request context used by read-path tests. + *

+ * The returned context enables caching, marks the request as non-repeat, and asks the cache to + * update cache metrics. These values match the read-path behavior covered by the topology-backed + * cache access service tests. + *

+ * @return cache request context for read-path tests + */ + private static CacheRequestContext requestContext() { + return CacheRequestContext.newBuilder().withCaching(true).withRepeat(false) + .withUpdateCacheMetrics(true).build(); + } + + /** + * Creates a cache write context used by cache population tests. + *

+ * The returned context uses the default non-in-memory and non-blocking write behavior expected by + * the existing topology-backed cache access service tests. + *

+ * @return cache write context for cache population tests + */ + private static CacheWriteContext writeContext() { + return CacheWriteContext.newBuilder().withInMemory(false).withWaitWhenCache(false).build(); + } + + /** + * Creates a placement policy that never promotes a block after a cache hit. + *

+ * This policy is useful for lookup tests that need to verify only lookup delegation and returned + * block behavior without introducing promotion side effects. + *

+ * @return cache placement and admission policy that disables promotion + */ + private static CachePlacementAdmissionPolicy noPromotionPolicy() { + CachePlacementAdmissionPolicy policy = mock(CachePlacementAdmissionPolicy.class); + when(policy.shouldPromote(any(), any(), any(), any(), any())) + .thenReturn(PromotionDecision.none()); + return policy; + } + + /** + * Creates a placement policy that admits writes to the supplied tiers. + *

+ * The returned policy admits every block and returns a multi-tier placement decision containing + * the tiers supplied by the caller. + *

+ * @param tiers cache tiers selected by the policy + * @return cache placement and admission policy that writes to the supplied tiers + */ + private static CachePlacementAdmissionPolicy admitToTiersPolicy(CacheTier... tiers) { + CachePlacementAdmissionPolicy policy = mock(CachePlacementAdmissionPolicy.class); + when(policy.shouldAdmit(any(), any(), any(), any(), any())) + .thenReturn(AdmissionDecision.admit()); + when(policy.selectTier(any(), any(), any(), any())) + .thenReturn(TierDecision.multiple(Arrays.asList(tiers))); + return policy; + } + + /** + * Creates a placement policy that rejects every block. + *

+ * The returned policy is used to verify that rejected cache writes are not delegated to the + * backing cache. + *

+ * @return cache placement and admission policy that rejects every block + */ + private static CachePlacementAdmissionPolicy rejectPolicy() { + CachePlacementAdmissionPolicy policy = mock(CachePlacementAdmissionPolicy.class); + when(policy.shouldAdmit(any(), any(), any(), any(), any())) + .thenReturn(AdmissionDecision.reject("test rejection")); + return policy; + } + + /** + * Copies an iterable of cached blocks into a list. + *

+ * The helper makes cached-block iterable assertions deterministic and easy to compare with the + * expected order. + *

+ * @param iterable cached-block iterable to copy + * @return list containing all cached blocks produced by the iterable + */ + private static List toList(Iterable iterable) { + List blocks = new ArrayList<>(); + for (CachedBlock block : iterable) { + blocks.add(block); + } + return blocks; + } +}