diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 4374e974db5bc..86e81caea1863 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -30,7 +30,6 @@ import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; -import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskAlignedChunkLoader; import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskChunkLoader; @@ -47,7 +46,6 @@ import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; -import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; @@ -61,7 +59,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -341,23 +341,35 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr // the order of timeSeriesMetadata list is same as subSensorList's order TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); List valueMeasurementList = alignedPath.getMeasurementList(); + // resolves the time column and every selected sensor in one call: the cache reads each leaf + // group that holds one of them once and answers the whole group from that read, so a wide + // projection does not walk the device metadata index once per column Set allSensors = alignedPath.getAllSensors(); - boolean isDebug = context.isDebug(); - String filePath = resource.getTsFilePath(); - IDeviceID deviceId = alignedPath.getDeviceId(); + if (!allSensors.contains("")) { + // the aligned loader always needs the time column, even if the caller's set omits it + allSensors = new HashSet<>(allSensors); + allSensors.add(""); + } // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex // we should not ignore the non-exist of device in TsFileMetadata - TimeseriesMetadata timeColumn = - cache.get( - filePath, - new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, ""), + boolean ignoreNotExists = + context.ignoreNotExistsDevice() + || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE; + + Map metadata = + cache.getTimeseriesMetadata( + resource.getTsFilePath(), + resource.getTsFileID(), + alignedPath.getDeviceId(), + allSensors, allSensors, - context.ignoreNotExistsDevice() - || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, - isDebug, + ignoreNotExists, + context.isDebug(), context, rootMeasurementMetadataIndexNodeOffset); + + TimeseriesMetadata timeColumn = metadata.get(""); if (timeColumn != null) { // only need time column, like count_time aggregation if (valueMeasurementList.isEmpty()) { @@ -375,38 +387,49 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr } else { List valueTimeSeriesMetadataList = new ArrayList<>(valueMeasurementList.size()); - // if all the queried aligned sensors does not exist, we will return null - boolean exist = false; for (String measurement : valueMeasurementList) { - TimeseriesMetadata valueColumn = - cache.get( - filePath, - new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement), - allSensors, - context.ignoreNotExistsDevice() - || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, - isDebug, - context, - rootMeasurementMetadataIndexNodeOffset); - exist = (exist || (valueColumn != null)); - valueTimeSeriesMetadataList.add(valueColumn); - } - if (!ignoreAllNullRows || exist) { - alignedTimeSeriesMetadata = - setModifications( - resource, - timeColumn, - valueTimeSeriesMetadataList, - alignedPath, - context, - globalTimeFilter, - ignoreAllNullRows); + valueTimeSeriesMetadataList.add(metadata.get(measurement)); } + alignedTimeSeriesMetadata = + buildAlignedTimeSeriesMetadata( + resource, + alignedPath, + timeColumn, + valueTimeSeriesMetadataList, + context, + globalTimeFilter, + ignoreAllNullRows); } } return alignedTimeSeriesMetadata; } + private static AbstractAlignedTimeSeriesMetadata buildAlignedTimeSeriesMetadata( + TsFileResource resource, + AlignedFullPath alignedPath, + TimeseriesMetadata timeColumn, + List valueTimeSeriesMetadataList, + FragmentInstanceContext context, + Filter globalTimeFilter, + boolean ignoreAllNullRows) { + // if all the queried aligned sensors does not exist, we will return null + boolean exist = false; + for (TimeseriesMetadata valueColumn : valueTimeSeriesMetadataList) { + exist = (exist || (valueColumn != null)); + } + if (!ignoreAllNullRows || exist) { + return setModifications( + resource, + timeColumn, + valueTimeSeriesMetadataList, + alignedPath, + context, + globalTimeFilter, + ignoreAllNullRows); + } + return null; + } + private static AbstractAlignedTimeSeriesMetadata setModifications( TsFileResource resource, TimeseriesMetadata timeColumnMetadata, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 68457df0ebbcd..d0a25b0541455 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; +import org.apache.iotdb.db.queryengine.execution.fragment.QueryStatistics; import org.apache.iotdb.db.queryengine.metric.TimeSeriesMetadataCacheMetrics; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; @@ -35,7 +36,6 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Weigher; -import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TimeseriesMetadata; @@ -48,13 +48,13 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.lang.ref.WeakReference; +import java.util.Collection; import java.util.Collections; -import java.util.List; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.WeakHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.function.LongConsumer; @@ -86,10 +86,17 @@ public class TimeSeriesMetadataCache { private final AtomicLong entryAverageSize = new AtomicLong(0); - private final Map> devices = - Collections.synchronizedMap(new WeakHashMap<>()); private static final String SEPARATOR = "$"; + private static final int DEVICE_LOCK_STRIPES = 1024; + private static final Object[] DEVICE_LOCKS = new Object[DEVICE_LOCK_STRIPES]; + + static { + for (int i = 0; i < DEVICE_LOCK_STRIPES; i++) { + DEVICE_LOCKS[i] = new Object(); + } + } + private final AtomicLong evictedExistingEntryCount = new AtomicLong(0); private final AtomicLong evictedNonExistingEntryCount = new AtomicLong(0); @@ -137,7 +144,6 @@ public static TimeSeriesMetadataCache getInstance() { return TimeSeriesMetadataCache.TimeSeriesMetadataCacheHolder.INSTANCE; } - @SuppressWarnings({"squid:S1860", "squid:S6541", "squid:S3776"}) // Suppress synchronize warning public TimeseriesMetadata get( String filePath, TimeSeriesMetadataCacheKey key, @@ -149,7 +155,6 @@ public TimeseriesMetadata get( return get(filePath, key, allSensors, ignoreNotExists, debug, queryContext, null); } - @SuppressWarnings({"squid:S1860", "squid:S6541", "squid:S3776"}) // Suppress synchronize warning public TimeseriesMetadata get( String filePath, TimeSeriesMetadataCacheKey key, @@ -159,163 +164,350 @@ public TimeseriesMetadata get( QueryContext queryContext, long[] deviceMetadataIndexNodeOffset) throws IOException { + return getTimeseriesMetadata( + filePath, + key.tsFileID, + key.device, + Collections.singleton(key.measurement), + allSensors, + ignoreNotExists, + debug, + queryContext, + deviceMetadataIndexNodeOffset) + .get(key.measurement); + } + + /** + * Resolve the metadata of {@code requested} measurements of a device in one file. + * + *

Cached measurements are returned directly. The remaining ones are checked against the file's + * bloom filter and then loaded leaf group by leaf group: one pending measurement locates the + * group it belongs to, that whole group is read and deserialized once, and every requested + * measurement found in it is resolved. For files that expose a bloom filter, each group is read + * at most once per call, independent of LRU cache pressure, and groups that contain no requested + * measurement are never read. Without one (external tsfiles), a measurement the bloom filter + * cannot rule out is read from its leaf, so a definitively absent measurement may read the leaf + * that holds its floor entry again; this matches the previous per-column loader. + * + *

{@code allSensors} is the full selected set; it filters each group read and warms the cache, + * so a later single-measurement {@link #get} can reuse the group. {@code requested} must be a + * subset of {@code allSensors}. + * + *

The returned map holds copies of the cached (or freshly read) metadata, so the caller may + * apply {@code setModified} and {@code setChunkMetadataLoader} without corrupting cached entries. + * Load statistics are counted per requested measurement. + * + * @param deviceMetadataIndexNodeOffset non-null for external tsfiles, which address the device + * metadata index by offset; bloom filtering is skipped in that case + * @return a map with an entry for every measurement in {@code requested}: a metadata copy when + * the series exists and is non-empty, null otherwise + */ + public Map getTimeseriesMetadata( + String filePath, + TsFileID tsFileID, + IDeviceID device, + Set requested, + Set allSensors, + boolean ignoreNotExists, + boolean debug, + QueryContext queryContext, + long[] deviceMetadataIndexNodeOffset) + throws IOException { long startTime = System.nanoTime(); + long cachePhaseEndTime = 0; long loadBloomFilterTime = 0; + long cacheHitCount = 0; + long diskLookupCount = 0; + boolean externalTsFile = queryContext.isExternalTsFileScan(); LongConsumer timeSeriesMetadataIoSizeRecorder = queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; LongConsumer bloomFilterIoSizeRecorder = queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet; - boolean cacheHit = true; - boolean externalTsFile = queryContext.isExternalTsFileScan(); + String devicePrefix = device.toString() + IoTDBConstant.PATH_SEPARATOR; + + Map result = new HashMap<>(requested.size()); + Set pending = new HashSet<>(requested.size()); + try { - if (!CACHE_ENABLE || externalTsFile) { - String deviceStringFormat = key.device.toString(); - cacheHit = false; - - // bloom filter part - TsFileSequenceReader reader = - FileReaderManager.getInstance() - .get(filePath, key.tsFileID, true, bloomFilterIoSizeRecorder, externalTsFile); - if (deviceMetadataIndexNodeOffset == null) { - BloomFilter bloomFilter = reader.readBloomFilter(bloomFilterIoSizeRecorder); - queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount().incrementAndGet(); - if (bloomFilter != null - && !bloomFilter.contains( - deviceStringFormat + IoTDBConstant.PATH_SEPARATOR + key.measurement)) { - loadBloomFilterTime = System.nanoTime() - startTime; - return null; + if (CACHE_ENABLE && !externalTsFile) { + collectCachedMetadata(tsFileID, device, requested, result, pending, filePath, debug); + cacheHitCount = result.size(); + if (pending.isEmpty()) { + cachePhaseEndTime = System.nanoTime(); + return result; + } + cachePhaseEndTime = System.nanoTime(); + // serialize reads of the same device+file (same lock key as get) so threads don't each + // walk the device metadata index and race on the shared reader + synchronized (deviceLock(device.toString(), filePath)) { + if (debug) { + for (String measurement : pending) { + DEBUG_LOGGER.info( + StorageEngineMessages.CACHE_MISS_IN_FILE, device, measurement, filePath); + } + DEBUG_LOGGER.info(StorageEngineMessages.DEVICE_ALL_SENSORS, device, allSensors); } - loadBloomFilterTime = System.nanoTime() - startTime; + BloomFilter bloomFilter = + loadBloomFilter(filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, false); + if (deviceAbsent(requested, result, bloomFilter, devicePrefix)) { + fillMissingWithNull(requested, result); + return result; + } + diskLookupCount = requested.size() - cacheHitCount; + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); + readAndMerge( + reader, + tsFileID, + device, + deviceMetadataIndexNodeOffset, + pending, + requested, + allSensors, + ignoreNotExists, + true, + debug, + result, + timeSeriesMetadataIoSizeRecorder); + return result; } + } - TimeseriesMetadata timeseriesMetadata = - reader.readTimeseriesMetadata( - key.device, - deviceMetadataIndexNodeOffset, - key.measurement, - ignoreNotExists, - timeSeriesMetadataIoSizeRecorder); - return (timeseriesMetadata == null || timeseriesMetadata.getStatistics().getCount() == 0) - ? null - : timeseriesMetadata; + // cache disabled or external tsfiles: nothing was resolved from the cache + pending.addAll(requested); + BloomFilter bloomFilter = + loadBloomFilter( + filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, externalTsFile); + if (deviceAbsent(requested, result, bloomFilter, devicePrefix)) { + fillMissingWithNull(requested, result); + return result; } + diskLookupCount = requested.size() - cacheHitCount; + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); + readAndMerge( + reader, + tsFileID, + device, + deviceMetadataIndexNodeOffset, + pending, + requested, + allSensors, + ignoreNotExists, + false, + debug, + result, + timeSeriesMetadataIoSizeRecorder); + return result; + } finally { + long endTime = System.nanoTime(); + long cacheLookupTime; + long diskLookupTime; + if (cacheHitCount == 0 || cachePhaseEndTime == 0) { + cacheLookupTime = 0; + diskLookupTime = endTime - startTime - loadBloomFilterTime; + } else if (diskLookupCount == 0) { + cacheLookupTime = endTime - startTime - loadBloomFilterTime; + diskLookupTime = 0; + } else { + cacheLookupTime = cachePhaseEndTime - startTime; + diskLookupTime = endTime - cachePhaseEndTime - loadBloomFilterTime; + } + recordLoadStatistics( + queryContext, + cacheHitCount, + cacheLookupTime, + diskLookupCount, + diskLookupTime, + loadBloomFilterTime); + } + } + + private BloomFilter loadBloomFilter( + String filePath, + TsFileID tsFileID, + QueryContext queryContext, + LongConsumer bloomFilterIoSizeRecorder, + boolean externalTsFile) + throws IOException { + return BloomFilterCache.getInstance() + .get( + new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID), + false, + bloomFilterIoSizeRecorder, + queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet, + queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet, + externalTsFile); + } - TimeseriesMetadata timeseriesMetadata = lruCache.getIfPresent(key); + /** + * An aligned device always has its time column. When the cache or the bloom filter proves it + * absent, no selected series of the device can be read from this file. For a single requested + * series, the bloom filter can rule that series out directly. + */ + private static boolean deviceAbsent( + Set requested, + Map result, + BloomFilter bloomFilter, + String devicePrefix) { + if (requested.contains("")) { + if (result.containsKey("")) { + return result.get("") == null; + } + return bloomFilter != null && !bloomFilter.contains(devicePrefix); + } + if (bloomFilter != null && requested.size() == 1) { + return !bloomFilter.contains(devicePrefix + requested.iterator().next()); + } + return false; + } - if (timeseriesMetadata == null) { + /** + * Reads the pending measurements from the leaf groups that hold them. When the cache is enabled + * the group siblings are returned as well, so a later single-column get() finds them in the LRU. + */ + private void readAndMerge( + TsFileSequenceReader reader, + TsFileID tsFileID, + IDeviceID device, + long[] deviceMetadataIndexNodeOffset, + Set pending, + Set requested, + Set allSensors, + boolean ignoreNotExists, + boolean cacheEnabled, + boolean debug, + Map result, + LongConsumer ioSizeRecorder) + throws IOException { + // cached measurements are already in the result: no need to deserialize them again + Set returned; + if (!cacheEnabled || allSensors.equals(requested)) { + // the aligned batch read requests exactly the set it caches, so only the misses remain + returned = pending; + } else { + returned = new HashSet<>(allSensors); + returned.removeAll(result.keySet()); + } + Map loaded = + reader.readTimeseriesMetadata( + device, + deviceMetadataIndexNodeOffset, + pending, + returned, + ignoreNotExists, + ioSizeRecorder); + for (Map.Entry entry : loaded.entrySet()) { + String measurement = entry.getKey(); + TimeseriesMetadata metadata = entry.getValue(); + if (metadata.getStatistics().getCount() == 0) { if (debug) { DEBUG_LOGGER.info( - StorageEngineMessages.CACHE_MISS_IN_FILE, key.device, key.measurement, filePath); - DEBUG_LOGGER.info(StorageEngineMessages.DEVICE_ALL_SENSORS, key.device, allSensors); + StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement)); } - String deviceStringFormat = key.device.toString(); - // allow for the parallelism of different devices - synchronized ( - devices.computeIfAbsent( - deviceStringFormat + SEPARATOR + filePath, WeakReference::new)) { - // double check - timeseriesMetadata = lruCache.getIfPresent(key); - if (timeseriesMetadata == null) { - cacheHit = false; - - if (deviceMetadataIndexNodeOffset == null) { - long loadBloomFilterStartTime = System.nanoTime(); - // bloom filter part - BloomFilter bloomFilter = - BloomFilterCache.getInstance() - .get( - new BloomFilterCache.BloomFilterCacheKey(filePath, key.tsFileID), - debug, - bloomFilterIoSizeRecorder, - queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount() - ::addAndGet, - queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount() - ::addAndGet, - false); - if (bloomFilter != null - && !bloomFilter.contains( - deviceStringFormat + TsFileConstant.PATH_SEPARATOR + key.measurement)) { - if (debug) { - DEBUG_LOGGER.info( - StorageEngineMessages.TS_METADATA_FILTERED_BY_BLOOM_FILTER, key); - } - loadBloomFilterTime = System.nanoTime() - loadBloomFilterStartTime; - return null; - } - - loadBloomFilterTime = System.nanoTime() - loadBloomFilterStartTime; - } - TsFileSequenceReader reader = - FileReaderManager.getInstance() - .get(filePath, key.tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); - List timeSeriesMetadataList = - reader.readTimeseriesMetadata( - key.device, - deviceMetadataIndexNodeOffset, - key.measurement, - allSensors, - ignoreNotExists, - timeSeriesMetadataIoSizeRecorder); - // put TimeSeriesMetadata of all sensors used in this read into cache - for (TimeseriesMetadata metadata : timeSeriesMetadataList) { - TimeSeriesMetadataCacheKey k = - new TimeSeriesMetadataCacheKey( - key.tsFileID, key.device, metadata.getMeasurementId()); - if (metadata.getStatistics().getCount() != 0) { - lruCache.put(k, metadata); - } - if (metadata.getMeasurementId().equals(key.measurement)) { - timeseriesMetadata = metadata.getStatistics().getCount() == 0 ? null : metadata; - } - } - if (timeseriesMetadata == null - && !ignoreNotExists - && memoryConfig.isMayCacheNonExistSeries()) { - lruCache.put(key, NULL_EXISTS_CACHE_PLACE_HOLDER); - } - } + if (requested.contains(measurement)) { + result.put(measurement, null); + } + cacheNonExistenceIfNeeded(tsFileID, device, measurement, cacheEnabled, ignoreNotExists); + } else { + if (cacheEnabled) { + lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), metadata); + } + if (requested.contains(measurement)) { + result.put(measurement, new TimeseriesMetadata(metadata)); } } - if (timeseriesMetadata == null || timeseriesMetadata == NULL_EXISTS_CACHE_PLACE_HOLDER) { + } + fillMissingWithNull(requested, result); + } + + private static void fillMissingWithNull( + Set requested, Map result) { + for (String measurement : requested) { + result.putIfAbsent(measurement, null); + } + } + + private void cacheNonExistenceIfNeeded( + TsFileID tsFileID, + IDeviceID device, + String measurement, + boolean cacheEnabled, + boolean ignoreNotExists) { + if (cacheEnabled && !ignoreNotExists && memoryConfig.isMayCacheNonExistSeries()) { + lruCache.put( + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), + NULL_EXISTS_CACHE_PLACE_HOLDER); + } + } + + private static void recordLoadStatistics( + QueryContext queryContext, + long cacheHitCount, + long cacheLookupTime, + long diskLookupCount, + long diskLookupTime, + long bloomFilterTime) { + QueryStatistics statistics = queryContext.getQueryStatistics(); + statistics.getLoadBloomFilterTime().getAndAdd(bloomFilterTime); + if (cacheHitCount > 0) { + statistics.getLoadTimeSeriesMetadataFromCacheCount().getAndAdd(cacheHitCount); + statistics.getLoadTimeSeriesMetadataFromCacheTime().getAndAdd(cacheLookupTime); + } + if (diskLookupCount > 0 || diskLookupTime > 0) { + statistics.getLoadTimeSeriesMetadataFromDiskCount().getAndAdd(diskLookupCount); + statistics.getLoadTimeSeriesMetadataFromDiskTime().getAndAdd(diskLookupTime); + } + } + + /** + * Resolve cached measurements into {@code resolved} and collect the rest into {@code missing}. + */ + private void collectCachedMetadata( + TsFileID tsFileID, + IDeviceID device, + Collection measurements, + Map resolved, + Collection missing, + String filePath, + boolean debug) { + for (String measurement : measurements) { + TimeSeriesMetadataCacheKey key = + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement); + TimeseriesMetadata metadata = lruCache.getIfPresent(key); + if (metadata == null) { + missing.add(measurement); + } else if (metadata == NULL_EXISTS_CACHE_PLACE_HOLDER) { if (debug) { DEBUG_LOGGER.info(StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, key); } - return null; + resolved.put(measurement, null); } else { if (debug) { DEBUG_LOGGER.info( StorageEngineMessages.STORAGE_LOG_GET_TIMESERIES_METADATA_IN_FILE_FROM_CACHE_36652729, - key.device, - key.measurement, + device, + measurement, filePath, - timeseriesMetadata); + metadata); } - return new TimeseriesMetadata(timeseriesMetadata); - } - } finally { - queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime); - if (cacheHit) { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); - } else { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); + resolved.put(measurement, new TimeseriesMetadata(metadata)); } } } + /** + * Striped, strongly referenced monitor for one device+file. A WeakHashMap entry can be reclaimed + * by GC between calls, which would let two threads hold "the same" lock simultaneously. + */ + private static Object deviceLock(String deviceString, String filePath) { + return DEVICE_LOCKS[ + Math.floorMod((deviceString + SEPARATOR + filePath).hashCode(), DEVICE_LOCK_STRIPES)]; + } + public double calculateTimeSeriesMetadataHitRatio() { return lruCache.stats().hitRate(); } @@ -373,11 +565,15 @@ public static class TimeSeriesMetadataCacheKey { private final TsFileID tsFileID; private final IDeviceID device; private final String measurement; + private final int hash; public TimeSeriesMetadataCacheKey(TsFileID tsFileID, IDeviceID device, String measurement) { this.tsFileID = tsFileID; this.device = device; this.measurement = measurement; + // a key is built for every lookup and every write: compute the hash once instead of + // allocating the Objects.hash argument array on each call + this.hash = Objects.hash(tsFileID, device, measurement); } public long getRetainedSizeInBytes() { @@ -400,7 +596,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(tsFileID, device, measurement); + return hash; } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 87526eb594b9f..db2462bd87de8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -22,17 +22,27 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSourceType; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.LazyTsFileDeviceIterator; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.BloomFilter; +import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Ignore; import org.junit.Test; @@ -41,8 +51,12 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -51,9 +65,11 @@ import java.util.function.LongConsumer; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; public class TimeSeriesMetadataCacheTest { @@ -244,17 +260,17 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { reader.readTimeseriesMetadata( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.eq(measurement), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(true), Mockito.any(LongConsumer.class))) - .thenReturn(Collections.emptyList()); + .thenReturn(Collections.emptyMap()); Mockito.when( reader.readTimeseriesMetadata( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.eq(measurement), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(false), Mockito.any(LongConsumer.class))) .thenThrow(new IOException("missing device")); @@ -267,8 +283,8 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { .readTimeseriesMetadata( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.eq(measurement), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(false), Mockito.any(LongConsumer.class)); } finally { @@ -280,4 +296,614 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { .setMayCacheNonExistSeries(mayCacheNonExistSeries); } } + + private static Map getTimeseriesMetadata( + TimeSeriesMetadataCache cache, + String filePath, + TsFileID tsFileID, + IDeviceID deviceID, + Set allSensors, + boolean ignoreNotExists) + throws IOException { + return cache.getTimeseriesMetadata( + filePath, + tsFileID, + deviceID, + allSensors, + allSensors, + ignoreNotExists, + false, + new QueryContext(false, false), + null); + } + + private static void stubGroupRead( + TsFileSequenceReader reader, + IDeviceID deviceID, + boolean ignoreNotExists, + List group) + throws IOException { + Map loaded = new HashMap<>(); + for (TimeseriesMetadata metadata : group) { + loaded.put(metadata.getMeasurementId(), metadata); + } + Mockito.when( + reader.readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(ignoreNotExists), + Mockito.any(LongConsumer.class))) + .thenReturn(loaded); + } + + @Test + public void testGetTimeseriesMetadata() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-device.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + TimeseriesMetadata timeColumn = + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS()); + + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + TimeseriesMetadata s1 = + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS()); + + // an empty series (count == 0) is mapped to null and cached as the null placeholder + TimeseriesMetadata emptyS2 = + new TimeseriesMetadata( + (byte) 0, + 0, + "s2", + TSDataType.INT32, + Statistics.getStatsByType(TSDataType.INT32), + new PublicBAOS()); + + List group = new ArrayList<>(); + group.add(timeColumn); + group.add(s1); + group.add(emptyS2); + // the group read must carry chunk metadata, otherwise the downstream chunk loader NPEs + s1.setChunkMetadataList(Collections.singletonList(new ChunkMetadata())); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, false, group); + + Map result = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + + assertNotNull(result.get("")); + assertEquals("", result.get("").getMeasurementId()); + assertNotNull(result.get("s1")); + assertEquals("s1", result.get("s1").getMeasurementId()); + // chunk metadata must flow through to the result (and the cached copy) unchanged + assertNotNull(result.get("s1").getChunkMetadataList()); + assertNull(result.get("s2")); + + // a second lookup of cached measurements must be served without touching the reader again + Set cachedSensors = new HashSet<>(Arrays.asList("", "s1")); + Map cachedResult = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, cachedSensors, false); + assertNotNull(cachedResult.get("s1")); + Mockito.verify(reader, Mockito.times(1)) + .readTimeseriesMetadata( + Mockito.any(), + Mockito.any(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); + + // the group read also populated the per-measurement cache + TimeseriesMetadata cached = + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s1"), + Collections.singleton("s1"), + true, + false, + new QueryContext(false, false)); + assertNotNull(cached); + assertEquals("s1", cached.getMeasurementId()); + + // the cache hands out copies: mutating a returned entry must not poison later reads + result.get("s1").setModified(true); + TimeseriesMetadata cachedAgain = + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s1"), + Collections.singleton("s1"), + true, + false, + new QueryContext(false, false)); + assertNotNull(cachedAgain); + assertFalse(cachedAgain.isModified()); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-absent-device-bloom.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = Collections.singleton(""); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + // the bloom filter proves the device is missing, so the device must not be read at all + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))) + .thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1)); + + Map result = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNull(result.get("")); + Mockito.verify(reader, Mockito.never()) + .readTimeseriesMetadata( + Mockito.any(), + Mockito.any(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-absent-device-throw.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, true, Collections.emptyList()); + Mockito.when( + reader.readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class))) + .thenThrow(new IOException("device absent")); + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); + bloomFilter.add(deviceID.toString() + "."); + bloomFilter.add(deviceID.toString() + ".s1"); + bloomFilter.add(deviceID.toString() + ".s2"); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); + + // ignoreNotExists keeps the missing device silent, but caches nothing + Map ignored = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, true); + assertNull(ignored.get("")); + assertNull(ignored.get("s1")); + assertNull(ignored.get("s2")); + + // a normal query must surface the reader's failure instead of a silently empty device + assertThrows( + IOException.class, + () -> getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testBatchMetadataMatchesPerColumn() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + int columnCount = 700; + File file = new File("target/batch-compare.tsfile"); + file.delete(); + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.compare.d1"); + List schemas = new ArrayList<>(columnCount); + for (int i = 0; i < columnCount; i++) { + schemas.add(new MeasurementSchema("s" + i, TSDataType.INT32)); + } + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerAlignedTimeseries(deviceID, schemas); + for (long time = 0; time < 3; time++) { + TSRecord record = new TSRecord(deviceID, time); + for (int i = 0; i < columnCount; i++) { + record.addPoint("s" + i, (int) time); + } + writer.writeRecord(record); + } + } + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = file.getAbsolutePath(); + Set allSensors = new HashSet<>(); + for (int i = 0; i < columnCount; i++) { + allSensors.add("s" + i); + } + allSensors.add(""); + allSensors.add("absent"); + + Map perColumn = new HashMap<>(); + for (String measurement : allSensors) { + perColumn.put( + measurement, + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, measurement), + allSensors, + false, + false, + new QueryContext(false, false))); + } + cache.clear(); + + Map batch = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + + assertEquals(allSensors.size(), batch.size()); + for (String measurement : allSensors) { + TimeseriesMetadata expected = perColumn.get(measurement); + TimeseriesMetadata actual = batch.get(measurement); + if (expected == null) { + assertNull(measurement, actual); + continue; + } + assertNotNull(measurement, actual); + assertEquals(measurement, actual.getMeasurementId()); + assertEquals( + measurement, expected.getStatistics().getCount(), actual.getStatistics().getCount()); + assertEquals( + measurement, + expected.getStatistics().getStartTime(), + actual.getStatistics().getStartTime()); + assertEquals( + measurement, + expected.getStatistics().getEndTime(), + actual.getStatistics().getEndTime()); + assertEquals( + measurement, + expected.getChunkMetadataList().size(), + actual.getChunkMetadataList().size()); + for (int i = 0; i < expected.getChunkMetadataList().size(); i++) { + ChunkMetadata expectedChunk = (ChunkMetadata) expected.getChunkMetadataList().get(i); + ChunkMetadata actualChunk = (ChunkMetadata) actual.getChunkMetadataList().get(i); + assertEquals( + measurement, + expectedChunk.getOffsetOfChunkHeader(), + actualChunk.getOffsetOfChunkHeader()); + assertEquals( + measurement, + expectedChunk.getStatistics().getStartTime(), + actualChunk.getStatistics().getStartTime()); + assertEquals( + measurement, + expectedChunk.getStatistics().getEndTime(), + actualChunk.getStatistics().getEndTime()); + } + } + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + assertTrue(file.delete()); + } + } + + @Test + public void testBloomFilterExcludedMeasurementsSkipRead() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-time-column-filtered.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + // both the time column and s1 are excluded, so no leaf group needs to be read + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); + + Map result = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + + assertNull(result.get("")); + assertNull(result.get("s1")); + Mockito.verify(reader, Mockito.never()) + .readTimeseriesMetadata( + Mockito.any(), + Mockito.any(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testEvictedMeasurementReloadsOnlyItsGroup() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-single-miss.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + Statistics statistics = Statistics.getStatsByType(TSDataType.INT32); + statistics.setCount(1); + statistics.setStartTime(0); + statistics.setEndTime(10); + TimeseriesMetadata s2 = + new TimeseriesMetadata((byte) 0, 0, "s2", TSDataType.INT32, statistics, new PublicBAOS()); + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + + List group = new ArrayList<>(); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS())); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS())); + group.add(s2); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, false, group); + + Map first = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNotNull(first.get("s2")); + + // evict one measurement: the next call reads only the group that holds it, and that one + // read also resolves the other members of the group + cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s2")); + Map second = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + + assertNotNull(second.get("s2")); + Mockito.verify(reader, Mockito.times(2)) + .readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testBatchIgnoreNotExistsDoesNotCachePlaceholders() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-ignore-not-exists.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, true, Collections.emptyList()); + Mockito.when( + reader.readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class))) + .thenThrow(new IOException("device absent")); + + Map ignored = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, true); + assertNull(ignored.get("")); + assertNull(ignored.get("s1")); + + // ignoreNotExists must not write NULL placeholders: a normal query still has to read + assertThrows( + IOException.class, + () -> getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false)); + // the empty group read resolves the time column as absent, which ends the load for the + // device, plus the failing normal query + Mockito.verify(reader, Mockito.times(2)) + .readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testEvictedMeasurementsShareOneGroupRead() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-few-missing.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2", "s3", "s4", "s5")); + + List group = new ArrayList<>(); + for (String measurement : allSensors) { + TSDataType dataType = measurement.isEmpty() ? TSDataType.INT64 : TSDataType.INT32; + Statistics statistics = Statistics.getStatsByType(dataType); + statistics.setCount(1); + statistics.setStartTime(0); + statistics.setEndTime(10); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, measurement, dataType, statistics, new PublicBAOS())); + } + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, false, group); + + Map first = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNotNull(first.get("s5")); + + // evict three measurements: they share one leaf group, so a single read must resolve them + for (String measurement : Arrays.asList("s2", "s3", "s4")) { + cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, measurement)); + } + + Map second = + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNotNull(second.get("s2")); + assertNotNull(second.get("s3")); + assertNotNull(second.get("s4")); + Mockito.verify(reader, Mockito.times(2)) + .readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testExternalTsFileReadsAllRequestedMeasurements() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + File file = new File("target/external-cache-disabled.tsfile"); + file.delete(); + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.external.d1"); + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerAlignedTimeseries( + deviceID, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32))); + for (int time = 0; time < 3; time++) { + TSRecord record = new TSRecord(deviceID, time); + record.addPoint("s1", time); + record.addPoint("s2", time); + writer.writeRecord(record); + } + } + cache.clear(); + bloomFilterCache.clear(); + long[] offset; + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) { + LazyTsFileDeviceIterator iterator = new LazyTsFileDeviceIterator(reader); + assertTrue(iterator.hasNext()); + iterator.next(); + offset = iterator.getCurrentDeviceMeasurementNodeOffset(); + } + // external tsfiles bypass the metadata cache: every requested measurement must still be read + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setQueryDataSourceType( + QueryDataSourceType.EXTERNAL_TSFILE_SCAN); + try { + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + TsFileResource resource = new TsFileResource(file); + resource.setStatusForTest(TsFileResourceStatus.NORMAL); + Map result = + cache.getTimeseriesMetadata( + file.getAbsolutePath(), + resource.getTsFileID(), + deviceID, + allSensors, + allSensors, + false, + false, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT, + offset); + assertEquals(allSensors.size(), result.size()); + for (String measurement : allSensors) { + assertNotNull(measurement, result.get(measurement)); + assertEquals(measurement, result.get(measurement).getMeasurementId()); + } + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setQueryDataSourceType( + QueryDataSourceType.SERIES_SCAN); + cache.clear(); + bloomFilterCache.clear(); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java new file mode 100644 index 0000000000000..2decb73ab6749 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.AlignedFullPath; +import org.apache.iotdb.commons.path.AlignedPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache; +import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; +import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.db.utils.constant.TestConstant; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; +import org.apache.tsfile.write.chunk.IChunkWriter; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; + +import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.createChunkWriter; +import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.writeAlignedChunk; + +/** + * Verifies metadata loading on a real device wider than one measurement leaf group: each selected + * column is resolved from the leaf group that holds it, and one group read serves every selected + * column of that group. + */ +public class AlignedSeriesScanBatchTest { + + private static final String DATABASE = "root.wide_pd"; + private static final IDeviceID DEVICE = + IDeviceID.Factory.DEFAULT_FACTORY.create(DATABASE + ".d1"); + private static final int COLUMN_COUNT = 300; + private static final int PRESENT_COLUMN_COUNT = 30; + private static final int ROW_COUNT = 10; + + @Test + public void testScanReadsWideAlignedDevice() throws Exception { + Assert.assertTrue( + "the fixture must span more than one measurement leaf group", + COLUMN_COUNT + 1 > TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode()); + File file = newFile(1); + TsFileResource resource = writeWideDevice(file, COLUMN_COUNT); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + metadataCache.clear(); + bloomFilterCache.clear(); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + // the unified path is only reachable from table dialect, which forces this flag to false + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); + try { + long cacheHitsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get(); + long diskReadsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get(); + + TsBlock block = scan(alignedPath(columnNames(COLUMN_COUNT)), resource); + + Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); + Assert.assertEquals(ROW_COUNT, block.getPositionCount()); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals(row, block.getTimeByIndex(row)); + } + for (int column = 0; column < COLUMN_COUNT; column++) { + Column valueColumn = block.getColumn(column); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertFalse("s" + column + " row " + row, valueColumn.isNull(row)); + Assert.assertEquals("s" + column + " row " + row, row, valueColumn.getInt(row)); + } + } + + // a cold cache: every selected series is a disk lookup, counted per series + Assert.assertEquals( + cacheHitsBefore, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get()); + // per-series counters: a cold scan counts one disk lookup per selected measurement + long diskReads = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get() + - diskReadsBefore; + Assert.assertTrue("disk lookups: " + diskReads, diskReads >= 1); + Assert.assertTrue("disk lookups: " + diskReads, diskReads <= COLUMN_COUNT + 1); + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + @Test + public void testSparseProjectionFiltersAbsentColumns() throws Exception { + File file = newFile(2); + TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + AlignedFullPath scanPath = alignedPath(columnNames(COLUMN_COUNT)); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); + try { + metadataCache.clear(); + bloomFilterCache.clear(); + long diskReadsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get(); + + TsBlock block = scan(scanPath, resource); + + Assert.assertEquals(ROW_COUNT, block.getPositionCount()); + Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals(row, block.getTimeByIndex(row)); + } + for (int column = 0; column < COLUMN_COUNT; column++) { + Column valueColumn = block.getColumn(column); + boolean present = column < PRESENT_COLUMN_COUNT; + for (int row = 0; row < ROW_COUNT; row++) { + if (present) { + Assert.assertFalse(valueColumn.isNull(row)); + Assert.assertEquals(row, valueColumn.getInt(row)); + } else { + Assert.assertTrue("s" + column + " row " + row, valueColumn.isNull(row)); + } + } + } + + // the bloom filter excludes the 270 absent columns; a few may survive as false positives, + // but the sparse projection must still look up far fewer series than it selects + long diskReads = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get() + - diskReadsBefore; + // per-series counters: bloom-excluded measurements count as disk lookups, as they did on the + // per-column path + Assert.assertTrue("disk lookups: " + diskReads, diskReads >= 1); + Assert.assertTrue("disk lookups: " + diskReads, diskReads <= COLUMN_COUNT + 1); + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + @Test + public void testAllNullProjection() throws Exception { + File file = newFile(3); + TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + // every selected field is absent from the file + AlignedFullPath absentPath = alignedPath(absentColumnNames(COLUMN_COUNT)); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + try { + // table dialect: all-null rows are kept + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); + metadataCache.clear(); + bloomFilterCache.clear(); + TsBlock block = scan(absentPath, resource); + Assert.assertEquals(ROW_COUNT, block.getPositionCount()); + Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); + for (int column = 0; column < COLUMN_COUNT; column++) { + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertTrue("absent" + column + " row " + row, block.getColumn(column).isNull(row)); + } + } + + // tree dialect: an all-null projection skips the file + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(true); + metadataCache.clear(); + bloomFilterCache.clear(); + Assert.assertFalse(hasFile(absentPath, resource)); + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + private static TsBlock scan(AlignedFullPath scanPath, TsFileResource resource) + throws IOException { + AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, resource); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + return seriesScanUtil.nextPage(); + } + + private static boolean hasFile(AlignedFullPath scanPath, TsFileResource resource) + throws IOException { + AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, resource); + while (true) { + Optional hasNextFile = seriesScanUtil.hasNextFile(); + if (hasNextFile.isPresent()) { + return hasNextFile.get(); + } + } + } + + private static AlignedSeriesScanUtil newScanUtil( + AlignedFullPath scanPath, TsFileResource resource) throws IOException { + SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); + builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); + SeriesScanOptions scanOptions = builder.build(); + AlignedSeriesScanUtil seriesScanUtil = + new AlignedSeriesScanUtil( + scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource( + new QueryDataSource(Collections.singletonList(resource), Collections.emptyList())); + return seriesScanUtil; + } + + private static AlignedFullPath alignedPath(List measurementNames) { + List schemas = new ArrayList<>(measurementNames.size()); + for (String measurement : measurementNames) { + schemas.add(new MeasurementSchema(measurement, TSDataType.INT32)); + } + return new AlignedFullPath(DEVICE, measurementNames, schemas); + } + + private static List columnNames(int count) { + List names = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + names.add("s" + i); + } + return names; + } + + private static List absentColumnNames(int count) { + List names = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + names.add("absent" + i); + } + return names; + } + + private static File newFile(int version) { + File file = new File(TestConstant.getTestTsFilePath(DATABASE, 0, 0, version)); + if (!file.getParentFile().exists()) { + Assert.assertTrue(file.getParentFile().mkdirs()); + } + return file; + } + + private static TsFileResource writeWideDevice(File file, int presentColumnCount) + throws IOException, IllegalPathException { + TsFileResource resource = new TsFileResource(file); + List writtenPaths = new ArrayList<>(presentColumnCount); + List dataTypes = new ArrayList<>(presentColumnCount); + List encodings = new ArrayList<>(presentColumnCount); + List compressionTypes = new ArrayList<>(presentColumnCount); + for (int i = 0; i < presentColumnCount; i++) { + writtenPaths.add(new AlignedPath(DEVICE.toString(), "s" + i)); + dataTypes.add(TSDataType.INT32); + encodings.add(TSEncoding.PLAIN); + compressionTypes.add(CompressionType.UNCOMPRESSED); + } + try (TsFileIOWriter writer = new TsFileIOWriter(file)) { + writer.startChunkGroup(DEVICE); + for (IChunkWriter chunkWriter : + createChunkWriter(writtenPaths, dataTypes, encodings, compressionTypes, true)) { + writeAlignedChunk( + (AlignedChunkWriterImpl) chunkWriter, + writer, + Collections.singletonList(new TimeRange(0L, ROW_COUNT - 1L)), + true); + } + writer.endChunkGroup(); + writer.endFile(); + } + resource.updateStartTime(DEVICE, 0); + resource.updateEndTime(DEVICE, ROW_COUNT - 1L); + resource.setStatusForTest(TsFileResourceStatus.NORMAL); + return resource; + } + + private static void cleanup( + File file, + TsFileResource resource, + TimeSeriesMetadataCache metadataCache, + BloomFilterCache bloomFilterCache) + throws IOException { + metadataCache.clear(); + bloomFilterCache.clear(); + FileReaderManager.getInstance().closeFileAndRemoveReader(resource.getTsFileID()); + Assert.assertTrue(file.delete()); + } +} diff --git a/pom.xml b/pom.xml index abd01e78010fc..ee37426af6fe2 100644 --- a/pom.xml +++ b/pom.xml @@ -143,7 +143,7 @@ 0.25.0 1.9 1.5.6-3 - 2.5.1-261023-SNAPSHOT + 2.5.1-261117-SNAPSHOT en