From 0579b197fe05c6fbb14f6d8a7fa1a07d8aa21c2b Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 10 Nov 2026 12:19:35 +0800 Subject: [PATCH 01/27] Batch scan TimeSeriesMetadata for wide-table queries A table scan that selects (nearly) all field columns of a device previously deserialized the device's TimeSeriesMetadata column by column, re-walking the device metadata index once per leaf node. Add a batch path that reads the whole device's metadata in a single pass via TsFileSequenceReader.readDeviceMetadata, populating the per-measurement cache and building the aligned metadata through the existing modification/loader logic. The batch path is taken when the selected field columns reach a configurable fraction (device_metadata_batch_scan_ratio, default 0.8) of the table's total field columns. --- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 14 +++ .../apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ++ .../source/AlignedSeriesScanUtil.java | 3 +- .../operator/source/FileLoaderUtils.java | 95 +++++++++++++++++-- .../ExternalTsFileSeriesScanUtil.java | 3 +- .../DataNodeTableOperatorGenerator.java | 12 +++ .../plan/parameter/SeriesScanOptions.java | 9 ++ .../buffer/TimeSeriesMetadataCache.java | 54 +++++++++++ .../buffer/TimeSeriesMetadataCacheTest.java | 76 +++++++++++++++ .../conf/iotdb-system.properties.template | 6 ++ 10 files changed, 269 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index c6a0d4878969f..c1d3aad8e8b37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -853,6 +853,12 @@ public class IoTDBConfig implements DefaultEncodingProvider { /** time cost(ms) threshold for slow query. Unit: millisecond */ private long slowQueryThreshold = 10000; + /** + * When a table scan queries at least this fraction of a table's field columns, deserialize all + * the device's TimeSeriesMetadata in a single batch read instead of column by column. + */ + private double deviceMetadataBatchScanRatio = 0.8; + /** time window threshold for record of history queries. Unit: minute */ private int queryCostStatWindow = 0; @@ -2962,6 +2968,14 @@ public void setSlowQueryThreshold(long slowQueryThreshold) { this.slowQueryThreshold = slowQueryThreshold; } + public double getDeviceMetadataBatchScanRatio() { + return deviceMetadataBatchScanRatio; + } + + public void setDeviceMetadataBatchScanRatio(double deviceMetadataBatchScanRatio) { + this.deviceMetadataBatchScanRatio = deviceMetadataBatchScanRatio; + } + public int getQueryCostStatWindow() { return queryCostStatWindow; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 04fe3381b7541..dae9ba26b0864 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -519,6 +519,12 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException properties.getProperty( "query_timeout_threshold", Long.toString(conf.getQueryTimeoutThreshold())))); + conf.setDeviceMetadataBatchScanRatio( + Double.parseDouble( + properties.getProperty( + "device_metadata_batch_scan_ratio", + Double.toString(conf.getDeviceMetadataBatchScanRatio())))); + conf.setCopyToAllowedExportDirs( Arrays.stream( properties diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java index 331a5a5b267d1..517a5e0718bfc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java @@ -101,7 +101,8 @@ protected AbstractAlignedTimeSeriesMetadata loadTimeSeriesMetadata( scanOptions.getGlobalTimeFilter(), isSeq, ignoreAllNullRows, - null); + null, + scanOptions.getTotalFieldColumnCount()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 4374e974db5bc..dd08c320d1f37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.commons.path.AlignedFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; import org.apache.iotdb.db.exception.CorruptedTsFileException; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; @@ -62,6 +63,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -222,7 +224,8 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata( Filter globalTimeFilter, boolean isSeq, boolean ignoreAllNullRows, - long[] rootMeasurementMetadataIndexNodeOffset) + long[] rootMeasurementMetadataIndexNodeOffset, + int totalFieldColumnCount) throws IOException { final long t1 = System.nanoTime(); boolean loadFromMem = false; @@ -241,7 +244,8 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata( context, globalTimeFilter, ignoreAllNullRows, - rootMeasurementMetadataIndexNodeOffset); + rootMeasurementMetadataIndexNodeOffset, + totalFieldColumnCount); } else { // if the tsfile is unclosed, we just get it directly from TsFileResource loadFromMem = true; alignedTimeSeriesMetadata = @@ -333,7 +337,8 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr FragmentInstanceContext context, Filter globalTimeFilter, boolean ignoreAllNullRows, - long[] rootMeasurementMetadataIndexNodeOffset) + long[] rootMeasurementMetadataIndexNodeOffset, + int totalFieldColumnCount) throws IOException { AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null; // load all the TimeseriesMetadata of vector, the first one is for time column and the @@ -348,13 +353,33 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex // we should not ignore the non-exist of device in TsFileMetadata + boolean ignoreNotExists = + context.ignoreNotExistsDevice() + || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE; + + // For a wide device (a table scan that selects almost all field columns), deserialize the + // whole device's TimeSeriesMetadata in a single batch read instead of walking the device + // metadata index once per column. + if (shouldUseDeviceMetadataBatchScan(valueMeasurementList.size(), totalFieldColumnCount) + && ignoreNotExists) { + return loadAlignedTimeSeriesMetadataBatch( + resource, + alignedPath, + valueMeasurementList, + context, + globalTimeFilter, + ignoreAllNullRows, + cache, + filePath, + deviceId); + } + TimeseriesMetadata timeColumn = cache.get( filePath, new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, ""), allSensors, - context.ignoreNotExistsDevice() - || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, + ignoreNotExists, isDebug, context, rootMeasurementMetadataIndexNodeOffset); @@ -383,8 +408,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr filePath, new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement), allSensors, - context.ignoreNotExistsDevice() - || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, + ignoreNotExists, isDebug, context, rootMeasurementMetadataIndexNodeOffset); @@ -407,6 +431,63 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr return alignedTimeSeriesMetadata; } + private static boolean shouldUseDeviceMetadataBatchScan( + int selectedColumnCount, int totalFieldColumnCount) { + return totalFieldColumnCount > 0 + && (double) selectedColumnCount / totalFieldColumnCount + >= IoTDBDescriptor.getInstance().getConfig().getDeviceMetadataBatchScanRatio(); + } + + private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBatch( + TsFileResource resource, + AlignedFullPath alignedPath, + List valueMeasurementList, + FragmentInstanceContext context, + Filter globalTimeFilter, + boolean ignoreAllNullRows, + TimeSeriesMetadataCache cache, + String filePath, + IDeviceID deviceId) + throws IOException { + Map deviceMetadata = + cache.getDeviceTimeSeriesMetadata(filePath, resource.getTsFileID(), deviceId, context); + TimeseriesMetadata timeColumn = deviceMetadata.get(""); + if (timeColumn == null) { + return null; + } + // only need time column, like count_time aggregation + if (valueMeasurementList.isEmpty()) { + return setModifications( + resource, + timeColumn, + Collections.emptyList(), + alignedPath, + context, + globalTimeFilter, + false); + } + List valueTimeSeriesMetadataList = + new ArrayList<>(valueMeasurementList.size()); + // if all the queried aligned sensors does not exist, we will return null + boolean exist = false; + for (String measurement : valueMeasurementList) { + TimeseriesMetadata valueColumn = deviceMetadata.get(measurement); + exist = (exist || (valueColumn != null)); + valueTimeSeriesMetadataList.add(valueColumn); + } + if (!ignoreAllNullRows || exist) { + return setModifications( + resource, + timeColumn, + valueTimeSeriesMetadataList, + alignedPath, + context, + globalTimeFilter, + ignoreAllNullRows); + } + return null; + } + private static AbstractAlignedTimeSeriesMetadata setModifications( TsFileResource resource, TimeseriesMetadata timeColumnMetadata, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java index e907218fe8a7b..61a2832e66b40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java @@ -83,7 +83,8 @@ static AbstractAlignedTimeSeriesMetadata loadTimeSeriesMetadata( globalTimeFilter, resource.isSeq(), context.isIgnoreAllNullRows(), - new long[] {currentDeviceOffset.getStartOffset(), currentDeviceOffset.getEndOffset()}); + new long[] {currentDeviceOffset.getStartOffset(), currentDeviceOffset.getEndOffset()}, + 0); } @FunctionalInterface diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index 01584fe230776..8b6b9b36bb88c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -1185,6 +1185,9 @@ private void addSource( topKRuntimeFilter); seriesScanOptions.setTTLForTableView(viewTTL); seriesScanOptions.setIsTableViewForTreeModel(node instanceof TreeDeviceViewScanNode); + if (!(node instanceof TreeDeviceViewScanNode) && !(node instanceof ExternalTsFileScanNode)) { + seriesScanOptions.setTotalFieldColumnCount(computeTotalFieldColumnCount(node)); + } OperatorContext operatorContext = addOperatorContext(context, node.getPlanNodeId(), className); @@ -1222,6 +1225,15 @@ private BatchDeviceEntrySource createDeviceEntrySource(DeviceTableScanNode node) .orElseGet(() -> new InMemoryDeviceEntrySource(node.getDeviceEntries())); } + private int computeTotalFieldColumnCount(DeviceTableScanNode node) { + QualifiedObjectName qualifiedObjectName = node.getQualifiedObjectName(); + TsTable tsTable = + DataNodeTableCache.getInstance() + .getTableInCache( + qualifiedObjectName.getDatabaseName(), qualifiedObjectName.getObjectName()); + return tsTable == null ? 0 : tsTable.getFieldNum(); + } + private BatchDeviceEntrySource createAggregationDeviceEntrySource(AggregationTableScanNode node) { if (node.getDeviceEntryCount() == 0 && node.getGroupingKeys().isEmpty()) { return new InMemoryDeviceEntrySource(Collections.singletonList(null)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java index 4d40bbd32c4d4..2a173e3b09e0f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java @@ -56,6 +56,7 @@ public class SeriesScanOptions implements Accountable { private PaginationController paginationController; private boolean isTableViewForTreeModel; private long ttlForTableView = Long.MAX_VALUE; + private int totalFieldColumnCount = 0; private final TopKRuntimeFilter topKRuntimeFilter; private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance( @@ -178,6 +179,14 @@ public void setIsTableViewForTreeModel(boolean isTableViewForTreeModel) { this.isTableViewForTreeModel = isTableViewForTreeModel; } + public int getTotalFieldColumnCount() { + return totalFieldColumnCount; + } + + public void setTotalFieldColumnCount(int totalFieldColumnCount) { + this.totalFieldColumnCount = totalFieldColumnCount; + } + /** * pushLimitToEachDevice==false means that all devices return total limit rows. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 68457df0ebbcd..28f7eab222964 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -50,6 +50,7 @@ import java.io.IOException; import java.lang.ref.WeakReference; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -316,6 +317,59 @@ public TimeseriesMetadata get( } } + /** + * Load all TimeseriesMetadata of a device in a single batch read and populate the per-measurement + * cache. This is used by wide-table scans (a device with many columns) to avoid walking the + * device metadata index once per column / leaf node. + * + *

The returned map holds copies of the cached metadata (or the freshly read objects when the + * cache is disabled / for external tsfiles), so the caller may apply {@code setModified} and + * {@code setChunkMetadataLoader} without corrupting cached entries — matching {@link #get}. + * + * @param filePath path of the tsfile + * @param tsFileID id of the tsfile + * @param device target device + * @param queryContext query context used for metrics and to detect external tsfiles + * @return map from measurementId (including the time column, keyed by "") to TimeseriesMetadata; + * entries whose series is empty (count == 0) are mapped to null. + * @throws IOException if the metadata cannot be read from disk + */ + public Map getDeviceTimeSeriesMetadata( + String filePath, TsFileID tsFileID, IDeviceID device, QueryContext queryContext) + throws IOException { + long startTime = System.nanoTime(); + boolean externalTsFile = queryContext.isExternalTsFileScan(); + LongConsumer timeSeriesMetadataIoSizeRecorder = + queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; + + try { + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); + Map deviceMetadata = reader.readDeviceMetadata(device); + + Map result = new HashMap<>(deviceMetadata.size()); + for (Map.Entry entry : deviceMetadata.entrySet()) { + TimeseriesMetadata metadata = entry.getValue(); + if (metadata.getStatistics().getCount() == 0) { + result.put(entry.getKey(), null); + continue; + } + if (CACHE_ENABLE && !externalTsFile) { + lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, entry.getKey()), metadata); + } + result.put(entry.getKey(), new TimeseriesMetadata(metadata)); + } + return result; + } finally { + queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromDiskCount().incrementAndGet(); + queryContext + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskTime() + .getAndAdd(System.nanoTime() - startTime); + } + } + public double calculateTimeSeriesMetadataHitRatio() { return lruCache.stats().hitRate(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 87526eb594b9f..de70d297b82fe 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -30,7 +30,9 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -42,7 +44,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -280,4 +284,76 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { .setMayCacheNonExistSeries(mayCacheNonExistSeries); } } + + @Test + public void testGetDeviceTimeSeriesMetadata() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + try { + String filePath = "target/batch-device.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + TimeseriesMetadata timeColumn = + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS()); + + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + TimeseriesMetadata s1 = + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS()); + + // an empty series (count == 0) must be mapped to null and not cached + TimeseriesMetadata emptyS2 = + new TimeseriesMetadata( + (byte) 0, + 0, + "s2", + TSDataType.INT32, + Statistics.getStatsByType(TSDataType.INT32), + new PublicBAOS()); + + Map deviceMetadata = new HashMap<>(); + deviceMetadata.put("", timeColumn); + deviceMetadata.put("s1", s1); + deviceMetadata.put("s2", emptyS2); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.readDeviceMetadata(deviceID)).thenReturn(deviceMetadata); + + Map result = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, new QueryContext(false, false)); + + assertNotNull(result.get("")); + assertEquals("", result.get("").getMeasurementId()); + assertNotNull(result.get("s1")); + assertEquals("s1", result.get("s1").getMeasurementId()); + assertNull(result.get("s2")); + + // the non-empty entries should now be cached for per-measurement lookups + TimeseriesMetadata cached = + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s1"), + Collections.singleton("s1"), + true, + false, + new QueryContext(false, false)); + assertNotNull(cached); + assertEquals("s1", cached.getMeasurementId()); + } finally { + cache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } } diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index a3fe0970318c8..798da5494cb5c 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1267,6 +1267,12 @@ max_tsblock_line_number=1000 # Datatype: long slow_query_threshold=10000 +# When a table scan queries at least this fraction of a table's field columns, deserialize all the +# device's TimeSeriesMetadata in a single batch read instead of column by column. +# effectiveMode: restart +# Datatype: double +device_metadata_batch_scan_ratio=0.8 + # Time window threshold(min) for record of history queries. # effectiveMode: hot_reload # Datatype: int From 1be4e0c85465c15183daa2013d2a9bf370ef5494 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 10 Nov 2026 16:48:21 +0800 Subject: [PATCH 02/27] Fix NPE in batch TimeSeriesMetadata read by loading chunk metadata The batch path read a device's TimeseriesMetadata via TsFileSequenceReader.readDeviceMetadata, which deserializes only the header and statistics and leaves chunkMetadataList null. The downstream DiskAlignedChunkMetadataLoader dereferences that list in getCopiedChunkMetadataList, so a wide-table scan NPE'd at query time, and the null-chunk entries also polluted the shared per-measurement LRU cache (breaking later narrow queries on the same device). Switch to getDeviceTimeseriesMetadata, which deserializes chunk metadata too, and adapt the unit test to the list-based API, asserting that chunk metadata flows through to the result. --- .../buffer/TimeSeriesMetadataCache.java | 19 +++++++++++-------- .../buffer/TimeSeriesMetadataCacheTest.java | 16 ++++++++++------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 28f7eab222964..7f4ba60ab1f72 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -346,19 +346,22 @@ public Map getDeviceTimeSeriesMetadata( TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); - Map deviceMetadata = reader.readDeviceMetadata(device); - - Map result = new HashMap<>(deviceMetadata.size()); - for (Map.Entry entry : deviceMetadata.entrySet()) { - TimeseriesMetadata metadata = entry.getValue(); + // Use the chunk-metadata-including variant: the returned TimeseriesMetadata are passed to + // setModifications and then to DiskAlignedChunkMetadataLoader, which dereferences the chunk + // metadata list, so readDeviceMetadata (which skips chunk metadata) would NPE at query time. + List deviceMetadataList = reader.getDeviceTimeseriesMetadata(device); + + Map result = new HashMap<>(deviceMetadataList.size()); + for (TimeseriesMetadata metadata : deviceMetadataList) { + String measurementId = metadata.getMeasurementId(); if (metadata.getStatistics().getCount() == 0) { - result.put(entry.getKey(), null); + result.put(measurementId, null); continue; } if (CACHE_ENABLE && !externalTsFile) { - lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, entry.getKey()), metadata); + lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId), metadata); } - result.put(entry.getKey(), new TimeseriesMetadata(metadata)); + result.put(measurementId, new TimeseriesMetadata(metadata)); } return result; } finally { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index de70d297b82fe..eec5a7a13af72 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -27,6 +27,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.TimeseriesMetadata; @@ -44,7 +45,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -321,14 +321,16 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { Statistics.getStatsByType(TSDataType.INT32), new PublicBAOS()); - Map deviceMetadata = new HashMap<>(); - deviceMetadata.put("", timeColumn); - deviceMetadata.put("s1", s1); - deviceMetadata.put("s2", emptyS2); + List deviceMetadata = new ArrayList<>(); + deviceMetadata.add(timeColumn); + deviceMetadata.add(s1); + deviceMetadata.add(emptyS2); + // the batch read must carry chunk metadata, otherwise the downstream chunk loader NPEs + s1.setChunkMetadataList(Collections.singletonList(new ChunkMetadata())); TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - Mockito.when(reader.readDeviceMetadata(deviceID)).thenReturn(deviceMetadata); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)).thenReturn(deviceMetadata); Map result = cache.getDeviceTimeSeriesMetadata( @@ -338,6 +340,8 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { assertEquals("", result.get("").getMeasurementId()); assertNotNull(result.get("s1")); assertEquals("s1", result.get("s1").getMeasurementId()); + // chunk metadata must flow through to the result (and the cached copy) unchanged + assertNotNull(result.get("s1").getChunkMetadataList()); assertNull(result.get("s2")); // the non-empty entries should now be cached for per-measurement lookups From 758f4aea543974d205422c4cc62d34e1d6dd5940 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 10 Nov 2026 17:17:25 +0800 Subject: [PATCH 03/27] Clean up batch TimeSeriesMetadata scan path Remove the unreachable time-column-only branch in the batch loader (the batch gate already requires a non-empty selection), and validate device_metadata_batch_scan_ratio so a non-positive value falls back to the default instead of routing every table scan through the batch path. Document the valid range in the properties template. --- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 11 +++++++++-- .../execution/operator/source/FileLoaderUtils.java | 11 ----------- .../resources/conf/iotdb-system.properties.template | 1 + 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index dae9ba26b0864..df91d783ddb7e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -519,11 +519,18 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException properties.getProperty( "query_timeout_threshold", Long.toString(conf.getQueryTimeoutThreshold())))); - conf.setDeviceMetadataBatchScanRatio( + double deviceMetadataBatchScanRatio = Double.parseDouble( properties.getProperty( "device_metadata_batch_scan_ratio", - Double.toString(conf.getDeviceMetadataBatchScanRatio())))); + Double.toString(conf.getDeviceMetadataBatchScanRatio()))); + // a non-positive ratio would route every table scan, even single-column ones, through the + // batch path; keep the current value instead + deviceMetadataBatchScanRatio = + deviceMetadataBatchScanRatio <= 0 + ? conf.getDeviceMetadataBatchScanRatio() + : deviceMetadataBatchScanRatio; + conf.setDeviceMetadataBatchScanRatio(deviceMetadataBatchScanRatio); conf.setCopyToAllowedExportDirs( Arrays.stream( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index dd08c320d1f37..857328a58838c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -455,17 +455,6 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBa if (timeColumn == null) { return null; } - // only need time column, like count_time aggregation - if (valueMeasurementList.isEmpty()) { - return setModifications( - resource, - timeColumn, - Collections.emptyList(), - alignedPath, - context, - globalTimeFilter, - false); - } List valueTimeSeriesMetadataList = new ArrayList<>(valueMeasurementList.size()); // if all the queried aligned sensors does not exist, we will return null diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 798da5494cb5c..e8021790bbe83 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1269,6 +1269,7 @@ slow_query_threshold=10000 # When a table scan queries at least this fraction of a table's field columns, deserialize all the # device's TimeSeriesMetadata in a single batch read instead of column by column. +# Value must be in (0, 1]; a non-positive value falls back to the default. # effectiveMode: restart # Datatype: double device_metadata_batch_scan_ratio=0.8 From 2563c508aeeeecf21d4de53a7c6ea46148fa886b Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 10 Nov 2026 18:15:07 +0800 Subject: [PATCH 04/27] Expand batch TimeSeriesMetadata scan to device-level time index files Remove the ignoreNotExists gate so the batch path is also taken for files with a device-level time index, where previously only file-level-index files (or recovery) used it. When the batch read finds the device is missing from the file, replicate the per-column path: skip silently when ignoreNotExists is set, otherwise throw CorruptedTsFileException to surface the time-index/metadata inconsistency. --- .../operator/source/FileLoaderUtils.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 857328a58838c..3f5f174da85d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -360,8 +360,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr // For a wide device (a table scan that selects almost all field columns), deserialize the // whole device's TimeSeriesMetadata in a single batch read instead of walking the device // metadata index once per column. - if (shouldUseDeviceMetadataBatchScan(valueMeasurementList.size(), totalFieldColumnCount) - && ignoreNotExists) { + if (shouldUseDeviceMetadataBatchScan(valueMeasurementList.size(), totalFieldColumnCount)) { return loadAlignedTimeSeriesMetadataBatch( resource, alignedPath, @@ -369,6 +368,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr context, globalTimeFilter, ignoreAllNullRows, + ignoreNotExists, cache, filePath, deviceId); @@ -445,12 +445,27 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBa FragmentInstanceContext context, Filter globalTimeFilter, boolean ignoreAllNullRows, + boolean ignoreNotExists, TimeSeriesMetadataCache cache, String filePath, IDeviceID deviceId) throws IOException { Map deviceMetadata = cache.getDeviceTimeSeriesMetadata(filePath, resource.getTsFileID(), deviceId, context); + if (deviceMetadata.isEmpty()) { + // device not found in this file. With a file-level time index (or during recovery) that is + // expected, so skip; with a device-level time index the device should be present, so surface + // the inconsistency instead of silently dropping the device's data. + if (ignoreNotExists) { + return null; + } + throw new CorruptedTsFileException( + resource.getTsFile(), + CorruptedTsFileException.Stage.READ_TIMESERIES_METADATA, + DataNodeQueryMessages + .EXCEPTION_FAILED_TO_READ_TIMESERIES_METADATA_THE_TSFILE_MAY_BE_CORRUPTED_PLEASE_CHECK_THE_LOGS_FOR_THE_CORRUPTED_FILE_PATH_0B9E652E, + null); + } TimeseriesMetadata timeColumn = deviceMetadata.get(""); if (timeColumn == null) { return null; From 31ecea85e5650d4612c37d8a15d375e058b57656 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 10 Nov 2026 18:22:02 +0800 Subject: [PATCH 05/27] Hardcode device metadata batch scan ratio to 0.6 Remove the device_metadata_batch_scan_ratio config parameter and hardcode the threshold at 0.6 (previously configurable, default 0.8) so the batch path covers a wider set of scans without requiring configuration. --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 14 -------------- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 13 ------------- .../execution/operator/source/FileLoaderUtils.java | 10 +++++++--- .../conf/iotdb-system.properties.template | 7 ------- 4 files changed, 7 insertions(+), 37 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index c1d3aad8e8b37..c6a0d4878969f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -853,12 +853,6 @@ public class IoTDBConfig implements DefaultEncodingProvider { /** time cost(ms) threshold for slow query. Unit: millisecond */ private long slowQueryThreshold = 10000; - /** - * When a table scan queries at least this fraction of a table's field columns, deserialize all - * the device's TimeSeriesMetadata in a single batch read instead of column by column. - */ - private double deviceMetadataBatchScanRatio = 0.8; - /** time window threshold for record of history queries. Unit: minute */ private int queryCostStatWindow = 0; @@ -2968,14 +2962,6 @@ public void setSlowQueryThreshold(long slowQueryThreshold) { this.slowQueryThreshold = slowQueryThreshold; } - public double getDeviceMetadataBatchScanRatio() { - return deviceMetadataBatchScanRatio; - } - - public void setDeviceMetadataBatchScanRatio(double deviceMetadataBatchScanRatio) { - this.deviceMetadataBatchScanRatio = deviceMetadataBatchScanRatio; - } - public int getQueryCostStatWindow() { return queryCostStatWindow; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index df91d783ddb7e..04fe3381b7541 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -519,19 +519,6 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException properties.getProperty( "query_timeout_threshold", Long.toString(conf.getQueryTimeoutThreshold())))); - double deviceMetadataBatchScanRatio = - Double.parseDouble( - properties.getProperty( - "device_metadata_batch_scan_ratio", - Double.toString(conf.getDeviceMetadataBatchScanRatio()))); - // a non-positive ratio would route every table scan, even single-column ones, through the - // batch path; keep the current value instead - deviceMetadataBatchScanRatio = - deviceMetadataBatchScanRatio <= 0 - ? conf.getDeviceMetadataBatchScanRatio() - : deviceMetadataBatchScanRatio; - conf.setDeviceMetadataBatchScanRatio(deviceMetadataBatchScanRatio); - conf.setCopyToAllowedExportDirs( Arrays.stream( properties diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 3f5f174da85d8..25f532af27017 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -22,7 +22,6 @@ import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.commons.path.AlignedFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; -import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; import org.apache.iotdb.db.exception.CorruptedTsFileException; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; @@ -74,6 +73,12 @@ public class FileLoaderUtils { private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET = SeriesScanCostMetricSet.getInstance(); + /** + * Fraction of a table's field columns that a scan must select for it to use the batch + * device-metadata read instead of the per-column path. + */ + private static final double DEVICE_METADATA_BATCH_SCAN_RATIO = 0.6; + private FileLoaderUtils() { // util class } @@ -434,8 +439,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr private static boolean shouldUseDeviceMetadataBatchScan( int selectedColumnCount, int totalFieldColumnCount) { return totalFieldColumnCount > 0 - && (double) selectedColumnCount / totalFieldColumnCount - >= IoTDBDescriptor.getInstance().getConfig().getDeviceMetadataBatchScanRatio(); + && (double) selectedColumnCount / totalFieldColumnCount >= DEVICE_METADATA_BATCH_SCAN_RATIO; } private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBatch( diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index e8021790bbe83..a3fe0970318c8 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1267,13 +1267,6 @@ max_tsblock_line_number=1000 # Datatype: long slow_query_threshold=10000 -# When a table scan queries at least this fraction of a table's field columns, deserialize all the -# device's TimeSeriesMetadata in a single batch read instead of column by column. -# Value must be in (0, 1]; a non-positive value falls back to the default. -# effectiveMode: restart -# Datatype: double -device_metadata_batch_scan_ratio=0.8 - # Time window threshold(min) for record of history queries. # effectiveMode: hot_reload # Datatype: int From 2cc96048b01da2531d30ba50c5ea0192f4a20c54 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 11 Nov 2026 09:56:32 +0800 Subject: [PATCH 06/27] Synchronize batch metadata read and include device in corruption error Serialize concurrent reads of the same device+file in getDeviceTimeSeriesMetadata (matching the per-column get) so threads don't each deserialize the whole device and race on the shared reader. Also attach the missing device id to the CorruptedTsFileException thrown when a device is absent from a device-level time index file. --- .../operator/source/FileLoaderUtils.java | 5 +- .../buffer/TimeSeriesMetadataCache.java | 46 ++++++++++++------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 25f532af27017..98dcf4c7991e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; import org.apache.iotdb.db.exception.CorruptedTsFileException; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; @@ -468,7 +469,9 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBa CorruptedTsFileException.Stage.READ_TIMESERIES_METADATA, DataNodeQueryMessages .EXCEPTION_FAILED_TO_READ_TIMESERIES_METADATA_THE_TSFILE_MAY_BE_CORRUPTED_PLEASE_CHECK_THE_LOGS_FOR_THE_CORRUPTED_FILE_PATH_0B9E652E, - null); + new IOException( + String.format( + StorageEngineMessages.DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT, deviceId))); } TimeseriesMetadata timeColumn = deviceMetadata.get(""); if (timeColumn == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 7f4ba60ab1f72..30457bf0570cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -346,24 +346,15 @@ public Map getDeviceTimeSeriesMetadata( TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); - // Use the chunk-metadata-including variant: the returned TimeseriesMetadata are passed to - // setModifications and then to DiskAlignedChunkMetadataLoader, which dereferences the chunk - // metadata list, so readDeviceMetadata (which skips chunk metadata) would NPE at query time. - List deviceMetadataList = reader.getDeviceTimeseriesMetadata(device); - - Map result = new HashMap<>(deviceMetadataList.size()); - for (TimeseriesMetadata metadata : deviceMetadataList) { - String measurementId = metadata.getMeasurementId(); - if (metadata.getStatistics().getCount() == 0) { - result.put(measurementId, null); - continue; - } - if (CACHE_ENABLE && !externalTsFile) { - lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId), metadata); + if (CACHE_ENABLE && !externalTsFile) { + // serialize concurrent reads of the same device+file (same lock key as get) so threads + // don't each deserialize the whole device and race on the shared reader + synchronized ( + devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, WeakReference::new)) { + return readDeviceTimeSeriesMetadata(reader, tsFileID, device, true); } - result.put(measurementId, new TimeseriesMetadata(metadata)); } - return result; + return readDeviceTimeSeriesMetadata(reader, tsFileID, device, false); } finally { queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromDiskCount().incrementAndGet(); queryContext @@ -373,6 +364,29 @@ public Map getDeviceTimeSeriesMetadata( } } + private Map readDeviceTimeSeriesMetadata( + TsFileSequenceReader reader, TsFileID tsFileID, IDeviceID device, boolean cacheEnabled) + throws IOException { + // Use the chunk-metadata-including variant: the returned TimeseriesMetadata are passed to + // setModifications and then to DiskAlignedChunkMetadataLoader, which dereferences the chunk + // metadata list, so readDeviceMetadata (which skips chunk metadata) would NPE at query time. + List deviceMetadataList = reader.getDeviceTimeseriesMetadata(device); + + Map result = new HashMap<>(deviceMetadataList.size()); + for (TimeseriesMetadata metadata : deviceMetadataList) { + String measurementId = metadata.getMeasurementId(); + if (metadata.getStatistics().getCount() == 0) { + result.put(measurementId, null); + continue; + } + if (cacheEnabled) { + lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId), metadata); + } + result.put(measurementId, new TimeseriesMetadata(metadata)); + } + return result; + } + public double calculateTimeSeriesMetadataHitRatio() { return lruCache.stats().hitRate(); } From 4bcb8b62e9fccc91dd5ff669ba84651ff06b87e2 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 11 Nov 2026 10:05:04 +0800 Subject: [PATCH 07/27] Fix batch scan ratio denominator and cache empty series Count every value column (FIELD + ATTRIBUTE) as the batch scan ratio denominator instead of only FIELD columns, matching the selected-column numerator. Cache empty (count == 0) series as the null placeholder in the batch read, matching the per-column get, so later per-column reads don't re-read them. Add a unit test for the absent-device (empty map) path. --- .../operator/source/FileLoaderUtils.java | 3 +- .../DataNodeTableOperatorGenerator.java | 5 ++- .../buffer/TimeSeriesMetadataCache.java | 21 ++++++++++--- .../buffer/TimeSeriesMetadataCacheTest.java | 31 ++++++++++++++++++- 4 files changed, 53 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 98dcf4c7991e3..a154789e30cfb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -456,7 +456,8 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBa IDeviceID deviceId) throws IOException { Map deviceMetadata = - cache.getDeviceTimeSeriesMetadata(filePath, resource.getTsFileID(), deviceId, context); + cache.getDeviceTimeSeriesMetadata( + filePath, resource.getTsFileID(), deviceId, context, ignoreNotExists); if (deviceMetadata.isEmpty()) { // device not found in this file. With a file-level time index (or during recovery) that is // expected, so skip; with a device-level time index the device should be present, so surface diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index 8b6b9b36bb88c..6d8b235bd1322 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -1231,7 +1231,10 @@ private int computeTotalFieldColumnCount(DeviceTableScanNode node) { DataNodeTableCache.getInstance() .getTableInCache( qualifiedObjectName.getDatabaseName(), qualifiedObjectName.getObjectName()); - return tsTable == null ? 0 : tsTable.getFieldNum(); + // count every value column (FIELD + ATTRIBUTE) the scan may select, matching the denominator + // used by shouldUseDeviceMetadataBatchScan; getColumnNum() also includes the tag (id) columns + // and the time column, hence the subtraction. + return tsTable == null ? 0 : tsTable.getColumnNum() - tsTable.getTagNum() - 1; } private BatchDeviceEntrySource createAggregationDeviceEntrySource(AggregationTableScanNode node) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 30457bf0570cf..c0d1eacee1837 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -335,7 +335,11 @@ public TimeseriesMetadata get( * @throws IOException if the metadata cannot be read from disk */ public Map getDeviceTimeSeriesMetadata( - String filePath, TsFileID tsFileID, IDeviceID device, QueryContext queryContext) + String filePath, + TsFileID tsFileID, + IDeviceID device, + QueryContext queryContext, + boolean ignoreNotExists) throws IOException { long startTime = System.nanoTime(); boolean externalTsFile = queryContext.isExternalTsFileScan(); @@ -351,10 +355,10 @@ public Map getDeviceTimeSeriesMetadata( // don't each deserialize the whole device and race on the shared reader synchronized ( devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, WeakReference::new)) { - return readDeviceTimeSeriesMetadata(reader, tsFileID, device, true); + return readDeviceTimeSeriesMetadata(reader, tsFileID, device, true, ignoreNotExists); } } - return readDeviceTimeSeriesMetadata(reader, tsFileID, device, false); + return readDeviceTimeSeriesMetadata(reader, tsFileID, device, false, ignoreNotExists); } finally { queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromDiskCount().incrementAndGet(); queryContext @@ -365,7 +369,11 @@ public Map getDeviceTimeSeriesMetadata( } private Map readDeviceTimeSeriesMetadata( - TsFileSequenceReader reader, TsFileID tsFileID, IDeviceID device, boolean cacheEnabled) + TsFileSequenceReader reader, + TsFileID tsFileID, + IDeviceID device, + boolean cacheEnabled, + boolean ignoreNotExists) throws IOException { // Use the chunk-metadata-including variant: the returned TimeseriesMetadata are passed to // setModifications and then to DiskAlignedChunkMetadataLoader, which dereferences the chunk @@ -377,6 +385,11 @@ private Map readDeviceTimeSeriesMetadata( String measurementId = metadata.getMeasurementId(); if (metadata.getStatistics().getCount() == 0) { result.put(measurementId, null); + if (cacheEnabled && !ignoreNotExists && memoryConfig.isMayCacheNonExistSeries()) { + lruCache.put( + new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId), + NULL_EXISTS_CACHE_PLACE_HOLDER); + } continue; } if (cacheEnabled) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index eec5a7a13af72..67351702f8162 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -58,6 +58,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; public class TimeSeriesMetadataCacheTest { @@ -334,7 +335,7 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { Map result = cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, new QueryContext(false, false)); + filePath, tsFileID, deviceID, new QueryContext(false, false), false); assertNotNull(result.get("")); assertEquals("", result.get("").getMeasurementId()); @@ -360,4 +361,32 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { fileReaderManager.closeFileAndRemoveReader(tsFileID); } } + + @Test + public void testGetDeviceTimeSeriesMetadataForAbsentDevice() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + try { + String filePath = "target/batch-absent-device.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)) + .thenReturn(Collections.emptyList()); + + Map result = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, new QueryContext(false, false), false); + + // an absent device yields an empty map, which loadAlignedTimeSeriesMetadataBatch then + // distinguishes from "present but empty time column" + assertTrue(result.isEmpty()); + } finally { + cache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } } From a6a563427f20d7826e03570329476eda79c629e2 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 11 Nov 2026 10:09:08 +0800 Subject: [PATCH 08/27] Add unit test for batch scan absent-device throw logic Cover the loadAlignedTimeSeriesMetadataBatch empty-map branch: a device absent from a device-level time index file throws CorruptedTsFileException, while with a file-level time index (or during recovery) it is skipped. --- .../operator/source/FileLoaderUtilsTest.java | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java new file mode 100644 index 0000000000000..9fab5e782a821 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source; + +import org.apache.iotdb.commons.path.AlignedFullPath; +import org.apache.iotdb.db.exception.CorruptedTsFileException; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; +import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; + +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; +import org.apache.tsfile.read.filter.basic.Filter; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class FileLoaderUtilsTest { + + @Test + public void testLoadAlignedTimeSeriesMetadataBatchAbsentDevice() throws Exception { + TsFileResource resource = Mockito.mock(TsFileResource.class); + Mockito.when(resource.getTsFileID()).thenReturn(new TsFileID()); + Mockito.when(resource.getTsFile()).thenReturn(new File("target/x.tsfile")); + + TimeSeriesMetadataCache cache = Mockito.mock(TimeSeriesMetadataCache.class); + Mockito.when( + cache.getDeviceTimeSeriesMetadata( + Mockito.anyString(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.anyBoolean())) + .thenReturn(Collections.emptyMap()); + + IDeviceID deviceId = Factory.DEFAULT_FACTORY.create("root.device"); + + Method method = + FileLoaderUtils.class.getDeclaredMethod( + "loadAlignedTimeSeriesMetadataBatch", + TsFileResource.class, + AlignedFullPath.class, + List.class, + FragmentInstanceContext.class, + Filter.class, + boolean.class, + boolean.class, + TimeSeriesMetadataCache.class, + String.class, + IDeviceID.class); + method.setAccessible(true); + + // a device absent from a device-level time index file is an inconsistency: surface it + try { + method.invoke( + null, + resource, + null, + Collections.emptyList(), + null, + null, + false, + false, + cache, + "target/x.tsfile", + deviceId); + fail("expected CorruptedTsFileException when the device is absent and ignoreNotExists=false"); + } catch (InvocationTargetException e) { + assertTrue(e.getCause() instanceof CorruptedTsFileException); + } + + // with a file-level time index (or during recovery) an absent device is expected: skip it + try { + Object result = + method.invoke( + null, + resource, + null, + Collections.emptyList(), + null, + null, + false, + true, + cache, + "target/x.tsfile", + deviceId); + assertNull(result); + } catch (InvocationTargetException e) { + fail("unexpected exception: " + e.getCause()); + } + } +} From 1199ab3f885a736a33428f7180b128ed659cf736 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 11 Nov 2026 10:12:47 +0800 Subject: [PATCH 09/27] Add end-to-end test for wide-table batch metadata scan Drive AlignedSeriesScanUtil with all value columns selected and a matching total field column count so the batch path is taken, and assert the aligned pages are still read correctly. --- .../series/AlignedSeriesScanBatchTest.java | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java new file mode 100644 index 0000000000000..abcfbb7c0b3ef --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.AlignedFullPath; +import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; +import org.apache.iotdb.db.utils.EnvironmentUtils; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; + +/** Verifies the wide-table batch metadata scan (see FileLoaderUtils) reads data correctly. */ +public class AlignedSeriesScanBatchTest extends AbstractAlignedSeriesScanTest { + + @Test + public void testBatchScanReadsAlignedData() throws IOException, IllegalPathException { + AlignedFullPath scanPath = + new AlignedFullPath( + TEST_DEVICE, + Arrays.asList("s1", "s2", "s3"), + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32), + new MeasurementSchema("s3", TSDataType.INT32))); + + SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); + builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); + SeriesScanOptions scanOptions = builder.build(); + // all 3 value columns are selected, so selected/total = 1.0 >= 0.6 and the batch path is taken + scanOptions.setTotalFieldColumnCount(3); + + AlignedSeriesScanUtil seriesScanUtil = + new AlignedSeriesScanUtil( + scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); + + // the batch path must still assemble the same aligned pages: file 1 has 10 points + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + } +} From 42749a444cdf2ccd8fb948381700a7deb8777377 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 11 Nov 2026 11:27:58 +0800 Subject: [PATCH 10/27] Don't use batch metadata scan for single-column devices The batch read offers no benefit for a single-column device (one leaf either way) and it bypasses the per-column cache's debug logging, which IoTDBDebugQueryIT.tableTest relies on. Require at least two value columns before taking the batch path. --- .../execution/operator/source/FileLoaderUtils.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index a154789e30cfb..9e8d522b3c02c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -439,7 +439,9 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr private static boolean shouldUseDeviceMetadataBatchScan( int selectedColumnCount, int totalFieldColumnCount) { - return totalFieldColumnCount > 0 + // a single-column device gains nothing from a batch read (one leaf either way), and it would + // bypass the per-column cache's debug logging + return totalFieldColumnCount > 1 && (double) selectedColumnCount / totalFieldColumnCount >= DEVICE_METADATA_BATCH_SCAN_RATIO; } From 98aba2c20aa9415399df8f0da09e2370a870bfd2 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 16:27:22 +0800 Subject: [PATCH 11/27] Reuse the metadata cache for batch device scans getDeviceTimeSeriesMetadata always deserialized the device's whole metadata index and then re-populated the cache, never reading it, so repeated wide-table scans were slower than the per-column path that it populates. Check the per-measurement cache first, read the device only when one of the requested measurements is missing, and cache just those measurements (plus the null placeholders for absent ones). Align the absent-device behavior with the per-column get(): skip the device when the file's bloom filter proves it is missing, otherwise raise the read IOException so the existing wrapper records the suspicious-disk metric. Require a null device offset for the batch path instead of relying on the zero sentinel used by external scans, and extract the aligned metadata assembly shared by both paths. Count only FIELD columns in the ratio denominator to match the smaller numerator, and extend the tests to cover the cache hit, the bloom-filter decisions, the gating boundary and the batch path actually being taken. --- .../operator/source/FileLoaderUtils.java | 83 ++++++------ .../DataNodeTableOperatorGenerator.java | 7 +- .../buffer/TimeSeriesMetadataCache.java | 126 +++++++++++++++--- .../operator/source/FileLoaderUtilsTest.java | 98 ++------------ .../buffer/TimeSeriesMetadataCacheTest.java | 71 ++++++++-- .../series/AlignedSeriesScanBatchTest.java | 29 ++++ 6 files changed, 256 insertions(+), 158 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 9e8d522b3c02c..ca34d9f9ba495 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -25,7 +25,6 @@ import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; import org.apache.iotdb.db.exception.CorruptedTsFileException; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; -import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; @@ -365,12 +364,15 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr // For a wide device (a table scan that selects almost all field columns), deserialize the // whole device's TimeSeriesMetadata in a single batch read instead of walking the device - // metadata index once per column. - if (shouldUseDeviceMetadataBatchScan(valueMeasurementList.size(), totalFieldColumnCount)) { + // metadata index once per column. External tsfiles address metadata by offset, so they must + // stay on the per-column path. + if (rootMeasurementMetadataIndexNodeOffset == null + && shouldUseDeviceMetadataBatchScan(valueMeasurementList.size(), totalFieldColumnCount)) { return loadAlignedTimeSeriesMetadataBatch( resource, alignedPath, valueMeasurementList, + allSensors, context, globalTimeFilter, ignoreAllNullRows, @@ -406,10 +408,8 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr } else { List valueTimeSeriesMetadataList = new ArrayList<>(valueMeasurementList.size()); - // if all the queried aligned sensors does not exist, we will return null - boolean exist = false; for (String measurement : valueMeasurementList) { - TimeseriesMetadata valueColumn = + valueTimeSeriesMetadataList.add( cache.get( filePath, new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement), @@ -417,27 +417,23 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr ignoreNotExists, isDebug, context, - rootMeasurementMetadataIndexNodeOffset); - exist = (exist || (valueColumn != null)); - valueTimeSeriesMetadataList.add(valueColumn); - } - if (!ignoreAllNullRows || exist) { - alignedTimeSeriesMetadata = - setModifications( - resource, - timeColumn, - valueTimeSeriesMetadataList, - alignedPath, - context, - globalTimeFilter, - ignoreAllNullRows); + rootMeasurementMetadataIndexNodeOffset)); } + alignedTimeSeriesMetadata = + buildAlignedTimeSeriesMetadata( + resource, + alignedPath, + timeColumn, + valueTimeSeriesMetadataList, + context, + globalTimeFilter, + ignoreAllNullRows); } } return alignedTimeSeriesMetadata; } - private static boolean shouldUseDeviceMetadataBatchScan( + static boolean shouldUseDeviceMetadataBatchScan( int selectedColumnCount, int totalFieldColumnCount) { // a single-column device gains nothing from a batch read (one leaf either way), and it would // bypass the per-column cache's debug logging @@ -449,6 +445,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBa TsFileResource resource, AlignedFullPath alignedPath, List valueMeasurementList, + Set allSensors, FragmentInstanceContext context, Filter globalTimeFilter, boolean ignoreAllNullRows, @@ -459,22 +456,11 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBa throws IOException { Map deviceMetadata = cache.getDeviceTimeSeriesMetadata( - filePath, resource.getTsFileID(), deviceId, context, ignoreNotExists); + filePath, resource.getTsFileID(), deviceId, allSensors, context, ignoreNotExists); if (deviceMetadata.isEmpty()) { - // device not found in this file. With a file-level time index (or during recovery) that is - // expected, so skip; with a device-level time index the device should be present, so surface - // the inconsistency instead of silently dropping the device's data. - if (ignoreNotExists) { - return null; - } - throw new CorruptedTsFileException( - resource.getTsFile(), - CorruptedTsFileException.Stage.READ_TIMESERIES_METADATA, - DataNodeQueryMessages - .EXCEPTION_FAILED_TO_READ_TIMESERIES_METADATA_THE_TSFILE_MAY_BE_CORRUPTED_PLEASE_CHECK_THE_LOGS_FOR_THE_CORRUPTED_FILE_PATH_0B9E652E, - new IOException( - String.format( - StorageEngineMessages.DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT, deviceId))); + // the cache applies the same bloom-filter / ignoreNotExists rules as the per-column get(), + // so an empty map means the device is expected to be absent from this file + return null; } TimeseriesMetadata timeColumn = deviceMetadata.get(""); if (timeColumn == null) { @@ -482,12 +468,31 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBa } List valueTimeSeriesMetadataList = new ArrayList<>(valueMeasurementList.size()); + for (String measurement : valueMeasurementList) { + valueTimeSeriesMetadataList.add(deviceMetadata.get(measurement)); + } + return buildAlignedTimeSeriesMetadata( + resource, + alignedPath, + timeColumn, + valueTimeSeriesMetadataList, + context, + globalTimeFilter, + ignoreAllNullRows); + } + + private static AbstractAlignedTimeSeriesMetadata buildAlignedTimeSeriesMetadata( + TsFileResource resource, + AlignedFullPath alignedPath, + TimeseriesMetadata timeColumn, + List valueTimeSeriesMetadataList, + FragmentInstanceContext context, + Filter globalTimeFilter, + boolean ignoreAllNullRows) { // if all the queried aligned sensors does not exist, we will return null boolean exist = false; - for (String measurement : valueMeasurementList) { - TimeseriesMetadata valueColumn = deviceMetadata.get(measurement); + for (TimeseriesMetadata valueColumn : valueTimeSeriesMetadataList) { exist = (exist || (valueColumn != null)); - valueTimeSeriesMetadataList.add(valueColumn); } if (!ignoreAllNullRows || exist) { return setModifications( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index 6d8b235bd1322..47679da161a24 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -1231,10 +1231,9 @@ private int computeTotalFieldColumnCount(DeviceTableScanNode node) { DataNodeTableCache.getInstance() .getTableInCache( qualifiedObjectName.getDatabaseName(), qualifiedObjectName.getObjectName()); - // count every value column (FIELD + ATTRIBUTE) the scan may select, matching the denominator - // used by shouldUseDeviceMetadataBatchScan; getColumnNum() also includes the tag (id) columns - // and the time column, hence the subtraction. - return tsTable == null ? 0 : tsTable.getColumnNum() - tsTable.getTagNum() - 1; + // the numerator only contains FIELD columns, so the denominator must be the field count too; + // getColumnNum() would also count tag, attribute and time columns + return tsTable == null ? 0 : tsTable.getFieldNum(); } private BatchDeviceEntrySource createAggregationDeviceEntrySource(AggregationTableScanNode node) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index c0d1eacee1837..922f2c4e4c959 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -318,26 +318,32 @@ public TimeseriesMetadata get( } /** - * Load all TimeseriesMetadata of a device in a single batch read and populate the per-measurement - * cache. This is used by wide-table scans (a device with many columns) to avoid walking the - * device metadata index once per column / leaf node. + * Load the TimeseriesMetadata of {@code allSensors} of a device, reading the device in a single + * batch when the per-measurement cache does not already hold all of them. This is used by + * wide-table scans (a device with many columns) to avoid walking the device metadata index once + * per column / leaf node. * - *

The returned map holds copies of the cached metadata (or the freshly read objects when the - * cache is disabled / for external tsfiles), so the caller may apply {@code setModified} and - * {@code setChunkMetadataLoader} without corrupting cached entries — matching {@link #get}. + *

The returned map always holds copies of the cached (or freshly read) metadata, so the caller + * may apply {@code setModified} and {@code setChunkMetadataLoader} without corrupting cached + * entries — matching {@link #get}. * * @param filePath path of the tsfile * @param tsFileID id of the tsfile * @param device target device + * @param allSensors measurements the caller needs, including the time column keyed by "" * @param queryContext query context used for metrics and to detect external tsfiles + * @param ignoreNotExists whether a device missing from a file that claims a device-level time + * index should be skipped instead of reported as an inconsistency * @return map from measurementId (including the time column, keyed by "") to TimeseriesMetadata; - * entries whose series is empty (count == 0) are mapped to null. - * @throws IOException if the metadata cannot be read from disk + * entries whose series is empty (count == 0) are mapped to null + * @throws IOException if the metadata cannot be read from disk or the device is unexpectedly + * absent */ public Map getDeviceTimeSeriesMetadata( String filePath, TsFileID tsFileID, IDeviceID device, + Set allSensors, QueryContext queryContext, boolean ignoreNotExists) throws IOException { @@ -345,33 +351,85 @@ public Map getDeviceTimeSeriesMetadata( boolean externalTsFile = queryContext.isExternalTsFileScan(); LongConsumer timeSeriesMetadataIoSizeRecorder = queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; + boolean cacheHit = true; try { - TsFileSequenceReader reader = - FileReaderManager.getInstance() - .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); if (CACHE_ENABLE && !externalTsFile) { + Map cached = + getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors); + if (cached != null) { + return cached; + } // serialize concurrent reads of the same device+file (same lock key as get) so threads // don't each deserialize the whole device and race on the shared reader synchronized ( devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, WeakReference::new)) { - return readDeviceTimeSeriesMetadata(reader, tsFileID, device, true, ignoreNotExists); + cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors); + if (cached != null) { + return cached; + } + cacheHit = false; + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); + return readDeviceTimeSeriesMetadata( + reader, tsFileID, device, allSensors, true, ignoreNotExists); } } - return readDeviceTimeSeriesMetadata(reader, tsFileID, device, false, ignoreNotExists); + cacheHit = false; + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); + return readDeviceTimeSeriesMetadata( + reader, tsFileID, device, allSensors, false, ignoreNotExists); } finally { - queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromDiskCount().incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskTime() - .getAndAdd(System.nanoTime() - startTime); + if (cacheHit) { + queryContext + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .incrementAndGet(); + queryContext + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheTime() + .getAndAdd(System.nanoTime() - startTime); + } else { + queryContext + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .incrementAndGet(); + queryContext + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskTime() + .getAndAdd(System.nanoTime() - startTime); + } + } + } + + /** + * @return copies of the cached metadata for every measurement in {@code allSensors}, or null as + * soon as one of them is not cached and the device therefore has to be read from disk + */ + private Map getCachedDeviceTimeSeriesMetadata( + TsFileID tsFileID, IDeviceID device, Set allSensors) { + Map result = new HashMap<>(allSensors.size()); + for (String measurement : allSensors) { + TimeseriesMetadata metadata = + lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID, device, measurement)); + if (metadata == null) { + return null; + } + result.put( + measurement, + metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new TimeseriesMetadata(metadata)); } + return result; } private Map readDeviceTimeSeriesMetadata( TsFileSequenceReader reader, TsFileID tsFileID, IDeviceID device, + Set allSensors, boolean cacheEnabled, boolean ignoreNotExists) throws IOException { @@ -379,10 +437,27 @@ private Map readDeviceTimeSeriesMetadata( // setModifications and then to DiskAlignedChunkMetadataLoader, which dereferences the chunk // metadata list, so readDeviceMetadata (which skips chunk metadata) would NPE at query time. List deviceMetadataList = reader.getDeviceTimeseriesMetadata(device); + if (deviceMetadataList.isEmpty()) { + // Match get(): an absent device is only an inconsistency when ignoreNotExists is false and + // the bloom filter does not already prove that the device is missing from this file + if (ignoreNotExists) { + return Collections.emptyMap(); + } + BloomFilter bloomFilter = reader.readBloomFilter(); + if (bloomFilter != null + && !bloomFilter.contains(device.toString() + IoTDBConstant.PATH_SEPARATOR)) { + return Collections.emptyMap(); + } + throw new IOException( + String.format(StorageEngineMessages.DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT, device)); + } - Map result = new HashMap<>(deviceMetadataList.size()); + Map result = new HashMap<>(allSensors.size()); for (TimeseriesMetadata metadata : deviceMetadataList) { String measurementId = metadata.getMeasurementId(); + if (!allSensors.contains(measurementId)) { + continue; + } if (metadata.getStatistics().getCount() == 0) { result.put(measurementId, null); if (cacheEnabled && !ignoreNotExists && memoryConfig.isMayCacheNonExistSeries()) { @@ -397,6 +472,19 @@ private Map readDeviceTimeSeriesMetadata( } result.put(measurementId, new TimeseriesMetadata(metadata)); } + // remember selected measurements that do not exist in this file, so a later lookup of the same + // device can be served from the cache instead of reading the device again + for (String measurement : allSensors) { + if (result.containsKey(measurement)) { + continue; + } + result.put(measurement, null); + if (cacheEnabled && !ignoreNotExists && memoryConfig.isMayCacheNonExistSeries()) { + lruCache.put( + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), + NULL_EXISTS_CACHE_PLACE_HOLDER); + } + } return result; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java index 9fab5e782a821..92431d1114b8c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java @@ -19,101 +19,27 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; -import org.apache.iotdb.commons.path.AlignedFullPath; -import org.apache.iotdb.db.exception.CorruptedTsFileException; -import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; -import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; - -import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.IDeviceID.Factory; -import org.apache.tsfile.read.filter.basic.Filter; import org.junit.Test; -import org.mockito.Mockito; - -import java.io.File; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.Collections; -import java.util.List; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; public class FileLoaderUtilsTest { @Test - public void testLoadAlignedTimeSeriesMetadataBatchAbsentDevice() throws Exception { - TsFileResource resource = Mockito.mock(TsFileResource.class); - Mockito.when(resource.getTsFileID()).thenReturn(new TsFileID()); - Mockito.when(resource.getTsFile()).thenReturn(new File("target/x.tsfile")); - - TimeSeriesMetadataCache cache = Mockito.mock(TimeSeriesMetadataCache.class); - Mockito.when( - cache.getDeviceTimeSeriesMetadata( - Mockito.anyString(), - Mockito.any(), - Mockito.any(), - Mockito.any(), - Mockito.anyBoolean())) - .thenReturn(Collections.emptyMap()); - - IDeviceID deviceId = Factory.DEFAULT_FACTORY.create("root.device"); + public void testShouldUseDeviceMetadataBatchScan() { + // no field count is known (tree model, external tsfiles, or a table missing from the cache) + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 0)); - Method method = - FileLoaderUtils.class.getDeclaredMethod( - "loadAlignedTimeSeriesMetadataBatch", - TsFileResource.class, - AlignedFullPath.class, - List.class, - FragmentInstanceContext.class, - Filter.class, - boolean.class, - boolean.class, - TimeSeriesMetadataCache.class, - String.class, - IDeviceID.class); - method.setAccessible(true); + // a single-column device gains nothing from a batch read + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(1, 1)); - // a device absent from a device-level time index file is an inconsistency: surface it - try { - method.invoke( - null, - resource, - null, - Collections.emptyList(), - null, - null, - false, - false, - cache, - "target/x.tsfile", - deviceId); - fail("expected CorruptedTsFileException when the device is absent and ignoreNotExists=false"); - } catch (InvocationTargetException e) { - assertTrue(e.getCause() instanceof CorruptedTsFileException); - } + // below, at and above the threshold + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(5, 10)); + assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(6, 10)); + assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10)); - // with a file-level time index (or during recovery) an absent device is expected: skip it - try { - Object result = - method.invoke( - null, - resource, - null, - Collections.emptyList(), - null, - null, - false, - true, - cache, - "target/x.tsfile", - deviceId); - assertNull(result); - } catch (InvocationTargetException e) { - fail("unexpected exception: " + e.getCause()); - } + // a scan can reference measurements that no longer exist, so selected > total is possible + assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(11, 10)); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 67351702f8162..fc2d72d8a393d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -33,6 +33,7 @@ import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.BloomFilter; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; @@ -44,7 +45,9 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -295,6 +298,7 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { try { String filePath = "target/batch-device.tsfile"; IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); timeStatistics.setCount(1); @@ -312,7 +316,7 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { new TimeseriesMetadata( (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS()); - // an empty series (count == 0) must be mapped to null and not cached + // an empty series (count == 0) is mapped to null and cached as the null placeholder TimeseriesMetadata emptyS2 = new TimeseriesMetadata( (byte) 0, @@ -335,7 +339,7 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { Map result = cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, new QueryContext(false, false), false); + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); assertNotNull(result.get("")); assertEquals("", result.get("").getMeasurementId()); @@ -345,7 +349,15 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { assertNotNull(result.get("s1").getChunkMetadataList()); assertNull(result.get("s2")); - // the non-empty entries should now be cached for per-measurement lookups + // a second lookup of cached measurements must be served without touching the reader again + Set cachedSensors = new HashSet<>(Arrays.asList("", "s1")); + Map cachedResult = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, cachedSensors, new QueryContext(false, false), false); + assertNotNull(cachedResult.get("s1")); + Mockito.verify(reader, Mockito.times(1)).getDeviceTimeseriesMetadata(deviceID); + + // the batch read also populated the per-measurement cache TimeseriesMetadata cached = cache.get( filePath, @@ -363,27 +375,66 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { } @Test - public void testGetDeviceTimeSeriesMetadataForAbsentDevice() throws Exception { + public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception { TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); FileReaderManager fileReaderManager = FileReaderManager.getInstance(); TsFileID tsFileID = new TsFileID(); cache.clear(); try { - String filePath = "target/batch-absent-device.tsfile"; + String filePath = "target/batch-absent-device-bloom.tsfile"; IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = Collections.singleton(""); TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)) .thenReturn(Collections.emptyList()); - - Map result = - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, new QueryContext(false, false), false); + // the bloom filter proves the device is missing, so the per-column path would skip it too + Mockito.when(reader.readBloomFilter()).thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1)); // an absent device yields an empty map, which loadAlignedTimeSeriesMetadataBatch then // distinguishes from "present but empty time column" - assertTrue(result.isEmpty()); + assertTrue( + cache + .getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false) + .isEmpty()); + } finally { + cache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + try { + String filePath = "target/batch-absent-device-throw.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = Collections.singleton(""); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)) + .thenReturn(Collections.emptyList()); + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); + bloomFilter.add(deviceID.toString() + "."); + Mockito.when(reader.readBloomFilter()).thenReturn(bloomFilter); + + assertThrows( + IOException.class, + () -> + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false)); + + assertTrue( + cache + .getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), true) + .isEmpty()); } finally { cache.clear(); fileReaderManager.closeFileAndRemoveReader(tsFileID); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java index abcfbb7c0b3ef..e4f9b880ab5d5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; import org.apache.iotdb.db.utils.EnvironmentUtils; @@ -62,11 +63,39 @@ public void testBatchScanReadsAlignedData() throws IOException, IllegalPathExcep scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); + TimeSeriesMetadataCache.getInstance().clear(); + long cacheHitsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get(); + long diskReadsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get(); + // the batch path must still assemble the same aligned pages: file 1 has 10 points SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); Assert.assertTrue(seriesScanUtil.hasNextPage()); TsBlock tsBlock = seriesScanUtil.nextPage(); Assert.assertEquals(10, tsBlock.getPositionCount()); + + // the device must have been read in one batch: no per-column cache lookups were needed + Assert.assertEquals( + "the batch path must not fall back to per-column cache loads", + cacheHitsBefore, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get()); + // hasNextFile unpacks the first seq file and the first unseq file, one batch read each + Assert.assertEquals( + diskReadsBefore + 2, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get()); } } From 2c9fbe45e3504b14317c8eccdd5ce24a0d0a2465 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 19:04:18 +0800 Subject: [PATCH 12/27] Keep DEBUG queries on the per-column metadata path The batch metadata scan bypasses TimeSeriesMetadataCache.get(..., debug=true) and does not emit equivalent hit/miss diagnostics, so a DEBUG SELECT on a wide table lost its metadata logs. Make the batch gate take the debug and offset-addressed state explicitly so DEBUG queries keep the per-column path, and cover both conditions in the gating unit test. --- .../operator/source/FileLoaderUtils.java | 21 ++++++++++++------- .../operator/source/FileLoaderUtilsTest.java | 18 ++++++++++------ 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index ca34d9f9ba495..7e34046d00634 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -365,9 +365,13 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr // For a wide device (a table scan that selects almost all field columns), deserialize the // whole device's TimeSeriesMetadata in a single batch read instead of walking the device // metadata index once per column. External tsfiles address metadata by offset, so they must - // stay on the per-column path. - if (rootMeasurementMetadataIndexNodeOffset == null - && shouldUseDeviceMetadataBatchScan(valueMeasurementList.size(), totalFieldColumnCount)) { + // stay on the per-column path; DEBUG queries also stay there so the per-column cache's + // diagnostic logs are preserved. + if (shouldUseDeviceMetadataBatchScan( + valueMeasurementList.size(), + totalFieldColumnCount, + isDebug, + rootMeasurementMetadataIndexNodeOffset != null)) { return loadAlignedTimeSeriesMetadataBatch( resource, alignedPath, @@ -434,10 +438,13 @@ && shouldUseDeviceMetadataBatchScan(valueMeasurementList.size(), totalFieldColum } static boolean shouldUseDeviceMetadataBatchScan( - int selectedColumnCount, int totalFieldColumnCount) { - // a single-column device gains nothing from a batch read (one leaf either way), and it would - // bypass the per-column cache's debug logging - return totalFieldColumnCount > 1 + int selectedColumnCount, int totalFieldColumnCount, boolean debug, boolean offsetAddressed) { + // DEBUG queries keep the per-column cache's diagnostic logs; offset-addressed external + // tsfiles cannot be resolved through the table metadata index; a single-column device gains + // nothing from a batch read (one leaf either way) + return !debug + && !offsetAddressed + && totalFieldColumnCount > 1 && (double) selectedColumnCount / totalFieldColumnCount >= DEVICE_METADATA_BATCH_SCAN_RATIO; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java index 92431d1114b8c..2ef0fdd211a4d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java @@ -29,17 +29,23 @@ public class FileLoaderUtilsTest { @Test public void testShouldUseDeviceMetadataBatchScan() { // no field count is known (tree model, external tsfiles, or a table missing from the cache) - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 0)); + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 0, false, false)); // a single-column device gains nothing from a batch read - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(1, 1)); + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(1, 1, false, false)); // below, at and above the threshold - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(5, 10)); - assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(6, 10)); - assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10)); + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(5, 10, false, false)); + assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(6, 10, false, false)); + assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10, false, false)); // a scan can reference measurements that no longer exist, so selected > total is possible - assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(11, 10)); + assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(11, 10, false, false)); + + // DEBUG queries keep the per-column cache's diagnostic logs + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10, true, false)); + + // offset-addressed external tsfiles cannot be resolved through the table metadata index + assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10, false, true)); } } From 3fba9804848fd0ce62970eb128e829e992196430 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 19:09:23 +0800 Subject: [PATCH 13/27] Check the bloom filter before the batch device read getDeviceTimeSeriesMetadata read the whole device metadata index and only then discovered the device was absent, while the per-column get() can skip that read by probing the file's bloom filter first. Probe the cached time column entry before the read, keep the same missing-device skip/throw rules, and account the bloom filter cache/disk/time statistics like get(). --- .../buffer/TimeSeriesMetadataCache.java | 57 ++++++++++++++++--- .../buffer/TimeSeriesMetadataCacheTest.java | 18 ++++-- 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 922f2c4e4c959..d1e026a569f2f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -348,9 +348,12 @@ public Map getDeviceTimeSeriesMetadata( boolean ignoreNotExists) throws IOException { long startTime = System.nanoTime(); + long loadBloomFilterTime = 0; boolean externalTsFile = queryContext.isExternalTsFileScan(); LongConsumer timeSeriesMetadataIoSizeRecorder = queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; + LongConsumer bloomFilterIoSizeRecorder = + queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet; boolean cacheHit = true; try { @@ -369,6 +372,14 @@ public Map getDeviceTimeSeriesMetadata( return cached; } cacheHit = false; + long bloomFilterStartTime = System.nanoTime(); + boolean devicePossiblyPresent = + isDevicePossiblyPresent( + filePath, tsFileID, device, queryContext, bloomFilterIoSizeRecorder); + loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; + if (!devicePossiblyPresent) { + return Collections.emptyMap(); + } TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); @@ -377,12 +388,21 @@ public Map getDeviceTimeSeriesMetadata( } } cacheHit = false; + long bloomFilterStartTime = System.nanoTime(); + boolean devicePossiblyPresent = + isDevicePossiblyPresent( + filePath, tsFileID, device, queryContext, bloomFilterIoSizeRecorder); + loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; + if (!devicePossiblyPresent) { + return Collections.emptyMap(); + } TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); return readDeviceTimeSeriesMetadata( reader, tsFileID, device, allSensors, false, ignoreNotExists); } finally { + queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime); if (cacheHit) { queryContext .getQueryStatistics() @@ -391,7 +411,7 @@ public Map getDeviceTimeSeriesMetadata( queryContext .getQueryStatistics() .getLoadTimeSeriesMetadataFromCacheTime() - .getAndAdd(System.nanoTime() - startTime); + .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); } else { queryContext .getQueryStatistics() @@ -400,11 +420,35 @@ public Map getDeviceTimeSeriesMetadata( queryContext .getQueryStatistics() .getLoadTimeSeriesMetadataFromDiskTime() - .getAndAdd(System.nanoTime() - startTime); + .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); } } } + /** + * The time column (keyed by "") is written for every aligned device, so its bloom filter entry is + * a cheap per-file presence probe. A null filter cannot rule the device out. + */ + private boolean isDevicePossiblyPresent( + String filePath, + TsFileID tsFileID, + IDeviceID device, + QueryContext queryContext, + LongConsumer bloomFilterIoSizeRecorder) + throws IOException { + BloomFilter bloomFilter = + BloomFilterCache.getInstance() + .get( + new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID), + false, + bloomFilterIoSizeRecorder, + queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet, + queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet, + false); + return bloomFilter == null + || bloomFilter.contains(device.toString() + IoTDBConstant.PATH_SEPARATOR); + } + /** * @return copies of the cached metadata for every measurement in {@code allSensors}, or null as * soon as one of them is not cached and the device therefore has to be read from disk @@ -438,16 +482,11 @@ private Map readDeviceTimeSeriesMetadata( // metadata list, so readDeviceMetadata (which skips chunk metadata) would NPE at query time. List deviceMetadataList = reader.getDeviceTimeseriesMetadata(device); if (deviceMetadataList.isEmpty()) { - // Match get(): an absent device is only an inconsistency when ignoreNotExists is false and - // the bloom filter does not already prove that the device is missing from this file + // the caller already consulted the bloom filter, so the device could not be ruled out: + // missing metadata is an inconsistency unless ignoreNotExists is set if (ignoreNotExists) { return Collections.emptyMap(); } - BloomFilter bloomFilter = reader.readBloomFilter(); - if (bloomFilter != null - && !bloomFilter.contains(device.toString() + IoTDBConstant.PATH_SEPARATOR)) { - return Collections.emptyMap(); - } throw new IOException( String.format(StorageEngineMessages.DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT, device)); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index fc2d72d8a393d..ef4d2ceb21796 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -292,9 +292,11 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { @Test public void testGetDeviceTimeSeriesMetadata() throws Exception { TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); FileReaderManager fileReaderManager = FileReaderManager.getInstance(); TsFileID tsFileID = new TsFileID(); cache.clear(); + bloomFilterCache.clear(); try { String filePath = "target/batch-device.tsfile"; IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); @@ -370,6 +372,7 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { assertEquals("s1", cached.getMeasurementId()); } finally { cache.clear(); + bloomFilterCache.clear(); fileReaderManager.closeFileAndRemoveReader(tsFileID); } } @@ -377,9 +380,11 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { @Test public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception { TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); FileReaderManager fileReaderManager = FileReaderManager.getInstance(); TsFileID tsFileID = new TsFileID(); cache.clear(); + bloomFilterCache.clear(); try { String filePath = "target/batch-absent-device-bloom.tsfile"; IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); @@ -387,10 +392,9 @@ public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)) - .thenReturn(Collections.emptyList()); - // the bloom filter proves the device is missing, so the per-column path would skip it too - Mockito.when(reader.readBloomFilter()).thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1)); + // the bloom filter proves the device is missing, so the device must not be read at all + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))) + .thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1)); // an absent device yields an empty map, which loadAlignedTimeSeriesMetadataBatch then // distinguishes from "present but empty time column" @@ -399,8 +403,10 @@ public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception .getDeviceTimeSeriesMetadata( filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false) .isEmpty()); + Mockito.verify(reader, Mockito.never()).getDeviceTimeseriesMetadata(deviceID); } finally { cache.clear(); + bloomFilterCache.clear(); fileReaderManager.closeFileAndRemoveReader(tsFileID); } } @@ -408,9 +414,11 @@ filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false) @Test public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Exception { TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); FileReaderManager fileReaderManager = FileReaderManager.getInstance(); TsFileID tsFileID = new TsFileID(); cache.clear(); + bloomFilterCache.clear(); try { String filePath = "target/batch-absent-device-throw.tsfile"; IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); @@ -422,7 +430,7 @@ public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Except .thenReturn(Collections.emptyList()); BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); bloomFilter.add(deviceID.toString() + "."); - Mockito.when(reader.readBloomFilter()).thenReturn(bloomFilter); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); assertThrows( IOException.class, From 90a2cb045bfb40ac4d32100a30131642cc3534cb Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 13 Nov 2026 10:37:45 +0800 Subject: [PATCH 14/27] Address wide-scan batch review: partial cache reuse and diagnostics Reuse cached measurements and batch-read only the missing ones, probing the per-measurement bloom filter first so absent columns do not force a whole-device read; the all-or-nothing cache check is gone. Count load statistics per series so they stay comparable with the per-column path, and document the batch read's missing IO-size accounting. Report a dedicated error naming the device and the TsFile when the resource records a device that the file metadata lacks (en + zh), share the load statistics recording between both paths, and pass the external flag through the bloom filter lookup. Only batch when the table has more field columns than the measurement index fan-out, so small devices stay on the per-column path. Add a real 700-column batch-vs-per-column comparison, a 300-column end-to-end batch scan spanning two leaf groups, and gate boundary cases. --- .../iotdb/db/i18n/StorageEngineMessages.java | 3 + .../iotdb/db/i18n/StorageEngineMessages.java | 3 + .../operator/source/FileLoaderUtils.java | 9 +- .../buffer/TimeSeriesMetadataCache.java | 251 +++++++++++------- .../operator/source/FileLoaderUtilsTest.java | 36 ++- .../buffer/TimeSeriesMetadataCacheTest.java | 102 ++++++- .../series/AlignedSeriesScanBatchTest.java | 176 ++++++++---- 7 files changed, 411 insertions(+), 169 deletions(-) diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java index 7133bf09abda5..b78adeed4da52 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -1710,6 +1710,9 @@ private StorageEngineMessages() {} public static final String COMPACTION_CROSS_SPACE = "cross"; public static final String DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT = "%s does not exist in the resource file"; + public static final String + EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374 = + "Device %s is recorded in the resource but missing from the metadata of TsFile %s"; public static final String TARGET_FILE_SMALLER_THAN_MAGIC_STRING_AND_VERSION_NUMBER_SIZE_FMT = "target file %s is smaller than magic string and version number size"; public static final String CURRENT_POINT_TIMESTAMP_SHOULD_BE_LATER_FMT = diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java index 1603065835bda..a700729990593 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -1620,6 +1620,9 @@ private StorageEngineMessages() {} public static final String COMPACTION_CROSS_SPACE = "跨空间"; public static final String DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT = "%s 在资源文件中不存在"; + public static final String + EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374 = + "设备 %s 记录在资源文件中,但在 TsFile %s 的元数据中不存在"; public static final String TARGET_FILE_SMALLER_THAN_MAGIC_STRING_AND_VERSION_NUMBER_SIZE_FMT = "目标文件 %s 小于 magic string 和版本号大小"; public static final String CURRENT_POINT_TIMESTAMP_SHOULD_BE_LATER_FMT = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 7e34046d00634..05e0a681ab2a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -43,6 +43,7 @@ import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.iotdb.db.utils.SchemaUtils; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; @@ -440,11 +441,13 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr static boolean shouldUseDeviceMetadataBatchScan( int selectedColumnCount, int totalFieldColumnCount, boolean debug, boolean offsetAddressed) { // DEBUG queries keep the per-column cache's diagnostic logs; offset-addressed external - // tsfiles cannot be resolved through the table metadata index; a single-column device gains - // nothing from a batch read (one leaf either way) + // tsfiles cannot be resolved through the table metadata index; and when the device's series + // fit in a single measurement leaf group, one per-column lookup already reads them all, so + // batching only pays off once the table has more field columns than the index fan-out return !debug && !offsetAddressed - && totalFieldColumnCount > 1 + && totalFieldColumnCount + > TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode() && (double) selectedColumnCount / totalFieldColumnCount >= DEVICE_METADATA_BATCH_SCAN_RATIO; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index d1e026a569f2f..17193a43d11cd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; +import org.apache.iotdb.db.queryengine.execution.fragment.QueryStatistics; import org.apache.iotdb.db.queryengine.metric.TimeSeriesMetadataCacheMetrics; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; @@ -49,8 +50,11 @@ import java.io.IOException; import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -294,39 +298,35 @@ public TimeseriesMetadata get( return new TimeseriesMetadata(timeseriesMetadata); } } finally { - queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime); - if (cacheHit) { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); - } else { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); - } + long lookupTime = System.nanoTime() - startTime - loadBloomFilterTime; + recordLoadStatistics( + queryContext, + cacheHit ? 1 : 0, + cacheHit ? lookupTime : 0, + cacheHit ? 0 : 1, + cacheHit ? 0 : lookupTime, + loadBloomFilterTime); } } /** - * Load the TimeseriesMetadata of {@code allSensors} of a device, reading the device in a single - * batch when the per-measurement cache does not already hold all of them. This is used by - * wide-table scans (a device with many columns) to avoid walking the device metadata index once - * per column / leaf node. + * Load the TimeseriesMetadata of {@code allSensors} of a device, reusing the per-measurement + * cache and reading the device in a single batch only for the measurements that are still + * missing. This is used by wide-table scans (a device with many columns) to avoid walking the + * device metadata index once per column / leaf group. + * + *

Measurements already cached are returned without touching the file. The remaining ones are + * first checked against the file's cached bloom filter, so columns absent from this file do not + * force a device read; only measurements that cannot be ruled out are loaded by the batch read. * *

The returned map always holds copies of the cached (or freshly read) metadata, so the caller * may apply {@code setModified} and {@code setChunkMetadataLoader} without corrupting cached * entries — matching {@link #get}. * + *

Load statistics are counted per series (not per device read) so that they stay comparable + * with {@link #get}; the bytes read by the batch read cannot be reported because {@link + * TsFileSequenceReader#getDeviceTimeseriesMetadata(IDeviceID)} has no IO-size recorder. + * * @param filePath path of the tsfile * @param tsFileID id of the tsfile * @param device target device @@ -348,129 +348,180 @@ public Map getDeviceTimeSeriesMetadata( boolean ignoreNotExists) throws IOException { long startTime = System.nanoTime(); + long cachePhaseEndTime = 0; long loadBloomFilterTime = 0; + long cacheHitCount = 0; + long diskLookupCount = 0; boolean externalTsFile = queryContext.isExternalTsFileScan(); LongConsumer timeSeriesMetadataIoSizeRecorder = queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; LongConsumer bloomFilterIoSizeRecorder = queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet; - boolean cacheHit = true; + + Map result = new HashMap<>(allSensors.size()); + List missing = new ArrayList<>(allSensors.size()); try { if (CACHE_ENABLE && !externalTsFile) { - Map cached = - getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors); - if (cached != null) { - return cached; + // fast path without locking: reuse everything the per-measurement cache already has + collectCachedMetadata(tsFileID, device, allSensors, result, missing); + cacheHitCount = result.size(); + if (missing.isEmpty()) { + cachePhaseEndTime = System.nanoTime(); + return result; } - // serialize concurrent reads of the same device+file (same lock key as get) so threads - // don't each deserialize the whole device and race on the shared reader + // serialize reads of the same device+file (same lock key as get) so threads don't each + // deserialize the whole device and race on the shared reader synchronized ( devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, WeakReference::new)) { - cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors); - if (cached != null) { - return cached; + List stillMissing = new ArrayList<>(missing.size()); + collectCachedMetadata(tsFileID, device, missing, result, stillMissing); + missing = stillMissing; + cacheHitCount = result.size(); + cachePhaseEndTime = System.nanoTime(); + if (missing.isEmpty()) { + return result; } - cacheHit = false; + + // measurements the cached bloom filter already rejects are known absent in this file, so + // they must not trigger a device read long bloomFilterStartTime = System.nanoTime(); - boolean devicePossiblyPresent = - isDevicePossiblyPresent( - filePath, tsFileID, device, queryContext, bloomFilterIoSizeRecorder); + BloomFilter bloomFilter = + loadBloomFilter(filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, false); + List possiblyPresent = new ArrayList<>(missing.size()); + for (String measurement : missing) { + if (bloomFilterMayContain(bloomFilter, device, measurement)) { + possiblyPresent.add(measurement); + } else { + result.put(measurement, null); + } + } loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; - if (!devicePossiblyPresent) { - return Collections.emptyMap(); + diskLookupCount = missing.size(); + if (possiblyPresent.isEmpty()) { + return result; } + TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); - return readDeviceTimeSeriesMetadata( - reader, tsFileID, device, allSensors, true, ignoreNotExists); + result.putAll( + readDeviceTimeSeriesMetadata( + reader, + filePath, + tsFileID, + device, + new HashSet<>(possiblyPresent), + true, + ignoreNotExists)); + return result; } } - cacheHit = false; + + // cache disabled / external tsfiles: probe the time column first so an absent device does + // not pay for the whole-device read long bloomFilterStartTime = System.nanoTime(); - boolean devicePossiblyPresent = - isDevicePossiblyPresent( - filePath, tsFileID, device, queryContext, bloomFilterIoSizeRecorder); + BloomFilter bloomFilter = + loadBloomFilter( + filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, externalTsFile); loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; - if (!devicePossiblyPresent) { - return Collections.emptyMap(); + diskLookupCount = allSensors.size(); + if (!bloomFilterMayContain(bloomFilter, device, "")) { + for (String measurement : allSensors) { + result.put(measurement, null); + } + return result; } TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); return readDeviceTimeSeriesMetadata( - reader, tsFileID, device, allSensors, false, ignoreNotExists); + reader, filePath, tsFileID, device, allSensors, false, ignoreNotExists); } finally { - queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime); - if (cacheHit) { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); - } else { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); - } + long endTime = System.nanoTime(); + long cacheLookupTime = cachePhaseEndTime == 0 ? 0 : cachePhaseEndTime - startTime; + long diskLookupTime = + cachePhaseEndTime == 0 + ? endTime - startTime - loadBloomFilterTime + : endTime - cachePhaseEndTime - loadBloomFilterTime; + recordLoadStatistics( + queryContext, + cacheHitCount, + cacheLookupTime, + diskLookupCount, + diskLookupTime, + loadBloomFilterTime); } } - /** - * The time column (keyed by "") is written for every aligned device, so its bloom filter entry is - * a cheap per-file presence probe. A null filter cannot rule the device out. - */ - private boolean isDevicePossiblyPresent( + private BloomFilter loadBloomFilter( String filePath, TsFileID tsFileID, - IDeviceID device, QueryContext queryContext, - LongConsumer bloomFilterIoSizeRecorder) + LongConsumer bloomFilterIoSizeRecorder, + boolean externalTsFile) throws IOException { - BloomFilter bloomFilter = - BloomFilterCache.getInstance() - .get( - new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID), - false, - bloomFilterIoSizeRecorder, - queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet, - queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet, - false); + return BloomFilterCache.getInstance() + .get( + new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID), + false, + bloomFilterIoSizeRecorder, + queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet, + queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet, + externalTsFile); + } + + /** A null filter cannot rule the measurement out. */ + private static boolean bloomFilterMayContain( + BloomFilter bloomFilter, IDeviceID device, String measurement) { return bloomFilter == null - || bloomFilter.contains(device.toString() + IoTDBConstant.PATH_SEPARATOR); + || bloomFilter.contains(device.toString() + IoTDBConstant.PATH_SEPARATOR + measurement); + } + + private static void recordLoadStatistics( + QueryContext queryContext, + long cacheHitCount, + long cacheLookupTime, + long diskLookupCount, + long diskLookupTime, + long bloomFilterTime) { + QueryStatistics statistics = queryContext.getQueryStatistics(); + statistics.getLoadBloomFilterTime().getAndAdd(bloomFilterTime); + if (cacheHitCount > 0) { + statistics.getLoadTimeSeriesMetadataFromCacheCount().getAndAdd(cacheHitCount); + statistics.getLoadTimeSeriesMetadataFromCacheTime().getAndAdd(cacheLookupTime); + } + if (diskLookupCount > 0) { + statistics.getLoadTimeSeriesMetadataFromDiskCount().getAndAdd(diskLookupCount); + statistics.getLoadTimeSeriesMetadataFromDiskTime().getAndAdd(diskLookupTime); + } } /** - * @return copies of the cached metadata for every measurement in {@code allSensors}, or null as - * soon as one of them is not cached and the device therefore has to be read from disk + * Resolve cached measurements into {@code resolved} and collect the rest into {@code missing}. */ - private Map getCachedDeviceTimeSeriesMetadata( - TsFileID tsFileID, IDeviceID device, Set allSensors) { - Map result = new HashMap<>(allSensors.size()); - for (String measurement : allSensors) { + private void collectCachedMetadata( + TsFileID tsFileID, + IDeviceID device, + Collection measurements, + Map resolved, + List missing) { + for (String measurement : measurements) { TimeseriesMetadata metadata = lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID, device, measurement)); if (metadata == null) { - return null; + missing.add(measurement); + } else { + resolved.put( + measurement, + metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new TimeseriesMetadata(metadata)); } - result.put( - measurement, - metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new TimeseriesMetadata(metadata)); } - return result; } private Map readDeviceTimeSeriesMetadata( TsFileSequenceReader reader, + String filePath, TsFileID tsFileID, IDeviceID device, Set allSensors, @@ -488,7 +539,11 @@ private Map readDeviceTimeSeriesMetadata( return Collections.emptyMap(); } throw new IOException( - String.format(StorageEngineMessages.DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT, device)); + String.format( + StorageEngineMessages + .EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374, + device, + filePath)); } Map result = new HashMap<>(allSensors.size()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java index 2ef0fdd211a4d..b62ba8fe7cdc3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.junit.Test; import static org.junit.Assert.assertFalse; @@ -28,24 +29,43 @@ public class FileLoaderUtilsTest { @Test public void testShouldUseDeviceMetadataBatchScan() { + int maxDegreeOfIndexNode = TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode(); + // a table wide enough to span more than one measurement leaf group + int wideTableFieldCount = maxDegreeOfIndexNode * 2; + int threshold = (int) Math.ceil(wideTableFieldCount * 0.6); + // no field count is known (tree model, external tsfiles, or a table missing from the cache) assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 0, false, false)); - // a single-column device gains nothing from a batch read - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(1, 1, false, false)); + // a device whose series fit in a single measurement leaf group gains nothing from batching + assertFalse( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + maxDegreeOfIndexNode, maxDegreeOfIndexNode, false, false)); // below, at and above the threshold - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(5, 10, false, false)); - assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(6, 10, false, false)); - assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10, false, false)); + assertFalse( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + threshold - 1, wideTableFieldCount, false, false)); + assertTrue( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + threshold, wideTableFieldCount, false, false)); + assertTrue( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + wideTableFieldCount, wideTableFieldCount, false, false)); // a scan can reference measurements that no longer exist, so selected > total is possible - assertTrue(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(11, 10, false, false)); + assertTrue( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + wideTableFieldCount + 1, wideTableFieldCount, false, false)); // DEBUG queries keep the per-column cache's diagnostic logs - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10, true, false)); + assertFalse( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + wideTableFieldCount, wideTableFieldCount, true, false)); // offset-addressed external tsfiles cannot be resolved through the table metadata index - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 10, false, true)); + assertFalse( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + wideTableFieldCount, wideTableFieldCount, false, true)); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index ef4d2ceb21796..90cd54b36b840 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -37,6 +37,7 @@ import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Ignore; import org.junit.Test; @@ -47,6 +48,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -396,13 +398,12 @@ public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))) .thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1)); - // an absent device yields an empty map, which loadAlignedTimeSeriesMetadataBatch then - // distinguishes from "present but empty time column" - assertTrue( - cache - .getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false) - .isEmpty()); + // an absent device must not be read: the filtered-out measurements are returned as null, + // which the caller distinguishes from "present but empty time column" + Map result = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + assertNull(result.get("")); Mockito.verify(reader, Mockito.never()).getDeviceTimeseriesMetadata(deviceID); } finally { cache.clear(); @@ -445,7 +446,94 @@ filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), true) .isEmpty()); } finally { cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testBatchMetadataMatchesPerColumn() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + int columnCount = 700; + File file = new File("target/batch-compare.tsfile"); + file.delete(); + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.compare.d1"); + List schemas = new ArrayList<>(columnCount); + for (int i = 0; i < columnCount; i++) { + schemas.add(new MeasurementSchema("s" + i, TSDataType.INT32)); + } + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerAlignedTimeseries(deviceID, schemas); + for (long time = 0; time < 3; time++) { + TSRecord record = new TSRecord(deviceID, time); + for (int i = 0; i < columnCount; i++) { + record.addPoint("s" + i, (int) time); + } + writer.writeRecord(record); + } + } + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = file.getAbsolutePath(); + Set allSensors = new HashSet<>(); + for (int i = 0; i < columnCount; i++) { + allSensors.add("s" + i); + } + allSensors.add(""); + allSensors.add("absent"); + + Map perColumn = new HashMap<>(); + for (String measurement : allSensors) { + perColumn.put( + measurement, + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, measurement), + allSensors, + false, + false, + new QueryContext(false, false))); + } + cache.clear(); + + Map batch = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + + assertEquals(allSensors.size(), batch.size()); + for (String measurement : allSensors) { + TimeseriesMetadata expected = perColumn.get(measurement); + TimeseriesMetadata actual = batch.get(measurement); + if (expected == null) { + assertNull(measurement, actual); + continue; + } + assertNotNull(measurement, actual); + assertEquals(measurement, actual.getMeasurementId()); + assertEquals( + measurement, expected.getStatistics().getCount(), actual.getStatistics().getCount()); + assertEquals( + measurement, + expected.getStatistics().getStartTime(), + actual.getStatistics().getStartTime()); + assertEquals( + measurement, + expected.getStatistics().getEndTime(), + actual.getStatistics().getEndTime()); + assertEquals( + measurement, + expected.getChunkMetadataList().size(), + actual.getChunkMetadataList().size()); + } + } finally { + cache.clear(); + bloomFilterCache.clear(); fileReaderManager.closeFileAndRemoveReader(tsFileID); + assertTrue(file.delete()); } } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java index e4f9b880ab5d5..f095bf436af72 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -21,81 +21,151 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.AlignedFullPath; +import org.apache.iotdb.commons.path.AlignedPath; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; +import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; +import org.apache.tsfile.write.chunk.IChunkWriter; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; import org.junit.Assert; import org.junit.Test; +import java.io.File; import java.io.IOException; -import java.util.Arrays; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; +import java.util.List; -/** Verifies the wide-table batch metadata scan (see FileLoaderUtils) reads data correctly. */ -public class AlignedSeriesScanBatchTest extends AbstractAlignedSeriesScanTest { +import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.createChunkWriter; +import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.writeAlignedChunk; + +/** + * Verifies that a real multi-leaf-group device (more field columns than the measurement index + * fan-out) is read correctly through the batch metadata path. + */ +public class AlignedSeriesScanBatchTest { + + private static final String DATABASE = "root.wide_pd"; + private static final IDeviceID DEVICE = + IDeviceID.Factory.DEFAULT_FACTORY.create(DATABASE + ".d1"); @Test - public void testBatchScanReadsAlignedData() throws IOException, IllegalPathException { - AlignedFullPath scanPath = - new AlignedFullPath( - TEST_DEVICE, - Arrays.asList("s1", "s2", "s3"), - Arrays.asList( - new MeasurementSchema("s1", TSDataType.INT32), - new MeasurementSchema("s2", TSDataType.INT32), - new MeasurementSchema("s3", TSDataType.INT32))); + public void testBatchScanReadsWideAlignedDevice() throws IOException, IllegalPathException { + int columnCount = 300; + File file = new File(TestConstant.getTestTsFilePath(DATABASE, 0, 0, 1)); + if (!file.getParentFile().exists()) { + Assert.assertTrue(file.getParentFile().mkdirs()); + } + TsFileResource resource = new TsFileResource(file); + List writtenPaths = new ArrayList<>(columnCount); + List dataTypes = new ArrayList<>(columnCount); + List encodings = new ArrayList<>(columnCount); + List compressionTypes = new ArrayList<>(columnCount); + List measurementNames = new ArrayList<>(columnCount); + List measurementSchemas = new ArrayList<>(columnCount); + + for (int i = 0; i < columnCount; i++) { + String measurement = "s" + i; + measurementNames.add(measurement); + measurementSchemas.add(new MeasurementSchema(measurement, TSDataType.INT32)); + writtenPaths.add(new AlignedPath(DEVICE.toString(), measurement)); + dataTypes.add(TSDataType.INT32); + encodings.add(TSEncoding.PLAIN); + compressionTypes.add(CompressionType.UNCOMPRESSED); + } + + try (TsFileIOWriter writer = new TsFileIOWriter(file)) { + writer.startChunkGroup(DEVICE); + for (IChunkWriter chunkWriter : + createChunkWriter(writtenPaths, dataTypes, encodings, compressionTypes, true)) { + writeAlignedChunk( + (AlignedChunkWriterImpl) chunkWriter, + writer, + Collections.singletonList(new TimeRange(0L, 9L)), + true); + } + writer.endChunkGroup(); + writer.endFile(); + } + resource.updateStartTime(DEVICE, 0); + resource.updateEndTime(DEVICE, 9); + resource.setStatusForTest(TsFileResourceStatus.NORMAL); - SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); - builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); - SeriesScanOptions scanOptions = builder.build(); - // all 3 value columns are selected, so selected/total = 1.0 >= 0.6 and the batch path is taken - scanOptions.setTotalFieldColumnCount(3); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + metadataCache.clear(); + bloomFilterCache.clear(); + try { + long cacheHitsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get(); + long diskReadsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get(); - AlignedSeriesScanUtil seriesScanUtil = - new AlignedSeriesScanUtil( - scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); - seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); + AlignedFullPath scanPath = new AlignedFullPath(DEVICE, measurementNames, measurementSchemas); + SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); + builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); + SeriesScanOptions scanOptions = builder.build(); + // all columns are selected and the table is wider than the index fan-out, so batching applies + scanOptions.setTotalFieldColumnCount(columnCount); - TimeSeriesMetadataCache.getInstance().clear(); - long cacheHitsBefore = - EnvironmentUtils.TEST_QUERY_FI_CONTEXT - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .get(); - long diskReadsBefore = - EnvironmentUtils.TEST_QUERY_FI_CONTEXT - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskCount() - .get(); + AlignedSeriesScanUtil seriesScanUtil = + new AlignedSeriesScanUtil( + scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource( + new QueryDataSource(Collections.singletonList(resource), Collections.emptyList())); - // the batch path must still assemble the same aligned pages: file 1 has 10 points - SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); - SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); - Assert.assertTrue(seriesScanUtil.hasNextPage()); - TsBlock tsBlock = seriesScanUtil.nextPage(); - Assert.assertEquals(10, tsBlock.getPositionCount()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); - // the device must have been read in one batch: no per-column cache lookups were needed - Assert.assertEquals( - "the batch path must not fall back to per-column cache loads", - cacheHitsBefore, - EnvironmentUtils.TEST_QUERY_FI_CONTEXT - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .get()); - // hasNextFile unpacks the first seq file and the first unseq file, one batch read each - Assert.assertEquals( - diskReadsBefore + 2, - EnvironmentUtils.TEST_QUERY_FI_CONTEXT - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskCount() - .get()); + // one batch read per device and file: no per-column cache lookups happened + Assert.assertEquals( + "the batch path must not fall back to per-column cache loads", + cacheHitsBefore, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get()); + // statistics are counted per series, like the per-column path (time column + all fields) + Assert.assertEquals( + diskReadsBefore + columnCount + 1, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get()); + } finally { + metadataCache.clear(); + bloomFilterCache.clear(); + FileReaderManager.getInstance().closeFileAndRemoveReader(resource.getTsFileID()); + Assert.assertTrue(file.delete()); + } } } From caf6df1ef1b164b9388a665f1b89874990aded61 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 13 Nov 2026 11:49:41 +0800 Subject: [PATCH 15/27] Refine the batch metadata path after review Skip the device read when the time column is known absent, so a bloom filter false positive on a field cannot force an absent-device read. Fall back to per-column lookups for a small remainder (one or two measurements), which reads a single leaf group instead of the whole device, and count the cold-call cache probe and lock wait in the disk lookup time. Strengthen the wide-scan tests: assert timestamps and every field value, compare a sparse batch scan against the per-column path, cover an all-null projection under ignoreAllNullRows, and add cache regressions for the filtered time column and the single-miss fallback. --- .../buffer/TimeSeriesMetadataCache.java | 52 +++- .../buffer/TimeSeriesMetadataCacheTest.java | 127 +++++++- .../series/AlignedSeriesScanBatchTest.java | 294 ++++++++++++++---- 3 files changed, 403 insertions(+), 70 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 17193a43d11cd..a44bf4016ed01 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -95,6 +95,9 @@ public class TimeSeriesMetadataCache { Collections.synchronizedMap(new WeakHashMap<>()); private static final String SEPARATOR = "$"; + /** A remainder this small is cheaper to load with per-column leaf-group lookups. */ + private static final int MAX_PER_COLUMN_FALLBACK = 2; + private final AtomicLong evictedExistingEntryCount = new AtomicLong(0); private final AtomicLong evictedNonExistingEntryCount = new AtomicLong(0); @@ -352,6 +355,7 @@ public Map getDeviceTimeSeriesMetadata( long loadBloomFilterTime = 0; long cacheHitCount = 0; long diskLookupCount = 0; + boolean delegated = false; boolean externalTsFile = queryContext.isExternalTsFileScan(); LongConsumer timeSeriesMetadataIoSizeRecorder = queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; @@ -402,6 +406,32 @@ public Map getDeviceTimeSeriesMetadata( return result; } + // the time column is always written for an aligned device, so a known-absent time + // column means the file will be skipped anyway; a bloom-filter false positive on a field + // must not force a device read and a possible absent-device error + if (allSensors.contains("") && result.get("") == null && !possiblyPresent.contains("")) { + return result; + } + + // for a small remainder, a per-column lookup reads one leaf group instead of the whole + // device; those lookups record their own statistics + if (possiblyPresent.size() <= MAX_PER_COLUMN_FALLBACK) { + delegated = true; + diskLookupCount = missing.size() - possiblyPresent.size(); + for (String measurement : possiblyPresent) { + result.put( + measurement, + get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), + allSensors, + ignoreNotExists, + false, + queryContext)); + } + return result; + } + TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); @@ -439,11 +469,23 @@ public Map getDeviceTimeSeriesMetadata( reader, filePath, tsFileID, device, allSensors, false, ignoreNotExists); } finally { long endTime = System.nanoTime(); - long cacheLookupTime = cachePhaseEndTime == 0 ? 0 : cachePhaseEndTime - startTime; - long diskLookupTime = - cachePhaseEndTime == 0 - ? endTime - startTime - loadBloomFilterTime - : endTime - cachePhaseEndTime - loadBloomFilterTime; + long cacheLookupTime; + long diskLookupTime; + if (delegated) { + // the per-column fallbacks recorded their own disk time + cacheLookupTime = cachePhaseEndTime == 0 ? 0 : cachePhaseEndTime - startTime; + diskLookupTime = 0; + } else if (cacheHitCount == 0 || cachePhaseEndTime == 0) { + // cold or cache-disabled call: cache probes and the lock wait belong to the disk lookup + cacheLookupTime = 0; + diskLookupTime = endTime - startTime - loadBloomFilterTime; + } else if (diskLookupCount == 0) { + cacheLookupTime = endTime - startTime - loadBloomFilterTime; + diskLookupTime = 0; + } else { + cacheLookupTime = cachePhaseEndTime - startTime; + diskLookupTime = endTime - cachePhaseEndTime - loadBloomFilterTime; + } recordLoadStatistics( queryContext, cacheHitCount, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 90cd54b36b840..32efa93539d31 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -423,7 +423,9 @@ public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Except try { String filePath = "target/batch-absent-device-throw.tsfile"; IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); - Set allSensors = Collections.singleton(""); + // more than MAX_PER_COLUMN_FALLBACK entries so the batch read (not the per-column fallback) + // is used + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); @@ -431,6 +433,8 @@ public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Except .thenReturn(Collections.emptyList()); BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); bloomFilter.add(deviceID.toString() + "."); + bloomFilter.add(deviceID.toString() + ".s1"); + bloomFilter.add(deviceID.toString() + ".s2"); Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); assertThrows( @@ -536,4 +540,125 @@ public void testBatchMetadataMatchesPerColumn() throws Exception { assertTrue(file.delete()); } } + + @Test + public void testAbsentTimeColumnSkipsDeviceRead() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-time-column-filtered.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + // the time column is filtered out, while s1 is a bloom filter false positive + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); + bloomFilter.add(deviceID.toString() + ".s1"); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); + + Map result = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + + assertNull(result.get("")); + assertNull(result.get("s1")); + Mockito.verify(reader, Mockito.never()).getDeviceTimeseriesMetadata(deviceID); + Mockito.verify(reader, Mockito.never()) + .readTimeseriesMetadata( + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.anyBoolean(), + Mockito.any()); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testSingleMissingMeasurementUsesPerColumnFallback() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-single-miss.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + Statistics statistics = Statistics.getStatsByType(TSDataType.INT32); + statistics.setCount(1); + statistics.setStartTime(0); + statistics.setEndTime(10); + TimeseriesMetadata s2 = + new TimeseriesMetadata((byte) 0, 0, "s2", TSDataType.INT32, statistics, new PublicBAOS()); + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + + List deviceMetadata = new ArrayList<>(); + deviceMetadata.add( + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS())); + deviceMetadata.add( + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS())); + deviceMetadata.add(s2); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)).thenReturn(deviceMetadata); + + Map first = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + assertNotNull(first.get("s2")); + + // evict one measurement: the next call must load only that one, not the whole device + cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s2")); + Mockito.when( + reader.readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.eq("s2"), + Mockito.eq(allSensors), + Mockito.eq(false), + Mockito.any(LongConsumer.class))) + .thenReturn(Collections.singletonList(s2)); + + Map second = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + + assertNotNull(second.get("s2")); + Mockito.verify(reader, Mockito.times(1)).getDeviceTimeseriesMetadata(deviceID); + Mockito.verify(reader, Mockito.times(1)) + .readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.eq("s2"), + Mockito.eq(allSensors), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java index f095bf436af72..017180dff82fb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -35,6 +35,7 @@ import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.iotdb.db.utils.constant.TestConstant; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; @@ -55,62 +56,28 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.createChunkWriter; import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.writeAlignedChunk; /** - * Verifies that a real multi-leaf-group device (more field columns than the measurement index - * fan-out) is read correctly through the batch metadata path. + * Verifies the batch metadata path on a real device wider than one measurement leaf group, and that + * it matches the per-column path for dense, sparse and all-null projections. */ public class AlignedSeriesScanBatchTest { private static final String DATABASE = "root.wide_pd"; private static final IDeviceID DEVICE = IDeviceID.Factory.DEFAULT_FACTORY.create(DATABASE + ".d1"); + private static final int COLUMN_COUNT = 300; + private static final int PRESENT_COLUMN_COUNT = 30; + private static final int ROW_COUNT = 10; @Test - public void testBatchScanReadsWideAlignedDevice() throws IOException, IllegalPathException { - int columnCount = 300; - File file = new File(TestConstant.getTestTsFilePath(DATABASE, 0, 0, 1)); - if (!file.getParentFile().exists()) { - Assert.assertTrue(file.getParentFile().mkdirs()); - } - TsFileResource resource = new TsFileResource(file); - List writtenPaths = new ArrayList<>(columnCount); - List dataTypes = new ArrayList<>(columnCount); - List encodings = new ArrayList<>(columnCount); - List compressionTypes = new ArrayList<>(columnCount); - List measurementNames = new ArrayList<>(columnCount); - List measurementSchemas = new ArrayList<>(columnCount); - - for (int i = 0; i < columnCount; i++) { - String measurement = "s" + i; - measurementNames.add(measurement); - measurementSchemas.add(new MeasurementSchema(measurement, TSDataType.INT32)); - writtenPaths.add(new AlignedPath(DEVICE.toString(), measurement)); - dataTypes.add(TSDataType.INT32); - encodings.add(TSEncoding.PLAIN); - compressionTypes.add(CompressionType.UNCOMPRESSED); - } - - try (TsFileIOWriter writer = new TsFileIOWriter(file)) { - writer.startChunkGroup(DEVICE); - for (IChunkWriter chunkWriter : - createChunkWriter(writtenPaths, dataTypes, encodings, compressionTypes, true)) { - writeAlignedChunk( - (AlignedChunkWriterImpl) chunkWriter, - writer, - Collections.singletonList(new TimeRange(0L, 9L)), - true); - } - writer.endChunkGroup(); - writer.endFile(); - } - resource.updateStartTime(DEVICE, 0); - resource.updateEndTime(DEVICE, 9); - resource.setStatusForTest(TsFileResourceStatus.NORMAL); - + public void testBatchScanReadsWideAlignedDevice() throws Exception { + File file = newFile(1); + TsFileResource resource = writeWideDevice(file, COLUMN_COUNT); TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); metadataCache.clear(); @@ -127,24 +94,20 @@ public void testBatchScanReadsWideAlignedDevice() throws IOException, IllegalPat .getLoadTimeSeriesMetadataFromDiskCount() .get(); - AlignedFullPath scanPath = new AlignedFullPath(DEVICE, measurementNames, measurementSchemas); - SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); - builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); - SeriesScanOptions scanOptions = builder.build(); - // all columns are selected and the table is wider than the index fan-out, so batching applies - scanOptions.setTotalFieldColumnCount(columnCount); - - AlignedSeriesScanUtil seriesScanUtil = - new AlignedSeriesScanUtil( - scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); - seriesScanUtil.initQueryDataSource( - new QueryDataSource(Collections.singletonList(resource), Collections.emptyList())); - - SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); - SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); - Assert.assertTrue(seriesScanUtil.hasNextPage()); - TsBlock tsBlock = seriesScanUtil.nextPage(); - Assert.assertEquals(10, tsBlock.getPositionCount()); + TsBlock block = scan(alignedPath(columnNames(COLUMN_COUNT)), COLUMN_COUNT, resource); + + Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); + Assert.assertEquals(ROW_COUNT, block.getPositionCount()); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals(row, block.getTimeByIndex(row)); + } + for (int column = 0; column < COLUMN_COUNT; column++) { + Column valueColumn = block.getColumn(column); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertFalse("s" + column + " row " + row, valueColumn.isNull(row)); + Assert.assertEquals("s" + column + " row " + row, row, valueColumn.getInt(row)); + } + } // one batch read per device and file: no per-column cache lookups happened Assert.assertEquals( @@ -156,16 +119,219 @@ public void testBatchScanReadsWideAlignedDevice() throws IOException, IllegalPat .get()); // statistics are counted per series, like the per-column path (time column + all fields) Assert.assertEquals( - diskReadsBefore + columnCount + 1, + diskReadsBefore + COLUMN_COUNT + 1, EnvironmentUtils.TEST_QUERY_FI_CONTEXT .getQueryStatistics() .getLoadTimeSeriesMetadataFromDiskCount() .get()); } finally { + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + @Test + public void testSparseBatchScanMatchesPerColumn() throws Exception { + File file = newFile(2); + TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + AlignedFullPath scanPath = alignedPath(columnNames(COLUMN_COUNT)); + try { metadataCache.clear(); bloomFilterCache.clear(); - FileReaderManager.getInstance().closeFileAndRemoveReader(resource.getTsFileID()); - Assert.assertTrue(file.delete()); + long cacheHitsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get(); + TsBlock batched = scan(scanPath, COLUMN_COUNT, resource); + long batchedCacheHits = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get() + - cacheHitsBefore; + + metadataCache.clear(); + long perColumnCacheHitsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get(); + TsBlock perColumn = scan(scanPath, 0, resource); + long perColumnCacheHits = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get() + - perColumnCacheHitsBefore; + + // the sparse projection is identical to the per-column scan, including every null field + Assert.assertEquals(perColumn.getPositionCount(), batched.getPositionCount()); + Assert.assertEquals(perColumn.getValueColumnCount(), batched.getValueColumnCount()); + for (int row = 0; row < batched.getPositionCount(); row++) { + Assert.assertEquals(perColumn.getTimeByIndex(row), batched.getTimeByIndex(row)); + } + for (int column = 0; column < batched.getValueColumnCount(); column++) { + Column batchedColumn = batched.getColumn(column); + Column perColumnColumn = perColumn.getColumn(column); + boolean present = column < PRESENT_COLUMN_COUNT; + for (int row = 0; row < batched.getPositionCount(); row++) { + Assert.assertEquals( + "s" + column + " row " + row, perColumnColumn.isNull(row), batchedColumn.isNull(row)); + if (present) { + Assert.assertFalse(batchedColumn.isNull(row)); + Assert.assertEquals(row, batchedColumn.getInt(row)); + Assert.assertEquals(perColumnColumn.getInt(row), batchedColumn.getInt(row)); + } else { + Assert.assertTrue(batchedColumn.isNull(row)); + } + } + } + + // the batch scan resolved everything from disk, the per-column scan reused leaf groups + Assert.assertEquals(0, batchedCacheHits); + Assert.assertTrue(perColumnCacheHits > 0); + } finally { + cleanup(file, resource, metadataCache, bloomFilterCache); } } + + @Test + public void testBatchScanSkipsAllNullRows() throws Exception { + File file = newFile(3); + TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + // every selected field is absent from the file: with ignoreAllNullRows the file must be + // skipped by both paths + AlignedFullPath absentPath = alignedPath(absentColumnNames(COLUMN_COUNT)); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(true); + try { + metadataCache.clear(); + bloomFilterCache.clear(); + Assert.assertFalse(hasFile(absentPath, COLUMN_COUNT, resource)); + + metadataCache.clear(); + Assert.assertFalse(hasFile(absentPath, 0, resource)); + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + private static TsBlock scan( + AlignedFullPath scanPath, int totalFieldColumnCount, TsFileResource resource) + throws IOException { + AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, totalFieldColumnCount, resource); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + return seriesScanUtil.nextPage(); + } + + private static boolean hasFile( + AlignedFullPath scanPath, int totalFieldColumnCount, TsFileResource resource) + throws IOException { + AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, totalFieldColumnCount, resource); + while (true) { + Optional hasNextFile = seriesScanUtil.hasNextFile(); + if (hasNextFile.isPresent()) { + return hasNextFile.get(); + } + } + } + + private static AlignedSeriesScanUtil newScanUtil( + AlignedFullPath scanPath, int totalFieldColumnCount, TsFileResource resource) + throws IOException { + SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); + builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); + SeriesScanOptions scanOptions = builder.build(); + scanOptions.setTotalFieldColumnCount(totalFieldColumnCount); + AlignedSeriesScanUtil seriesScanUtil = + new AlignedSeriesScanUtil( + scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource( + new QueryDataSource(Collections.singletonList(resource), Collections.emptyList())); + return seriesScanUtil; + } + + private static AlignedFullPath alignedPath(List measurementNames) { + List schemas = new ArrayList<>(measurementNames.size()); + for (String measurement : measurementNames) { + schemas.add(new MeasurementSchema(measurement, TSDataType.INT32)); + } + return new AlignedFullPath(DEVICE, measurementNames, schemas); + } + + private static List columnNames(int count) { + List names = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + names.add("s" + i); + } + return names; + } + + private static List absentColumnNames(int count) { + List names = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + names.add("absent" + i); + } + return names; + } + + private static File newFile(int version) { + File file = new File(TestConstant.getTestTsFilePath(DATABASE, 0, 0, version)); + if (!file.getParentFile().exists()) { + Assert.assertTrue(file.getParentFile().mkdirs()); + } + return file; + } + + private static TsFileResource writeWideDevice(File file, int presentColumnCount) + throws IOException, IllegalPathException { + TsFileResource resource = new TsFileResource(file); + List writtenPaths = new ArrayList<>(presentColumnCount); + List dataTypes = new ArrayList<>(presentColumnCount); + List encodings = new ArrayList<>(presentColumnCount); + List compressionTypes = new ArrayList<>(presentColumnCount); + for (int i = 0; i < presentColumnCount; i++) { + writtenPaths.add(new AlignedPath(DEVICE.toString(), "s" + i)); + dataTypes.add(TSDataType.INT32); + encodings.add(TSEncoding.PLAIN); + compressionTypes.add(CompressionType.UNCOMPRESSED); + } + try (TsFileIOWriter writer = new TsFileIOWriter(file)) { + writer.startChunkGroup(DEVICE); + for (IChunkWriter chunkWriter : + createChunkWriter(writtenPaths, dataTypes, encodings, compressionTypes, true)) { + writeAlignedChunk( + (AlignedChunkWriterImpl) chunkWriter, + writer, + Collections.singletonList(new TimeRange(0L, ROW_COUNT - 1L)), + true); + } + writer.endChunkGroup(); + writer.endFile(); + } + resource.updateStartTime(DEVICE, 0); + resource.updateEndTime(DEVICE, ROW_COUNT - 1L); + resource.setStatusForTest(TsFileResourceStatus.NORMAL); + return resource; + } + + private static void cleanup( + File file, + TsFileResource resource, + TimeSeriesMetadataCache metadataCache, + BloomFilterCache bloomFilterCache) + throws IOException { + metadataCache.clear(); + bloomFilterCache.clear(); + FileReaderManager.getInstance().closeFileAndRemoveReader(resource.getTsFileID()); + Assert.assertTrue(file.delete()); + } } From e9275076bab9c6478ce02a54f1f66127aa147a1d Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 13 Nov 2026 11:55:39 +0800 Subject: [PATCH 16/27] Document the selective-read limitation of the batch path TsFileSequenceReader is an external apache/tsfile dependency and has no batch API that accepts a measurement set, so the batch path still reads the whole device and filters afterwards. Note the constraint and the existing mitigations (bloom filter, per-column fallback) at the call site; a truly selective reader needs an upstream change. --- .../db/storageengine/buffer/TimeSeriesMetadataCache.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index a44bf4016ed01..f2bdddf890192 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -432,6 +432,12 @@ public Map getDeviceTimeSeriesMetadata( return result; } + // TsFileSequenceReader has no selective batch-read API that accepts a measurement set, + // so this call loads every measurement of the device and filters afterwards. A truly + // selective read requires an upstream apache/tsfile change; the bloom filter and the + // per-column fallback above keep this branch to measurements that are plausibly present. + // The measurement-level readTimeseriesMetadata API is group-selective already and is what + // the per-column path and the fallback use. TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); From f8b9c831371d2abeed6580803c4cad22b0144af9 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 11:35:16 +0800 Subject: [PATCH 17/27] Address the follow-up review of the wide-scan batch path Load the remaining misses through the per-column get() when the cache already holds most of the request: it touches only their leaf groups instead of re-reading the whole device. Replace the GC-sensitive WeakHashMap monitor with fixed lock stripes, and make the returned map complete for allSensors on every early return. Give aggregation table scans a totalFieldColumnCount so they can use the batch path, and make the fan-out gate span two leaf groups (>=), fixing its off-by-one and comment. Tests: production ignoreAllNullRows=false coverage including all-null projections, batch negative-caching parity, copy-on-return, a three-miss eviction case, the multi-leaf premise, and per-chunk offset/statistics comparison. --- .../operator/source/FileLoaderUtils.java | 9 +- .../DataNodeTableOperatorGenerator.java | 4 + .../buffer/TimeSeriesMetadataCache.java | 55 +++++-- .../operator/source/FileLoaderUtilsTest.java | 9 +- .../buffer/TimeSeriesMetadataCacheTest.java | 150 +++++++++++++++++- .../series/AlignedSeriesScanBatchTest.java | 49 +++++- 6 files changed, 246 insertions(+), 30 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 05e0a681ab2a7..aa6df29f1ba67 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -441,13 +441,14 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr static boolean shouldUseDeviceMetadataBatchScan( int selectedColumnCount, int totalFieldColumnCount, boolean debug, boolean offsetAddressed) { // DEBUG queries keep the per-column cache's diagnostic logs; offset-addressed external - // tsfiles cannot be resolved through the table metadata index; and when the device's series - // fit in a single measurement leaf group, one per-column lookup already reads them all, so - // batching only pays off once the table has more field columns than the index fan-out + // tsfiles cannot be resolved through the table metadata index; and a table with at least + // max_degree_of_index_node field columns has a dense device of max_degree+1 series (time column + // included), which spans more than one measurement leaf group. A sparse device may hold fewer + // in-file series; the bloom filter and the cache-aware load keep that case cheap. return !debug && !offsetAddressed && totalFieldColumnCount - > TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode() + >= TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode() && (double) selectedColumnCount / totalFieldColumnCount >= DEVICE_METADATA_BATCH_SCAN_RATIO; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index 47679da161a24..b6a763fd9f0e0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -1783,6 +1783,10 @@ private boolean ensureFieldColumnForTreeNonAlignedDeviceViewScan( node.getPushDownPredicate()); seriesScanOptions.setTTLForTableView(tableViewTTL); seriesScanOptions.setIsTableViewForTreeModel(node instanceof AggregationTreeDeviceViewScanNode); + if (!(node instanceof AggregationTreeDeviceViewScanNode) + && !(node instanceof ExternalTsFileAggregationScanNode)) { + seriesScanOptions.setTotalFieldColumnCount(computeTotalFieldColumnCount(node)); + } Set allSensors = new HashSet<>(measurementColumnNames); allSensors.add(""); // for time column diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index f2bdddf890192..b20d3dd5286dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -49,7 +49,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -59,7 +58,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.WeakHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.function.LongConsumer; @@ -91,11 +89,18 @@ public class TimeSeriesMetadataCache { private final AtomicLong entryAverageSize = new AtomicLong(0); - private final Map> devices = - Collections.synchronizedMap(new WeakHashMap<>()); private static final String SEPARATOR = "$"; - /** A remainder this small is cheaper to load with per-column leaf-group lookups. */ + private static final int DEVICE_LOCK_STRIPES = 1024; + private static final Object[] DEVICE_LOCKS = new Object[DEVICE_LOCK_STRIPES]; + + static { + for (int i = 0; i < DEVICE_LOCK_STRIPES; i++) { + DEVICE_LOCKS[i] = new Object(); + } + } + + /** A cold-cache remainder this small is cheaper to load with per-column leaf-group lookups. */ private static final int MAX_PER_COLUMN_FALLBACK = 2; private final AtomicLong evictedExistingEntryCount = new AtomicLong(0); @@ -218,9 +223,7 @@ public TimeseriesMetadata get( } String deviceStringFormat = key.device.toString(); // allow for the parallelism of different devices - synchronized ( - devices.computeIfAbsent( - deviceStringFormat + SEPARATOR + filePath, WeakReference::new)) { + synchronized (deviceLock(deviceStringFormat, filePath)) { // double check timeseriesMetadata = lruCache.getIfPresent(key); if (timeseriesMetadata == null) { @@ -376,8 +379,7 @@ public Map getDeviceTimeSeriesMetadata( } // serialize reads of the same device+file (same lock key as get) so threads don't each // deserialize the whole device and race on the shared reader - synchronized ( - devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, WeakReference::new)) { + synchronized (deviceLock(device.toString(), filePath)) { List stillMissing = new ArrayList<>(missing.size()); collectCachedMetadata(tsFileID, device, missing, result, stillMissing); missing = stillMissing; @@ -410,12 +412,18 @@ public Map getDeviceTimeSeriesMetadata( // column means the file will be skipped anyway; a bloom-filter false positive on a field // must not force a device read and a possible absent-device error if (allSensors.contains("") && result.get("") == null && !possiblyPresent.contains("")) { + for (String measurement : possiblyPresent) { + result.put(measurement, null); + } return result; } - // for a small remainder, a per-column lookup reads one leaf group instead of the whole - // device; those lookups record their own statistics - if (possiblyPresent.size() <= MAX_PER_COLUMN_FALLBACK) { + // when the cache already holds most of the request, the remaining misses are likely + // evicted present columns; loading them through get() touches only their leaf groups + // (later misses in the same group hit the cache) instead of re-reading the whole device. + // Those lookups record their own statistics. + if (possiblyPresent.size() <= MAX_PER_COLUMN_FALLBACK + || (cacheHitCount > 0 && possiblyPresent.size() <= cacheHitCount)) { delegated = true; diskLookupCount = missing.size() - possiblyPresent.size(); for (String measurement : possiblyPresent) { @@ -450,6 +458,9 @@ public Map getDeviceTimeSeriesMetadata( new HashSet<>(possiblyPresent), true, ignoreNotExists)); + for (String measurement : possiblyPresent) { + result.putIfAbsent(measurement, null); + } return result; } } @@ -471,8 +482,13 @@ public Map getDeviceTimeSeriesMetadata( TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); - return readDeviceTimeSeriesMetadata( - reader, filePath, tsFileID, device, allSensors, false, ignoreNotExists); + result.putAll( + readDeviceTimeSeriesMetadata( + reader, filePath, tsFileID, device, allSensors, false, ignoreNotExists)); + for (String measurement : allSensors) { + result.putIfAbsent(measurement, null); + } + return result; } finally { long endTime = System.nanoTime(); long cacheLookupTime; @@ -630,6 +646,15 @@ private Map readDeviceTimeSeriesMetadata( return result; } + /** + * Striped, strongly referenced monitor for one device+file. A WeakHashMap entry can be reclaimed + * by GC between calls, which would let two threads hold "the same" lock simultaneously. + */ + private static Object deviceLock(String deviceString, String filePath) { + return DEVICE_LOCKS[ + Math.floorMod((deviceString + SEPARATOR + filePath).hashCode(), DEVICE_LOCK_STRIPES)]; + } + public double calculateTimeSeriesMetadataHitRatio() { return lruCache.stats().hitRate(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java index b62ba8fe7cdc3..2db7cafcbc6db 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java @@ -37,12 +37,17 @@ public void testShouldUseDeviceMetadataBatchScan() { // no field count is known (tree model, external tsfiles, or a table missing from the cache) assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 0, false, false)); - // a device whose series fit in a single measurement leaf group gains nothing from batching + // a table with maxDegree - 1 fields has at most maxDegree device series (time included), which + // still fit in a single measurement leaf group assertFalse( + FileLoaderUtils.shouldUseDeviceMetadataBatchScan( + maxDegreeOfIndexNode - 1, maxDegreeOfIndexNode - 1, false, false)); + // maxDegree fields span maxDegree + 1 device series (time included), i.e. two leaf groups + assertTrue( FileLoaderUtils.shouldUseDeviceMetadataBatchScan( maxDegreeOfIndexNode, maxDegreeOfIndexNode, false, false)); - // below, at and above the threshold + // below, at and above the ratio threshold on a wider table assertFalse( FileLoaderUtils.shouldUseDeviceMetadataBatchScan( threshold - 1, wideTableFieldCount, false, false)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 32efa93539d31..2ebdf4375b35c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -60,6 +60,7 @@ import java.util.function.LongConsumer; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; @@ -372,6 +373,19 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { new QueryContext(false, false)); assertNotNull(cached); assertEquals("s1", cached.getMeasurementId()); + + // the cache hands out copies: mutating a returned entry must not poison later reads + result.get("s1").setModified(true); + TimeseriesMetadata cachedAgain = + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s1"), + Collections.singleton("s1"), + true, + false, + new QueryContext(false, false)); + assertNotNull(cachedAgain); + assertFalse(cachedAgain.isModified()); } finally { cache.clear(); bloomFilterCache.clear(); @@ -443,11 +457,12 @@ public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Except cache.getDeviceTimeSeriesMetadata( filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false)); - assertTrue( - cache - .getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), true) - .isEmpty()); + Map ignored = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), true); + assertNull(ignored.get("")); + assertNull(ignored.get("s1")); + assertNull(ignored.get("s2")); } finally { cache.clear(); bloomFilterCache.clear(); @@ -532,6 +547,22 @@ public void testBatchMetadataMatchesPerColumn() throws Exception { measurement, expected.getChunkMetadataList().size(), actual.getChunkMetadataList().size()); + for (int i = 0; i < expected.getChunkMetadataList().size(); i++) { + ChunkMetadata expectedChunk = (ChunkMetadata) expected.getChunkMetadataList().get(i); + ChunkMetadata actualChunk = (ChunkMetadata) actual.getChunkMetadataList().get(i); + assertEquals( + measurement, + expectedChunk.getOffsetOfChunkHeader(), + actualChunk.getOffsetOfChunkHeader()); + assertEquals( + measurement, + expectedChunk.getStatistics().getStartTime(), + actualChunk.getStatistics().getStartTime()); + assertEquals( + measurement, + expectedChunk.getStatistics().getEndTime(), + actualChunk.getStatistics().getEndTime()); + } } } finally { cache.clear(); @@ -661,4 +692,113 @@ public void testSingleMissingMeasurementUsesPerColumnFallback() throws Exception fileReaderManager.closeFileAndRemoveReader(tsFileID); } } + + @Test + public void testBatchIgnoreNotExistsDoesNotCachePlaceholders() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-ignore-not-exists.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)) + .thenReturn(Collections.emptyList()); + + Map ignored = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), true); + assertNull(ignored.get("")); + assertNull(ignored.get("s1")); + + // ignoreNotExists must not write NULL placeholders: a normal query still has to read + assertThrows( + IOException.class, + () -> + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false)); + Mockito.verify(reader, Mockito.times(2)).getDeviceTimeseriesMetadata(deviceID); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testFewMissingMeasurementsUsePerColumnFallbackAfterEviction() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-few-missing.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2", "s3", "s4", "s5")); + + List deviceMetadata = new ArrayList<>(); + for (String measurement : allSensors) { + TSDataType dataType = measurement.isEmpty() ? TSDataType.INT64 : TSDataType.INT32; + Statistics statistics = Statistics.getStatsByType(dataType); + statistics.setCount(1); + statistics.setStartTime(0); + statistics.setEndTime(10); + deviceMetadata.add( + new TimeseriesMetadata( + (byte) 0, 0, measurement, dataType, statistics, new PublicBAOS())); + } + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)).thenReturn(deviceMetadata); + + Map first = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + assertNotNull(first.get("s5")); + + // evict three measurements: the cache still holds the majority, so the remaining misses + // must be loaded per column instead of re-reading the whole device + for (String measurement : Arrays.asList("s2", "s3", "s4")) { + cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, measurement)); + } + Mockito.when( + reader.readTimeseriesMetadata( + Mockito.any(), + Mockito.isNull(), + Mockito.anyString(), + Mockito.eq(allSensors), + Mockito.eq(false), + Mockito.any(LongConsumer.class))) + .thenReturn(deviceMetadata); + + Map second = + cache.getDeviceTimeSeriesMetadata( + filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + assertNotNull(second.get("s2")); + assertNotNull(second.get("s3")); + assertNotNull(second.get("s4")); + Mockito.verify(reader, Mockito.times(1)).getDeviceTimeseriesMetadata(deviceID); + Mockito.verify(reader, Mockito.times(1)) + .readTimeseriesMetadata( + Mockito.any(), + Mockito.isNull(), + Mockito.anyString(), + Mockito.eq(allSensors), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java index 017180dff82fb..fed93e643a639 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; @@ -76,12 +77,19 @@ public class AlignedSeriesScanBatchTest { @Test public void testBatchScanReadsWideAlignedDevice() throws Exception { + Assert.assertTrue( + "the fixture must span more than one measurement leaf group", + COLUMN_COUNT + 1 > TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode()); File file = newFile(1); TsFileResource resource = writeWideDevice(file, COLUMN_COUNT); TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); metadataCache.clear(); bloomFilterCache.clear(); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + // the batch path is only reachable from table dialect, which forces this flag to false + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); try { long cacheHitsBefore = EnvironmentUtils.TEST_QUERY_FI_CONTEXT @@ -125,6 +133,7 @@ public void testBatchScanReadsWideAlignedDevice() throws Exception { .getLoadTimeSeriesMetadataFromDiskCount() .get()); } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); cleanup(file, resource, metadataCache, bloomFilterCache); } } @@ -136,6 +145,9 @@ public void testSparseBatchScanMatchesPerColumn() throws Exception { TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); AlignedFullPath scanPath = alignedPath(columnNames(COLUMN_COUNT)); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); try { metadataCache.clear(); bloomFilterCache.clear(); @@ -193,23 +205,52 @@ public void testSparseBatchScanMatchesPerColumn() throws Exception { Assert.assertEquals(0, batchedCacheHits); Assert.assertTrue(perColumnCacheHits > 0); } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); cleanup(file, resource, metadataCache, bloomFilterCache); } } @Test - public void testBatchScanSkipsAllNullRows() throws Exception { + public void testAllNullProjectionMatchesPerColumn() throws Exception { File file = newFile(3); TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); - // every selected field is absent from the file: with ignoreAllNullRows the file must be - // skipped by both paths + // every selected field is absent from the file AlignedFullPath absentPath = alignedPath(absentColumnNames(COLUMN_COUNT)); boolean previousIgnoreAllNullRows = EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); - EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(true); try { + // table dialect (the only dialect that reaches the batch path): all-null rows are kept + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); + metadataCache.clear(); + bloomFilterCache.clear(); + TsBlock batched = scan(absentPath, COLUMN_COUNT, resource); + Assert.assertEquals(ROW_COUNT, batched.getPositionCount()); + Assert.assertEquals(COLUMN_COUNT, batched.getValueColumnCount()); + for (int column = 0; column < COLUMN_COUNT; column++) { + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertTrue( + "absent" + column + " row " + row, batched.getColumn(column).isNull(row)); + } + } + + metadataCache.clear(); + TsBlock perColumn = scan(absentPath, 0, resource); + Assert.assertEquals(perColumn.getPositionCount(), batched.getPositionCount()); + Assert.assertEquals(perColumn.getValueColumnCount(), batched.getValueColumnCount()); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals(perColumn.getTimeByIndex(row), batched.getTimeByIndex(row)); + } + for (int column = 0; column < COLUMN_COUNT; column++) { + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals( + perColumn.getColumn(column).isNull(row), batched.getColumn(column).isNull(row)); + } + } + + // tree dialect: an all-null projection skips the file in both paths + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(true); metadataCache.clear(); bloomFilterCache.clear(); Assert.assertFalse(hasFile(absentPath, COLUMN_COUNT, resource)); From 469394fe32fa19fdf868007d702ab7f98d230770 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 18:09:04 +0800 Subject: [PATCH 18/27] Account the pre-fallback phase as disk time on a cold batch miss When a cold batch call delegates the remaining measurements to the per-column get(), the cache probes and lock wait were assigned to cache time while the cache hit count was zero, so neither branch reported them. Attribute the phase to disk time when there are no cache hits and allow a disk time with a zero count to be recorded. --- .../buffer/TimeSeriesMetadataCache.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index b20d3dd5286dd..a8df912d871b9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -494,9 +494,16 @@ public Map getDeviceTimeSeriesMetadata( long cacheLookupTime; long diskLookupTime; if (delegated) { - // the per-column fallbacks recorded their own disk time - cacheLookupTime = cachePhaseEndTime == 0 ? 0 : cachePhaseEndTime - startTime; - diskLookupTime = 0; + // the per-column fallbacks recorded their own time and counts; the pre-fallback phase + // (cache probes, lock wait) is cache time when it produced hits and disk time on a cold + // call, where no cache hit would otherwise report it + if (cacheHitCount > 0) { + cacheLookupTime = cachePhaseEndTime == 0 ? 0 : cachePhaseEndTime - startTime; + diskLookupTime = 0; + } else { + cacheLookupTime = 0; + diskLookupTime = cachePhaseEndTime == 0 ? 0 : cachePhaseEndTime - startTime; + } } else if (cacheHitCount == 0 || cachePhaseEndTime == 0) { // cold or cache-disabled call: cache probes and the lock wait belong to the disk lookup cacheLookupTime = 0; @@ -555,7 +562,7 @@ private static void recordLoadStatistics( statistics.getLoadTimeSeriesMetadataFromCacheCount().getAndAdd(cacheHitCount); statistics.getLoadTimeSeriesMetadataFromCacheTime().getAndAdd(cacheLookupTime); } - if (diskLookupCount > 0) { + if (diskLookupCount > 0 || diskLookupTime > 0) { statistics.getLoadTimeSeriesMetadataFromDiskCount().getAndAdd(diskLookupCount); statistics.getLoadTimeSeriesMetadataFromDiskTime().getAndAdd(diskLookupTime); } From 7f053207c834ca8e0a9db504da2350328f0068a6 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 12:21:04 +0800 Subject: [PATCH 19/27] Refactor wide-device metadata loading into a set-based cache API Replace the device-wide batch read and its 0.6 projection gate with one getTimeseriesMetadata(Set) entry point: resolve cached columns first, then read each measurement leaf group that holds a selected column once, bloom-probing only the group probe and answering every requested column of the group from that read. get() becomes a single-column wrapper over it. This removes the static ratio, the parallel batch branch, the totalFieldColumnCount planner plumbing and the now-unused i18n exception constant. The cache and wide-device scan tests are rewritten for the new semantics. --- .../iotdb/db/i18n/StorageEngineMessages.java | 3 - .../iotdb/db/i18n/StorageEngineMessages.java | 3 - .../source/AlignedSeriesScanUtil.java | 3 +- .../operator/source/FileLoaderUtils.java | 126 +--- .../ExternalTsFileSeriesScanUtil.java | 3 +- .../DataNodeTableOperatorGenerator.java | 18 - .../plan/parameter/SeriesScanOptions.java | 9 - .../buffer/TimeSeriesMetadataCache.java | 627 ++++++++---------- .../operator/source/FileLoaderUtilsTest.java | 76 --- .../buffer/TimeSeriesMetadataCacheTest.java | 225 ++++--- .../series/AlignedSeriesScanBatchTest.java | 143 ++-- 11 files changed, 468 insertions(+), 768 deletions(-) delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java index 4e3de1433bd1f..91802ff30b0d1 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -1712,9 +1712,6 @@ private StorageEngineMessages() {} public static final String COMPACTION_CROSS_SPACE = "cross"; public static final String DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT = "%s does not exist in the resource file"; - public static final String - EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374 = - "Device %s is recorded in the resource but missing from the metadata of TsFile %s"; public static final String TARGET_FILE_SMALLER_THAN_MAGIC_STRING_AND_VERSION_NUMBER_SIZE_FMT = "target file %s is smaller than magic string and version number size"; public static final String CURRENT_POINT_TIMESTAMP_SHOULD_BE_LATER_FMT = diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java index b57ab3b824437..a2429b5438cd3 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -1622,9 +1622,6 @@ private StorageEngineMessages() {} public static final String COMPACTION_CROSS_SPACE = "跨空间"; public static final String DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT = "%s 在资源文件中不存在"; - public static final String - EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374 = - "设备 %s 记录在资源文件中,但在 TsFile %s 的元数据中不存在"; public static final String TARGET_FILE_SMALLER_THAN_MAGIC_STRING_AND_VERSION_NUMBER_SIZE_FMT = "目标文件 %s 小于 magic string 和版本号大小"; public static final String CURRENT_POINT_TIMESTAMP_SHOULD_BE_LATER_FMT = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java index 517a5e0718bfc..331a5a5b267d1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java @@ -101,8 +101,7 @@ protected AbstractAlignedTimeSeriesMetadata loadTimeSeriesMetadata( scanOptions.getGlobalTimeFilter(), isSeq, ignoreAllNullRows, - null, - scanOptions.getTotalFieldColumnCount()); + null); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index aa6df29f1ba67..d5c592cdde505 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -30,7 +30,6 @@ import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; -import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskAlignedChunkLoader; import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskChunkLoader; @@ -43,12 +42,10 @@ import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.iotdb.db.utils.SchemaUtils; -import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; -import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; @@ -74,12 +71,6 @@ public class FileLoaderUtils { private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET = SeriesScanCostMetricSet.getInstance(); - /** - * Fraction of a table's field columns that a scan must select for it to use the batch - * device-metadata read instead of the per-column path. - */ - private static final double DEVICE_METADATA_BATCH_SCAN_RATIO = 0.6; - private FileLoaderUtils() { // util class } @@ -230,8 +221,7 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata( Filter globalTimeFilter, boolean isSeq, boolean ignoreAllNullRows, - long[] rootMeasurementMetadataIndexNodeOffset, - int totalFieldColumnCount) + long[] rootMeasurementMetadataIndexNodeOffset) throws IOException { final long t1 = System.nanoTime(); boolean loadFromMem = false; @@ -250,8 +240,7 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata( context, globalTimeFilter, ignoreAllNullRows, - rootMeasurementMetadataIndexNodeOffset, - totalFieldColumnCount); + rootMeasurementMetadataIndexNodeOffset); } else { // if the tsfile is unclosed, we just get it directly from TsFileResource loadFromMem = true; alignedTimeSeriesMetadata = @@ -343,8 +332,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr FragmentInstanceContext context, Filter globalTimeFilter, boolean ignoreAllNullRows, - long[] rootMeasurementMetadataIndexNodeOffset, - int totalFieldColumnCount) + long[] rootMeasurementMetadataIndexNodeOffset) throws IOException { AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null; // load all the TimeseriesMetadata of vector, the first one is for time column and the @@ -352,10 +340,10 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr // the order of timeSeriesMetadata list is same as subSensorList's order TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); List valueMeasurementList = alignedPath.getMeasurementList(); + // resolves the time column and every selected sensor in one call: the cache reads each leaf + // group that holds one of them once and answers the whole group from that read, so a wide + // projection does not walk the device metadata index once per column Set allSensors = alignedPath.getAllSensors(); - boolean isDebug = context.isDebug(); - String filePath = resource.getTsFilePath(); - IDeviceID deviceId = alignedPath.getDeviceId(); // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex // we should not ignore the non-exist of device in TsFileMetadata @@ -363,39 +351,19 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr context.ignoreNotExistsDevice() || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE; - // For a wide device (a table scan that selects almost all field columns), deserialize the - // whole device's TimeSeriesMetadata in a single batch read instead of walking the device - // metadata index once per column. External tsfiles address metadata by offset, so they must - // stay on the per-column path; DEBUG queries also stay there so the per-column cache's - // diagnostic logs are preserved. - if (shouldUseDeviceMetadataBatchScan( - valueMeasurementList.size(), - totalFieldColumnCount, - isDebug, - rootMeasurementMetadataIndexNodeOffset != null)) { - return loadAlignedTimeSeriesMetadataBatch( - resource, - alignedPath, - valueMeasurementList, - allSensors, - context, - globalTimeFilter, - ignoreAllNullRows, - ignoreNotExists, - cache, - filePath, - deviceId); - } - - TimeseriesMetadata timeColumn = - cache.get( - filePath, - new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, ""), + Map metadata = + cache.getTimeseriesMetadata( + resource.getTsFilePath(), + resource.getTsFileID(), + alignedPath.getDeviceId(), + allSensors, allSensors, ignoreNotExists, - isDebug, + context.isDebug(), context, rootMeasurementMetadataIndexNodeOffset); + + TimeseriesMetadata timeColumn = metadata.get(""); if (timeColumn != null) { // only need time column, like count_time aggregation if (valueMeasurementList.isEmpty()) { @@ -414,15 +382,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr List valueTimeSeriesMetadataList = new ArrayList<>(valueMeasurementList.size()); for (String measurement : valueMeasurementList) { - valueTimeSeriesMetadataList.add( - cache.get( - filePath, - new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement), - allSensors, - ignoreNotExists, - isDebug, - context, - rootMeasurementMetadataIndexNodeOffset)); + valueTimeSeriesMetadataList.add(metadata.get(measurement)); } alignedTimeSeriesMetadata = buildAlignedTimeSeriesMetadata( @@ -438,60 +398,6 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr return alignedTimeSeriesMetadata; } - static boolean shouldUseDeviceMetadataBatchScan( - int selectedColumnCount, int totalFieldColumnCount, boolean debug, boolean offsetAddressed) { - // DEBUG queries keep the per-column cache's diagnostic logs; offset-addressed external - // tsfiles cannot be resolved through the table metadata index; and a table with at least - // max_degree_of_index_node field columns has a dense device of max_degree+1 series (time column - // included), which spans more than one measurement leaf group. A sparse device may hold fewer - // in-file series; the bloom filter and the cache-aware load keep that case cheap. - return !debug - && !offsetAddressed - && totalFieldColumnCount - >= TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode() - && (double) selectedColumnCount / totalFieldColumnCount >= DEVICE_METADATA_BATCH_SCAN_RATIO; - } - - private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataBatch( - TsFileResource resource, - AlignedFullPath alignedPath, - List valueMeasurementList, - Set allSensors, - FragmentInstanceContext context, - Filter globalTimeFilter, - boolean ignoreAllNullRows, - boolean ignoreNotExists, - TimeSeriesMetadataCache cache, - String filePath, - IDeviceID deviceId) - throws IOException { - Map deviceMetadata = - cache.getDeviceTimeSeriesMetadata( - filePath, resource.getTsFileID(), deviceId, allSensors, context, ignoreNotExists); - if (deviceMetadata.isEmpty()) { - // the cache applies the same bloom-filter / ignoreNotExists rules as the per-column get(), - // so an empty map means the device is expected to be absent from this file - return null; - } - TimeseriesMetadata timeColumn = deviceMetadata.get(""); - if (timeColumn == null) { - return null; - } - List valueTimeSeriesMetadataList = - new ArrayList<>(valueMeasurementList.size()); - for (String measurement : valueMeasurementList) { - valueTimeSeriesMetadataList.add(deviceMetadata.get(measurement)); - } - return buildAlignedTimeSeriesMetadata( - resource, - alignedPath, - timeColumn, - valueTimeSeriesMetadataList, - context, - globalTimeFilter, - ignoreAllNullRows); - } - private static AbstractAlignedTimeSeriesMetadata buildAlignedTimeSeriesMetadata( TsFileResource resource, AlignedFullPath alignedPath, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java index 61a2832e66b40..e907218fe8a7b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/ExternalTsFileSeriesScanUtil.java @@ -83,8 +83,7 @@ static AbstractAlignedTimeSeriesMetadata loadTimeSeriesMetadata( globalTimeFilter, resource.isSeq(), context.isIgnoreAllNullRows(), - new long[] {currentDeviceOffset.getStartOffset(), currentDeviceOffset.getEndOffset()}, - 0); + new long[] {currentDeviceOffset.getStartOffset(), currentDeviceOffset.getEndOffset()}); } @FunctionalInterface diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index 3d21f99208b68..44ca68490690f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -1185,9 +1185,6 @@ private void addSource( topKRuntimeFilter); seriesScanOptions.setTTLForTableView(viewTTL); seriesScanOptions.setIsTableViewForTreeModel(node instanceof TreeDeviceViewScanNode); - if (!(node instanceof TreeDeviceViewScanNode) && !(node instanceof ExternalTsFileScanNode)) { - seriesScanOptions.setTotalFieldColumnCount(computeTotalFieldColumnCount(node)); - } OperatorContext operatorContext = addOperatorContext(context, node.getPlanNodeId(), className); @@ -1225,17 +1222,6 @@ private BatchDeviceEntrySource createDeviceEntrySource(DeviceTableScanNode node) .orElseGet(() -> new InMemoryDeviceEntrySource(node.getDeviceEntries())); } - private int computeTotalFieldColumnCount(DeviceTableScanNode node) { - QualifiedObjectName qualifiedObjectName = node.getQualifiedObjectName(); - TsTable tsTable = - DataNodeTableCache.getInstance() - .getTableInCache( - qualifiedObjectName.getDatabaseName(), qualifiedObjectName.getObjectName()); - // the numerator only contains FIELD columns, so the denominator must be the field count too; - // getColumnNum() would also count tag, attribute and time columns - return tsTable == null ? 0 : tsTable.getFieldNum(); - } - private BatchDeviceEntrySource createAggregationDeviceEntrySource(AggregationTableScanNode node) { if (node.getDeviceEntryCount() == 0 && node.getGroupingKeys().isEmpty()) { return new InMemoryDeviceEntrySource(Collections.singletonList(null)); @@ -1787,10 +1773,6 @@ private boolean ensureFieldColumnForTreeNonAlignedDeviceViewScan( node.getPushDownPredicate()); seriesScanOptions.setTTLForTableView(tableViewTTL); seriesScanOptions.setIsTableViewForTreeModel(node instanceof AggregationTreeDeviceViewScanNode); - if (!(node instanceof AggregationTreeDeviceViewScanNode) - && !(node instanceof ExternalTsFileAggregationScanNode)) { - seriesScanOptions.setTotalFieldColumnCount(computeTotalFieldColumnCount(node)); - } Set allSensors = new HashSet<>(measurementColumnNames); allSensors.add(""); // for time column diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java index 2a173e3b09e0f..4d40bbd32c4d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java @@ -56,7 +56,6 @@ public class SeriesScanOptions implements Accountable { private PaginationController paginationController; private boolean isTableViewForTreeModel; private long ttlForTableView = Long.MAX_VALUE; - private int totalFieldColumnCount = 0; private final TopKRuntimeFilter topKRuntimeFilter; private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance( @@ -179,14 +178,6 @@ public void setIsTableViewForTreeModel(boolean isTableViewForTreeModel) { this.isTableViewForTreeModel = isTableViewForTreeModel; } - public int getTotalFieldColumnCount() { - return totalFieldColumnCount; - } - - public void setTotalFieldColumnCount(int totalFieldColumnCount) { - this.totalFieldColumnCount = totalFieldColumnCount; - } - /** * pushLimitToEachDevice==false means that all devices return total limit rows. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index a8df912d871b9..15a1aa7a427ca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -36,7 +36,6 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Weigher; -import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TimeseriesMetadata; @@ -49,11 +48,10 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -100,9 +98,6 @@ public class TimeSeriesMetadataCache { } } - /** A cold-cache remainder this small is cheaper to load with per-column leaf-group lookups. */ - private static final int MAX_PER_COLUMN_FALLBACK = 2; - private final AtomicLong evictedExistingEntryCount = new AtomicLong(0); private final AtomicLong evictedNonExistingEntryCount = new AtomicLong(0); @@ -150,7 +145,6 @@ public static TimeSeriesMetadataCache getInstance() { return TimeSeriesMetadataCache.TimeSeriesMetadataCacheHolder.INSTANCE; } - @SuppressWarnings({"squid:S1860", "squid:S6541", "squid:S3776"}) // Suppress synchronize warning public TimeseriesMetadata get( String filePath, TimeSeriesMetadataCacheKey key, @@ -162,7 +156,6 @@ public TimeseriesMetadata get( return get(filePath, key, allSensors, ignoreNotExists, debug, queryContext, null); } - @SuppressWarnings({"squid:S1860", "squid:S6541", "squid:S3776"}) // Suppress synchronize warning public TimeseriesMetadata get( String filePath, TimeSeriesMetadataCacheKey key, @@ -172,340 +165,163 @@ public TimeseriesMetadata get( QueryContext queryContext, long[] deviceMetadataIndexNodeOffset) throws IOException { - long startTime = System.nanoTime(); - long loadBloomFilterTime = 0; - LongConsumer timeSeriesMetadataIoSizeRecorder = - queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; - LongConsumer bloomFilterIoSizeRecorder = - queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet; - boolean cacheHit = true; - boolean externalTsFile = queryContext.isExternalTsFileScan(); - try { - if (!CACHE_ENABLE || externalTsFile) { - String deviceStringFormat = key.device.toString(); - cacheHit = false; - - // bloom filter part - TsFileSequenceReader reader = - FileReaderManager.getInstance() - .get(filePath, key.tsFileID, true, bloomFilterIoSizeRecorder, externalTsFile); - if (deviceMetadataIndexNodeOffset == null) { - BloomFilter bloomFilter = reader.readBloomFilter(bloomFilterIoSizeRecorder); - queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount().incrementAndGet(); - if (bloomFilter != null - && !bloomFilter.contains( - deviceStringFormat + IoTDBConstant.PATH_SEPARATOR + key.measurement)) { - loadBloomFilterTime = System.nanoTime() - startTime; - return null; - } - loadBloomFilterTime = System.nanoTime() - startTime; - } - - TimeseriesMetadata timeseriesMetadata = - reader.readTimeseriesMetadata( - key.device, - deviceMetadataIndexNodeOffset, - key.measurement, - ignoreNotExists, - timeSeriesMetadataIoSizeRecorder); - return (timeseriesMetadata == null || timeseriesMetadata.getStatistics().getCount() == 0) - ? null - : timeseriesMetadata; - } - - TimeseriesMetadata timeseriesMetadata = lruCache.getIfPresent(key); - - if (timeseriesMetadata == null) { - if (debug) { - DEBUG_LOGGER.info( - StorageEngineMessages.CACHE_MISS_IN_FILE, key.device, key.measurement, filePath); - DEBUG_LOGGER.info(StorageEngineMessages.DEVICE_ALL_SENSORS, key.device, allSensors); - } - String deviceStringFormat = key.device.toString(); - // allow for the parallelism of different devices - synchronized (deviceLock(deviceStringFormat, filePath)) { - // double check - timeseriesMetadata = lruCache.getIfPresent(key); - if (timeseriesMetadata == null) { - cacheHit = false; - - if (deviceMetadataIndexNodeOffset == null) { - long loadBloomFilterStartTime = System.nanoTime(); - // bloom filter part - BloomFilter bloomFilter = - BloomFilterCache.getInstance() - .get( - new BloomFilterCache.BloomFilterCacheKey(filePath, key.tsFileID), - debug, - bloomFilterIoSizeRecorder, - queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount() - ::addAndGet, - queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount() - ::addAndGet, - false); - if (bloomFilter != null - && !bloomFilter.contains( - deviceStringFormat + TsFileConstant.PATH_SEPARATOR + key.measurement)) { - if (debug) { - DEBUG_LOGGER.info( - StorageEngineMessages.TS_METADATA_FILTERED_BY_BLOOM_FILTER, key); - } - loadBloomFilterTime = System.nanoTime() - loadBloomFilterStartTime; - return null; - } - - loadBloomFilterTime = System.nanoTime() - loadBloomFilterStartTime; - } - TsFileSequenceReader reader = - FileReaderManager.getInstance() - .get(filePath, key.tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); - List timeSeriesMetadataList = - reader.readTimeseriesMetadata( - key.device, - deviceMetadataIndexNodeOffset, - key.measurement, - allSensors, - ignoreNotExists, - timeSeriesMetadataIoSizeRecorder); - // put TimeSeriesMetadata of all sensors used in this read into cache - for (TimeseriesMetadata metadata : timeSeriesMetadataList) { - TimeSeriesMetadataCacheKey k = - new TimeSeriesMetadataCacheKey( - key.tsFileID, key.device, metadata.getMeasurementId()); - if (metadata.getStatistics().getCount() != 0) { - lruCache.put(k, metadata); - } - if (metadata.getMeasurementId().equals(key.measurement)) { - timeseriesMetadata = metadata.getStatistics().getCount() == 0 ? null : metadata; - } - } - if (timeseriesMetadata == null - && !ignoreNotExists - && memoryConfig.isMayCacheNonExistSeries()) { - lruCache.put(key, NULL_EXISTS_CACHE_PLACE_HOLDER); - } - } - } - } - if (timeseriesMetadata == null || timeseriesMetadata == NULL_EXISTS_CACHE_PLACE_HOLDER) { - if (debug) { - DEBUG_LOGGER.info(StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, key); - } - return null; - } else { - if (debug) { - DEBUG_LOGGER.info( - StorageEngineMessages.STORAGE_LOG_GET_TIMESERIES_METADATA_IN_FILE_FROM_CACHE_36652729, - key.device, - key.measurement, - filePath, - timeseriesMetadata); - } - return new TimeseriesMetadata(timeseriesMetadata); - } - } finally { - long lookupTime = System.nanoTime() - startTime - loadBloomFilterTime; - recordLoadStatistics( - queryContext, - cacheHit ? 1 : 0, - cacheHit ? lookupTime : 0, - cacheHit ? 0 : 1, - cacheHit ? 0 : lookupTime, - loadBloomFilterTime); - } + return getTimeseriesMetadata( + filePath, + key.tsFileID, + key.device, + Collections.singleton(key.measurement), + allSensors, + ignoreNotExists, + debug, + queryContext, + deviceMetadataIndexNodeOffset) + .get(key.measurement); } /** - * Load the TimeseriesMetadata of {@code allSensors} of a device, reusing the per-measurement - * cache and reading the device in a single batch only for the measurements that are still - * missing. This is used by wide-table scans (a device with many columns) to avoid walking the - * device metadata index once per column / leaf group. + * Resolve the metadata of {@code requested} measurements of a device in one file. * - *

Measurements already cached are returned without touching the file. The remaining ones are - * first checked against the file's cached bloom filter, so columns absent from this file do not - * force a device read; only measurements that cannot be ruled out are loaded by the batch read. + *

Cached measurements are returned directly. The remaining ones are checked against the file's + * bloom filter and then loaded leaf group by leaf group: one pending measurement locates the + * group it belongs to, that whole group is read and deserialized once, and every requested + * measurement found in it is resolved. Each group is therefore read at most once per call, + * independent of LRU cache pressure, and groups that contain no requested measurement are never + * read. * - *

The returned map always holds copies of the cached (or freshly read) metadata, so the caller - * may apply {@code setModified} and {@code setChunkMetadataLoader} without corrupting cached - * entries — matching {@link #get}. + *

{@code allSensors} is the full selected set; it filters each group read and warms the cache, + * so a later single-measurement {@link #get} can reuse the group. {@code requested} must be a + * subset of {@code allSensors}. * - *

Load statistics are counted per series (not per device read) so that they stay comparable - * with {@link #get}; the bytes read by the batch read cannot be reported because {@link - * TsFileSequenceReader#getDeviceTimeseriesMetadata(IDeviceID)} has no IO-size recorder. + *

The returned map holds copies of the cached (or freshly read) metadata, so the caller may + * apply {@code setModified} and {@code setChunkMetadataLoader} without corrupting cached entries. + * Load statistics are counted per requested measurement. * - * @param filePath path of the tsfile - * @param tsFileID id of the tsfile - * @param device target device - * @param allSensors measurements the caller needs, including the time column keyed by "" - * @param queryContext query context used for metrics and to detect external tsfiles - * @param ignoreNotExists whether a device missing from a file that claims a device-level time - * index should be skipped instead of reported as an inconsistency - * @return map from measurementId (including the time column, keyed by "") to TimeseriesMetadata; - * entries whose series is empty (count == 0) are mapped to null - * @throws IOException if the metadata cannot be read from disk or the device is unexpectedly - * absent + * @param deviceMetadataIndexNodeOffset non-null for external tsfiles, which address the device + * metadata index by offset; bloom filtering is skipped in that case + * @return a map with an entry for every measurement in {@code requested}: a metadata copy when + * the series exists and is non-empty, null otherwise */ - public Map getDeviceTimeSeriesMetadata( + public Map getTimeseriesMetadata( String filePath, TsFileID tsFileID, IDeviceID device, + Set requested, Set allSensors, + boolean ignoreNotExists, + boolean debug, QueryContext queryContext, - boolean ignoreNotExists) + long[] deviceMetadataIndexNodeOffset) throws IOException { long startTime = System.nanoTime(); long cachePhaseEndTime = 0; long loadBloomFilterTime = 0; long cacheHitCount = 0; long diskLookupCount = 0; - boolean delegated = false; boolean externalTsFile = queryContext.isExternalTsFileScan(); LongConsumer timeSeriesMetadataIoSizeRecorder = queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; LongConsumer bloomFilterIoSizeRecorder = queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet; + String devicePrefix = device.toString() + IoTDBConstant.PATH_SEPARATOR; - Map result = new HashMap<>(allSensors.size()); - List missing = new ArrayList<>(allSensors.size()); + Map result = new HashMap<>(requested.size()); + // ordered set: the first element is the probe of the next leaf group, and resolving a whole + // group removes up to max_degree_of_index_node members in O(1) each + Set pending = new LinkedHashSet<>(requested.size()); try { if (CACHE_ENABLE && !externalTsFile) { - // fast path without locking: reuse everything the per-measurement cache already has - collectCachedMetadata(tsFileID, device, allSensors, result, missing); + collectCachedMetadata(tsFileID, device, requested, result, pending, filePath, debug); cacheHitCount = result.size(); - if (missing.isEmpty()) { + if (pending.isEmpty()) { cachePhaseEndTime = System.nanoTime(); return result; } // serialize reads of the same device+file (same lock key as get) so threads don't each - // deserialize the whole device and race on the shared reader + // walk the device metadata index and race on the shared reader synchronized (deviceLock(device.toString(), filePath)) { - List stillMissing = new ArrayList<>(missing.size()); - collectCachedMetadata(tsFileID, device, missing, result, stillMissing); - missing = stillMissing; - cacheHitCount = result.size(); cachePhaseEndTime = System.nanoTime(); - if (missing.isEmpty()) { - return result; - } - - // measurements the cached bloom filter already rejects are known absent in this file, so - // they must not trigger a device read - long bloomFilterStartTime = System.nanoTime(); - BloomFilter bloomFilter = - loadBloomFilter(filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, false); - List possiblyPresent = new ArrayList<>(missing.size()); - for (String measurement : missing) { - if (bloomFilterMayContain(bloomFilter, device, measurement)) { - possiblyPresent.add(measurement); - } else { - result.put(measurement, null); + if (debug) { + for (String measurement : pending) { + DEBUG_LOGGER.info( + StorageEngineMessages.CACHE_MISS_IN_FILE, device, measurement, filePath); } + DEBUG_LOGGER.info(StorageEngineMessages.DEVICE_ALL_SENSORS, device, allSensors); } - loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; - diskLookupCount = missing.size(); - if (possiblyPresent.isEmpty()) { - return result; + BloomFilter bloomFilter = null; + if (deviceMetadataIndexNodeOffset == null) { + long bloomFilterStartTime = System.nanoTime(); + bloomFilter = + loadBloomFilter(filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, false); + loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; } - - // the time column is always written for an aligned device, so a known-absent time - // column means the file will be skipped anyway; a bloom-filter false positive on a field - // must not force a device read and a possible absent-device error - if (allSensors.contains("") && result.get("") == null && !possiblyPresent.contains("")) { - for (String measurement : possiblyPresent) { - result.put(measurement, null); - } - return result; - } - - // when the cache already holds most of the request, the remaining misses are likely - // evicted present columns; loading them through get() touches only their leaf groups - // (later misses in the same group hit the cache) instead of re-reading the whole device. - // Those lookups record their own statistics. - if (possiblyPresent.size() <= MAX_PER_COLUMN_FALLBACK - || (cacheHitCount > 0 && possiblyPresent.size() <= cacheHitCount)) { - delegated = true; - diskLookupCount = missing.size() - possiblyPresent.size(); - for (String measurement : possiblyPresent) { - result.put( - measurement, - get( - filePath, - new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), - allSensors, - ignoreNotExists, - false, - queryContext)); - } - return result; - } - - // TsFileSequenceReader has no selective batch-read API that accepts a measurement set, - // so this call loads every measurement of the device and filters afterwards. A truly - // selective read requires an upstream apache/tsfile change; the bloom filter and the - // per-column fallback above keep this branch to measurements that are plausibly present. - // The measurement-level readTimeseriesMetadata API is group-selective already and is what - // the per-column path and the fallback use. - TsFileSequenceReader reader = - FileReaderManager.getInstance() - .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); - result.putAll( - readDeviceTimeSeriesMetadata( - reader, - filePath, - tsFileID, - device, - new HashSet<>(possiblyPresent), - true, - ignoreNotExists)); - for (String measurement : possiblyPresent) { - result.putIfAbsent(measurement, null); + if (!pending.isEmpty()) { + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); + LeafLoadStatistics loaded = + loadLeafGroups( + reader, + tsFileID, + device, + deviceMetadataIndexNodeOffset, + filePath, + bloomFilter, + devicePrefix, + pending, + requested, + allSensors, + ignoreNotExists, + true, + debug, + result, + timeSeriesMetadataIoSizeRecorder); + diskLookupCount = loaded.diskLookups; + cacheHitCount += loaded.cacheHits; } return result; } } - // cache disabled / external tsfiles: probe the time column first so an absent device does - // not pay for the whole-device read - long bloomFilterStartTime = System.nanoTime(); - BloomFilter bloomFilter = - loadBloomFilter( - filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, externalTsFile); - loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; - diskLookupCount = allSensors.size(); - if (!bloomFilterMayContain(bloomFilter, device, "")) { - for (String measurement : allSensors) { - result.put(measurement, null); - } - return result; + // cache disabled or external tsfiles + BloomFilter bloomFilter = null; + if (deviceMetadataIndexNodeOffset == null) { + long bloomFilterStartTime = System.nanoTime(); + bloomFilter = + loadBloomFilter( + filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, externalTsFile); + loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; } - TsFileSequenceReader reader = - FileReaderManager.getInstance() - .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); - result.putAll( - readDeviceTimeSeriesMetadata( - reader, filePath, tsFileID, device, allSensors, false, ignoreNotExists)); - for (String measurement : allSensors) { - result.putIfAbsent(measurement, null); + if (!pending.isEmpty()) { + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); + LeafLoadStatistics loaded = + loadLeafGroups( + reader, + tsFileID, + device, + deviceMetadataIndexNodeOffset, + filePath, + bloomFilter, + devicePrefix, + pending, + requested, + allSensors, + ignoreNotExists, + false, + debug, + result, + timeSeriesMetadataIoSizeRecorder); + diskLookupCount = loaded.diskLookups; + cacheHitCount += loaded.cacheHits; } return result; } finally { long endTime = System.nanoTime(); long cacheLookupTime; long diskLookupTime; - if (delegated) { - // the per-column fallbacks recorded their own time and counts; the pre-fallback phase - // (cache probes, lock wait) is cache time when it produced hits and disk time on a cold - // call, where no cache hit would otherwise report it - if (cacheHitCount > 0) { - cacheLookupTime = cachePhaseEndTime == 0 ? 0 : cachePhaseEndTime - startTime; - diskLookupTime = 0; - } else { - cacheLookupTime = 0; - diskLookupTime = cachePhaseEndTime == 0 ? 0 : cachePhaseEndTime - startTime; - } - } else if (cacheHitCount == 0 || cachePhaseEndTime == 0) { - // cold or cache-disabled call: cache probes and the lock wait belong to the disk lookup + if (cacheHitCount == 0 || cachePhaseEndTime == 0) { cacheLookupTime = 0; diskLookupTime = endTime - startTime - loadBloomFilterTime; } else if (diskLookupCount == 0) { @@ -525,6 +341,143 @@ public Map getDeviceTimeSeriesMetadata( } } + /** + * Read leaf groups for the pending measurements, one group per iteration. The group returned by + * {@code readTimeseriesMetadata} contains every {@code allSensors} entry of the group, so all + * pending measurements it covers are resolved without relying on the LRU retaining the group. + */ + private LeafLoadStatistics loadLeafGroups( + TsFileSequenceReader reader, + TsFileID tsFileID, + IDeviceID device, + long[] deviceMetadataIndexNodeOffset, + String filePath, + BloomFilter bloomFilter, + String devicePrefix, + Set pending, + Set requested, + Set allSensors, + boolean ignoreNotExists, + boolean cacheEnabled, + boolean debug, + Map result, + LongConsumer ioSizeRecorder) + throws IOException { + LeafLoadStatistics statistics = new LeafLoadStatistics(); + while (!pending.isEmpty()) { + String probe = pending.iterator().next(); + // another thread may have loaded this probe's group while we waited for the lock; checking + // only the probe keeps the cold path free of a per-column cache pass + TimeseriesMetadata cachedProbe = + cacheEnabled + ? lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID, device, probe)) + : null; + if (cachedProbe != null) { + pending.remove(probe); + statistics.cacheHits++; + if (cachedProbe == NULL_EXISTS_CACHE_PLACE_HOLDER) { + if (debug) { + DEBUG_LOGGER.info( + StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, + new TimeSeriesMetadataCacheKey(tsFileID, device, probe)); + } + result.put(probe, null); + } else { + if (debug) { + DEBUG_LOGGER.info( + StorageEngineMessages + .STORAGE_LOG_GET_TIMESERIES_METADATA_IN_FILE_FROM_CACHE_36652729, + device, + probe, + filePath, + cachedProbe); + } + result.put(probe, new TimeseriesMetadata(cachedProbe)); + } + continue; + } + // bloom-probe only the probe: the group read resolves the other selected measurements of + // that group, so probing each of them up front would add a per-column cost to every call + if (bloomFilter != null && !bloomFilterMayContain(bloomFilter, devicePrefix, probe)) { + pending.remove(probe); + if (requested.contains(probe)) { + if (debug) { + DEBUG_LOGGER.info( + StorageEngineMessages.TS_METADATA_FILTERED_BY_BLOOM_FILTER, + new TimeSeriesMetadataCacheKey(tsFileID, device, probe)); + } + result.put(probe, null); + } + continue; + } + statistics.diskLookups++; + List group = + reader.readTimeseriesMetadata( + device, + deviceMetadataIndexNodeOffset, + probe, + allSensors, + ignoreNotExists, + ioSizeRecorder); + boolean probeFound = false; + for (TimeseriesMetadata metadata : group) { + String measurementId = metadata.getMeasurementId(); + pending.remove(measurementId); + probeFound = probeFound || measurementId.equals(probe); + if (metadata.getStatistics().getCount() != 0) { + if (cacheEnabled) { + lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId), metadata); + } + if (requested.contains(measurementId)) { + result.put(measurementId, new TimeseriesMetadata(metadata)); + } + } else if (requested.contains(measurementId)) { + if (debug) { + DEBUG_LOGGER.info( + StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, + new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId)); + } + result.put(measurementId, null); + cacheNonExistenceIfNeeded(tsFileID, device, measurementId, cacheEnabled, ignoreNotExists); + } + } + if (!probeFound) { + pending.remove(probe); + if (requested.contains(probe)) { + if (debug) { + DEBUG_LOGGER.info( + StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, + new TimeSeriesMetadataCacheKey(tsFileID, device, probe)); + } + result.put(probe, null); + cacheNonExistenceIfNeeded(tsFileID, device, probe, cacheEnabled, ignoreNotExists); + } + } + } + for (String measurement : requested) { + result.putIfAbsent(measurement, null); + } + return statistics; + } + + private static final class LeafLoadStatistics { + private int diskLookups; + private int cacheHits; + } + + private void cacheNonExistenceIfNeeded( + TsFileID tsFileID, + IDeviceID device, + String measurement, + boolean cacheEnabled, + boolean ignoreNotExists) { + if (cacheEnabled && !ignoreNotExists && memoryConfig.isMayCacheNonExistSeries()) { + lruCache.put( + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), + NULL_EXISTS_CACHE_PLACE_HOLDER); + } + } + private BloomFilter loadBloomFilter( String filePath, TsFileID tsFileID, @@ -544,9 +497,8 @@ private BloomFilter loadBloomFilter( /** A null filter cannot rule the measurement out. */ private static boolean bloomFilterMayContain( - BloomFilter bloomFilter, IDeviceID device, String measurement) { - return bloomFilter == null - || bloomFilter.contains(device.toString() + IoTDBConstant.PATH_SEPARATOR + measurement); + BloomFilter bloomFilter, String devicePrefix, String measurement) { + return bloomFilter == null || bloomFilter.contains(devicePrefix + measurement); } private static void recordLoadStatistics( @@ -576,81 +528,32 @@ private void collectCachedMetadata( IDeviceID device, Collection measurements, Map resolved, - List missing) { + Collection missing, + String filePath, + boolean debug) { for (String measurement : measurements) { - TimeseriesMetadata metadata = - lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID, device, measurement)); + TimeSeriesMetadataCacheKey key = + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement); + TimeseriesMetadata metadata = lruCache.getIfPresent(key); if (metadata == null) { missing.add(measurement); + } else if (metadata == NULL_EXISTS_CACHE_PLACE_HOLDER) { + if (debug) { + DEBUG_LOGGER.info(StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, key); + } + resolved.put(measurement, null); } else { - resolved.put( - measurement, - metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new TimeseriesMetadata(metadata)); - } - } - } - - private Map readDeviceTimeSeriesMetadata( - TsFileSequenceReader reader, - String filePath, - TsFileID tsFileID, - IDeviceID device, - Set allSensors, - boolean cacheEnabled, - boolean ignoreNotExists) - throws IOException { - // Use the chunk-metadata-including variant: the returned TimeseriesMetadata are passed to - // setModifications and then to DiskAlignedChunkMetadataLoader, which dereferences the chunk - // metadata list, so readDeviceMetadata (which skips chunk metadata) would NPE at query time. - List deviceMetadataList = reader.getDeviceTimeseriesMetadata(device); - if (deviceMetadataList.isEmpty()) { - // the caller already consulted the bloom filter, so the device could not be ruled out: - // missing metadata is an inconsistency unless ignoreNotExists is set - if (ignoreNotExists) { - return Collections.emptyMap(); - } - throw new IOException( - String.format( - StorageEngineMessages - .EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374, + if (debug) { + DEBUG_LOGGER.info( + StorageEngineMessages.STORAGE_LOG_GET_TIMESERIES_METADATA_IN_FILE_FROM_CACHE_36652729, device, - filePath)); - } - - Map result = new HashMap<>(allSensors.size()); - for (TimeseriesMetadata metadata : deviceMetadataList) { - String measurementId = metadata.getMeasurementId(); - if (!allSensors.contains(measurementId)) { - continue; - } - if (metadata.getStatistics().getCount() == 0) { - result.put(measurementId, null); - if (cacheEnabled && !ignoreNotExists && memoryConfig.isMayCacheNonExistSeries()) { - lruCache.put( - new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId), - NULL_EXISTS_CACHE_PLACE_HOLDER); + measurement, + filePath, + metadata); } - continue; - } - if (cacheEnabled) { - lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId), metadata); - } - result.put(measurementId, new TimeseriesMetadata(metadata)); - } - // remember selected measurements that do not exist in this file, so a later lookup of the same - // device can be served from the cache instead of reading the device again - for (String measurement : allSensors) { - if (result.containsKey(measurement)) { - continue; - } - result.put(measurement, null); - if (cacheEnabled && !ignoreNotExists && memoryConfig.isMayCacheNonExistSeries()) { - lruCache.put( - new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), - NULL_EXISTS_CACHE_PLACE_HOLDER); + resolved.put(measurement, new TimeseriesMetadata(metadata)); } } - return result; } /** diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java deleted file mode 100644 index 2db7cafcbc6db..0000000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtilsTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.execution.operator.source; - -import org.apache.tsfile.common.conf.TSFileDescriptor; -import org.junit.Test; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class FileLoaderUtilsTest { - - @Test - public void testShouldUseDeviceMetadataBatchScan() { - int maxDegreeOfIndexNode = TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode(); - // a table wide enough to span more than one measurement leaf group - int wideTableFieldCount = maxDegreeOfIndexNode * 2; - int threshold = (int) Math.ceil(wideTableFieldCount * 0.6); - - // no field count is known (tree model, external tsfiles, or a table missing from the cache) - assertFalse(FileLoaderUtils.shouldUseDeviceMetadataBatchScan(10, 0, false, false)); - - // a table with maxDegree - 1 fields has at most maxDegree device series (time included), which - // still fit in a single measurement leaf group - assertFalse( - FileLoaderUtils.shouldUseDeviceMetadataBatchScan( - maxDegreeOfIndexNode - 1, maxDegreeOfIndexNode - 1, false, false)); - // maxDegree fields span maxDegree + 1 device series (time included), i.e. two leaf groups - assertTrue( - FileLoaderUtils.shouldUseDeviceMetadataBatchScan( - maxDegreeOfIndexNode, maxDegreeOfIndexNode, false, false)); - - // below, at and above the ratio threshold on a wider table - assertFalse( - FileLoaderUtils.shouldUseDeviceMetadataBatchScan( - threshold - 1, wideTableFieldCount, false, false)); - assertTrue( - FileLoaderUtils.shouldUseDeviceMetadataBatchScan( - threshold, wideTableFieldCount, false, false)); - assertTrue( - FileLoaderUtils.shouldUseDeviceMetadataBatchScan( - wideTableFieldCount, wideTableFieldCount, false, false)); - - // a scan can reference measurements that no longer exist, so selected > total is possible - assertTrue( - FileLoaderUtils.shouldUseDeviceMetadataBatchScan( - wideTableFieldCount + 1, wideTableFieldCount, false, false)); - - // DEBUG queries keep the per-column cache's diagnostic logs - assertFalse( - FileLoaderUtils.shouldUseDeviceMetadataBatchScan( - wideTableFieldCount, wideTableFieldCount, true, false)); - - // offset-addressed external tsfiles cannot be resolved through the table metadata index - assertFalse( - FileLoaderUtils.shouldUseDeviceMetadataBatchScan( - wideTableFieldCount, wideTableFieldCount, false, true)); - } -} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 2ebdf4375b35c..6764446d4ed05 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -292,8 +292,46 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { } } + private static Map getTimeseriesMetadata( + TimeSeriesMetadataCache cache, + String filePath, + TsFileID tsFileID, + IDeviceID deviceID, + Set allSensors, + boolean ignoreNotExists) + throws IOException { + return cache.getTimeseriesMetadata( + filePath, + tsFileID, + deviceID, + allSensors, + allSensors, + ignoreNotExists, + false, + new QueryContext(false, false), + null); + } + + private static void stubGroupRead( + TsFileSequenceReader reader, + IDeviceID deviceID, + Set allSensors, + boolean ignoreNotExists, + List group) + throws IOException { + Mockito.when( + reader.readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyString(), + Mockito.eq(allSensors), + Mockito.eq(ignoreNotExists), + Mockito.any(LongConsumer.class))) + .thenReturn(group); + } + @Test - public void testGetDeviceTimeSeriesMetadata() throws Exception { + public void testGetTimeseriesMetadata() throws Exception { TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); FileReaderManager fileReaderManager = FileReaderManager.getInstance(); @@ -331,20 +369,19 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { Statistics.getStatsByType(TSDataType.INT32), new PublicBAOS()); - List deviceMetadata = new ArrayList<>(); - deviceMetadata.add(timeColumn); - deviceMetadata.add(s1); - deviceMetadata.add(emptyS2); - // the batch read must carry chunk metadata, otherwise the downstream chunk loader NPEs + List group = new ArrayList<>(); + group.add(timeColumn); + group.add(s1); + group.add(emptyS2); + // the group read must carry chunk metadata, otherwise the downstream chunk loader NPEs s1.setChunkMetadataList(Collections.singletonList(new ChunkMetadata())); TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)).thenReturn(deviceMetadata); + stubGroupRead(reader, deviceID, allSensors, false, group); Map result = - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); assertNotNull(result.get("")); assertEquals("", result.get("").getMeasurementId()); @@ -357,12 +394,18 @@ public void testGetDeviceTimeSeriesMetadata() throws Exception { // a second lookup of cached measurements must be served without touching the reader again Set cachedSensors = new HashSet<>(Arrays.asList("", "s1")); Map cachedResult = - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, cachedSensors, new QueryContext(false, false), false); + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, cachedSensors, false); assertNotNull(cachedResult.get("s1")); - Mockito.verify(reader, Mockito.times(1)).getDeviceTimeseriesMetadata(deviceID); + Mockito.verify(reader, Mockito.times(1)) + .readTimeseriesMetadata( + Mockito.any(), + Mockito.any(), + Mockito.anyString(), + Mockito.any(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); - // the batch read also populated the per-measurement cache + // the group read also populated the per-measurement cache TimeseriesMetadata cached = cache.get( filePath, @@ -412,13 +455,17 @@ public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))) .thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1)); - // an absent device must not be read: the filtered-out measurements are returned as null, - // which the caller distinguishes from "present but empty time column" Map result = - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); assertNull(result.get("")); - Mockito.verify(reader, Mockito.never()).getDeviceTimeseriesMetadata(deviceID); + Mockito.verify(reader, Mockito.never()) + .readTimeseriesMetadata( + Mockito.any(), + Mockito.any(), + Mockito.anyString(), + Mockito.any(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); } finally { cache.clear(); bloomFilterCache.clear(); @@ -437,32 +484,37 @@ public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Except try { String filePath = "target/batch-absent-device-throw.tsfile"; IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); - // more than MAX_PER_COLUMN_FALLBACK entries so the batch read (not the per-column fallback) - // is used Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)) - .thenReturn(Collections.emptyList()); + stubGroupRead(reader, deviceID, allSensors, true, Collections.emptyList()); + Mockito.when( + reader.readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyString(), + Mockito.eq(allSensors), + Mockito.eq(false), + Mockito.any(LongConsumer.class))) + .thenThrow(new IOException("device absent")); BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); bloomFilter.add(deviceID.toString() + "."); bloomFilter.add(deviceID.toString() + ".s1"); bloomFilter.add(deviceID.toString() + ".s2"); Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); - assertThrows( - IOException.class, - () -> - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false)); - + // ignoreNotExists keeps the missing device silent, but caches nothing Map ignored = - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), true); + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, true); assertNull(ignored.get("")); assertNull(ignored.get("s1")); assertNull(ignored.get("s2")); + + // a normal query must surface the reader's failure instead of a silently empty device + assertThrows( + IOException.class, + () -> getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false)); } finally { cache.clear(); bloomFilterCache.clear(); @@ -520,8 +572,7 @@ public void testBatchMetadataMatchesPerColumn() throws Exception { cache.clear(); Map batch = - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); assertEquals(allSensors.size(), batch.size()); for (String measurement : allSensors) { @@ -573,7 +624,7 @@ public void testBatchMetadataMatchesPerColumn() throws Exception { } @Test - public void testAbsentTimeColumnSkipsDeviceRead() throws Exception { + public void testBloomFilterExcludedMeasurementsSkipRead() throws Exception { TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); FileReaderManager fileReaderManager = FileReaderManager.getInstance(); @@ -587,26 +638,23 @@ public void testAbsentTimeColumnSkipsDeviceRead() throws Exception { TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - // the time column is filtered out, while s1 is a bloom filter false positive + // both the time column and s1 are excluded, so no leaf group needs to be read BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); - bloomFilter.add(deviceID.toString() + ".s1"); Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); Map result = - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); assertNull(result.get("")); assertNull(result.get("s1")); - Mockito.verify(reader, Mockito.never()).getDeviceTimeseriesMetadata(deviceID); Mockito.verify(reader, Mockito.never()) .readTimeseriesMetadata( Mockito.any(), Mockito.any(), - Mockito.any(), + Mockito.anyString(), Mockito.any(), Mockito.anyBoolean(), - Mockito.any()); + Mockito.any(LongConsumer.class)); } finally { cache.clear(); bloomFilterCache.clear(); @@ -615,7 +663,7 @@ public void testAbsentTimeColumnSkipsDeviceRead() throws Exception { } @Test - public void testSingleMissingMeasurementUsesPerColumnFallback() throws Exception { + public void testEvictedMeasurementReloadsOnlyItsGroup() throws Exception { TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); FileReaderManager fileReaderManager = FileReaderManager.getInstance(); @@ -642,47 +690,35 @@ public void testSingleMissingMeasurementUsesPerColumnFallback() throws Exception s1Statistics.setStartTime(0); s1Statistics.setEndTime(10); - List deviceMetadata = new ArrayList<>(); - deviceMetadata.add( + List group = new ArrayList<>(); + group.add( new TimeseriesMetadata( (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS())); - deviceMetadata.add( + group.add( new TimeseriesMetadata( (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS())); - deviceMetadata.add(s2); + group.add(s2); TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)).thenReturn(deviceMetadata); + stubGroupRead(reader, deviceID, allSensors, false, group); Map first = - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); assertNotNull(first.get("s2")); - // evict one measurement: the next call must load only that one, not the whole device + // evict one measurement: the next call reads only the group that holds it, and that one + // read also resolves the other members of the group cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s2")); - Mockito.when( - reader.readTimeseriesMetadata( - Mockito.eq(deviceID), - Mockito.isNull(), - Mockito.eq("s2"), - Mockito.eq(allSensors), - Mockito.eq(false), - Mockito.any(LongConsumer.class))) - .thenReturn(Collections.singletonList(s2)); - Map second = - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); assertNotNull(second.get("s2")); - Mockito.verify(reader, Mockito.times(1)).getDeviceTimeseriesMetadata(deviceID); - Mockito.verify(reader, Mockito.times(1)) + Mockito.verify(reader, Mockito.times(2)) .readTimeseriesMetadata( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.eq("s2"), + Mockito.anyString(), Mockito.eq(allSensors), Mockito.eq(false), Mockito.any(LongConsumer.class)); @@ -708,22 +744,36 @@ public void testBatchIgnoreNotExistsDoesNotCachePlaceholders() throws Exception TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)) - .thenReturn(Collections.emptyList()); + stubGroupRead(reader, deviceID, allSensors, true, Collections.emptyList()); + Mockito.when( + reader.readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyString(), + Mockito.eq(allSensors), + Mockito.eq(false), + Mockito.any(LongConsumer.class))) + .thenThrow(new IOException("device absent")); Map ignored = - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), true); + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, true); assertNull(ignored.get("")); assertNull(ignored.get("s1")); // ignoreNotExists must not write NULL placeholders: a normal query still has to read assertThrows( IOException.class, - () -> - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false)); - Mockito.verify(reader, Mockito.times(2)).getDeviceTimeseriesMetadata(deviceID); + () -> getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false)); + // one group read per pending measurement when the group comes back empty, plus the failing + // normal query + Mockito.verify(reader, Mockito.times(4)) + .readTimeseriesMetadata( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyString(), + Mockito.eq(allSensors), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); } finally { cache.clear(); bloomFilterCache.clear(); @@ -732,7 +782,7 @@ public void testBatchIgnoreNotExistsDoesNotCachePlaceholders() throws Exception } @Test - public void testFewMissingMeasurementsUsePerColumnFallbackAfterEviction() throws Exception { + public void testEvictedMeasurementsShareOneGroupRead() throws Exception { TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); FileReaderManager fileReaderManager = FileReaderManager.getInstance(); @@ -744,52 +794,39 @@ public void testFewMissingMeasurementsUsePerColumnFallbackAfterEviction() throws IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2", "s3", "s4", "s5")); - List deviceMetadata = new ArrayList<>(); + List group = new ArrayList<>(); for (String measurement : allSensors) { TSDataType dataType = measurement.isEmpty() ? TSDataType.INT64 : TSDataType.INT32; Statistics statistics = Statistics.getStatsByType(dataType); statistics.setCount(1); statistics.setStartTime(0); statistics.setEndTime(10); - deviceMetadata.add( + group.add( new TimeseriesMetadata( (byte) 0, 0, measurement, dataType, statistics, new PublicBAOS())); } TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)).thenReturn(deviceMetadata); + stubGroupRead(reader, deviceID, allSensors, false, group); Map first = - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); assertNotNull(first.get("s5")); - // evict three measurements: the cache still holds the majority, so the remaining misses - // must be loaded per column instead of re-reading the whole device + // evict three measurements: they share one leaf group, so a single read must resolve them for (String measurement : Arrays.asList("s2", "s3", "s4")) { cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, measurement)); } - Mockito.when( - reader.readTimeseriesMetadata( - Mockito.any(), - Mockito.isNull(), - Mockito.anyString(), - Mockito.eq(allSensors), - Mockito.eq(false), - Mockito.any(LongConsumer.class))) - .thenReturn(deviceMetadata); Map second = - cache.getDeviceTimeSeriesMetadata( - filePath, tsFileID, deviceID, allSensors, new QueryContext(false, false), false); + getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); assertNotNull(second.get("s2")); assertNotNull(second.get("s3")); assertNotNull(second.get("s4")); - Mockito.verify(reader, Mockito.times(1)).getDeviceTimeseriesMetadata(deviceID); - Mockito.verify(reader, Mockito.times(1)) + Mockito.verify(reader, Mockito.times(2)) .readTimeseriesMetadata( - Mockito.any(), + Mockito.eq(deviceID), Mockito.isNull(), Mockito.anyString(), Mockito.eq(allSensors), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java index fed93e643a639..16927deca2534 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -63,8 +63,9 @@ import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.writeAlignedChunk; /** - * Verifies the batch metadata path on a real device wider than one measurement leaf group, and that - * it matches the per-column path for dense, sparse and all-null projections. + * Verifies metadata loading on a real device wider than one measurement leaf group: each selected + * column is resolved from the leaf group that holds it, and one group read serves every selected + * column of that group. */ public class AlignedSeriesScanBatchTest { @@ -76,7 +77,7 @@ public class AlignedSeriesScanBatchTest { private static final int ROW_COUNT = 10; @Test - public void testBatchScanReadsWideAlignedDevice() throws Exception { + public void testScanReadsWideAlignedDevice() throws Exception { Assert.assertTrue( "the fixture must span more than one measurement leaf group", COLUMN_COUNT + 1 > TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode()); @@ -88,7 +89,7 @@ public void testBatchScanReadsWideAlignedDevice() throws Exception { bloomFilterCache.clear(); boolean previousIgnoreAllNullRows = EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); - // the batch path is only reachable from table dialect, which forces this flag to false + // the unified path is only reachable from table dialect, which forces this flag to false EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); try { long cacheHitsBefore = @@ -102,7 +103,7 @@ public void testBatchScanReadsWideAlignedDevice() throws Exception { .getLoadTimeSeriesMetadataFromDiskCount() .get(); - TsBlock block = scan(alignedPath(columnNames(COLUMN_COUNT)), COLUMN_COUNT, resource); + TsBlock block = scan(alignedPath(columnNames(COLUMN_COUNT)), resource); Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); Assert.assertEquals(ROW_COUNT, block.getPositionCount()); @@ -117,21 +118,23 @@ public void testBatchScanReadsWideAlignedDevice() throws Exception { } } - // one batch read per device and file: no per-column cache lookups happened + // a cold cache: every selected series is a disk lookup, counted per series Assert.assertEquals( - "the batch path must not fall back to per-column cache loads", cacheHitsBefore, EnvironmentUtils.TEST_QUERY_FI_CONTEXT .getQueryStatistics() .getLoadTimeSeriesMetadataFromCacheCount() .get()); - // statistics are counted per series, like the per-column path (time column + all fields) - Assert.assertEquals( - diskReadsBefore + COLUMN_COUNT + 1, + // one read serves a whole leaf group, so a cold scan issues far fewer metadata reads than + // the number of selected series + long diskReads = EnvironmentUtils.TEST_QUERY_FI_CONTEXT - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskCount() - .get()); + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get() + - diskReadsBefore; + Assert.assertTrue("disk lookups: " + diskReads, diskReads >= 1); + Assert.assertTrue("disk lookups: " + diskReads, diskReads <= COLUMN_COUNT + 1); } finally { EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); cleanup(file, resource, metadataCache, bloomFilterCache); @@ -139,7 +142,7 @@ public void testBatchScanReadsWideAlignedDevice() throws Exception { } @Test - public void testSparseBatchScanMatchesPerColumn() throws Exception { + public void testSparseProjectionFiltersAbsentColumns() throws Exception { File file = newFile(2); TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); @@ -151,59 +154,43 @@ public void testSparseBatchScanMatchesPerColumn() throws Exception { try { metadataCache.clear(); bloomFilterCache.clear(); - long cacheHitsBefore = + long diskReadsBefore = EnvironmentUtils.TEST_QUERY_FI_CONTEXT .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() + .getLoadTimeSeriesMetadataFromDiskCount() .get(); - TsBlock batched = scan(scanPath, COLUMN_COUNT, resource); - long batchedCacheHits = - EnvironmentUtils.TEST_QUERY_FI_CONTEXT - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .get() - - cacheHitsBefore; - metadataCache.clear(); - long perColumnCacheHitsBefore = - EnvironmentUtils.TEST_QUERY_FI_CONTEXT - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .get(); - TsBlock perColumn = scan(scanPath, 0, resource); - long perColumnCacheHits = - EnvironmentUtils.TEST_QUERY_FI_CONTEXT - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .get() - - perColumnCacheHitsBefore; + TsBlock block = scan(scanPath, resource); - // the sparse projection is identical to the per-column scan, including every null field - Assert.assertEquals(perColumn.getPositionCount(), batched.getPositionCount()); - Assert.assertEquals(perColumn.getValueColumnCount(), batched.getValueColumnCount()); - for (int row = 0; row < batched.getPositionCount(); row++) { - Assert.assertEquals(perColumn.getTimeByIndex(row), batched.getTimeByIndex(row)); + Assert.assertEquals(ROW_COUNT, block.getPositionCount()); + Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals(row, block.getTimeByIndex(row)); } - for (int column = 0; column < batched.getValueColumnCount(); column++) { - Column batchedColumn = batched.getColumn(column); - Column perColumnColumn = perColumn.getColumn(column); + for (int column = 0; column < COLUMN_COUNT; column++) { + Column valueColumn = block.getColumn(column); boolean present = column < PRESENT_COLUMN_COUNT; - for (int row = 0; row < batched.getPositionCount(); row++) { - Assert.assertEquals( - "s" + column + " row " + row, perColumnColumn.isNull(row), batchedColumn.isNull(row)); + for (int row = 0; row < ROW_COUNT; row++) { if (present) { - Assert.assertFalse(batchedColumn.isNull(row)); - Assert.assertEquals(row, batchedColumn.getInt(row)); - Assert.assertEquals(perColumnColumn.getInt(row), batchedColumn.getInt(row)); + Assert.assertFalse(valueColumn.isNull(row)); + Assert.assertEquals(row, valueColumn.getInt(row)); } else { - Assert.assertTrue(batchedColumn.isNull(row)); + Assert.assertTrue("s" + column + " row " + row, valueColumn.isNull(row)); } } } - // the batch scan resolved everything from disk, the per-column scan reused leaf groups - Assert.assertEquals(0, batchedCacheHits); - Assert.assertTrue(perColumnCacheHits > 0); + // the bloom filter excludes the 270 absent columns; a few may survive as false positives, + // but the sparse projection must still look up far fewer series than it selects + long diskReads = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get() + - diskReadsBefore; + // at most one group read per present series plus the time column + Assert.assertTrue("disk lookups: " + diskReads, diskReads >= 1); + Assert.assertTrue("disk lookups: " + diskReads, diskReads <= PRESENT_COLUMN_COUNT + 1); } finally { EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); cleanup(file, resource, metadataCache, bloomFilterCache); @@ -211,7 +198,7 @@ public void testSparseBatchScanMatchesPerColumn() throws Exception { } @Test - public void testAllNullProjectionMatchesPerColumn() throws Exception { + public void testAllNullProjection() throws Exception { File file = newFile(3); TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); @@ -221,62 +208,42 @@ public void testAllNullProjectionMatchesPerColumn() throws Exception { boolean previousIgnoreAllNullRows = EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); try { - // table dialect (the only dialect that reaches the batch path): all-null rows are kept + // table dialect: all-null rows are kept EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); metadataCache.clear(); bloomFilterCache.clear(); - TsBlock batched = scan(absentPath, COLUMN_COUNT, resource); - Assert.assertEquals(ROW_COUNT, batched.getPositionCount()); - Assert.assertEquals(COLUMN_COUNT, batched.getValueColumnCount()); - for (int column = 0; column < COLUMN_COUNT; column++) { - for (int row = 0; row < ROW_COUNT; row++) { - Assert.assertTrue( - "absent" + column + " row " + row, batched.getColumn(column).isNull(row)); - } - } - - metadataCache.clear(); - TsBlock perColumn = scan(absentPath, 0, resource); - Assert.assertEquals(perColumn.getPositionCount(), batched.getPositionCount()); - Assert.assertEquals(perColumn.getValueColumnCount(), batched.getValueColumnCount()); - for (int row = 0; row < ROW_COUNT; row++) { - Assert.assertEquals(perColumn.getTimeByIndex(row), batched.getTimeByIndex(row)); - } + TsBlock block = scan(absentPath, resource); + Assert.assertEquals(ROW_COUNT, block.getPositionCount()); + Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); for (int column = 0; column < COLUMN_COUNT; column++) { for (int row = 0; row < ROW_COUNT; row++) { - Assert.assertEquals( - perColumn.getColumn(column).isNull(row), batched.getColumn(column).isNull(row)); + Assert.assertTrue("absent" + column + " row " + row, block.getColumn(column).isNull(row)); } } - // tree dialect: an all-null projection skips the file in both paths + // tree dialect: an all-null projection skips the file EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(true); metadataCache.clear(); bloomFilterCache.clear(); - Assert.assertFalse(hasFile(absentPath, COLUMN_COUNT, resource)); - - metadataCache.clear(); - Assert.assertFalse(hasFile(absentPath, 0, resource)); + Assert.assertFalse(hasFile(absentPath, resource)); } finally { EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); cleanup(file, resource, metadataCache, bloomFilterCache); } } - private static TsBlock scan( - AlignedFullPath scanPath, int totalFieldColumnCount, TsFileResource resource) + private static TsBlock scan(AlignedFullPath scanPath, TsFileResource resource) throws IOException { - AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, totalFieldColumnCount, resource); + AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, resource); SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); Assert.assertTrue(seriesScanUtil.hasNextPage()); return seriesScanUtil.nextPage(); } - private static boolean hasFile( - AlignedFullPath scanPath, int totalFieldColumnCount, TsFileResource resource) + private static boolean hasFile(AlignedFullPath scanPath, TsFileResource resource) throws IOException { - AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, totalFieldColumnCount, resource); + AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, resource); while (true) { Optional hasNextFile = seriesScanUtil.hasNextFile(); if (hasNextFile.isPresent()) { @@ -286,12 +253,10 @@ private static boolean hasFile( } private static AlignedSeriesScanUtil newScanUtil( - AlignedFullPath scanPath, int totalFieldColumnCount, TsFileResource resource) - throws IOException { + AlignedFullPath scanPath, TsFileResource resource) throws IOException { SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); SeriesScanOptions scanOptions = builder.build(); - scanOptions.setTotalFieldColumnCount(totalFieldColumnCount); AlignedSeriesScanUtil seriesScanUtil = new AlignedSeriesScanUtil( scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); From 7871781fdf1603a86ded5080603742f7c6f714dd Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 13:17:59 +0800 Subject: [PATCH 20/27] Skip a device once its time column is proven absent An aligned device always has its time column, so when the requested set contains it and the cache or the bloom filter proves it absent, no selected series of that device can be read from this file. The per-column loader returned null at that point; the set-based loader kept probing the remaining measurements, and a bloom-filter false positive on any of them sent the device metadata index lookup into the reader, which throws when the device is not in the file (e.g. a seq file that only holds a subset of the devices read by cross compaction). Probe the time column first and stop the load when it resolves to null. Found by ReadPointAlignedCrossCompactionTest.test21. --- .../buffer/TimeSeriesMetadataCache.java | 64 +++++++++++++++++++ .../buffer/TimeSeriesMetadataCacheTest.java | 6 +- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 15a1aa7a427ca..bd63caef31278 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -255,6 +255,10 @@ public Map getTimeseriesMetadata( loadBloomFilter(filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, false); loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; } + if (timeColumnAbsent(requested, result, pending, bloomFilter, devicePrefix)) { + fillMissingWithNull(requested, result); + return result; + } if (!pending.isEmpty()) { TsFileSequenceReader reader = FileReaderManager.getInstance() @@ -292,6 +296,10 @@ public Map getTimeseriesMetadata( filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, externalTsFile); loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; } + if (timeColumnAbsent(requested, result, pending, bloomFilter, devicePrefix)) { + fillMissingWithNull(requested, result); + return result; + } if (!pending.isEmpty()) { TsFileSequenceReader reader = FileReaderManager.getInstance() @@ -364,7 +372,16 @@ private LeafLoadStatistics loadLeafGroups( LongConsumer ioSizeRecorder) throws IOException { LeafLoadStatistics statistics = new LeafLoadStatistics(); + // probe the time column first: when it is absent, no selected series of this device can be + // read from this file, and the per-column path returned without touching any other column + if (whereFirst(pending, requested)) { + pending = withFirst(pending, ""); + } while (!pending.isEmpty()) { + if (timeColumnResolvedToNull(requested, result)) { + fillMissingWithNull(requested, result); + return statistics; + } String probe = pending.iterator().next(); // another thread may have loaded this probe's group while we waited for the lock; checking // only the probe keeps the cold path free of a per-column cache pass @@ -465,6 +482,53 @@ private static final class LeafLoadStatistics { private int cacheHits; } + /** + * An aligned device always has its time column. When the requested set contains it and the cache + * or the bloom filter proves it absent, every selected series of the device is absent from this + * file, so the per-column loader returned null without reading any other column. + */ + private static boolean timeColumnAbsent( + Set requested, + Map result, + Set pending, + BloomFilter bloomFilter, + String devicePrefix) { + if (!requested.contains("")) { + return false; + } + if (result.containsKey("")) { + return result.get("") == null; + } + if (bloomFilter != null && !bloomFilterMayContain(bloomFilter, devicePrefix, "")) { + result.put("", null); + return true; + } + return false; + } + + private static boolean timeColumnResolvedToNull( + Set requested, Map result) { + return requested.contains("") && result.containsKey("") && result.get("") == null; + } + + private static boolean whereFirst(Set pending, Set requested) { + return requested.contains("") && pending.contains(""); + } + + private static Set withFirst(Set pending, String first) { + Set ordered = new LinkedHashSet<>(pending.size()); + ordered.add(first); + ordered.addAll(pending); + return ordered; + } + + private static void fillMissingWithNull( + Set requested, Map result) { + for (String measurement : requested) { + result.putIfAbsent(measurement, null); + } + } + private void cacheNonExistenceIfNeeded( TsFileID tsFileID, IDeviceID device, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 6764446d4ed05..25b90a37544e6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -764,9 +764,9 @@ public void testBatchIgnoreNotExistsDoesNotCachePlaceholders() throws Exception assertThrows( IOException.class, () -> getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false)); - // one group read per pending measurement when the group comes back empty, plus the failing - // normal query - Mockito.verify(reader, Mockito.times(4)) + // the empty group read resolves the time column as absent, which ends the load for the + // device, plus the failing normal query + Mockito.verify(reader, Mockito.times(2)) .readTimeseriesMetadata( Mockito.eq(deviceID), Mockito.isNull(), From aa8a964130ac8c4b25f41089a4f2745bc58a8e64 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 13:34:47 +0800 Subject: [PATCH 21/27] Seed pending measurements in the cache-disabled metadata path The requested measurements were only added to the pending set by the cache pass. The cache-disabled and external-TsFile branch skipped that pass without seeding the set, so it stayed empty, no leaf group was read and the call returned nothing: read_tsfile queries returned zero rows on every external TsFile. Seed pending from the requested set in that branch and add a regression test that loads an external file through getTimeseriesMetadata with the device metadata offset. --- .../buffer/TimeSeriesMetadataCache.java | 4 +- .../buffer/TimeSeriesMetadataCacheTest.java | 63 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index bd63caef31278..a3c8a2528bd09 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -287,7 +287,9 @@ public Map getTimeseriesMetadata( } } - // cache disabled or external tsfiles + // cache disabled or external tsfiles: nothing was resolved from the cache, so every + // requested measurement is still pending + pending.addAll(requested); BloomFilter bloomFilter = null; if (deviceMetadataIndexNodeOffset == null) { long bloomFilterStartTime = System.nanoTime(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 25b90a37544e6..bc086b2b150b3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -22,8 +22,12 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSourceType; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; @@ -32,6 +36,7 @@ import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.LazyTsFileDeviceIterator; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.utils.BloomFilter; import org.apache.tsfile.utils.PublicBAOS; @@ -838,4 +843,62 @@ public void testEvictedMeasurementsShareOneGroupRead() throws Exception { fileReaderManager.closeFileAndRemoveReader(tsFileID); } } + + @Test + public void testExternalTsFileReadsAllRequestedMeasurements() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + File file = new File("target/external-cache-disabled.tsfile"); + file.delete(); + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.external.d1"); + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerAlignedTimeseries( + deviceID, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32))); + for (int time = 0; time < 3; time++) { + TSRecord record = new TSRecord(deviceID, time); + record.addPoint("s1", time); + record.addPoint("s2", time); + writer.writeRecord(record); + } + } + cache.clear(); + bloomFilterCache.clear(); + long[] offset; + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) { + LazyTsFileDeviceIterator iterator = new LazyTsFileDeviceIterator(reader); + assertTrue(iterator.hasNext()); + iterator.next(); + offset = iterator.getCurrentDeviceMeasurementNodeOffset(); + } + // external tsfiles bypass the metadata cache: every requested measurement must still be read + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setQueryDataSourceType( + QueryDataSourceType.EXTERNAL_TSFILE_SCAN); + try { + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + TsFileResource resource = new TsFileResource(file); + resource.setStatusForTest(TsFileResourceStatus.NORMAL); + Map result = + cache.getTimeseriesMetadata( + file.getAbsolutePath(), + resource.getTsFileID(), + deviceID, + allSensors, + allSensors, + false, + false, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT, + offset); + assertEquals(allSensors.size(), result.size()); + for (String measurement : allSensors) { + assertNotNull(measurement, result.get(measurement)); + assertEquals(measurement, result.get(measurement).getMeasurementId()); + } + } finally { + cache.clear(); + bloomFilterCache.clear(); + } + } } From 991ddb53a6669f720aa3186e6cedddf817ebdda1 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 14:41:15 +0800 Subject: [PATCH 22/27] Restore per-series load counters and trim disabled-cache reads Count disk lookups per requested measurement, cache hits included in the same total, as the per-column path did, so EXPLAIN ANALYZE and the monitoring counters stay comparable; leaf-read behavior stays covered by the cache tests rather than by the counter. When the metadata cache is disabled, filter the group read by the requested measurements instead of allSensors, so a non-aligned get() does not deserialize and drop its siblings' chunk metadata. Restore the shared test context to SERIES_SCAN after the external-file test, and document that the once-per-group guarantee needs a bloom filter. --- .../buffer/TimeSeriesMetadataCache.java | 21 ++++++++++++------- .../buffer/TimeSeriesMetadataCacheTest.java | 2 ++ .../series/AlignedSeriesScanBatchTest.java | 8 +++---- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index a3c8a2528bd09..bd041aebda720 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -184,9 +184,11 @@ public TimeseriesMetadata get( *

Cached measurements are returned directly. The remaining ones are checked against the file's * bloom filter and then loaded leaf group by leaf group: one pending measurement locates the * group it belongs to, that whole group is read and deserialized once, and every requested - * measurement found in it is resolved. Each group is therefore read at most once per call, - * independent of LRU cache pressure, and groups that contain no requested measurement are never - * read. + * measurement found in it is resolved. For files that expose a bloom filter, each group is read + * at most once per call, independent of LRU cache pressure, and groups that contain no requested + * measurement are never read. Without one (external tsfiles), a measurement the bloom filter + * cannot rule out is read from its leaf, so a definitively absent measurement may read the leaf + * that holds its floor entry again; this matches the previous per-column loader. * *

{@code allSensors} is the full selected set; it filters each group read and warms the cache, * so a later single-measurement {@link #get} can reuse the group. {@code requested} must be a @@ -280,8 +282,10 @@ public Map getTimeseriesMetadata( debug, result, timeSeriesMetadataIoSizeRecorder); - diskLookupCount = loaded.diskLookups; cacheHitCount += loaded.cacheHits; + // per-series counters: every requested measurement that was not resolved from the + // cache is a disk lookup, including the ones the bloom filter excluded (as before) + diskLookupCount = requested.size() - cacheHitCount; } return result; } @@ -323,8 +327,8 @@ public Map getTimeseriesMetadata( debug, result, timeSeriesMetadataIoSizeRecorder); - diskLookupCount = loaded.diskLookups; cacheHitCount += loaded.cacheHits; + diskLookupCount = requested.size() - cacheHitCount; } return result; } finally { @@ -374,6 +378,9 @@ private LeafLoadStatistics loadLeafGroups( LongConsumer ioSizeRecorder) throws IOException { LeafLoadStatistics statistics = new LeafLoadStatistics(); + // without a cache to warm, only the measurements this call needs are worth deserializing; + // with the cache enabled the siblings keep the group warm for later single-column gets + Set groupFilter = cacheEnabled ? allSensors : requested; // probe the time column first: when it is absent, no selected series of this device can be // read from this file, and the per-column path returned without touching any other column if (whereFirst(pending, requested)) { @@ -429,13 +436,12 @@ private LeafLoadStatistics loadLeafGroups( } continue; } - statistics.diskLookups++; List group = reader.readTimeseriesMetadata( device, deviceMetadataIndexNodeOffset, probe, - allSensors, + groupFilter, ignoreNotExists, ioSizeRecorder); boolean probeFound = false; @@ -480,7 +486,6 @@ private LeafLoadStatistics loadLeafGroups( } private static final class LeafLoadStatistics { - private int diskLookups; private int cacheHits; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index bc086b2b150b3..024492b5bf52b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -897,6 +897,8 @@ public void testExternalTsFileReadsAllRequestedMeasurements() throws Exception { assertEquals(measurement, result.get(measurement).getMeasurementId()); } } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setQueryDataSourceType( + QueryDataSourceType.SERIES_SCAN); cache.clear(); bloomFilterCache.clear(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java index 16927deca2534..2decb73ab6749 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -125,8 +125,7 @@ public void testScanReadsWideAlignedDevice() throws Exception { .getQueryStatistics() .getLoadTimeSeriesMetadataFromCacheCount() .get()); - // one read serves a whole leaf group, so a cold scan issues far fewer metadata reads than - // the number of selected series + // per-series counters: a cold scan counts one disk lookup per selected measurement long diskReads = EnvironmentUtils.TEST_QUERY_FI_CONTEXT .getQueryStatistics() @@ -188,9 +187,10 @@ public void testSparseProjectionFiltersAbsentColumns() throws Exception { .getLoadTimeSeriesMetadataFromDiskCount() .get() - diskReadsBefore; - // at most one group read per present series plus the time column + // per-series counters: bloom-excluded measurements count as disk lookups, as they did on the + // per-column path Assert.assertTrue("disk lookups: " + diskReads, diskReads >= 1); - Assert.assertTrue("disk lookups: " + diskReads, diskReads <= PRESENT_COLUMN_COUNT + 1); + Assert.assertTrue("disk lookups: " + diskReads, diskReads <= COLUMN_COUNT + 1); } finally { EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); cleanup(file, resource, metadataCache, bloomFilterCache); From ddd2e3466e5066cbeb2ea3cb42860ff16bcb7bcc Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 14:45:11 +0800 Subject: [PATCH 23/27] Inline one-use metadata load helpers Keep only the helpers with several call sites or real logic: inline the time-column and bloom predicates, rebuild the probe order where it is used, and return the cache-hit count from loadLeafGroups instead of wrapping it in a one-field class. --- .../buffer/TimeSeriesMetadataCache.java | 62 ++++++------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index bd041aebda720..99f3087e7b37d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -257,7 +257,7 @@ public Map getTimeseriesMetadata( loadBloomFilter(filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, false); loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; } - if (timeColumnAbsent(requested, result, pending, bloomFilter, devicePrefix)) { + if (timeColumnAbsent(requested, result, bloomFilter, devicePrefix)) { fillMissingWithNull(requested, result); return result; } @@ -265,7 +265,7 @@ public Map getTimeseriesMetadata( TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); - LeafLoadStatistics loaded = + cacheHitCount += loadLeafGroups( reader, tsFileID, @@ -282,7 +282,6 @@ public Map getTimeseriesMetadata( debug, result, timeSeriesMetadataIoSizeRecorder); - cacheHitCount += loaded.cacheHits; // per-series counters: every requested measurement that was not resolved from the // cache is a disk lookup, including the ones the bloom filter excluded (as before) diskLookupCount = requested.size() - cacheHitCount; @@ -302,7 +301,7 @@ public Map getTimeseriesMetadata( filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, externalTsFile); loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; } - if (timeColumnAbsent(requested, result, pending, bloomFilter, devicePrefix)) { + if (timeColumnAbsent(requested, result, bloomFilter, devicePrefix)) { fillMissingWithNull(requested, result); return result; } @@ -310,7 +309,7 @@ public Map getTimeseriesMetadata( TsFileSequenceReader reader = FileReaderManager.getInstance() .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); - LeafLoadStatistics loaded = + cacheHitCount += loadLeafGroups( reader, tsFileID, @@ -327,7 +326,6 @@ public Map getTimeseriesMetadata( debug, result, timeSeriesMetadataIoSizeRecorder); - cacheHitCount += loaded.cacheHits; diskLookupCount = requested.size() - cacheHitCount; } return result; @@ -360,7 +358,7 @@ public Map getTimeseriesMetadata( * {@code readTimeseriesMetadata} contains every {@code allSensors} entry of the group, so all * pending measurements it covers are resolved without relying on the LRU retaining the group. */ - private LeafLoadStatistics loadLeafGroups( + private int loadLeafGroups( TsFileSequenceReader reader, TsFileID tsFileID, IDeviceID device, @@ -377,19 +375,23 @@ private LeafLoadStatistics loadLeafGroups( Map result, LongConsumer ioSizeRecorder) throws IOException { - LeafLoadStatistics statistics = new LeafLoadStatistics(); + int cacheHits = 0; // without a cache to warm, only the measurements this call needs are worth deserializing; // with the cache enabled the siblings keep the group warm for later single-column gets Set groupFilter = cacheEnabled ? allSensors : requested; // probe the time column first: when it is absent, no selected series of this device can be // read from this file, and the per-column path returned without touching any other column - if (whereFirst(pending, requested)) { - pending = withFirst(pending, ""); + if (requested.contains("") && pending.contains("")) { + Set ordered = new LinkedHashSet<>(pending.size()); + ordered.add(""); + ordered.addAll(pending); + pending = ordered; } while (!pending.isEmpty()) { - if (timeColumnResolvedToNull(requested, result)) { + // an empty time column means no selected series of this device is readable from this file + if (requested.contains("") && result.containsKey("") && result.get("") == null) { fillMissingWithNull(requested, result); - return statistics; + return cacheHits; } String probe = pending.iterator().next(); // another thread may have loaded this probe's group while we waited for the lock; checking @@ -400,7 +402,7 @@ private LeafLoadStatistics loadLeafGroups( : null; if (cachedProbe != null) { pending.remove(probe); - statistics.cacheHits++; + cacheHits++; if (cachedProbe == NULL_EXISTS_CACHE_PLACE_HOLDER) { if (debug) { DEBUG_LOGGER.info( @@ -424,7 +426,8 @@ private LeafLoadStatistics loadLeafGroups( } // bloom-probe only the probe: the group read resolves the other selected measurements of // that group, so probing each of them up front would add a per-column cost to every call - if (bloomFilter != null && !bloomFilterMayContain(bloomFilter, devicePrefix, probe)) { + // a null filter cannot rule the measurement out + if (bloomFilter != null && !bloomFilter.contains(devicePrefix + probe)) { pending.remove(probe); if (requested.contains(probe)) { if (debug) { @@ -482,11 +485,7 @@ private LeafLoadStatistics loadLeafGroups( for (String measurement : requested) { result.putIfAbsent(measurement, null); } - return statistics; - } - - private static final class LeafLoadStatistics { - private int cacheHits; + return cacheHits; } /** @@ -497,7 +496,6 @@ private static final class LeafLoadStatistics { private static boolean timeColumnAbsent( Set requested, Map result, - Set pending, BloomFilter bloomFilter, String devicePrefix) { if (!requested.contains("")) { @@ -506,29 +504,13 @@ private static boolean timeColumnAbsent( if (result.containsKey("")) { return result.get("") == null; } - if (bloomFilter != null && !bloomFilterMayContain(bloomFilter, devicePrefix, "")) { + if (bloomFilter != null && !bloomFilter.contains(devicePrefix)) { result.put("", null); return true; } return false; } - private static boolean timeColumnResolvedToNull( - Set requested, Map result) { - return requested.contains("") && result.containsKey("") && result.get("") == null; - } - - private static boolean whereFirst(Set pending, Set requested) { - return requested.contains("") && pending.contains(""); - } - - private static Set withFirst(Set pending, String first) { - Set ordered = new LinkedHashSet<>(pending.size()); - ordered.add(first); - ordered.addAll(pending); - return ordered; - } - private static void fillMissingWithNull( Set requested, Map result) { for (String measurement : requested) { @@ -566,12 +548,6 @@ private BloomFilter loadBloomFilter( externalTsFile); } - /** A null filter cannot rule the measurement out. */ - private static boolean bloomFilterMayContain( - BloomFilter bloomFilter, String devicePrefix, String measurement) { - return bloomFilter == null || bloomFilter.contains(devicePrefix + measurement); - } - private static void recordLoadStatistics( QueryContext queryContext, long cacheHitCount, From f10e3327909a0a6e8050945503cfd4223dff16ab Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 16:45:42 +0800 Subject: [PATCH 24/27] Use the selective tsfile metadata reader Replace the per-probe leaf-group loop with one call to TsFileSequenceReader.readTimeseriesMetadata(device, offset, measurements, returnedMeasurements, ...), which resolves the pending measurements against the leaf groups that hold them, reads each visited group once, and also returns the group siblings a single-column call needs for cache warming. The cache keeps the cache pass, the bloom-filter device gate, the negative placeholder and the per-series counters; the probe loop, probeFound, the group filter and the per-column bloom probes are gone. The aligned loader now always requests the time column. This needs the tsfile change that adds the set-based overload; the tsfile dependency must be bumped to a version containing it before this branch can be pushed. --- .../operator/source/FileLoaderUtils.java | 6 + .../buffer/TimeSeriesMetadataCache.java | 340 ++++++------------ .../buffer/TimeSeriesMetadataCacheTest.java | 67 ++-- 3 files changed, 156 insertions(+), 257 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index d5c592cdde505..86e81caea1863 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -59,6 +59,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -344,6 +345,11 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr // group that holds one of them once and answers the whole group from that read, so a wide // projection does not walk the device metadata index once per column Set allSensors = alignedPath.getAllSensors(); + if (!allSensors.contains("")) { + // the aligned loader always needs the time column, even if the caller's set omits it + allSensors = new HashSet<>(allSensors); + allSensors.add(""); + } // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex // we should not ignore the non-exist of device in TsFileMetadata diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 99f3087e7b37d..958880cf8525e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -51,8 +51,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; +import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -227,9 +226,7 @@ public Map getTimeseriesMetadata( String devicePrefix = device.toString() + IoTDBConstant.PATH_SEPARATOR; Map result = new HashMap<>(requested.size()); - // ordered set: the first element is the probe of the next leaf group, and resolving a whole - // group removes up to max_degree_of_index_node members in O(1) each - Set pending = new LinkedHashSet<>(requested.size()); + Set pending = new HashSet<>(requested.size()); try { if (CACHE_ENABLE && !externalTsFile) { @@ -239,10 +236,10 @@ public Map getTimeseriesMetadata( cachePhaseEndTime = System.nanoTime(); return result; } + cachePhaseEndTime = System.nanoTime(); // serialize reads of the same device+file (same lock key as get) so threads don't each // walk the device metadata index and race on the shared reader synchronized (deviceLock(device.toString(), filePath)) { - cachePhaseEndTime = System.nanoTime(); if (debug) { for (String measurement : pending) { DEBUG_LOGGER.info( @@ -250,84 +247,59 @@ public Map getTimeseriesMetadata( } DEBUG_LOGGER.info(StorageEngineMessages.DEVICE_ALL_SENSORS, device, allSensors); } - BloomFilter bloomFilter = null; - if (deviceMetadataIndexNodeOffset == null) { - long bloomFilterStartTime = System.nanoTime(); - bloomFilter = - loadBloomFilter(filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, false); - loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; - } - if (timeColumnAbsent(requested, result, bloomFilter, devicePrefix)) { + BloomFilter bloomFilter = + loadBloomFilter(filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, false); + if (deviceAbsent(requested, result, bloomFilter, devicePrefix)) { fillMissingWithNull(requested, result); return result; } - if (!pending.isEmpty()) { - TsFileSequenceReader reader = - FileReaderManager.getInstance() - .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); - cacheHitCount += - loadLeafGroups( - reader, - tsFileID, - device, - deviceMetadataIndexNodeOffset, - filePath, - bloomFilter, - devicePrefix, - pending, - requested, - allSensors, - ignoreNotExists, - true, - debug, - result, - timeSeriesMetadataIoSizeRecorder); - // per-series counters: every requested measurement that was not resolved from the - // cache is a disk lookup, including the ones the bloom filter excluded (as before) - diskLookupCount = requested.size() - cacheHitCount; - } + diskLookupCount = requested.size() - cacheHitCount; + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); + readAndMerge( + reader, + tsFileID, + device, + deviceMetadataIndexNodeOffset, + pending, + requested, + allSensors, + ignoreNotExists, + true, + debug, + result, + timeSeriesMetadataIoSizeRecorder); return result; } } - // cache disabled or external tsfiles: nothing was resolved from the cache, so every - // requested measurement is still pending + // cache disabled or external tsfiles: nothing was resolved from the cache pending.addAll(requested); - BloomFilter bloomFilter = null; - if (deviceMetadataIndexNodeOffset == null) { - long bloomFilterStartTime = System.nanoTime(); - bloomFilter = - loadBloomFilter( - filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, externalTsFile); - loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; - } - if (timeColumnAbsent(requested, result, bloomFilter, devicePrefix)) { + BloomFilter bloomFilter = + loadBloomFilter( + filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder, externalTsFile); + if (deviceAbsent(requested, result, bloomFilter, devicePrefix)) { fillMissingWithNull(requested, result); return result; } - if (!pending.isEmpty()) { - TsFileSequenceReader reader = - FileReaderManager.getInstance() - .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); - cacheHitCount += - loadLeafGroups( - reader, - tsFileID, - device, - deviceMetadataIndexNodeOffset, - filePath, - bloomFilter, - devicePrefix, - pending, - requested, - allSensors, - ignoreNotExists, - false, - debug, - result, - timeSeriesMetadataIoSizeRecorder); - diskLookupCount = requested.size() - cacheHitCount; - } + diskLookupCount = requested.size() - cacheHitCount; + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); + readAndMerge( + reader, + tsFileID, + device, + deviceMetadataIndexNodeOffset, + pending, + requested, + allSensors, + ignoreNotExists, + false, + debug, + result, + timeSeriesMetadataIoSizeRecorder); return result; } finally { long endTime = System.nanoTime(); @@ -353,19 +325,54 @@ public Map getTimeseriesMetadata( } } + private BloomFilter loadBloomFilter( + String filePath, + TsFileID tsFileID, + QueryContext queryContext, + LongConsumer bloomFilterIoSizeRecorder, + boolean externalTsFile) + throws IOException { + return BloomFilterCache.getInstance() + .get( + new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID), + false, + bloomFilterIoSizeRecorder, + queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet, + queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet, + externalTsFile); + } + + /** + * An aligned device always has its time column. When the cache or the bloom filter proves it + * absent, no selected series of the device can be read from this file. For a single requested + * series, the bloom filter can rule that series out directly. + */ + private static boolean deviceAbsent( + Set requested, + Map result, + BloomFilter bloomFilter, + String devicePrefix) { + if (requested.contains("")) { + if (result.containsKey("")) { + return result.get("") == null; + } + return bloomFilter != null && !bloomFilter.contains(devicePrefix); + } + if (bloomFilter != null && requested.size() == 1) { + return !bloomFilter.contains(devicePrefix + requested.iterator().next()); + } + return false; + } + /** - * Read leaf groups for the pending measurements, one group per iteration. The group returned by - * {@code readTimeseriesMetadata} contains every {@code allSensors} entry of the group, so all - * pending measurements it covers are resolved without relying on the LRU retaining the group. + * Reads the pending measurements from the leaf groups that hold them. When the cache is enabled + * the group siblings are returned as well, so a later single-column get() finds them in the LRU. */ - private int loadLeafGroups( + private void readAndMerge( TsFileSequenceReader reader, TsFileID tsFileID, IDeviceID device, long[] deviceMetadataIndexNodeOffset, - String filePath, - BloomFilter bloomFilter, - String devicePrefix, Set pending, Set requested, Set allSensors, @@ -375,140 +382,40 @@ private int loadLeafGroups( Map result, LongConsumer ioSizeRecorder) throws IOException { - int cacheHits = 0; - // without a cache to warm, only the measurements this call needs are worth deserializing; - // with the cache enabled the siblings keep the group warm for later single-column gets - Set groupFilter = cacheEnabled ? allSensors : requested; - // probe the time column first: when it is absent, no selected series of this device can be - // read from this file, and the per-column path returned without touching any other column - if (requested.contains("") && pending.contains("")) { - Set ordered = new LinkedHashSet<>(pending.size()); - ordered.add(""); - ordered.addAll(pending); - pending = ordered; - } - while (!pending.isEmpty()) { - // an empty time column means no selected series of this device is readable from this file - if (requested.contains("") && result.containsKey("") && result.get("") == null) { - fillMissingWithNull(requested, result); - return cacheHits; - } - String probe = pending.iterator().next(); - // another thread may have loaded this probe's group while we waited for the lock; checking - // only the probe keeps the cold path free of a per-column cache pass - TimeseriesMetadata cachedProbe = - cacheEnabled - ? lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID, device, probe)) - : null; - if (cachedProbe != null) { - pending.remove(probe); - cacheHits++; - if (cachedProbe == NULL_EXISTS_CACHE_PLACE_HOLDER) { - if (debug) { - DEBUG_LOGGER.info( - StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, - new TimeSeriesMetadataCacheKey(tsFileID, device, probe)); - } - result.put(probe, null); - } else { - if (debug) { - DEBUG_LOGGER.info( - StorageEngineMessages - .STORAGE_LOG_GET_TIMESERIES_METADATA_IN_FILE_FROM_CACHE_36652729, - device, - probe, - filePath, - cachedProbe); - } - result.put(probe, new TimeseriesMetadata(cachedProbe)); + // cached measurements are already in the result: no need to deserialize them again + Set returned = new HashSet<>(cacheEnabled ? allSensors : requested); + returned.removeAll(result.keySet()); + Map loaded = + reader.readTimeseriesMetadata( + device, + deviceMetadataIndexNodeOffset, + pending, + returned, + ignoreNotExists, + ioSizeRecorder); + for (Map.Entry entry : loaded.entrySet()) { + String measurement = entry.getKey(); + TimeseriesMetadata metadata = entry.getValue(); + if (metadata.getStatistics().getCount() == 0) { + if (debug) { + DEBUG_LOGGER.info( + StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement)); } - continue; - } - // bloom-probe only the probe: the group read resolves the other selected measurements of - // that group, so probing each of them up front would add a per-column cost to every call - // a null filter cannot rule the measurement out - if (bloomFilter != null && !bloomFilter.contains(devicePrefix + probe)) { - pending.remove(probe); - if (requested.contains(probe)) { - if (debug) { - DEBUG_LOGGER.info( - StorageEngineMessages.TS_METADATA_FILTERED_BY_BLOOM_FILTER, - new TimeSeriesMetadataCacheKey(tsFileID, device, probe)); - } - result.put(probe, null); + if (requested.contains(measurement)) { + result.put(measurement, null); } - continue; - } - List group = - reader.readTimeseriesMetadata( - device, - deviceMetadataIndexNodeOffset, - probe, - groupFilter, - ignoreNotExists, - ioSizeRecorder); - boolean probeFound = false; - for (TimeseriesMetadata metadata : group) { - String measurementId = metadata.getMeasurementId(); - pending.remove(measurementId); - probeFound = probeFound || measurementId.equals(probe); - if (metadata.getStatistics().getCount() != 0) { - if (cacheEnabled) { - lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId), metadata); - } - if (requested.contains(measurementId)) { - result.put(measurementId, new TimeseriesMetadata(metadata)); - } - } else if (requested.contains(measurementId)) { - if (debug) { - DEBUG_LOGGER.info( - StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, - new TimeSeriesMetadataCacheKey(tsFileID, device, measurementId)); - } - result.put(measurementId, null); - cacheNonExistenceIfNeeded(tsFileID, device, measurementId, cacheEnabled, ignoreNotExists); + cacheNonExistenceIfNeeded(tsFileID, device, measurement, cacheEnabled, ignoreNotExists); + } else { + if (cacheEnabled) { + lruCache.put(new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), metadata); } - } - if (!probeFound) { - pending.remove(probe); - if (requested.contains(probe)) { - if (debug) { - DEBUG_LOGGER.info( - StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, - new TimeSeriesMetadataCacheKey(tsFileID, device, probe)); - } - result.put(probe, null); - cacheNonExistenceIfNeeded(tsFileID, device, probe, cacheEnabled, ignoreNotExists); + if (requested.contains(measurement)) { + result.put(measurement, new TimeseriesMetadata(metadata)); } } } - for (String measurement : requested) { - result.putIfAbsent(measurement, null); - } - return cacheHits; - } - - /** - * An aligned device always has its time column. When the requested set contains it and the cache - * or the bloom filter proves it absent, every selected series of the device is absent from this - * file, so the per-column loader returned null without reading any other column. - */ - private static boolean timeColumnAbsent( - Set requested, - Map result, - BloomFilter bloomFilter, - String devicePrefix) { - if (!requested.contains("")) { - return false; - } - if (result.containsKey("")) { - return result.get("") == null; - } - if (bloomFilter != null && !bloomFilter.contains(devicePrefix)) { - result.put("", null); - return true; - } - return false; + fillMissingWithNull(requested, result); } private static void fillMissingWithNull( @@ -531,23 +438,6 @@ private void cacheNonExistenceIfNeeded( } } - private BloomFilter loadBloomFilter( - String filePath, - TsFileID tsFileID, - QueryContext queryContext, - LongConsumer bloomFilterIoSizeRecorder, - boolean externalTsFile) - throws IOException { - return BloomFilterCache.getInstance() - .get( - new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID), - false, - bloomFilterIoSizeRecorder, - queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet, - queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet, - externalTsFile); - } - private static void recordLoadStatistics( QueryContext queryContext, long cacheHitCount, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 024492b5bf52b..db2462bd87de8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -260,17 +260,17 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { reader.readTimeseriesMetadata( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.eq(measurement), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(true), Mockito.any(LongConsumer.class))) - .thenReturn(Collections.emptyList()); + .thenReturn(Collections.emptyMap()); Mockito.when( reader.readTimeseriesMetadata( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.eq(measurement), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(false), Mockito.any(LongConsumer.class))) .thenThrow(new IOException("missing device")); @@ -283,8 +283,8 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { .readTimeseriesMetadata( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.eq(measurement), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(false), Mockito.any(LongConsumer.class)); } finally { @@ -320,19 +320,22 @@ private static Map getTimeseriesMetadata( private static void stubGroupRead( TsFileSequenceReader reader, IDeviceID deviceID, - Set allSensors, boolean ignoreNotExists, List group) throws IOException { + Map loaded = new HashMap<>(); + for (TimeseriesMetadata metadata : group) { + loaded.put(metadata.getMeasurementId(), metadata); + } Mockito.when( reader.readTimeseriesMetadata( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.anyString(), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(ignoreNotExists), Mockito.any(LongConsumer.class))) - .thenReturn(group); + .thenReturn(loaded); } @Test @@ -383,7 +386,7 @@ public void testGetTimeseriesMetadata() throws Exception { TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - stubGroupRead(reader, deviceID, allSensors, false, group); + stubGroupRead(reader, deviceID, false, group); Map result = getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); @@ -405,8 +408,8 @@ public void testGetTimeseriesMetadata() throws Exception { .readTimeseriesMetadata( Mockito.any(), Mockito.any(), - Mockito.anyString(), - Mockito.any(), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.anyBoolean(), Mockito.any(LongConsumer.class)); @@ -467,8 +470,8 @@ public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception .readTimeseriesMetadata( Mockito.any(), Mockito.any(), - Mockito.anyString(), - Mockito.any(), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.anyBoolean(), Mockito.any(LongConsumer.class)); } finally { @@ -493,13 +496,13 @@ public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Except TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - stubGroupRead(reader, deviceID, allSensors, true, Collections.emptyList()); + stubGroupRead(reader, deviceID, true, Collections.emptyList()); Mockito.when( reader.readTimeseriesMetadata( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.anyString(), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(false), Mockito.any(LongConsumer.class))) .thenThrow(new IOException("device absent")); @@ -656,8 +659,8 @@ public void testBloomFilterExcludedMeasurementsSkipRead() throws Exception { .readTimeseriesMetadata( Mockito.any(), Mockito.any(), - Mockito.anyString(), - Mockito.any(), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.anyBoolean(), Mockito.any(LongConsumer.class)); } finally { @@ -706,7 +709,7 @@ public void testEvictedMeasurementReloadsOnlyItsGroup() throws Exception { TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - stubGroupRead(reader, deviceID, allSensors, false, group); + stubGroupRead(reader, deviceID, false, group); Map first = getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); @@ -723,8 +726,8 @@ public void testEvictedMeasurementReloadsOnlyItsGroup() throws Exception { .readTimeseriesMetadata( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.anyString(), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(false), Mockito.any(LongConsumer.class)); } finally { @@ -749,13 +752,13 @@ public void testBatchIgnoreNotExistsDoesNotCachePlaceholders() throws Exception TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - stubGroupRead(reader, deviceID, allSensors, true, Collections.emptyList()); + stubGroupRead(reader, deviceID, true, Collections.emptyList()); Mockito.when( reader.readTimeseriesMetadata( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.anyString(), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(false), Mockito.any(LongConsumer.class))) .thenThrow(new IOException("device absent")); @@ -775,8 +778,8 @@ public void testBatchIgnoreNotExistsDoesNotCachePlaceholders() throws Exception .readTimeseriesMetadata( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.anyString(), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.anyBoolean(), Mockito.any(LongConsumer.class)); } finally { @@ -813,7 +816,7 @@ public void testEvictedMeasurementsShareOneGroupRead() throws Exception { TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); fileReaderManager.setReaderForTest(tsFileID, true, reader); - stubGroupRead(reader, deviceID, allSensors, false, group); + stubGroupRead(reader, deviceID, false, group); Map first = getTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); @@ -833,8 +836,8 @@ public void testEvictedMeasurementsShareOneGroupRead() throws Exception { .readTimeseriesMetadata( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.anyString(), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(false), Mockito.any(LongConsumer.class)); } finally { From fe90930321e823b48a491a96e7a4299264ad1ea9 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 16:51:53 +0800 Subject: [PATCH 25/27] Depend on the tsfile snapshot that carries the selective reader The set-based readTimeseriesMetadata overload is not in the dated tsfile snapshot the build used before. Point the dependency at the 2.5.1 snapshot, which will contain it once apache/tsfile#1123 is merged and published. The IoTDB change that uses the new API must not be pushed until that snapshot exists in the configured repositories. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index abd01e78010fc..ce545c989735e 100644 --- a/pom.xml +++ b/pom.xml @@ -143,7 +143,7 @@ 0.25.0 1.9 1.5.6-3 - 2.5.1-261023-SNAPSHOT + 2.5.1-SNAPSHOT en From 913a4e8769ee02e0e2d9cc0c5eba4b747d0297f5 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 17:32:28 +0800 Subject: [PATCH 26/27] Pin the dated tsfile snapshot instead of the moving one The build must not resolve the shared 2.5.1-SNAPSHOT, whose content can change under it. Depend on the dated 2.5.1-261117-SNAPSHOT that carries the selective metadata reader, and publish that snapshot before pushing the branch so CI no longer resolves a shared moving artifact. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ce545c989735e..ee37426af6fe2 100644 --- a/pom.xml +++ b/pom.xml @@ -143,7 +143,7 @@ 0.25.0 1.9 1.5.6-3 - 2.5.1-SNAPSHOT + 2.5.1-261117-SNAPSHOT en From aa52c51607b15d3f04e4dc7ab0e8487872ebddda Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 18 Nov 2026 10:13:33 +0800 Subject: [PATCH 27/27] Cut cache-key and returned-set allocations on the batch path Cache the TimeSeriesMetadataCacheKey hash in the constructor instead of allocating the Objects.hash argument array on every lookup and write, and pass the pending set straight to the tsfile reader when the aligned batch path requests exactly the set it caches. --- .../buffer/TimeSeriesMetadataCache.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 958880cf8525e..d0a25b0541455 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -383,8 +383,14 @@ private void readAndMerge( LongConsumer ioSizeRecorder) throws IOException { // cached measurements are already in the result: no need to deserialize them again - Set returned = new HashSet<>(cacheEnabled ? allSensors : requested); - returned.removeAll(result.keySet()); + Set returned; + if (!cacheEnabled || allSensors.equals(requested)) { + // the aligned batch read requests exactly the set it caches, so only the misses remain + returned = pending; + } else { + returned = new HashSet<>(allSensors); + returned.removeAll(result.keySet()); + } Map loaded = reader.readTimeseriesMetadata( device, @@ -559,11 +565,15 @@ public static class TimeSeriesMetadataCacheKey { private final TsFileID tsFileID; private final IDeviceID device; private final String measurement; + private final int hash; public TimeSeriesMetadataCacheKey(TsFileID tsFileID, IDeviceID device, String measurement) { this.tsFileID = tsFileID; this.device = device; this.measurement = measurement; + // a key is built for every lookup and every write: compute the hash once instead of + // allocating the Objects.hash argument array on each call + this.hash = Objects.hash(tsFileID, device, measurement); } public long getRetainedSizeInBytes() { @@ -586,7 +596,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(tsFileID, device, measurement); + return hash; } @Override