Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
0579b19
Batch scan TimeSeriesMetadata for wide-table queries
ColinLeeo Nov 10, 2026
1be4e0c
Fix NPE in batch TimeSeriesMetadata read by loading chunk metadata
ColinLeeo Nov 10, 2026
758f4ae
Clean up batch TimeSeriesMetadata scan path
ColinLeeo Nov 10, 2026
2563c50
Expand batch TimeSeriesMetadata scan to device-level time index files
ColinLeeo Nov 10, 2026
31ecea8
Hardcode device metadata batch scan ratio to 0.6
ColinLeeo Nov 10, 2026
2cc9604
Synchronize batch metadata read and include device in corruption error
ColinLeeo Nov 11, 2026
4bcb8b6
Fix batch scan ratio denominator and cache empty series
ColinLeeo Nov 11, 2026
a6a5634
Add unit test for batch scan absent-device throw logic
ColinLeeo Nov 11, 2026
1199ab3
Add end-to-end test for wide-table batch metadata scan
ColinLeeo Nov 11, 2026
42749a4
Don't use batch metadata scan for single-column devices
ColinLeeo Nov 11, 2026
98aba2c
Reuse the metadata cache for batch device scans
ColinLeeo Nov 12, 2026
2c9fbe4
Keep DEBUG queries on the per-column metadata path
ColinLeeo Nov 12, 2026
3fba980
Check the bloom filter before the batch device read
ColinLeeo Nov 12, 2026
90a2cb0
Address wide-scan batch review: partial cache reuse and diagnostics
ColinLeeo Nov 13, 2026
caf6df1
Refine the batch metadata path after review
ColinLeeo Nov 13, 2026
e927507
Document the selective-read limitation of the batch path
ColinLeeo Nov 13, 2026
f8b9c83
Address the follow-up review of the wide-scan batch path
ColinLeeo Nov 16, 2026
469394f
Account the pre-fallback phase as disk time on a cold batch miss
ColinLeeo Nov 16, 2026
6ab4b8e
Merge branch 'master' of github.com:apache/iotdb into review/pr-19198
ColinLeeo Nov 16, 2026
7f05320
Refactor wide-device metadata loading into a set-based cache API
ColinLeeo Nov 17, 2026
7871781
Skip a device once its time column is proven absent
ColinLeeo Nov 17, 2026
aa8a964
Seed pending measurements in the cache-disabled metadata path
ColinLeeo Nov 17, 2026
991ddb5
Restore per-series load counters and trim disabled-cache reads
ColinLeeo Nov 17, 2026
ddd2e34
Inline one-use metadata load helpers
ColinLeeo Nov 17, 2026
f10e332
Use the selective tsfile metadata reader
ColinLeeo Nov 17, 2026
fe90930
Depend on the tsfile snapshot that carries the selective reader
ColinLeeo Nov 17, 2026
913a4e8
Pin the dated tsfile snapshot instead of the moving one
ColinLeeo Nov 17, 2026
aa52c51
Cut cache-key and returned-set allocations on the batch path
ColinLeeo Nov 18, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskAlignedChunkLoader;
import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskChunkLoader;
Expand All @@ -47,7 +46,6 @@
import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.ITimeSeriesMetadata;
import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.TimeseriesMetadata;
Expand All @@ -61,7 +59,9 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -341,23 +341,35 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr
// the order of timeSeriesMetadata list is same as subSensorList's order
TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
List<String> valueMeasurementList = alignedPath.getMeasurementList();
// resolves the time column and every selected sensor in one call: the cache reads each leaf
// group that holds one of them once and answers the whole group from that read, so a wide
// projection does not walk the device metadata index once per column
Set<String> allSensors = alignedPath.getAllSensors();
boolean isDebug = context.isDebug();
String filePath = resource.getTsFilePath();
IDeviceID deviceId = alignedPath.getDeviceId();
if (!allSensors.contains("")) {
// the aligned loader always needs the time column, even if the caller's set omits it
allSensors = new HashSet<>(allSensors);
allSensors.add("");
}
Comment on lines +348 to +352

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

allSensors will never contain "", so just allSensors.add("") is enough.


// when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
// we should not ignore the non-exist of device in TsFileMetadata
TimeseriesMetadata timeColumn =
cache.get(
filePath,
new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, ""),
boolean ignoreNotExists =
context.ignoreNotExistsDevice()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE;

Map<String, TimeseriesMetadata> metadata =
cache.getTimeseriesMetadata(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Dense cold loads are still 15–32% slower than base

For a 700-field device, select-all cold measured 6.6–7.5 ms against base's 5.7 ms, and the first-60% projection 4.0–4.2 ms against 3.5–3.8 ms. Bytes are equal, and this matches the 17–37% in the description.

The extra work appears to be per call, repeated for every device and file:

  • building the pending and result sets for 701 names;
  • the reader turning measurements into a TreeSet and back into a list, i.e. sorting 701 strings;
  • getChildIndexEntries over that list.

The selected set is identical for every device and file of a scan. Sorting it once per scan and passing the sorted list through (which needs the tsfile API to accept a pre-sorted list) should remove most of the repeated cost. A profile would confirm which part dominates.

resource.getTsFilePath(),
resource.getTsFileID(),
alignedPath.getDeviceId(),
allSensors,
allSensors,
context.ignoreNotExistsDevice()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
isDebug,
ignoreNotExists,
context.isDebug(),
context,
rootMeasurementMetadataIndexNodeOffset);

TimeseriesMetadata timeColumn = metadata.get("");
if (timeColumn != null) {
// only need time column, like count_time aggregation
if (valueMeasurementList.isEmpty()) {
Expand All @@ -375,38 +387,49 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr
} else {
List<TimeseriesMetadata> valueTimeSeriesMetadataList =
new ArrayList<>(valueMeasurementList.size());
// if all the queried aligned sensors does not exist, we will return null
boolean exist = false;
for (String measurement : valueMeasurementList) {
TimeseriesMetadata valueColumn =
cache.get(
filePath,
new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement),
allSensors,
context.ignoreNotExistsDevice()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
isDebug,
context,
rootMeasurementMetadataIndexNodeOffset);
exist = (exist || (valueColumn != null));
valueTimeSeriesMetadataList.add(valueColumn);
}
if (!ignoreAllNullRows || exist) {
alignedTimeSeriesMetadata =
setModifications(
resource,
timeColumn,
valueTimeSeriesMetadataList,
alignedPath,
context,
globalTimeFilter,
ignoreAllNullRows);
valueTimeSeriesMetadataList.add(metadata.get(measurement));
}
alignedTimeSeriesMetadata =
buildAlignedTimeSeriesMetadata(
resource,
alignedPath,
timeColumn,
valueTimeSeriesMetadataList,
context,
globalTimeFilter,
ignoreAllNullRows);
}
}
return alignedTimeSeriesMetadata;
}

private static AbstractAlignedTimeSeriesMetadata buildAlignedTimeSeriesMetadata(
TsFileResource resource,
AlignedFullPath alignedPath,
TimeseriesMetadata timeColumn,
List<TimeseriesMetadata> valueTimeSeriesMetadataList,
FragmentInstanceContext context,
Filter globalTimeFilter,
boolean ignoreAllNullRows) {
// if all the queried aligned sensors does not exist, we will return null
boolean exist = false;
for (TimeseriesMetadata valueColumn : valueTimeSeriesMetadataList) {
exist = (exist || (valueColumn != null));
}
if (!ignoreAllNullRows || exist) {
return setModifications(
resource,
timeColumn,
valueTimeSeriesMetadataList,
alignedPath,
context,
globalTimeFilter,
ignoreAllNullRows);
}
return null;
}

private static AbstractAlignedTimeSeriesMetadata setModifications(
TsFileResource resource,
TimeseriesMetadata timeColumnMetadata,
Expand Down
Loading
Loading