Repository navigation
Batch scan TimeSeriesMetadata for wide-table queries #19198
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
0579b19
1be4e0c
758f4ae
2563c50
31ecea8
2cc9604
4bcb8b6
a6a5634
1199ab3
42749a4
98aba2c
2c9fbe4
3fba980
90a2cb0
caf6df1
e927507
f8b9c83
469394f
6ab4b8e
7f05320
7871781
aa8a964
991ddb5
ddd2e34
f10e332
fe90930
913a4e8
aa52c51
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -30,7 +30,6 @@ | |
| import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; | ||
| import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; | ||
| import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; | ||
| import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey; | ||
| import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; | ||
| import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskAlignedChunkLoader; | ||
| import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskChunkLoader; | ||
|
|
@@ -47,7 +46,6 @@ | |
| import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; | ||
| import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; | ||
| import org.apache.tsfile.file.metadata.IChunkMetadata; | ||
| import org.apache.tsfile.file.metadata.IDeviceID; | ||
| import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; | ||
| import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; | ||
| import org.apache.tsfile.file.metadata.TimeseriesMetadata; | ||
|
|
@@ -61,7 +59,9 @@ | |
| import java.io.IOException; | ||
| import java.util.ArrayList; | ||
| import java.util.Collections; | ||
| import java.util.HashSet; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Set; | ||
| import java.util.stream.Collectors; | ||
|
|
||
|
|
@@ -341,23 +341,35 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr | |
| // the order of timeSeriesMetadata list is same as subSensorList's order | ||
| TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); | ||
| List<String> valueMeasurementList = alignedPath.getMeasurementList(); | ||
| // resolves the time column and every selected sensor in one call: the cache reads each leaf | ||
| // group that holds one of them once and answers the whole group from that read, so a wide | ||
| // projection does not walk the device metadata index once per column | ||
| Set<String> allSensors = alignedPath.getAllSensors(); | ||
| boolean isDebug = context.isDebug(); | ||
| String filePath = resource.getTsFilePath(); | ||
| IDeviceID deviceId = alignedPath.getDeviceId(); | ||
| if (!allSensors.contains("")) { | ||
| // the aligned loader always needs the time column, even if the caller's set omits it | ||
| allSensors = new HashSet<>(allSensors); | ||
| allSensors.add(""); | ||
| } | ||
|
|
||
| // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex | ||
| // we should not ignore the non-exist of device in TsFileMetadata | ||
| TimeseriesMetadata timeColumn = | ||
| cache.get( | ||
| filePath, | ||
| new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, ""), | ||
| boolean ignoreNotExists = | ||
| context.ignoreNotExistsDevice() | ||
| || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE; | ||
|
|
||
| Map<String, TimeseriesMetadata> metadata = | ||
| cache.getTimeseriesMetadata( | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P2] Dense cold loads are still 15–32% slower than base For a 700-field device, select-all cold measured 6.6–7.5 ms against base's 5.7 ms, and the first-60% projection 4.0–4.2 ms against 3.5–3.8 ms. Bytes are equal, and this matches the 17–37% in the description. The extra work appears to be per call, repeated for every device and file:
The selected set is identical for every device and file of a scan. Sorting it once per scan and passing the sorted list through (which needs the tsfile API to accept a pre-sorted list) should remove most of the repeated cost. A profile would confirm which part dominates. |
||
| resource.getTsFilePath(), | ||
| resource.getTsFileID(), | ||
| alignedPath.getDeviceId(), | ||
| allSensors, | ||
| allSensors, | ||
| context.ignoreNotExistsDevice() | ||
| || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, | ||
| isDebug, | ||
| ignoreNotExists, | ||
| context.isDebug(), | ||
| context, | ||
| rootMeasurementMetadataIndexNodeOffset); | ||
|
|
||
| TimeseriesMetadata timeColumn = metadata.get(""); | ||
| if (timeColumn != null) { | ||
| // only need time column, like count_time aggregation | ||
| if (valueMeasurementList.isEmpty()) { | ||
|
|
@@ -375,38 +387,49 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr | |
| } else { | ||
| List<TimeseriesMetadata> valueTimeSeriesMetadataList = | ||
| new ArrayList<>(valueMeasurementList.size()); | ||
| // if all the queried aligned sensors does not exist, we will return null | ||
| boolean exist = false; | ||
| for (String measurement : valueMeasurementList) { | ||
| TimeseriesMetadata valueColumn = | ||
| cache.get( | ||
| filePath, | ||
| new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement), | ||
| allSensors, | ||
| context.ignoreNotExistsDevice() | ||
| || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, | ||
| isDebug, | ||
| context, | ||
| rootMeasurementMetadataIndexNodeOffset); | ||
| exist = (exist || (valueColumn != null)); | ||
| valueTimeSeriesMetadataList.add(valueColumn); | ||
| } | ||
| if (!ignoreAllNullRows || exist) { | ||
| alignedTimeSeriesMetadata = | ||
| setModifications( | ||
| resource, | ||
| timeColumn, | ||
| valueTimeSeriesMetadataList, | ||
| alignedPath, | ||
| context, | ||
| globalTimeFilter, | ||
| ignoreAllNullRows); | ||
| valueTimeSeriesMetadataList.add(metadata.get(measurement)); | ||
| } | ||
| alignedTimeSeriesMetadata = | ||
| buildAlignedTimeSeriesMetadata( | ||
| resource, | ||
| alignedPath, | ||
| timeColumn, | ||
| valueTimeSeriesMetadataList, | ||
| context, | ||
| globalTimeFilter, | ||
| ignoreAllNullRows); | ||
| } | ||
| } | ||
| return alignedTimeSeriesMetadata; | ||
| } | ||
|
|
||
| private static AbstractAlignedTimeSeriesMetadata buildAlignedTimeSeriesMetadata( | ||
| TsFileResource resource, | ||
| AlignedFullPath alignedPath, | ||
| TimeseriesMetadata timeColumn, | ||
| List<TimeseriesMetadata> valueTimeSeriesMetadataList, | ||
| FragmentInstanceContext context, | ||
| Filter globalTimeFilter, | ||
| boolean ignoreAllNullRows) { | ||
| // if all the queried aligned sensors does not exist, we will return null | ||
| boolean exist = false; | ||
| for (TimeseriesMetadata valueColumn : valueTimeSeriesMetadataList) { | ||
| exist = (exist || (valueColumn != null)); | ||
| } | ||
| if (!ignoreAllNullRows || exist) { | ||
| return setModifications( | ||
| resource, | ||
| timeColumn, | ||
| valueTimeSeriesMetadataList, | ||
| alignedPath, | ||
| context, | ||
| globalTimeFilter, | ||
| ignoreAllNullRows); | ||
| } | ||
| return null; | ||
| } | ||
|
|
||
| private static AbstractAlignedTimeSeriesMetadata setModifications( | ||
| TsFileResource resource, | ||
| TimeseriesMetadata timeColumnMetadata, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
allSensors will never contain "", so just allSensors.add("") is enough.