Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@

import org.apache.iotdb.db.i18n.DataNodePipeMessages;
import org.apache.iotdb.db.utils.BitMapUtils;
import org.apache.iotdb.db.utils.TypeServices;

import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.Field;
import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
Expand Down Expand Up @@ -171,6 +173,25 @@ public static void putValue(
unmarkNullValue(tablet, rowIndex, columnIndex);
}

/** Copy a query field without boxing primitive values on the way to the tablet. */
public static void putFieldValue(
final Tablet tablet,
final int rowIndex,
final int columnIndex,
final TSDataType dataType,
final Field field) {
if (field.getDataType() == null) {
putValue(tablet, rowIndex, columnIndex, dataType, null);
return;
}

TypeServices.Pipe.PIPE_FIELD_TABLET_VALUE_WRITER_SERVICE
.call(Type.fromTsDataType(dataType))
.write(field, tablet.getValues()[columnIndex], rowIndex);
// A row discarded after applying mods may have marked this slot before it is reused.
unmarkNullValue(tablet, rowIndex, columnIndex);
}

public static void unmarkNullValue(
final Tablet tablet, final int rowIndex, final int columnIndex) {
final BitMap[] bitMaps = tablet.getBitMaps();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,6 @@ private Tablet buildNextTablet() throws IOException {
final int fieldSize = fields.size();
for (int i = 0; i < fieldSize; i++) {
final Field field = fields.get(i);
final String measurement = measurements.get(i);
final TSDataType dataType = schemas.get(i).getType();
// Check if this value is deleted by mods
if (field == null
Expand All @@ -194,8 +193,7 @@ private Tablet buildNextTablet() throws IOException {
}
PipeTabletUtils.markNullValue(tablet, rowIndex, i);
} else {
PipeTabletUtils.putValue(
tablet, rowIndex, i, dataType, field.getObjectValue(schemas.get(i).getType()));
PipeTabletUtils.putFieldValue(tablet, rowIndex, i, dataType, field);
isNeedFillTime = true;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,10 @@
import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.PublicBAOS;
import org.apache.tsfile.utils.RamUsageEstimator;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.write.record.Tablet;

import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
Expand Down Expand Up @@ -200,14 +198,7 @@
}
}
for (final Pair<Boolean, Tablet> tabletPair : batchTablets) {
try (final PublicBAOS byteArrayOutputStream =
new PublicBAOS(calculateTabletSerializedSize(tabletPair.getRight()));
final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
tabletPair.getRight().serialize(outputStream);
ReadWriteIOUtils.write(true, outputStream);
tabletBuffers.add(
ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()));
}
tabletBuffers.add(serializeTabletWithAlignment(tabletPair.getRight(), true));
tabletDataBases.add(databaseName);
}
}
Expand All @@ -227,7 +218,7 @@
return pipe2BytesAccumulated;
}

private long buildTabletInsertionBuffer(final TabletInsertionEvent event) throws IOException {

Check warning on line 221 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Remove the declaration of thrown exception 'java.io.IOException', as it cannot be thrown from method's body.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHs6ca-p1qD7ThqXF9M&open=AaHs6ca-p1qD7ThqXF9M&pullRequest=19282
long estimateSize = 0;
final ByteBuffer buffer;
if (event instanceof PipeInsertNodeTabletInsertionEvent) {
Expand Down Expand Up @@ -268,13 +259,7 @@
pipeRawTabletInsertionEvent.getTableModelDatabaseName());
} else {
final Tablet tablet = pipeRawTabletInsertionEvent.convertToTablet();
try (final PublicBAOS byteArrayOutputStream =
new PublicBAOS(calculateTabletSerializedSize(tablet));
final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
tablet.serialize(outputStream);
ReadWriteIOUtils.write(pipeRawTabletInsertionEvent.isAligned(), outputStream);
buffer = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
}
buffer = serializeTabletWithAlignment(tablet, pipeRawTabletInsertionEvent.isAligned());
final String databaseName = pipeRawTabletInsertionEvent.getTreeModelDatabaseName();
estimateSize = RamUsageEstimator.sizeOf(databaseName) + buffer.limit();
increaseTotalBufferSizeAndUpdateMemoryBlock(estimateSize);
Expand Down Expand Up @@ -307,8 +292,14 @@
return PipeMemoryWeightUtil.calculateTabletSizeInBytes(tablet) + 4;
}

private static int calculateTabletSerializedSize(final Tablet tablet) {
return tablet.serializedSize() + Byte.BYTES;
private static ByteBuffer serializeTabletWithAlignment(
final Tablet tablet, final boolean isAligned) {
final ByteBuffer buffer =
ByteBuffer.allocate(Math.addExact(tablet.serializedSize(), Byte.BYTES));
tablet.serialize(buffer);
ReadWriteIOUtils.write(isAligned, buffer);
buffer.flip();
return buffer;
}

static boolean mayAppendTablet(final Tablet target, final Tablet source) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -272,15 +272,12 @@ public byte[] toTPipeTransferBytes() throws IOException {
throw new IOException(DataNodePipeMessages.CANNOT_SERIALIZE_BOTH_TABLET_AND_STATEMENT_ARE);
}

try (final PublicBAOS byteArrayOutputStream =
new PublicBAOS(calculateAirGapSerializedSize(tabletToSerialize));
final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), outputStream);
ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW.getType(), outputStream);
tabletToSerialize.serialize(outputStream);
ReadWriteIOUtils.write(isAlignedToSerialize, outputStream);
return byteArrayOutputStream.toByteArray();
}
final ByteBuffer buffer = ByteBuffer.allocate(calculateAirGapSerializedSize(tabletToSerialize));
ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), buffer);
ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW.getType(), buffer);
tabletToSerialize.serialize(buffer);
ReadWriteIOUtils.write(isAlignedToSerialize, buffer);
return buffer.array();
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -173,17 +173,15 @@
/////////////////////////////// Air Gap ///////////////////////////////

public static byte[] toTPipeTransferBytes(
final Tablet tablet, final boolean isAligned, final String dataBaseName) throws IOException {

Check warning on line 176 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Remove the declaration of thrown exception 'java.io.IOException', as it cannot be thrown from method's body.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHtpkqXGcGtb3bTOliM&open=AaHtpkqXGcGtb3bTOliM&pullRequest=19282
try (final PublicBAOS byteArrayOutputStream =
new PublicBAOS(calculateAirGapSerializedSize(tablet, dataBaseName));
final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), outputStream);
ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), outputStream);
tablet.serialize(outputStream);
ReadWriteIOUtils.write(isAligned, outputStream);
ReadWriteIOUtils.write(dataBaseName, outputStream);
return byteArrayOutputStream.toByteArray();
}
final ByteBuffer buffer =
ByteBuffer.allocate(calculateAirGapSerializedSize(tablet, dataBaseName));
ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), buffer);
ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), buffer);
tablet.serialize(buffer);
ReadWriteIOUtils.write(isAligned, buffer);
ReadWriteIOUtils.write(dataBaseName, buffer);
return buffer.array();
}

static int calculateSerializedSize(final Tablet tablet, final String dataBaseName) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@
import org.apache.tsfile.file.metadata.statistics.DateStatistics;
import org.apache.tsfile.file.metadata.statistics.Statistics;
import org.apache.tsfile.read.common.BatchData;
import org.apache.tsfile.read.common.Field;
import org.apache.tsfile.read.common.block.TsBlockBuilder;
import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder;
import org.apache.tsfile.read.common.type.Type;
Expand Down Expand Up @@ -3957,6 +3958,34 @@
};
};

public static final TypeService<FieldTabletValueWriter> PIPE_FIELD_TABLET_VALUE_WRITER_SERVICE =
type ->
switch (type.getTypeEnum()) {
case BOOLEAN ->

Check warning on line 3964 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeServices.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

'case' child has incorrect indentation level 14, expected level should be 10.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHtggN_Ly9860H4m2Td&open=AaHtggN_Ly9860H4m2Td&pullRequest=19282
(field, column, index) -> ((boolean[]) column)[index] = field.getBoolV();
case INT32 -> (field, column, index) -> ((int[]) column)[index] = field.getIntV();

Check warning on line 3966 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeServices.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

'case' child has incorrect indentation level 14, expected level should be 10.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHtggN_Ly9860H4m2Te&open=AaHtggN_Ly9860H4m2Te&pullRequest=19282
case INT64, TIMESTAMP ->

Check warning on line 3967 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeServices.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

'case' child has incorrect indentation level 14, expected level should be 10.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHtggN_Ly9860H4m2Tf&open=AaHtggN_Ly9860H4m2Tf&pullRequest=19282
(field, column, index) -> ((long[]) column)[index] = field.getLongV();
case FLOAT -> (field, column, index) -> ((float[]) column)[index] = field.getFloatV();

Check warning on line 3969 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeServices.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

'case' child has incorrect indentation level 14, expected level should be 10.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHtggN_Ly9860H4m2Tg&open=AaHtggN_Ly9860H4m2Tg&pullRequest=19282
case DOUBLE ->

Check warning on line 3970 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeServices.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

'case' child has incorrect indentation level 14, expected level should be 10.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHtggN_Ly9860H4m2Th&open=AaHtggN_Ly9860H4m2Th&pullRequest=19282
(field, column, index) -> ((double[]) column)[index] = field.getDoubleV();
case DATE ->

Check warning on line 3972 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeServices.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

'case' child has incorrect indentation level 14, expected level should be 10.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHtggN_Ly9860H4m2Ti&open=AaHtggN_Ly9860H4m2Ti&pullRequest=19282
(field, column, index) -> {
if (column instanceof LocalDate[] dates) {
dates[index] = field.getDateV();
} else {
type.addValue(index, type.getValue(field), column);
}
};
case TEXT, STRING, BLOB ->

Check warning on line 3980 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeServices.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

'case' child has incorrect indentation level 14, expected level should be 10.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHtggN_Ly9860H4m2Tj&open=AaHtggN_Ly9860H4m2Tj&pullRequest=19282
(field, column, index) -> {
final Binary value = field.getBinaryV();
((Binary[]) column)[index] = value == null ? Binary.EMPTY_VALUE : value;
};
case OBJECT, ROW, UNKNOWN, VECTOR ->

Check warning on line 3985 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeServices.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

'case' child has incorrect indentation level 14, expected level should be 10.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHtggN_Ly9860H4m2Tk&open=AaHtggN_Ly9860H4m2Tk&pullRequest=19282
(field, column, index) -> type.addValue(index, type.getValue(field), column);
};

Check warning on line 3987 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeServices.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

'switch rcurly' has incorrect indentation level 12, expected level should be 8.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHtggN_Ly9860H4m2Tl&open=AaHtggN_Ly9860H4m2Tl&pullRequest=19282

public static final TypeService<TsPrimitiveTabletValueWriter>
PIPE_TS_PRIMITIVE_TABLET_VALUE_WRITER_SERVICE =
type ->
Expand Down Expand Up @@ -4002,7 +4031,7 @@
public static final TypeService<TabletValueColumnFilter>
PIPE_TABLET_VALUE_COLUMN_FILTER_SERVICE =
type ->
switch (type.getTypeEnum()) {

Check warning on line 4034 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeServices.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

'switch' has incorrect indentation level 16, expected level should be 12.

See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AaHtggN_Ly9860H4m2Tm&open=AaHtggN_Ly9860H4m2Tm&pullRequest=19282
case BOOLEAN, INT32, INT64, TIMESTAMP, FLOAT, DOUBLE ->
(originValueColumn,
rowIndexList,
Expand Down Expand Up @@ -4141,6 +4170,7 @@
// internal TsFile types, so the generic service check cannot be applied to it.
PIPE_ROW_OBJECT_GETTER_SERVICE.check();
AGGREGATE_ROW_VALUE_UPDATER_SERVICE.check();
PIPE_FIELD_TABLET_VALUE_WRITER_SERVICE.check();
PIPE_TS_PRIMITIVE_TABLET_VALUE_WRITER_SERVICE.check();
PIPE_BATCH_DATA_TABLET_VALUE_WRITER_SERVICE.check();
PIPE_TABLET_VALUE_COLUMN_FILTER_SERVICE.check();
Expand Down Expand Up @@ -4291,6 +4321,11 @@
void write(TsPrimitiveType value, Object column, int index);
}

@FunctionalInterface
public interface FieldTabletValueWriter {
void write(Field field, Object column, int index);
}

@FunctionalInterface
public interface BatchDataTabletValueWriter {
void write(BatchData value, Object column, int index);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,19 @@
package org.apache.iotdb.db.pipe.event.common.tablet;

import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.Field;
import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.DateUtils;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.Assert;
import org.junit.Test;

import java.time.LocalDate;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class PipeTabletUtilsTest {
Expand All @@ -49,4 +55,66 @@ public void testPutValueUnmarksReusedNullRow() {
Assert.assertFalse(tablet.isNull(0, 0));
Assert.assertTrue(tablet.isNull(0, 1));
}

@Test
public void testPutFieldValueMatchesObjectConversion() {
final TSDataType[] types = {
TSDataType.BOOLEAN,
TSDataType.INT32,
TSDataType.INT64,
TSDataType.TIMESTAMP,
TSDataType.FLOAT,
TSDataType.DOUBLE,
TSDataType.DATE,
TSDataType.TEXT,
TSDataType.STRING,
TSDataType.BLOB,
TSDataType.OBJECT
};
final Object[] values = {
true,
123456789,
9876543210L,
Long.MIN_VALUE,
-0.0f,
Double.longBitsToDouble(0x7ff0000000000001L),
DateUtils.parseDateExpressionToInt(LocalDate.of(2024, 2, 29)),
new Binary(new byte[] {1, 2, 3}),
new Binary(new byte[] {4, 5}),
new Binary(new byte[0]),
new Binary(new byte[16])
};
for (int i = 0; i < types.length; i++) {
final List<IMeasurementSchema> schemas =
Collections.singletonList(new MeasurementSchema("s1", types[i]));
final Tablet expected = new Tablet("root.sg.d1", schemas, 1);
final Tablet actual = new Tablet("root.sg.d1", schemas, 1);
final Field field = Type.fromTsDataType(types[i]).getField(values[i]);
PipeTabletUtils.markNullValue(actual, 0, 0);

PipeTabletUtils.putValue(expected, 0, 0, types[i], field.getObjectValue(types[i]));
PipeTabletUtils.putFieldValue(actual, 0, 0, types[i], field);

Assert.assertFalse(actual.isNull(0, 0));
Assert.assertEquals(expected.getValue(0, 0), actual.getValue(0, 0));
}
}

@Test
public void testPutFieldValuePreservesNullFieldDefaults() {
for (final TSDataType type :
Arrays.asList(TSDataType.INT32, TSDataType.DATE, TSDataType.STRING, TSDataType.BLOB)) {
final List<IMeasurementSchema> schemas =
Collections.singletonList(new MeasurementSchema("s1", type));
final Tablet expected = new Tablet("root.sg.d1", schemas, 1);
final Tablet actual = new Tablet("root.sg.d1", schemas, 1);
PipeTabletUtils.markNullValue(actual, 0, 0);

PipeTabletUtils.putValue(expected, 0, 0, type, null);
PipeTabletUtils.putFieldValue(actual, 0, 0, type, new Field(null));

Assert.assertFalse(actual.isNull(0, 0));
Assert.assertEquals(expected.getValue(0, 0), actual.getValue(0, 0));
}
}
}
Loading
Loading