From eaaad099d6a29bff60cc374ad0ff30fad6b40e3d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:05:54 +0800 Subject: [PATCH 1/2] Reduce allocations in aligned page reads and tablet serialization --- .../tsfile/read/common/type/DoubleType.java | 3 +- .../tsfile/read/common/type/FloatType.java | 3 +- .../page/AbstractAlignedPageReader.java | 14 ++- .../apache/tsfile/utils/ReadWriteIOUtils.java | 37 ++++++ .../apache/tsfile/write/record/Tablet.java | 85 +++++++++++++- .../reader/AlignedPageReaderPushDownTest.java | 106 ++++++++++++++++++ .../tsfile/utils/ReadWriteIOUtilsTest.java | 57 ++++++++++ .../tsfile/write/record/TabletTest.java | 89 +++++++++++++++ 8 files changed, 381 insertions(+), 13 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 1fe10746bc..14c6160be6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -449,7 +449,8 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) public void serializeArray(Object array, int length, ByteBuffer buffer) { double[] values = (double[]) array; for (int i = 0; i < length; i++) { - buffer.putDouble(values[i]); + // Match the canonical NaN representation of the stream serialization. + buffer.putLong(Double.doubleToLongBits(values[i])); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 14458dde9e..0300b16292 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -447,7 +447,8 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) public void serializeArray(Object array, int length, ByteBuffer buffer) { float[] values = (float[]) array; for (int i = 0; i < length; i++) { - buffer.putFloat(values[i]); + // Match the canonical NaN representation of the stream serialization. + buffer.putInt(Float.floatToIntBits(values[i])); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/AbstractAlignedPageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/AbstractAlignedPageReader.java index 2a95472b61..d990a88500 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/AbstractAlignedPageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/AbstractAlignedPageReader.java @@ -129,7 +129,10 @@ public abstract class AbstractAlignedPageReader implements IPageReader { public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException { BatchData pageData = BatchDataFactory.createBatchData(TSDataType.VECTOR, ascending, false); int timeIndex = -1; - Object[] rowValues = new Object[valueCount]; + // Without record filters, only the presence of each value is needed. Avoid boxing every + // primitive just to check whether the row is all null. + Object[] rowValues = + globalTimeFilter == null && pushDownFilter == null ? null : new Object[valueCount]; while (timePageReader.hasNextTime()) { long timestamp = timePageReader.nextTime(); timeIndex++; @@ -141,13 +144,14 @@ public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException { ValuePageReader pageReader = valuePageReaderList.get(i); if (pageReader != null) { v[i] = pageReader.nextValue(timestamp, timeIndex); - rowValues[i] = (v[i] == null) ? null : v[i].getValue(); } else { v[i] = null; - rowValues[i] = null; } - if (rowValues[i] != null) { - hasNotNullValues = true; + if (rowValues == null) { + hasNotNullValues |= v[i] != null; + } else { + rowValues[i] = v[i] == null ? null : v[i].getValue(); + hasNotNullValues |= rowValues[i] != null; } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java index 187cc616cf..8b6eefe083 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java @@ -257,6 +257,11 @@ public static int write(byte n, OutputStream outputStream) throws IOException { return Byte.BYTES; } + public static int write(byte n, DataOutputStream outputStream) throws IOException { + outputStream.writeByte(n); + return Byte.BYTES; + } + /** * write a short n. * @@ -268,6 +273,11 @@ public static int write(short n, OutputStream outputStream) throws IOException { return bytes.length; } + public static int write(short n, DataOutputStream outputStream) throws IOException { + outputStream.writeShort(n); + return SHORT_LEN; + } + /** * write a byte n to byteBuffer. * @@ -311,6 +321,11 @@ public static int write(int n, OutputStream outputStream) throws IOException { return INT_LEN; } + public static int write(int n, DataOutputStream outputStream) throws IOException { + outputStream.writeInt(n); + return INT_LEN; + } + /** write the size (int) of the binary and then the bytes in binary */ public static int write(Binary binary, OutputStream outputStream) throws IOException { byte[] size = BytesUtils.intToBytes(binary.getValues().length); @@ -319,6 +334,13 @@ public static int write(Binary binary, OutputStream outputStream) throws IOExcep return size.length + binary.getValues().length; } + public static int write(Binary binary, DataOutputStream outputStream) throws IOException { + byte[] values = binary.getValues(); + outputStream.writeInt(values.length); + outputStream.write(values); + return INT_LEN + values.length; + } + /** * write a int n to byteBuffer. * @@ -340,6 +362,11 @@ public static int write(float n, OutputStream outputStream) throws IOException { return FLOAT_LEN; } + public static int write(float n, DataOutputStream outputStream) throws IOException { + outputStream.writeFloat(n); + return FLOAT_LEN; + } + /** * Write a double n to outputStream. * @@ -351,6 +378,11 @@ public static int write(double n, OutputStream outputStream) throws IOException return DOUBLE_LEN; } + public static int write(double n, DataOutputStream outputStream) throws IOException { + outputStream.writeDouble(n); + return DOUBLE_LEN; + } + /** * write a long n to outputStream. * @@ -362,6 +394,11 @@ public static int write(long n, OutputStream outputStream) throws IOException { return LONG_LEN; } + public static int write(long n, DataOutputStream outputStream) throws IOException { + outputStream.writeLong(n); + return LONG_LEN; + } + /** write a long n to byteBuffer. */ public static int write(long n, ByteBuffer buffer) { buffer.putLong(n); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 843588c593..81ac940f94 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -33,7 +33,6 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; -import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -42,6 +41,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; @@ -601,11 +601,29 @@ private Object createValueColumnOfDataType(TSDataType dataType, int capacity) { /** Serialize {@link Tablet} */ public ByteBuffer serialize() throws IOException { - final int serializedSize = serializedSize(); - try (PublicBAOS byteArrayOutputStream = new PublicBAOS(serializedSize); - DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { - serialize(outputStream); - return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + final ByteBuffer buffer = ByteBuffer.allocate(serializedSize()); + serialize(buffer); + buffer.flip(); + return buffer; + } + + /** + * Serialize this tablet at the buffer's current position using the same big-endian format as + * {@link #serialize(DataOutputStream)}. The buffer's position advances and its byte order is + * preserved. The caller must provide at least {@link #serializedSize()} remaining bytes. + */ + public void serialize(ByteBuffer buffer) { + final ByteOrder previousOrder = buffer.order(); + buffer.order(ByteOrder.BIG_ENDIAN); + try { + ReadWriteIOUtils.write(insertTargetName, buffer); + ReadWriteIOUtils.write(rowSize, buffer); + writeMeasurementSchemas(buffer); + writeTimes(buffer); + writeBitMaps(buffer); + writeValues(buffer); + } finally { + buffer.order(previousOrder); } } @@ -713,6 +731,21 @@ private void writeMeasurementSchemas(DataOutputStream stream) throws IOException } } + private void writeMeasurementSchemas(ByteBuffer buffer) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(schemas != null), buffer); + if (schemas != null) { + ReadWriteIOUtils.write(schemas.size(), buffer); + for (int i = 0; i < schemas.size(); i++) { + IMeasurementSchema schema = schemas.get(i); + ReadWriteIOUtils.write(BytesUtils.boolToByte(schema != null), buffer); + if (schema != null) { + schema.serializeTo(buffer); + ReadWriteIOUtils.write((byte) columnCategories.get(i).ordinal(), buffer); + } + } + } + } + private void writeTimes(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(BytesUtils.boolToByte(timestamps != null), stream); if (timestamps != null) { @@ -722,6 +755,15 @@ private void writeTimes(DataOutputStream stream) throws IOException { } } + private void writeTimes(ByteBuffer buffer) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(timestamps != null), buffer); + if (timestamps != null) { + for (int i = 0; i < rowSize; i++) { + buffer.putLong(timestamps[i]); + } + } + } + /** Serialize {@link BitMap}s */ private void writeBitMaps(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(BytesUtils.boolToByte(bitMaps != null), stream); @@ -739,6 +781,23 @@ private void writeBitMaps(DataOutputStream stream) throws IOException { } } + private void writeBitMaps(ByteBuffer buffer) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(bitMaps != null), buffer); + if (bitMaps != null) { + int size = schemas == null ? 0 : schemas.size(); + for (int i = 0; i < size; i++) { + final boolean hasNull = bitMaps[i] != null && !bitMaps[i].isAllUnmarked(rowSize); + ReadWriteIOUtils.write(BytesUtils.boolToByte(hasNull), buffer); + if (hasNull) { + ReadWriteIOUtils.write(rowSize, buffer); + final byte[] bytes = bitMaps[i].getTruncatedByteArray(rowSize); + ReadWriteIOUtils.write(bytes.length, buffer); + buffer.put(bytes); + } + } + } + } + /** Serialize values */ private void writeValues(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(BytesUtils.boolToByte(values != null), stream); @@ -750,6 +809,20 @@ private void writeValues(DataOutputStream stream) throws IOException { } } + private void writeValues(ByteBuffer buffer) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(values != null), buffer); + if (values != null) { + int size = schemas == null ? 0 : schemas.size(); + for (int i = 0; i < size; i++) { + final Object column = values[i]; + ReadWriteIOUtils.write(BytesUtils.boolToByte(column != null), buffer); + if (column != null) { + Type.fromTsDataType(schemas.get(i).getType()).serializeArray(column, rowSize, buffer); + } + } + } + } + private void serializeColumn(Type type, Object column, DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(BytesUtils.boolToByte(column != null), stream); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/AlignedPageReaderPushDownTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/AlignedPageReaderPushDownTest.java index c529d89175..90a64dce8f 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/AlignedPageReaderPushDownTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/AlignedPageReaderPushDownTest.java @@ -29,6 +29,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.header.PageHeader; import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.filter.basic.Filter; @@ -36,6 +37,7 @@ import org.apache.tsfile.read.filter.factory.ValueFilterApi; import org.apache.tsfile.read.reader.page.AlignedPageReader; import org.apache.tsfile.read.reader.page.LazyLoadPageData; +import org.apache.tsfile.read.reader.page.TablePageReader; import org.apache.tsfile.read.reader.series.PaginationController; import org.apache.tsfile.write.page.TimePageWriter; import org.apache.tsfile.write.page.ValuePageWriter; @@ -62,6 +64,110 @@ public class AlignedPageReaderPushDownTest { private static final PageHeader testValuePageHeader2; private static ByteBuffer testValuePageData2; + @Test + public void testBatchDataWithoutRecordFilters() throws IOException { + for (boolean lazy : new boolean[] {false, true}) { + AlignedPageReader reader = + lazy ? generateAlignedPageReaderUsingLazyLoad(null) : generateAlignedPageReader(null); + BatchData data = reader.getAllSatisfiedPageData(true); + for (int time = 0; time < 100; time++, data.next()) { + Assert.assertTrue(data.hasCurrent()); + Assert.assertEquals(time, data.currentTime()); + Assert.assertEquals(time, data.getVector()[0].getInt()); + if (time >= 10 && time < 90) { + Assert.assertEquals(time, data.getVector()[1].getInt()); + } else { + Assert.assertNull(data.getVector()[1]); + } + } + Assert.assertFalse(data.hasCurrent()); + + reader = + lazy + ? generateSingleColumnAlignedPageReaderUsingLazyLoad(null, false) + : generateSingleColumnAlignedPageReader(null, false); + data = reader.getAllSatisfiedPageData(true); + for (int time = 10; time < 90; time++, data.next()) { + Assert.assertTrue(data.hasCurrent()); + Assert.assertEquals(time, data.currentTime()); + Assert.assertEquals(time, data.getVector()[0].getInt()); + } + Assert.assertFalse(data.hasCurrent()); + } + } + + @Test + public void testBatchDataWithoutRecordFiltersHonorsDeletes() throws IOException { + AlignedPageReader reader = generateAlignedPageReader(null, Arrays.asList(true, true)); + reader.setDeleteIntervalList( + Arrays.asList( + Collections.singletonList(new TimeRange(0, 29)), + Collections.singletonList(new TimeRange(20, 39)))); + BatchData data = reader.getAllSatisfiedPageData(true); + for (int time = 0; time < 100; time++) { + if (time < 10 || (time >= 20 && time < 30)) { + continue; + } + Assert.assertTrue(data.hasCurrent()); + Assert.assertEquals(time, data.currentTime()); + if (time < 30) { + Assert.assertNull(data.getVector()[0]); + } else { + Assert.assertEquals(time, data.getVector()[0].getInt()); + } + if ((time >= 10 && time < 20) || (time >= 40 && time < 90)) { + Assert.assertEquals(time, data.getVector()[1].getInt()); + } else { + Assert.assertNull(data.getVector()[1]); + } + data.next(); + } + Assert.assertFalse(data.hasCurrent()); + } + + @Test + public void testBatchDataWithRecordFilters() throws IOException { + for (Filter timeFilter : new Filter[] {null, TimeFilterApi.between(30, 69)}) { + AlignedPageReader reader = generateAlignedPageReader(timeFilter); + reader.addRecordFilter(ValueFilterApi.gtEq(DEFAULT_MEASUREMENT_INDEX, 50, TSDataType.INT32)); + BatchData data = reader.getAllSatisfiedPageData(true); + int end = timeFilter == null ? 100 : 70; + for (int time = 50; time < end; time++, data.next()) { + Assert.assertTrue(data.hasCurrent()); + Assert.assertEquals(time, data.currentTime()); + Assert.assertEquals(time, data.getVector()[0].getInt()); + } + Assert.assertFalse(data.hasCurrent()); + } + } + + @Test + public void testTableBatchDataKeepsAllNullRowsWithoutRecordFilters() throws IOException { + resetDataBuffer(); + TablePageReader reader = + new TablePageReader( + testTimePageHeader, + testTimePageData, + new DeltaBinaryDecoder.LongDeltaDecoder(), + Arrays.asList(null, testValuePageHeader2), + Arrays.asList(null, testValuePageData2), + Arrays.asList(TSDataType.INT32, TSDataType.INT32), + Arrays.asList(new IntRleDecoder(), new IntRleDecoder()), + null); + BatchData data = reader.getAllSatisfiedPageData(true); + for (int time = 0; time < 100; time++, data.next()) { + Assert.assertTrue(data.hasCurrent()); + Assert.assertEquals(time, data.currentTime()); + Assert.assertNull(data.getVector()[0]); + if (time >= 10 && time < 90) { + Assert.assertEquals(time, data.getVector()[1].getInt()); + } else { + Assert.assertNull(data.getVector()[1]); + } + } + Assert.assertFalse(data.hasCurrent()); + } + static { TimePageWriter timePageWriter = new TimePageWriter( diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java index 01f747c6c4..950f6296e0 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java @@ -47,6 +47,63 @@ public class ReadWriteIOUtilsTest { protected static final int DEFAULT_BUFFER_SIZE = 4096; + @Test + public void testDataOutputStreamWritesMatchOutputStreamFormat() throws IOException { + ByteArrayOutputStream reference = new ByteArrayOutputStream(); + ByteArrayOutputStream actual = new ByteArrayOutputStream(); + try (DataOutputStream stream = new DataOutputStream(actual)) { + for (byte value : new byte[] {Byte.MIN_VALUE, -1, 0, Byte.MAX_VALUE}) { + ReadWriteIOUtils.write(value, reference); + Assert.assertEquals(Byte.BYTES, ReadWriteIOUtils.write(value, stream)); + } + for (short value : new short[] {Short.MIN_VALUE, -1, 0, Short.MAX_VALUE}) { + ReadWriteIOUtils.write(value, reference); + Assert.assertEquals(Short.BYTES, ReadWriteIOUtils.write(value, stream)); + } + for (int value : new int[] {Integer.MIN_VALUE, -1, 0, Integer.MAX_VALUE}) { + ReadWriteIOUtils.write(value, reference); + Assert.assertEquals(Integer.BYTES, ReadWriteIOUtils.write(value, stream)); + } + for (long value : new long[] {Long.MIN_VALUE, -1, 0, Long.MAX_VALUE}) { + ReadWriteIOUtils.write(value, reference); + Assert.assertEquals(Long.BYTES, ReadWriteIOUtils.write(value, stream)); + } + for (float value : + new float[] { + -0.0f, + 0.0f, + Float.MIN_VALUE, + Float.MAX_VALUE, + Float.NEGATIVE_INFINITY, + Float.POSITIVE_INFINITY, + Float.intBitsToFloat(0x7fa12345) + }) { + ReadWriteIOUtils.write(value, reference); + Assert.assertEquals(Float.BYTES, ReadWriteIOUtils.write(value, stream)); + } + for (double value : + new double[] { + -0.0, + 0.0, + Double.MIN_VALUE, + Double.MAX_VALUE, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + Double.longBitsToDouble(0x7ff123456789abcdL) + }) { + ReadWriteIOUtils.write(value, reference); + Assert.assertEquals(Double.BYTES, ReadWriteIOUtils.write(value, stream)); + } + for (Binary value : new Binary[] {Binary.EMPTY_VALUE, new Binary(new byte[] {0, -1, 1, 0})}) { + ReadWriteIOUtils.write(value, reference); + Assert.assertEquals( + Integer.BYTES + value.getLength(), ReadWriteIOUtils.write(value, stream)); + } + Assert.assertEquals(reference.size(), stream.size()); + } + Assert.assertArrayEquals(reference.toByteArray(), actual.toByteArray()); + } + @Test public void testReadLongFromFragmentedStream() throws IOException { long expected = 0x8123456789ABCDEFL; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java index 660fe85345..358f2d85f5 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java @@ -39,7 +39,9 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.BufferOverflowException; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.util.ArrayList; @@ -823,11 +825,98 @@ private void assertSerializedSizeMatches(Tablet tablet, String scenario) throws DataOutputStream outputStream = new DataOutputStream(baos)) { tablet.serialize(outputStream); assertEquals(scenario + ": serialize(stream) size", expectedSize, baos.size()); + Assert.assertArrayEquals( + scenario + ": buffer and stream wire format", + Arrays.copyOf(baos.getBuf(), baos.size()), + buffer.array()); } buffer.rewind(); assertEquals(scenario + ": deserialize roundtrip", tablet, Tablet.deserialize(buffer)); } + @Test + public void testSerializeToExistingByteBuffer() throws IOException { + final List types = + Arrays.asList( + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.STRING, + TSDataType.BLOB, + TSDataType.DATE, + TSDataType.TIMESTAMP, + TSDataType.OBJECT); + final Tablet tablet = + createAndFillTreeTablet("root.设备.d1", buildColumnNames(types), types, 9, 0, true, true); + final byte[] expected = tablet.serialize().array(); + final int size = expected.length; + final ByteBuffer sliced = ByteBuffer.allocate(size + 13); + sliced.position(7); + for (ByteBuffer buffer : + new ByteBuffer[] { + ByteBuffer.allocate(size + 6), ByteBuffer.allocateDirect(size + 6), sliced.slice() + }) { + while (buffer.hasRemaining()) { + buffer.put((byte) 0x5a); + } + buffer.position(2); + buffer.limit(2 + size); + buffer.order(ByteOrder.LITTLE_ENDIAN); + tablet.serialize(buffer); + assertEquals(2 + size, buffer.position()); + assertEquals(2 + size, buffer.limit()); + assertEquals(ByteOrder.LITTLE_ENDIAN, buffer.order()); + buffer.position(2); + final byte[] actual = new byte[size]; + buffer.get(actual); + Assert.assertArrayEquals(expected, actual); + assertEquals(tablet, Tablet.deserialize(ByteBuffer.wrap(actual))); + buffer.clear(); + assertEquals((byte) 0x5a, buffer.get(0)); + assertEquals((byte) 0x5a, buffer.get(1)); + for (int i = 2 + size; i < buffer.capacity(); i++) { + assertEquals((byte) 0x5a, buffer.get(i)); + } + } + final ByteBuffer tooSmall = ByteBuffer.allocate(size - 1).order(ByteOrder.LITTLE_ENDIAN); + Assert.assertThrows(BufferOverflowException.class, () -> tablet.serialize(tooSmall)); + assertEquals(ByteOrder.LITTLE_ENDIAN, tooSmall.order()); + } + + @Test + public void testSerializationCanonicalizesNaNs() throws IOException { + final Tablet tablet = + new Tablet( + "root.sg.d1", + Arrays.asList( + new MeasurementSchema("f", TSDataType.FLOAT), + new MeasurementSchema("d", TSDataType.DOUBLE)), + 5); + final float[] floats = { + Float.intBitsToFloat(0x7fa12345), + Float.intBitsToFloat(0xffc12345), + -0.0f, + Float.POSITIVE_INFINITY, + Float.MIN_VALUE + }; + final double[] doubles = { + Double.longBitsToDouble(0x7ff123456789abcdL), + Double.longBitsToDouble(0xfff123456789abcdL), + -0.0, + Double.NEGATIVE_INFINITY, + Double.MIN_VALUE + }; + for (int i = 0; i < floats.length; i++) { + tablet.addTimestamp(i, i); + tablet.addValue("f", i, floats[i]); + tablet.addValue("d", i, doubles[i]); + } + assertSerializedSizeMatches(tablet, "floating-point wire format"); + } + @Test public void testAppendInconsistent() { Tablet t1 = From a1d3547f525c9a58f4c59eebb1757930672401dd Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:25:15 +0800 Subject: [PATCH 2/2] perf: delegate Tablet bitmap serialization --- .../java/org/apache/tsfile/utils/BitMap.java | 17 ++++++ .../apache/tsfile/utils/BitMapArrayImpl.java | 11 ++++ .../org/apache/tsfile/utils/BitMapImpl.java | 4 ++ .../apache/tsfile/utils/BitMapLongImpl.java | 14 +++++ .../tsfile/utils/BitMapSerializationTest.java | 58 +++++++++++++++++++ .../apache/tsfile/write/record/Tablet.java | 7 +-- 6 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 java/common/src/test/java/org/apache/tsfile/utils/BitMapSerializationTest.java diff --git a/java/common/src/main/java/org/apache/tsfile/utils/BitMap.java b/java/common/src/main/java/org/apache/tsfile/utils/BitMap.java index 337bec90fe..25bd39ac1e 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/BitMap.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/BitMap.java @@ -21,6 +21,7 @@ import org.apache.tsfile.i18n.Messages; +import java.nio.ByteBuffer; import java.util.Arrays; public class BitMap implements Accountable { @@ -235,6 +236,22 @@ public byte[] getTruncatedByteArray(int size) { return Arrays.copyOf(getByteArray(), getSizeOfBytes(size)); } + /** Return the serialized size of this bitmap for the specified number of bits. */ + public int serializedSize(int size) { + return Math.addExact(Integer.BYTES, getSizeOfBytes(size)); + } + + /** + * Write this bitmap's length-prefixed bytes to the buffer's current position. The length prefix + * uses the buffer's byte order. + */ + public int serializeTo(ByteBuffer buffer, int size) { + final int byteCount = getSizeOfBytes(size); + buffer.putInt(byteCount); + implementation.serializeTo(buffer, byteCount); + return Integer.BYTES + byteCount; + } + public void append(BitMap another, int position, int length) { for (int i = 0; i < length; i++) { if (another.isMarked(i)) { diff --git a/java/common/src/main/java/org/apache/tsfile/utils/BitMapArrayImpl.java b/java/common/src/main/java/org/apache/tsfile/utils/BitMapArrayImpl.java index 2a8a4d3f1c..ddf69594e1 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/BitMapArrayImpl.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/BitMapArrayImpl.java @@ -19,6 +19,7 @@ package org.apache.tsfile.utils; +import java.nio.ByteBuffer; import java.util.Arrays; class BitMapArrayImpl extends BitMapImpl { @@ -69,6 +70,16 @@ byte getByte(int index) { return bits[index]; } + @Override + void serializeTo(ByteBuffer buffer, int byteCount) { + int copiedByteCount = Math.min(byteCount, bits.length); + buffer.put(bits, 0, copiedByteCount); + while (copiedByteCount < byteCount) { + buffer.put((byte) 0); + copiedByteCount++; + } + } + @Override boolean isMarked(int position) { return (bits[position / Byte.SIZE] & BIT_UTIL[position % Byte.SIZE]) != 0; diff --git a/java/common/src/main/java/org/apache/tsfile/utils/BitMapImpl.java b/java/common/src/main/java/org/apache/tsfile/utils/BitMapImpl.java index a798af56a5..6ba5f3d0ec 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/BitMapImpl.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/BitMapImpl.java @@ -21,6 +21,8 @@ import org.apache.tsfile.i18n.Messages; +import java.nio.ByteBuffer; + abstract class BitMapImpl { protected int size; @@ -39,6 +41,8 @@ int getSize() { abstract byte getByte(int index); + abstract void serializeTo(ByteBuffer buffer, int byteCount); + abstract boolean isMarked(int position); abstract boolean isRangeAnyMarked(int start, int length); diff --git a/java/common/src/main/java/org/apache/tsfile/utils/BitMapLongImpl.java b/java/common/src/main/java/org/apache/tsfile/utils/BitMapLongImpl.java index 633a6ea972..66fb50e951 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/BitMapLongImpl.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/BitMapLongImpl.java @@ -19,6 +19,8 @@ package org.apache.tsfile.utils; +import java.nio.ByteBuffer; + class BitMapLongImpl extends BitMapImpl { private static final long ALL_BITS_MARKED = -1L; @@ -72,6 +74,18 @@ byte getByte(int index) { return index < Long.BYTES ? (byte) (bits >>> (index * Byte.SIZE)) : paddingByte; } + @Override + void serializeTo(ByteBuffer buffer, int byteCount) { + int copiedByteCount = Math.min(byteCount, getByteArrayLength()); + for (int i = 0; i < copiedByteCount; i++) { + buffer.put(getByte(i)); + } + while (copiedByteCount < byteCount) { + buffer.put((byte) 0); + copiedByteCount++; + } + } + @Override boolean isMarked(int position) { return (bits & (1L << position)) != 0; diff --git a/java/common/src/test/java/org/apache/tsfile/utils/BitMapSerializationTest.java b/java/common/src/test/java/org/apache/tsfile/utils/BitMapSerializationTest.java new file mode 100644 index 0000000000..ff4132ebde --- /dev/null +++ b/java/common/src/test/java/org/apache/tsfile/utils/BitMapSerializationTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.utils; + +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class BitMapSerializationTest { + + @Test + public void testSerializeToMatchesTruncatedBytes() { + for (int size : new int[] {1, 7, 8, 9, 63, 64, 65, 128}) { + BitMap bitMap = BitMap.createBitMapDynamically(size); + bitMap.mark(0); + if (size > 1) { + bitMap.mark(size - 1); + } + + byte[] bitmapBytes = bitMap.getTruncatedByteArray(size); + ByteBuffer expected = + ByteBuffer.allocate(Integer.BYTES + bitmapBytes.length).order(ByteOrder.LITTLE_ENDIAN); + expected.putInt(bitmapBytes.length).put(bitmapBytes); + + ByteBuffer actual = + ByteBuffer.allocateDirect(bitMap.serializedSize(size) + 2).order(ByteOrder.LITTLE_ENDIAN); + actual.position(2); + assertEquals(bitMap.serializedSize(size), bitMap.serializeTo(actual, size)); + assertEquals(2 + bitMap.serializedSize(size), actual.position()); + + byte[] actualBytes = new byte[bitMap.serializedSize(size)]; + actual.position(2); + actual.get(actualBytes); + assertArrayEquals(expected.array(), actualBytes); + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 81ac940f94..953dbf6d6e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -682,8 +682,7 @@ private int serializedSizeOfBitMaps() { } else { size = Math.addExact(size, Byte.BYTES); size = Math.addExact(size, Integer.BYTES); - size = Math.addExact(size, Integer.BYTES); - size = Math.addExact(size, BitMap.getSizeOfBytes(rowSize)); + size = Math.addExact(size, bitMaps[i].serializedSize(rowSize)); } } } @@ -790,9 +789,7 @@ private void writeBitMaps(ByteBuffer buffer) { ReadWriteIOUtils.write(BytesUtils.boolToByte(hasNull), buffer); if (hasNull) { ReadWriteIOUtils.write(rowSize, buffer); - final byte[] bytes = bitMaps[i].getTruncatedByteArray(rowSize); - ReadWriteIOUtils.write(bytes.length, buffer); - buffer.put(bytes); + bitMaps[i].serializeTo(buffer, rowSize); } } }