Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions java/common/src/main/java/org/apache/tsfile/utils/BitMap.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import org.apache.tsfile.i18n.Messages;

import java.nio.ByteBuffer;
import java.util.Arrays;

public class BitMap implements Accountable {
Expand Down Expand Up @@ -235,6 +236,22 @@ public byte[] getTruncatedByteArray(int size) {
return Arrays.copyOf(getByteArray(), getSizeOfBytes(size));
}

/** Return the serialized size of this bitmap for the specified number of bits. */
public int serializedSize(int size) {
return Math.addExact(Integer.BYTES, getSizeOfBytes(size));
}

/**
* Write this bitmap's length-prefixed bytes to the buffer's current position. The length prefix
* uses the buffer's byte order.
*/
public int serializeTo(ByteBuffer buffer, int size) {
final int byteCount = getSizeOfBytes(size);
buffer.putInt(byteCount);
implementation.serializeTo(buffer, byteCount);
return Integer.BYTES + byteCount;
}

public void append(BitMap another, int position, int length) {
for (int i = 0; i < length; i++) {
if (another.isMarked(i)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.tsfile.utils;

import java.nio.ByteBuffer;
import java.util.Arrays;

class BitMapArrayImpl extends BitMapImpl {
Expand Down Expand Up @@ -69,6 +70,16 @@ byte getByte(int index) {
return bits[index];
}

@Override
void serializeTo(ByteBuffer buffer, int byteCount) {
int copiedByteCount = Math.min(byteCount, bits.length);
buffer.put(bits, 0, copiedByteCount);
while (copiedByteCount < byteCount) {
buffer.put((byte) 0);
copiedByteCount++;
}
}

@Override
boolean isMarked(int position) {
return (bits[position / Byte.SIZE] & BIT_UTIL[position % Byte.SIZE]) != 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@

import org.apache.tsfile.i18n.Messages;

import java.nio.ByteBuffer;

abstract class BitMapImpl {

protected int size;
Expand All @@ -39,6 +41,8 @@ int getSize() {

abstract byte getByte(int index);

abstract void serializeTo(ByteBuffer buffer, int byteCount);

abstract boolean isMarked(int position);

abstract boolean isRangeAnyMarked(int start, int length);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@

package org.apache.tsfile.utils;

import java.nio.ByteBuffer;

class BitMapLongImpl extends BitMapImpl {

private static final long ALL_BITS_MARKED = -1L;
Expand Down Expand Up @@ -72,6 +74,18 @@ byte getByte(int index) {
return index < Long.BYTES ? (byte) (bits >>> (index * Byte.SIZE)) : paddingByte;
}

@Override
void serializeTo(ByteBuffer buffer, int byteCount) {
int copiedByteCount = Math.min(byteCount, getByteArrayLength());
for (int i = 0; i < copiedByteCount; i++) {
buffer.put(getByte(i));
}
while (copiedByteCount < byteCount) {
buffer.put((byte) 0);
copiedByteCount++;
}
}

@Override
boolean isMarked(int position) {
return (bits & (1L << position)) != 0;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.tsfile.utils;

import org.junit.Test;

import java.nio.ByteBuffer;
import java.nio.ByteOrder;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;

public class BitMapSerializationTest {

@Test
public void testSerializeToMatchesTruncatedBytes() {
for (int size : new int[] {1, 7, 8, 9, 63, 64, 65, 128}) {
BitMap bitMap = BitMap.createBitMapDynamically(size);
bitMap.mark(0);
if (size > 1) {
bitMap.mark(size - 1);
}

byte[] bitmapBytes = bitMap.getTruncatedByteArray(size);
ByteBuffer expected =
ByteBuffer.allocate(Integer.BYTES + bitmapBytes.length).order(ByteOrder.LITTLE_ENDIAN);
expected.putInt(bitmapBytes.length).put(bitmapBytes);

ByteBuffer actual =
ByteBuffer.allocateDirect(bitMap.serializedSize(size) + 2).order(ByteOrder.LITTLE_ENDIAN);
actual.position(2);
assertEquals(bitMap.serializedSize(size), bitMap.serializeTo(actual, size));
assertEquals(2 + bitMap.serializedSize(size), actual.position());

byte[] actualBytes = new byte[bitMap.serializedSize(size)];
actual.position(2);
actual.get(actualBytes);
assertArrayEquals(expected.array(), actualBytes);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,8 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream)
public void serializeArray(Object array, int length, ByteBuffer buffer) {
double[] values = (double[]) array;
for (int i = 0; i < length; i++) {
buffer.putDouble(values[i]);
// Match the canonical NaN representation of the stream serialization.
buffer.putLong(Double.doubleToLongBits(values[i]));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -447,7 +447,8 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream)
public void serializeArray(Object array, int length, ByteBuffer buffer) {
float[] values = (float[]) array;
for (int i = 0; i < length; i++) {
buffer.putFloat(values[i]);
// Match the canonical NaN representation of the stream serialization.
buffer.putInt(Float.floatToIntBits(values[i]));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,10 @@ public abstract class AbstractAlignedPageReader implements IPageReader {
public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException {
BatchData pageData = BatchDataFactory.createBatchData(TSDataType.VECTOR, ascending, false);
int timeIndex = -1;
Object[] rowValues = new Object[valueCount];
// Without record filters, only the presence of each value is needed. Avoid boxing every
// primitive just to check whether the row is all null.
Object[] rowValues =
globalTimeFilter == null && pushDownFilter == null ? null : new Object[valueCount];
while (timePageReader.hasNextTime()) {
long timestamp = timePageReader.nextTime();
timeIndex++;
Expand All @@ -141,13 +144,14 @@ public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException {
ValuePageReader pageReader = valuePageReaderList.get(i);
if (pageReader != null) {
v[i] = pageReader.nextValue(timestamp, timeIndex);
rowValues[i] = (v[i] == null) ? null : v[i].getValue();
} else {
v[i] = null;
rowValues[i] = null;
}
if (rowValues[i] != null) {
hasNotNullValues = true;
if (rowValues == null) {
hasNotNullValues |= v[i] != null;
} else {
rowValues[i] = v[i] == null ? null : v[i].getValue();
hasNotNullValues |= rowValues[i] != null;
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,11 @@ public static int write(byte n, OutputStream outputStream) throws IOException {
return Byte.BYTES;
}

public static int write(byte n, DataOutputStream outputStream) throws IOException {
outputStream.writeByte(n);
return Byte.BYTES;
}

/**
* write a short n.
*
Expand All @@ -268,6 +273,11 @@ public static int write(short n, OutputStream outputStream) throws IOException {
return bytes.length;
}

public static int write(short n, DataOutputStream outputStream) throws IOException {
outputStream.writeShort(n);
return SHORT_LEN;
}

/**
* write a byte n to byteBuffer.
*
Expand Down Expand Up @@ -311,6 +321,11 @@ public static int write(int n, OutputStream outputStream) throws IOException {
return INT_LEN;
}

public static int write(int n, DataOutputStream outputStream) throws IOException {
outputStream.writeInt(n);
return INT_LEN;
}

/** write the size (int) of the binary and then the bytes in binary */
public static int write(Binary binary, OutputStream outputStream) throws IOException {
byte[] size = BytesUtils.intToBytes(binary.getValues().length);
Expand All @@ -319,6 +334,13 @@ public static int write(Binary binary, OutputStream outputStream) throws IOExcep
return size.length + binary.getValues().length;
}

public static int write(Binary binary, DataOutputStream outputStream) throws IOException {
byte[] values = binary.getValues();
outputStream.writeInt(values.length);
outputStream.write(values);
return INT_LEN + values.length;
}

/**
* write a int n to byteBuffer.
*
Expand All @@ -340,6 +362,11 @@ public static int write(float n, OutputStream outputStream) throws IOException {
return FLOAT_LEN;
}

public static int write(float n, DataOutputStream outputStream) throws IOException {
outputStream.writeFloat(n);
return FLOAT_LEN;
}

/**
* Write a double n to outputStream.
*
Expand All @@ -351,6 +378,11 @@ public static int write(double n, OutputStream outputStream) throws IOException
return DOUBLE_LEN;
}

public static int write(double n, DataOutputStream outputStream) throws IOException {
outputStream.writeDouble(n);
return DOUBLE_LEN;
}

/**
* write a long n to outputStream.
*
Expand All @@ -362,6 +394,11 @@ public static int write(long n, OutputStream outputStream) throws IOException {
return LONG_LEN;
}

public static int write(long n, DataOutputStream outputStream) throws IOException {
outputStream.writeLong(n);
return LONG_LEN;
}

/** write a long n to byteBuffer. */
public static int write(long n, ByteBuffer buffer) {
buffer.putLong(n);
Expand Down
Loading
Loading