Each invocation decodes {@value #VALUE_COUNT} values; throughput is reported per-value via + * {@link OperationsPerInvocation}. + */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@Fork(1) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +@State(Scope.Thread) +public class AlpDecodingBenchmark { + + static final int VALUE_COUNT = 100_000; + private static final int INIT_SLAB_SIZE = 64 * 1024; + private static final int PAGE_SIZE = 4 * 1024 * 1024; + + @Param({"decimal", "random"}) + public String data; + + private byte[] alpDoublePage; + private byte[] plainDoublePage; + private byte[] alpFloatPage; + private byte[] plainFloatPage; + + @Setup(Level.Trial) + public void setup() throws IOException { + double[] doubles = AlpEncodingBenchmark.doubles(data, VALUE_COUNT); + float[] floats = AlpEncodingBenchmark.floats(data, VALUE_COUNT); + alpDoublePage = page( + new AlpValuesWriter.DoubleAlpValuesWriter(INIT_SLAB_SIZE, PAGE_SIZE, new HeapByteBufferAllocator()), + doubles); + plainDoublePage = + page(new PlainValuesWriter(INIT_SLAB_SIZE, PAGE_SIZE, new HeapByteBufferAllocator()), doubles); + alpFloatPage = page( + new AlpValuesWriter.FloatAlpValuesWriter(INIT_SLAB_SIZE, PAGE_SIZE, new HeapByteBufferAllocator()), + floats); + plainFloatPage = page(new PlainValuesWriter(INIT_SLAB_SIZE, PAGE_SIZE, new HeapByteBufferAllocator()), floats); + } + + private static byte[] page(ValuesWriter w, double[] values) throws IOException { + for (double v : values) { + w.writeDouble(v); + } + byte[] bytes = w.getBytes().toByteArray(); + w.close(); + return bytes; + } + + private static byte[] page(ValuesWriter w, float[] values) throws IOException { + for (float v : values) { + w.writeFloat(v); + } + byte[] bytes = w.getBytes().toByteArray(); + w.close(); + return bytes; + } + + private static void init(ValuesReader r, byte[] page) throws IOException { + r.initFromPage(VALUE_COUNT, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))); + } + + @Benchmark + @OperationsPerInvocation(VALUE_COUNT) + public void decodeDouble(Blackhole bh) throws IOException { + AlpValuesReaderForDouble r = new AlpValuesReaderForDouble(); + init(r, alpDoublePage); + for (int i = 0; i < VALUE_COUNT; i++) { + bh.consume(r.readDouble()); + } + } + + @Benchmark + @OperationsPerInvocation(VALUE_COUNT) + public void decodeDoublePlain(Blackhole bh) throws IOException { + PlainValuesReader.DoublePlainValuesReader r = new PlainValuesReader.DoublePlainValuesReader(); + init(r, plainDoublePage); + for (int i = 0; i < VALUE_COUNT; i++) { + bh.consume(r.readDouble()); + } + } + + @Benchmark + @OperationsPerInvocation(VALUE_COUNT) + public void decodeFloat(Blackhole bh) throws IOException { + AlpValuesReaderForFloat r = new AlpValuesReaderForFloat(); + init(r, alpFloatPage); + for (int i = 0; i < VALUE_COUNT; i++) { + bh.consume(r.readFloat()); + } + } + + @Benchmark + @OperationsPerInvocation(VALUE_COUNT) + public void decodeFloatPlain(Blackhole bh) throws IOException { + PlainValuesReader.FloatPlainValuesReader r = new PlainValuesReader.FloatPlainValuesReader(); + init(r, plainFloatPage); + for (int i = 0; i < VALUE_COUNT; i++) { + bh.consume(r.readFloat()); + } + } +} diff --git a/parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/AlpEncodingBenchmark.java b/parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/AlpEncodingBenchmark.java new file mode 100644 index 0000000000..09a6fc5b37 --- /dev/null +++ b/parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/AlpEncodingBenchmark.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.benchmarks; + +import java.io.IOException; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.apache.parquet.bytes.HeapByteBufferAllocator; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.alp.AlpValuesWriter; +import org.apache.parquet.column.values.plain.PlainValuesWriter; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Encoding-level micro-benchmarks for ALP, with PLAIN on the same data as a baseline. + * + *
{@code decimal} data is what ALP is designed for; {@code random} data is almost all + * exceptions, its worst case. + * + *
Each invocation encodes {@value #VALUE_COUNT} values; throughput is reported per-value via
+ * {@link OperationsPerInvocation}.
+ */
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Fork(1)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@State(Scope.Thread)
+public class AlpEncodingBenchmark {
+
+ // Enough values that each run includes both the full parameter search, used until enough
+ // vectors are sampled, and the preset search that follows, as a column chunk would.
+ static final int VALUE_COUNT = 1_000_000;
+ private static final int INIT_SLAB_SIZE = 64 * 1024;
+ private static final int PAGE_SIZE = 16 * 1024 * 1024;
+
+ @Param({"decimal", "random"})
+ public String data;
+
+ private double[] doubleData;
+ private float[] floatData;
+
+ @Setup(Level.Trial)
+ public void setup() {
+ doubleData = doubles(data, VALUE_COUNT);
+ floatData = floats(data, VALUE_COUNT);
+ }
+
+ static double[] doubles(String data, int count) {
+ Random random = new Random(42);
+ double[] values = new double[count];
+ for (int i = 0; i < count; i++) {
+ values[i] = "decimal".equals(data) ? Math.round(random.nextDouble() * 1e6) / 100.0 : random.nextDouble();
+ }
+ return values;
+ }
+
+ static float[] floats(String data, int count) {
+ Random random = new Random(42);
+ float[] values = new float[count];
+ for (int i = 0; i < count; i++) {
+ values[i] = "decimal".equals(data) ? Math.round(random.nextFloat() * 1e6f) / 100.0f : random.nextFloat();
+ }
+ return values;
+ }
+
+ @Benchmark
+ @OperationsPerInvocation(VALUE_COUNT)
+ public byte[] encodeDouble() throws IOException {
+ return encode(
+ new AlpValuesWriter.DoubleAlpValuesWriter(INIT_SLAB_SIZE, PAGE_SIZE, new HeapByteBufferAllocator()));
+ }
+
+ @Benchmark
+ @OperationsPerInvocation(VALUE_COUNT)
+ public byte[] encodeDoublePlain() throws IOException {
+ return encode(new PlainValuesWriter(INIT_SLAB_SIZE, PAGE_SIZE, new HeapByteBufferAllocator()));
+ }
+
+ @Benchmark
+ @OperationsPerInvocation(VALUE_COUNT)
+ public byte[] encodeFloat() throws IOException {
+ return encodeFloats(
+ new AlpValuesWriter.FloatAlpValuesWriter(INIT_SLAB_SIZE, PAGE_SIZE, new HeapByteBufferAllocator()));
+ }
+
+ @Benchmark
+ @OperationsPerInvocation(VALUE_COUNT)
+ public byte[] encodeFloatPlain() throws IOException {
+ return encodeFloats(new PlainValuesWriter(INIT_SLAB_SIZE, PAGE_SIZE, new HeapByteBufferAllocator()));
+ }
+
+ private byte[] encode(ValuesWriter w) throws IOException {
+ for (double v : doubleData) {
+ w.writeDouble(v);
+ }
+ byte[] bytes = w.getBytes().toByteArray();
+ w.close();
+ return bytes;
+ }
+
+ private byte[] encodeFloats(ValuesWriter w) throws IOException {
+ for (float v : floatData) {
+ w.writeFloat(v);
+ }
+ byte[] bytes = w.getBytes().toByteArray();
+ w.close();
+ return bytes;
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/ColumnProperty.java b/parquet-column/src/main/java/org/apache/parquet/column/ColumnProperty.java
index e89cbc9bdb..c2491ba2d5 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/ColumnProperty.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/ColumnProperty.java
@@ -19,9 +19,11 @@
package org.apache.parquet.column;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import org.apache.parquet.hadoop.metadata.ColumnPath;
/**
@@ -62,13 +64,19 @@ private MultipleColumnProperty(T defaultValue, Map Estimated size (in bits) = {@code length * bitWidth + exceptions * (Float.SIZE + Short.SIZE)},
+ * where bitWidth is the number of bits needed to represent the signed range (max - min) of
+ * non-exception encoded values after frame-of-reference subtraction, matching the writer's FOR
+ * packing. This produces better compression ratios than minimizing exception count alone. Ties in
+ * size are broken toward the higher exponent, then the higher factor. The first pair (in {@code
+ * pairs} order) that encodes every value into a single FOR value with zero exceptions wins
+ * outright. When no pair yields any non-exception values, {@code pairs[0]} is returned.
+ */
+ static EncodingParams findBestFloatParams(float[] values, int offset, int length, int[][] pairs) {
+ int bestExponent = pairs[0][E];
+ int bestFactor = pairs[0][F];
+ int bestExceptions = length;
+ long bestEstimatedSize = Long.MAX_VALUE;
+
+ // One reusable result for the whole search: every value below is encoded exactly once.
+ EncodeResult result = new EncodeResult();
+ for (int[] pair : pairs) {
+ int e = pair[E];
+ int f = pair[F];
+ int exceptions = 0;
+ int minEncoded = Integer.MAX_VALUE;
+ int maxEncoded = Integer.MIN_VALUE;
+ for (int i = 0; i < length; i++) {
+ tryEncodeFloat(values[offset + i], e, f, result);
+ if (result.isException) {
+ exceptions++;
+ } else {
+ int encoded = (int) result.encoded;
+ if (encoded < minEncoded) minEncoded = encoded;
+ if (encoded > maxEncoded) maxEncoded = encoded;
+ }
+ }
+ int nonExceptions = length - exceptions;
+ if (nonExceptions == 0) continue;
+ // Signed subtraction gives the true FOR span; getWidthFromMaxLong yields the correct
+ // unsigned bit width, and an overflowing large range is penalized with 64 bits. This matches
+ // the writer's FOR packing.
+ long delta = (nonExceptions < 2) ? 0 : ((long) maxEncoded - (long) minEncoded);
+ int bitsPerValue = BytesUtils.getWidthFromMaxLong(delta);
+ long estimatedSize = (long) length * bitsPerValue + (long) exceptions * (Float.SIZE + Short.SIZE);
+ if (estimatedSize < bestEstimatedSize
+ || (estimatedSize == bestEstimatedSize
+ && (e > bestExponent || (e == bestExponent && f > bestFactor)))) {
+ bestEstimatedSize = estimatedSize;
+ bestExponent = e;
+ bestFactor = f;
+ bestExceptions = exceptions;
+ if (bestExceptions == 0 && bitsPerValue == 0) {
+ return new EncodingParams(bestExponent, bestFactor, 0);
+ }
+ }
+ }
+ return new EncodingParams(bestExponent, bestFactor, bestExceptions);
+ }
+
+ /** Double counterpart to {@link #findBestFloatParams}; see it for the scoring and tie-break rules. */
+ static EncodingParams findBestDoubleParams(double[] values, int offset, int length, int[][] pairs) {
+ int bestExponent = pairs[0][E];
+ int bestFactor = pairs[0][F];
+ int bestExceptions = length;
+ long bestEstimatedSize = Long.MAX_VALUE;
+
+ // One reusable result for the whole search: every value below is encoded exactly once.
+ EncodeResult result = new EncodeResult();
+ for (int[] pair : pairs) {
+ int e = pair[E];
+ int f = pair[F];
+ int exceptions = 0;
+ long minEncoded = Long.MAX_VALUE;
+ long maxEncoded = Long.MIN_VALUE;
+ for (int i = 0; i < length; i++) {
+ tryEncodeDouble(values[offset + i], e, f, result);
+ if (result.isException) {
+ exceptions++;
+ } else {
+ long encoded = result.encoded;
+ if (encoded < minEncoded) minEncoded = encoded;
+ if (encoded > maxEncoded) maxEncoded = encoded;
+ }
+ }
+ int nonExceptions = length - exceptions;
+ if (nonExceptions == 0) continue;
+ long delta = (nonExceptions < 2) ? 0 : (maxEncoded - minEncoded);
+ int bitsPerValue = BytesUtils.getWidthFromMaxLong(delta);
+ long estimatedSize = (long) length * bitsPerValue + (long) exceptions * (Double.SIZE + Short.SIZE);
+ if (estimatedSize < bestEstimatedSize
+ || (estimatedSize == bestEstimatedSize
+ && (e > bestExponent || (e == bestExponent && f > bestFactor)))) {
+ bestEstimatedSize = estimatedSize;
+ bestExponent = e;
+ bestFactor = f;
+ bestExceptions = exceptions;
+ if (bestExceptions == 0 && bitsPerValue == 0) {
+ return new EncodingParams(bestExponent, bestFactor, 0);
+ }
+ }
+ }
+ return new EncodingParams(bestExponent, bestFactor, bestExceptions);
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpConfig.java b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpConfig.java
new file mode 100644
index 0000000000..d7f46f2876
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpConfig.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+/**
+ * Immutable per-column ALP encoding configuration.
+ *
+ * Whether a column is ALP encoded at all is not part of this configuration: a column is ALP
+ * encoded when it has an {@code AlpConfig} and is not when it has none.
+ */
+public final class AlpConfig {
+
+ /** Default values per encoded vector. */
+ public static final int DEFAULT_VECTOR_SIZE = AlpConstants.DEFAULT_VECTOR_SIZE;
+
+ /** Configuration using the default vector size. */
+ public static final AlpConfig DEFAULT = new AlpConfig(DEFAULT_VECTOR_SIZE);
+
+ private final int vectorSize;
+
+ /**
+ * @param vectorSize values per encoded vector; must be a power of 2 in the supported range
+ * @throws IllegalArgumentException if {@code vectorSize} is not a supported vector size
+ */
+ public AlpConfig(int vectorSize) {
+ AlpConstants.validateVectorSize(vectorSize);
+ this.vectorSize = vectorSize;
+ }
+
+ public int getVectorSize() {
+ return vectorSize;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ return vectorSize == ((AlpConfig) o).vectorSize;
+ }
+
+ @Override
+ public int hashCode() {
+ return Integer.hashCode(vectorSize);
+ }
+
+ @Override
+ public String toString() {
+ return "AlpConfig{vectorSize=" + vectorSize + '}';
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpConstants.java b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpConstants.java
new file mode 100644
index 0000000000..bdabd2e069
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpConstants.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import org.apache.parquet.Preconditions;
+
+/** Constants for the ALP (Adaptive Lossless floating-Point) encoding. */
+final class AlpConstants {
+
+ private AlpConstants() {
+ // Utility class
+ }
+
+ // Page header fields
+ static final int ALP_COMPRESSION_MODE = 0;
+ static final int ALP_INTEGER_ENCODING_FOR = 0;
+ static final int ALP_HEADER_SIZE = 7;
+
+ static final int DEFAULT_VECTOR_SIZE = 1024;
+
+ // BytePacker packs/unpacks 8 values at a time (pack8Values/unpack8Values).
+ static final int PACK_GROUP_SIZE = 8;
+
+ // At 2^15, a vector of all exceptions still fits num_exceptions, a uint16.
+ static final int MAX_LOG_VECTOR_SIZE = 15;
+ static final int MIN_LOG_VECTOR_SIZE = 3;
+
+ static final int FLOAT_MAX_EXPONENT = 10;
+ static final int DOUBLE_MAX_EXPONENT = 18;
+
+ // Per-vector metadata sizes in bytes
+ static final int ALP_INFO_SIZE = 4; // exponent(1) + factor(1) + num_exceptions(2)
+ static final int FLOAT_FOR_INFO_SIZE = 5; // frame_of_reference(4) + bit_width(1)
+ static final int DOUBLE_FOR_INFO_SIZE = 9; // frame_of_reference(8) + bit_width(1)
+
+ /** Validates vector size: must be a power of 2 in [2^MIN_LOG .. 2^MAX_LOG]. */
+ static void validateVectorSize(int vectorSize) {
+ Preconditions.checkArgument(
+ vectorSize > 0 && Integer.bitCount(vectorSize) == 1,
+ "Vector size must be a power of 2, got: %s",
+ vectorSize);
+ int logSize = Integer.numberOfTrailingZeros(vectorSize);
+ Preconditions.checkArgument(
+ logSize >= MIN_LOG_VECTOR_SIZE && logSize <= MAX_LOG_VECTOR_SIZE,
+ "Vector size log2 must be between %s and %s, got: %s (vectorSize=%s)",
+ MIN_LOG_VECTOR_SIZE,
+ MAX_LOG_VECTOR_SIZE,
+ logSize,
+ vectorSize);
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReader.java
new file mode 100644
index 0000000000..e47832335e
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReader.java
@@ -0,0 +1,401 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.apache.parquet.column.values.alp.AlpConstants.ALP_COMPRESSION_MODE;
+import static org.apache.parquet.column.values.alp.AlpConstants.ALP_HEADER_SIZE;
+import static org.apache.parquet.column.values.alp.AlpConstants.ALP_INFO_SIZE;
+import static org.apache.parquet.column.values.alp.AlpConstants.ALP_INTEGER_ENCODING_FOR;
+import static org.apache.parquet.column.values.alp.AlpConstants.MAX_LOG_VECTOR_SIZE;
+import static org.apache.parquet.column.values.alp.AlpConstants.MIN_LOG_VECTOR_SIZE;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.io.ParquetDecodingException;
+
+/**
+ * Abstract base class for ALP values readers with lazy per-vector decoding.
+ *
+ * Page layout:
+ * Each vector is decoded lazily on first access. Skipping values does not
+ * trigger decoding of intermediate vectors.
+ */
+abstract class AlpValuesReader extends ValuesReader {
+
+ private static final int BEFORE_FIRST_VECTOR = -1;
+
+ protected int valuesPerVector;
+ protected int totalValueCount = 0;
+ protected int numVectors;
+ protected int pageValueIndex = 0;
+ protected int currentVectorIndex = BEFORE_FIRST_VECTOR;
+
+ // One entry per vector plus a trailing entry for the end of the vector data, so the entry after
+ // any vector's offset is always where that vector ends.
+ protected int[] vectorOffsets;
+ protected ByteBuffer vectorsData;
+
+ // Scratch buffer for exception positions within a vector; shared by both readers (int[] in each).
+ protected int[] excPositionsBuffer;
+
+ /** Reused holder so readForInfo hands its result to decodePackedValues explicitly. */
+ protected final ForInfo forInfo = new ForInfo();
+
+ /**
+ * A vector's ForInfo header: the frame-of-reference base and the bit width of the packed values.
+ * The base is held as a long for both value types; the float reader narrows it on use.
+ */
+ protected static final class ForInfo {
+ long base;
+ int bitWidth;
+ }
+
+ @Override
+ public void initFromPage(int valuesCount, ByteBufferInputStream stream)
+ throws ParquetDecodingException, IOException {
+ readPageHeader(stream, valuesCount);
+ int minVectorBytes = ALP_INFO_SIZE + forInfoSize();
+ readOffsetArray(stream, minVectorBytes);
+ int vectorDataSize = sliceVectorData(stream);
+ // The trailing entry is filled here rather than in readOffsetArray because the vector data
+ // size is only known once the data has been sliced.
+ vectorOffsets[numVectors] = offsetArraySize() + vectorDataSize;
+ validateVectorOffsets(
+ vectorOffsets, minVectorBytes, maxVectorBytes(valuesPerVector, minVectorBytes, exceptionValueSize()));
+
+ ensureDecodedCapacity(valuesPerVector);
+ if (excPositionsBuffer == null || excPositionsBuffer.length < valuesPerVector) {
+ this.excPositionsBuffer = new int[valuesPerVector];
+ }
+ }
+
+ /**
+ * Reads and validates the fixed 7-byte page header, setting {@link #valuesPerVector} and
+ * {@link #totalValueCount}.
+ *
+ * The reads are relative on purpose. {@link ByteBufferInputStream#slice} does not promise a
+ * buffer positioned at 0, so the absolute {@code get(index)} form would read the wrong bytes for
+ * a page backed by a larger buffer.
+ */
+ private void readPageHeader(ByteBufferInputStream stream, int valuesCount) throws IOException {
+ ByteBuffer headerBuf = stream.slice(ALP_HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN);
+ int compressionMode = headerBuf.get() & 0xFF;
+ int integerEncoding = headerBuf.get() & 0xFF;
+ int logVectorSize = headerBuf.get() & 0xFF;
+ int numElements = headerBuf.getInt();
+
+ if (compressionMode != ALP_COMPRESSION_MODE) {
+ throw new ParquetDecodingException("Unsupported ALP compression mode: " + compressionMode);
+ }
+ if (integerEncoding != ALP_INTEGER_ENCODING_FOR) {
+ throw new ParquetDecodingException("Unsupported ALP integer encoding: " + integerEncoding);
+ }
+ if (logVectorSize < MIN_LOG_VECTOR_SIZE || logVectorSize > MAX_LOG_VECTOR_SIZE) {
+ throw new ParquetDecodingException("Invalid ALP log vector size: " + logVectorSize + ", must be between "
+ + MIN_LOG_VECTOR_SIZE + " and " + MAX_LOG_VECTOR_SIZE);
+ }
+ if (numElements < 0) {
+ throw new ParquetDecodingException("Invalid ALP element count: " + numElements);
+ }
+ // ALP's num_elements is the count of non-null values that went through encoding;
+ // valuesCount is the page row count, which is larger when the column has nulls.
+ // The two are equal only for required (non-null) columns.
+ if (numElements > valuesCount) {
+ throw new ParquetDecodingException(
+ "ALP header element count " + numElements + " exceeds page valuesCount " + valuesCount);
+ }
+
+ this.valuesPerVector = 1 << logVectorSize;
+ this.totalValueCount = numElements;
+ this.pageValueIndex = 0;
+ this.currentVectorIndex = BEFORE_FIRST_VECTOR;
+ }
+
+ /**
+ * Reads the offset array, one entry per vector, after bounding the vector count and checking the
+ * first offset. Leaves room for the trailing end-of-data entry, which the caller fills in.
+ */
+ private void readOffsetArray(ByteBufferInputStream stream, int minVectorBytes) throws IOException {
+ // Bound the vector count against the bytes actually present before allocating anything.
+ // numElements is only capped by valuesCount, which a forged page can set as high as
+ // Integer.MAX_VALUE, so computing this in int would overflow to a negative count, and even
+ // without overflow a large count would drive a huge int[] and a huge stream.slice allocation.
+ // Every vector costs a 4-byte offset entry plus at least its ALP and FOR headers, which gives
+ // a hard ceiling on how many vectors the remaining bytes can possibly describe.
+ long remainingBytes = stream.available();
+ this.numVectors = Math.toIntExact(((long) totalValueCount + valuesPerVector - 1) / valuesPerVector);
+ long minRemainingBytes = (long) numVectors * (Integer.BYTES + minVectorBytes);
+ if (minRemainingBytes > remainingBytes) {
+ throw new ParquetDecodingException("ALP header element count " + totalValueCount
+ + " requires at least " + minRemainingBytes + " bytes but only " + remainingBytes
+ + " remain in the page");
+ }
+
+ int offsetArraySize = offsetArraySize();
+ ByteBuffer offsetBuf = stream.slice(offsetArraySize).order(ByteOrder.LITTLE_ENDIAN);
+ if (numVectors == 0) {
+ this.vectorOffsets = new int[1];
+ return;
+ }
+ // Offsets are measured from the start of the offset array, so the first vector must begin
+ // exactly where the array ends. Check that before allocating and decoding the rest.
+ int firstOffset = offsetBuf.getInt();
+ if (firstOffset != offsetArraySize) {
+ throw new ParquetDecodingException(
+ "ALP first vector offset " + firstOffset + " must equal the offset array size " + offsetArraySize);
+ }
+ this.vectorOffsets = new int[numVectors + 1];
+ vectorOffsets[0] = firstOffset;
+ for (int v = 1; v < numVectors; v++) {
+ vectorOffsets[v] = offsetBuf.getInt();
+ }
+ }
+
+ /**
+ * Slices the remaining page bytes into a view positioned at 0, so the decode path can use
+ * absolute {@code get(pos)} reads.
+ */
+ private int sliceVectorData(ByteBufferInputStream stream) throws IOException {
+ int vectorDataSize = (int) stream.available();
+ ByteBuffer rawSlice = stream.slice(vectorDataSize);
+ this.vectorsData = rawSlice.slice().order(ByteOrder.LITTLE_ENDIAN);
+ return vectorDataSize;
+ }
+
+ private int offsetArraySize() {
+ return numVectors * Integer.BYTES;
+ }
+
+ /**
+ * Validates the offset array before any vector is decoded. {@code vectorOffsets} holds one entry
+ * per vector followed by the end of the vector data, all measured from the start of the offset
+ * array. The first entry is checked when the array is read, so this starts at entry 1: each
+ * vector, including the last, must span at least its fixed headers and no more than
+ * {@code maxVectorBytes}. Without this a forged offset can point into another vector and
+ * silently decode the wrong values.
+ */
+ static void validateVectorOffsets(int[] vectorOffsets, int minVectorBytes, int maxVectorBytes) {
+ int numVectors = vectorOffsets.length - 1;
+ for (int v = 1; v <= numVectors; v++) {
+ long span = (long) vectorOffsets[v] - vectorOffsets[v - 1];
+ if (span < minVectorBytes) {
+ throw new ParquetDecodingException("ALP vector " + (v - 1) + " at offset " + vectorOffsets[v - 1]
+ + " spans " + span + " bytes, fewer than the " + minVectorBytes + " its headers need");
+ }
+ if (span > maxVectorBytes) {
+ throw new ParquetDecodingException("ALP vector " + (v - 1) + " at offset " + vectorOffsets[v - 1]
+ + " spans " + span + " bytes, more than the " + maxVectorBytes + " any vector can occupy");
+ }
+ }
+ }
+
+ /**
+ * The most bytes one vector can occupy: its fixed headers, every value bit-packed at the full
+ * width of the type, and every value also stored as an exception. Vector sizes are multiples of
+ * the 8-value packing group, so the packed section needs no padding allowance.
+ */
+ static int maxVectorBytes(int valuesPerVector, int minVectorBytes, int valueSize) {
+ int packed = valuesPerVector * valueSize;
+ int exceptions = valuesPerVector * (Short.BYTES + valueSize);
+ return minVectorBytes + packed + exceptions;
+ }
+
+ /**
+ * The position in the vector data just past the last byte vector {@code vectorIndex} may read,
+ * taken from the next entry of the offset array; the trailing entry makes this valid for the last
+ * vector too.
+ */
+ int getVectorEndPosition(int vectorIndex) {
+ if (vectorIndex < 0 || vectorIndex >= numVectors) {
+ throw new IllegalArgumentException(
+ "ALP vector index " + vectorIndex + " is out of range for " + numVectors + " vectors");
+ }
+ return vectorOffsets[vectorIndex + 1] - offsetArraySize();
+ }
+
+ /**
+ * The position of vector {@code vectorIndex} in the vector data. This is not an offset as the spec
+ * defines one: those are measured from the start of the offset array, which precedes the vector
+ * data.
+ */
+ protected int getVectorDataPosition(int vectorIndex) {
+ return vectorOffsets[vectorIndex] - offsetArraySize();
+ }
+
+ /** Whether {@code vectorIndex} is the last vector in the page. */
+ static boolean isLast(int vectorIndex, int numVectors) {
+ return vectorIndex == numVectors - 1;
+ }
+
+ /**
+ * The number of values held by vector {@code vectorIndex}. Every vector holds
+ * {@code valuesPerVector} values except the last, which holds whatever remains and so may be
+ * partial. This is a value count, not a byte count.
+ */
+ static int getVectorSize(int vectorIndex, int valuesPerVector, int totalValueCount, int numVectors) {
+ if (isLast(vectorIndex, numVectors)) {
+ int lastVectorSize = totalValueCount % valuesPerVector;
+ return lastVectorSize == 0 ? valuesPerVector : lastVectorSize;
+ }
+ return valuesPerVector;
+ }
+
+ @Override
+ public void skip() {
+ skip(1);
+ }
+
+ @Override
+ public void skip(int n) {
+ // Compare against the remaining count rather than adding to pageValueIndex: a large n would
+ // overflow the sum, pass the check, and leave pageValueIndex negative.
+ if (n < 0 || n > totalValueCount - pageValueIndex) {
+ throw new ParquetDecodingException(String.format(
+ "Cannot skip this many elements. Current index: %d. Skip %d. Total count: %d",
+ pageValueIndex, n, totalValueCount));
+ }
+ pageValueIndex += n;
+ }
+
+ protected void ensureVectorDecoded() {
+ int vectorIndex = pageValueIndex / valuesPerVector;
+ if (vectorIndex != currentVectorIndex) {
+ decodeVector(vectorIndex);
+ currentVectorIndex = vectorIndex;
+ }
+ }
+
+ /**
+ * Decodes one vector, following the division the ALP spec makes: a vector header of AlpInfo and
+ * ForInfo, then a data section of PackedValues, ExceptionPositions and ExceptionValues. Runs once
+ * per vector, so the virtual hook calls stay off the per-value hot path.
+ */
+ protected final void decodeVector(int vectorIndex) {
+ int vectorSize = getVectorSize(vectorIndex, valuesPerVector, totalValueCount, numVectors);
+ int vectorEnd = getVectorEndPosition(vectorIndex);
+ int pos = getVectorDataPosition(vectorIndex);
+
+ // Vector header, part one: AlpInfo, which is the same for both value types.
+ int exponent = BytesUtils.readUnsignedByte(vectorsData, pos);
+ int factor = BytesUtils.readUnsignedByte(vectorsData, pos + 1);
+ int numExceptions = BytesUtils.readUnsignedShortLittleEndian(vectorsData, pos + 2);
+ pos += ALP_INFO_SIZE;
+ validateAlpInfo(vectorIndex, exponent, factor, numExceptions, vectorSize);
+
+ // Vector header, part two: ForInfo, whose width depends on the value type.
+ pos = readForInfo(pos, vectorIndex, forInfo);
+
+ // Data section, part one: PackedValues. Their width comes from this vector's own ForInfo, so
+ // confirm they fit before unpacking; otherwise this vector would decode the next one's bytes,
+ // or for the last vector read past the end of the page.
+ int packedEnd = pos + BytesUtils.paddedByteCountFromBits(vectorSize * forInfo.bitWidth);
+ if (packedEnd > vectorEnd) {
+ throw new ParquetDecodingException("ALP vector " + vectorIndex + " packed values end at " + packedEnd
+ + ", past the end of the vector at " + vectorEnd);
+ }
+ pos = decodePackedValues(pos, vectorSize, exponent, factor, forInfo);
+
+ // Data section, parts two and three: ExceptionPositions then ExceptionValues.
+ if (numExceptions > 0) {
+ long exceptionBytes = (long) numExceptions * (Short.BYTES + exceptionValueSize());
+ if (pos + exceptionBytes > vectorEnd) {
+ throw new ParquetDecodingException(
+ "ALP vector " + vectorIndex + " declares " + numExceptions + " exceptions needing "
+ + exceptionBytes + " bytes, past the end of the vector at " + vectorEnd);
+ }
+ applyExceptions(pos, numExceptions, vectorSize);
+ }
+ }
+
+ /** Validates the AlpInfo fields of a vector header against this type's limits. */
+ private void validateAlpInfo(int vectorIndex, int exponent, int factor, int numExceptions, int vectorSize) {
+ if (exponent > maxExponent()) {
+ throw new ParquetDecodingException("Invalid ALP " + typeName() + " exponent " + exponent + " in vector "
+ + vectorIndex + ", max is " + maxExponent());
+ }
+ if (factor > exponent) {
+ throw new ParquetDecodingException("Invalid ALP " + typeName() + " factor " + factor + " > exponent "
+ + exponent + " in vector " + vectorIndex);
+ }
+ if (numExceptions > vectorSize) {
+ throw new ParquetDecodingException("Invalid ALP numExceptions " + numExceptions + " > vectorSize "
+ + vectorSize + " in vector " + vectorIndex);
+ }
+ }
+
+ /**
+ * Reads {@code numExceptions} little-endian uint16 exception positions into {@code positionsOut}
+ * and returns the position just past them. Static so the exception block's two halves can be
+ * followed without tracing fields.
+ */
+ static int readExceptionPositions(ByteBuffer data, int pos, int numExceptions, int vectorSize, int[] positionsOut) {
+ for (int e = 0; e < numExceptions; e++) {
+ positionsOut[e] = BytesUtils.readUnsignedShortLittleEndian(data, pos);
+ if (positionsOut[e] >= vectorSize) {
+ throw new ParquetDecodingException(
+ "ALP exception position " + positionsOut[e] + " out of bounds for vectorSize " + vectorSize);
+ }
+ pos += Short.BYTES;
+ }
+ return pos;
+ }
+
+ /** Grows the decode buffers to hold {@code capacity} values, keeping them if they already do. */
+ protected abstract void ensureDecodedCapacity(int capacity);
+
+ /** The maximum exponent for this value type (used to validate the header). */
+ protected abstract int maxExponent();
+
+ /** The value type name ("float" / "double") used in decoding error messages. */
+ protected abstract String typeName();
+
+ /** Size in bytes of this type's frame-of-reference header (base + bit width). */
+ protected abstract int forInfoSize();
+
+ /** Size in bytes of one exception value of this type, as stored in the exception block. */
+ protected abstract int exceptionValueSize();
+
+ /**
+ * Reads this vector's ForInfo header into {@code out} and returns the position just past it.
+ */
+ protected abstract int readForInfo(int pos, int vectorIndex, ForInfo out);
+
+ /**
+ * Decodes the PackedValues of the data section into the type-specific decoded buffer and returns
+ * the position just past them.
+ */
+ protected abstract int decodePackedValues(int pos, int vectorSize, int exponent, int factor, ForInfo forInfo);
+
+ /**
+ * Reads the ExceptionPositions and ExceptionValues of the data section and applies them over the
+ * decoded values.
+ */
+ protected abstract void applyExceptions(int pos, int numExceptions, int vectorSize);
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReaderForDouble.java b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReaderForDouble.java
new file mode 100644
index 0000000000..7c5b688bd2
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReaderForDouble.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.apache.parquet.column.values.alp.AlpConstants.DOUBLE_FOR_INFO_SIZE;
+import static org.apache.parquet.column.values.alp.AlpConstants.DOUBLE_MAX_EXPONENT;
+import static org.apache.parquet.column.values.alp.AlpConstants.PACK_GROUP_SIZE;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.bitpacking.BytePackerForLong;
+import org.apache.parquet.column.values.bitpacking.Packer;
+import org.apache.parquet.io.ParquetDecodingException;
+
+/** ALP values reader for DOUBLE type with lazy per-vector decoding. */
+public class AlpValuesReaderForDouble extends AlpValuesReader {
+
+ private double[] decodedValues;
+ private long[] deltasBuffer;
+ private final long[] unpackPadBuf = new long[PACK_GROUP_SIZE];
+ private final byte[] unpackByteBuf = new byte[Long.SIZE]; // 8 values x 64 bits = 64 bytes
+
+ @Override
+ protected void ensureDecodedCapacity(int capacity) {
+ if (decodedValues == null || decodedValues.length < capacity) {
+ this.decodedValues = new double[capacity];
+ this.deltasBuffer = new long[capacity];
+ }
+ }
+
+ @Override
+ protected int maxExponent() {
+ return DOUBLE_MAX_EXPONENT;
+ }
+
+ @Override
+ protected String typeName() {
+ return "double";
+ }
+
+ @Override
+ protected int forInfoSize() {
+ return DOUBLE_FOR_INFO_SIZE;
+ }
+
+ @Override
+ protected int exceptionValueSize() {
+ return Double.BYTES;
+ }
+
+ @Override
+ public double readDouble() {
+ if (pageValueIndex >= totalValueCount) {
+ throw new ParquetDecodingException("ALP double data was already exhausted.");
+ }
+ ensureVectorDecoded();
+ int vectorValueIndex = pageValueIndex % valuesPerVector;
+ pageValueIndex++;
+ return decodedValues[vectorValueIndex];
+ }
+
+ @Override
+ protected int readForInfo(int pos, int vectorIndex, ForInfo out) {
+ out.base = vectorsData.getLong(pos);
+ out.bitWidth = BytesUtils.readUnsignedByte(vectorsData, pos + Long.BYTES);
+ if (out.bitWidth > Long.SIZE) {
+ throw new ParquetDecodingException(
+ "Invalid ALP double bitWidth " + out.bitWidth + " > 64 in vector " + vectorIndex);
+ }
+ return pos + DOUBLE_FOR_INFO_SIZE;
+ }
+
+ @Override
+ protected int decodePackedValues(int pos, int vectorSize, int exponent, int factor, ForInfo forInfo) {
+ if (forInfo.bitWidth > 0) {
+ pos = unpackLongsWithBytePacker(vectorsData, pos, deltasBuffer, vectorSize, forInfo.bitWidth);
+ } else {
+ Arrays.fill(deltasBuffer, 0, vectorSize, 0L);
+ }
+
+ for (int i = 0; i < vectorSize; i++) {
+ long encoded = deltasBuffer[i] + forInfo.base;
+ decodedValues[i] = AlpCodec.decodeDouble(encoded, exponent, factor);
+ }
+ return pos;
+ }
+
+ @Override
+ protected void applyExceptions(int pos, int numExceptions, int vectorSize) {
+ pos = readExceptionPositions(vectorsData, pos, numExceptions, vectorSize, excPositionsBuffer);
+ applyExceptionValues(vectorsData, pos, numExceptions, excPositionsBuffer, decodedValues);
+ }
+
+ /**
+ * Writes {@code numExceptions} raw double values over the decoded values at the given positions.
+ * Static and fully parameterised so it can be exercised directly.
+ */
+ static void applyExceptionValues(
+ ByteBuffer data, int pos, int numExceptions, int[] positions, double[] decodedValues) {
+ for (int e = 0; e < numExceptions; e++) {
+ decodedValues[positions[e]] = data.getDouble(pos);
+ pos += Double.BYTES;
+ }
+ }
+
+ private int unpackLongsWithBytePacker(ByteBuffer buf, int pos, long[] output, int count, int bitWidth) {
+ BytePackerForLong packer = Packer.LITTLE_ENDIAN.newBytePackerForLong(bitWidth);
+ int numFullGroups = count / PACK_GROUP_SIZE;
+ int remaining = count % PACK_GROUP_SIZE;
+
+ for (int g = 0; g < numFullGroups; g++) {
+ packer.unpack8Values(buf, pos, output, g * PACK_GROUP_SIZE);
+ pos += bitWidth;
+ }
+
+ // Last group might have fewer than 8 values; zero-pad and unpack,
+ // but only advance pos by the actual bytes in the page.
+ if (remaining > 0) {
+ int totalPackedBytes = BytesUtils.paddedByteCountFromBits(count * bitWidth);
+ int alreadyRead = numFullGroups * bitWidth;
+ int partialBytes = totalPackedBytes - alreadyRead;
+
+ for (int i = 0; i < partialBytes; i++) {
+ unpackByteBuf[i] = buf.get(pos + i);
+ }
+ for (int i = partialBytes; i < bitWidth; i++) {
+ unpackByteBuf[i] = 0;
+ }
+
+ packer.unpack8Values(unpackByteBuf, 0, unpackPadBuf, 0);
+ System.arraycopy(unpackPadBuf, 0, output, numFullGroups * PACK_GROUP_SIZE, remaining);
+ pos += partialBytes;
+ }
+
+ return pos;
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReaderForFloat.java b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReaderForFloat.java
new file mode 100644
index 0000000000..0a0cc90349
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesReaderForFloat.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.apache.parquet.column.values.alp.AlpConstants.FLOAT_FOR_INFO_SIZE;
+import static org.apache.parquet.column.values.alp.AlpConstants.FLOAT_MAX_EXPONENT;
+import static org.apache.parquet.column.values.alp.AlpConstants.PACK_GROUP_SIZE;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.bitpacking.BytePacker;
+import org.apache.parquet.column.values.bitpacking.Packer;
+import org.apache.parquet.io.ParquetDecodingException;
+
+/** ALP values reader for FLOAT type with lazy per-vector decoding. */
+public class AlpValuesReaderForFloat extends AlpValuesReader {
+
+ private float[] decodedValues;
+ private int[] deltasBuffer;
+ private final int[] unpackPadBuf = new int[PACK_GROUP_SIZE];
+ private final byte[] unpackByteBuf = new byte[Integer.SIZE]; // 8 values x 32 bits = 32 bytes
+
+ @Override
+ protected void ensureDecodedCapacity(int capacity) {
+ if (decodedValues == null || decodedValues.length < capacity) {
+ this.decodedValues = new float[capacity];
+ this.deltasBuffer = new int[capacity];
+ }
+ }
+
+ @Override
+ protected int maxExponent() {
+ return FLOAT_MAX_EXPONENT;
+ }
+
+ @Override
+ protected String typeName() {
+ return "float";
+ }
+
+ @Override
+ protected int forInfoSize() {
+ return FLOAT_FOR_INFO_SIZE;
+ }
+
+ @Override
+ protected int exceptionValueSize() {
+ return Float.BYTES;
+ }
+
+ @Override
+ public float readFloat() {
+ if (pageValueIndex >= totalValueCount) {
+ throw new ParquetDecodingException("ALP float data was already exhausted.");
+ }
+ ensureVectorDecoded();
+ int vectorValueIndex = pageValueIndex % valuesPerVector;
+ pageValueIndex++;
+ return decodedValues[vectorValueIndex];
+ }
+
+ @Override
+ protected int readForInfo(int pos, int vectorIndex, ForInfo out) {
+ out.base = vectorsData.getInt(pos);
+ out.bitWidth = BytesUtils.readUnsignedByte(vectorsData, pos + Integer.BYTES);
+ if (out.bitWidth > Integer.SIZE) {
+ throw new ParquetDecodingException(
+ "Invalid ALP float bitWidth " + out.bitWidth + " > 32 in vector " + vectorIndex);
+ }
+ return pos + FLOAT_FOR_INFO_SIZE;
+ }
+
+ @Override
+ protected int decodePackedValues(int pos, int vectorSize, int exponent, int factor, ForInfo forInfo) {
+ // Narrow the base before adding: the sum must wrap in 32 bits exactly as the writer's did.
+ int frameOfReference = (int) forInfo.base;
+ if (forInfo.bitWidth > 0) {
+ pos = unpackIntsWithBytePacker(vectorsData, pos, deltasBuffer, vectorSize, forInfo.bitWidth);
+ } else {
+ Arrays.fill(deltasBuffer, 0, vectorSize, 0);
+ }
+
+ for (int i = 0; i < vectorSize; i++) {
+ int encoded = deltasBuffer[i] + frameOfReference;
+ decodedValues[i] = AlpCodec.decodeFloat(encoded, exponent, factor);
+ }
+ return pos;
+ }
+
+ @Override
+ protected void applyExceptions(int pos, int numExceptions, int vectorSize) {
+ pos = readExceptionPositions(vectorsData, pos, numExceptions, vectorSize, excPositionsBuffer);
+ applyExceptionValues(vectorsData, pos, numExceptions, excPositionsBuffer, decodedValues);
+ }
+
+ /**
+ * Writes {@code numExceptions} raw float values over the decoded values at the given positions.
+ * Static and fully parameterised so it can be exercised directly.
+ */
+ static void applyExceptionValues(
+ ByteBuffer data, int pos, int numExceptions, int[] positions, float[] decodedValues) {
+ for (int e = 0; e < numExceptions; e++) {
+ decodedValues[positions[e]] = data.getFloat(pos);
+ pos += Float.BYTES;
+ }
+ }
+
+ private int unpackIntsWithBytePacker(ByteBuffer buf, int pos, int[] output, int count, int bitWidth) {
+ BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
+ int numFullGroups = count / PACK_GROUP_SIZE;
+ int remaining = count % PACK_GROUP_SIZE;
+
+ for (int g = 0; g < numFullGroups; g++) {
+ packer.unpack8Values(buf, pos, output, g * PACK_GROUP_SIZE);
+ pos += bitWidth;
+ }
+
+ if (remaining > 0) {
+ int totalPackedBytes = BytesUtils.paddedByteCountFromBits(count * bitWidth);
+ int alreadyRead = numFullGroups * bitWidth;
+ int partialBytes = totalPackedBytes - alreadyRead;
+
+ for (int i = 0; i < partialBytes; i++) {
+ unpackByteBuf[i] = buf.get(pos + i);
+ }
+ for (int i = partialBytes; i < bitWidth; i++) {
+ unpackByteBuf[i] = 0;
+ }
+
+ packer.unpack8Values(unpackByteBuf, 0, unpackPadBuf, 0);
+ System.arraycopy(unpackPadBuf, 0, output, numFullGroups * PACK_GROUP_SIZE, remaining);
+ pos += partialBytes;
+ }
+
+ return pos;
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesWriter.java
new file mode 100644
index 0000000000..95e46583f1
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesWriter.java
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.apache.parquet.column.values.alp.AlpConstants.ALP_COMPRESSION_MODE;
+import static org.apache.parquet.column.values.alp.AlpConstants.ALP_HEADER_SIZE;
+import static org.apache.parquet.column.values.alp.AlpConstants.ALP_INTEGER_ENCODING_FOR;
+import static org.apache.parquet.column.values.alp.AlpConstants.DEFAULT_VECTOR_SIZE;
+import static org.apache.parquet.column.values.alp.AlpConstants.PACK_GROUP_SIZE;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.parquet.bytes.ByteBufferAllocator;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.bytes.CapacityByteArrayOutputStream;
+import org.apache.parquet.column.Encoding;
+import org.apache.parquet.column.values.ValuesWriter;
+import org.apache.parquet.column.values.bitpacking.BytePacker;
+import org.apache.parquet.column.values.bitpacking.BytePackerForLong;
+import org.apache.parquet.column.values.bitpacking.Packer;
+
+/**
+ * ALP (Adaptive Lossless floating-Point) values writer.
+ *
+ * Writing is incremental: values are buffered in a fixed-size vector buffer,
+ * and each full vector is encoded and flushed to the output stream immediately.
+ * On {@link #getBytes()}, any remaining partial vector is flushed, and the
+ * final page bytes are assembled.
+ *
+ * Page layout:
+ * Each vector holds AlpInfo(4B) + ForInfo(5B/9B) + PackedValues + ExceptionPositions +
+ * ExceptionValues.
+ */
+public abstract class AlpValuesWriter extends ValuesWriter {
+
+ // The spec's sampling parameters: 8 sampled vectors per row group and up to 5 preset
+ // combinations. The row group size comes from the C++ implementation.
+ static final int SAMPLER_ROWGROUP_SIZE = 122_880;
+ static final int SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP = 8;
+ static final int MAX_PRESET_COMBINATIONS = 5;
+
+ protected final int valuesPerVector;
+ protected int bufferCount = 0;
+ private int totalValueCount = 0;
+ private final CapacityByteArrayOutputStream encodedVectors;
+ private final List "Fails cleanly" means raising a meaningful exception — preferably
+ * {@link ParquetDecodingException}, but at minimum a typed exception (not a JVM-level
+ * crash, infinite loop, wrong answer, or an allocation large enough to exhaust the heap).
+ * The tests cover both:
+ * The accepted types are deliberately narrow. An {@link OutOfMemoryError} means a corrupt
+ * length drove an unbounded allocation, and an {@link AssertionError} means an assertion inside
+ * {@code action} failed; accepting any {@link Throwable} here would let either count as a
+ * successful rejection.
+ *
+ * No message is asserted because these failures come from two sources: the reader's own
+ * validation, which carries an explanatory message, and the underlying {@link ByteBuffer}, whose
+ * {@link IndexOutOfBoundsException} and {@link BufferUnderflowException} have none. What these
+ * tests pin down is that the failure is loud and typed, not what it says.
+ */
+ private static void assertFailsCleanly(ThrowingCallable action) {
+ assertThatThrownBy(action)
+ .isInstanceOfAny(
+ ParquetDecodingException.class,
+ IOException.class,
+ IndexOutOfBoundsException.class,
+ BufferUnderflowException.class,
+ IllegalArgumentException.class);
+ }
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpBitPackingTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpBitPackingTest.java
new file mode 100644
index 0000000000..c26bb50d11
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpBitPackingTest.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.DirectByteBufferAllocator;
+import org.apache.parquet.column.values.bitpacking.BytePacker;
+import org.apache.parquet.column.values.bitpacking.BytePackerForLong;
+import org.apache.parquet.column.values.bitpacking.Packer;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for bit-packing behavior in the ALP encoding pipeline.
+ */
+public class AlpBitPackingTest {
+
+ @Test
+ public void testBytePackerIntRoundTrip() {
+ // Verify BytePacker pack/unpack round-trip for every bit width, including the
+ // full 32-bit width where the frame-of-reference delta uses all int bits.
+ for (int bitWidth = 1; bitWidth <= 32; bitWidth++) {
+ BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
+ // Largest value representable in bitWidth bits (unsigned). At width 32 that is
+ // all bits set (-1); (1L << 32) - 1 would not fit in an int, so special-case it.
+ int maxVal = (bitWidth == 32) ? -1 : (int) ((1L << bitWidth) - 1);
+
+ int[] input = new int[8];
+ for (int i = 0; i < 7; i++) {
+ input[i] = (maxVal >>> 3) * i; // spread of values, each fitting in bitWidth bits
+ }
+ input[7] = maxVal; // exercise the top bit of this width
+
+ byte[] packed = new byte[bitWidth];
+ packer.pack8Values(input, 0, packed, 0);
+
+ int[] unpacked = new int[8];
+ ByteBuffer buf = ByteBuffer.wrap(packed);
+ packer.unpack8Values(buf, 0, unpacked, 0);
+
+ for (int i = 0; i < 8; i++) {
+ assertThat(unpacked[i])
+ .as("BitWidth=" + bitWidth + " index=" + i)
+ .isEqualTo(input[i]);
+ }
+ }
+ }
+
+ @Test
+ public void testBytePackerForLongRoundTrip() {
+ // Verify BytePackerForLong pack/unpack round-trip for every bit width, including the
+ // full 64-bit width where the frame-of-reference delta uses all long bits.
+ for (int bitWidth = 1; bitWidth <= 64; bitWidth++) {
+ BytePackerForLong packer = Packer.LITTLE_ENDIAN.newBytePackerForLong(bitWidth);
+ // Largest value representable in bitWidth bits (unsigned). At width 64 that is
+ // all bits set (-1L); (1L << 64) - 1 would wrap to 0, so special-case it.
+ long maxVal = (bitWidth == 64) ? -1L : (1L << bitWidth) - 1;
+
+ long[] input = new long[8];
+ for (int i = 0; i < 7; i++) {
+ input[i] = (maxVal >>> 3) * i; // spread of values, each fitting in bitWidth bits
+ }
+ input[7] = maxVal; // exercise the top bit of this width
+
+ byte[] packed = new byte[bitWidth];
+ packer.pack8Values(input, 0, packed, 0);
+
+ long[] unpacked = new long[8];
+ ByteBuffer buf = ByteBuffer.wrap(packed);
+ packer.unpack8Values(buf, 0, unpacked, 0);
+
+ for (int i = 0; i < 8; i++) {
+ assertThat(unpacked[i])
+ .as("BitWidth=" + bitWidth + " index=" + i)
+ .isEqualTo(input[i]);
+ }
+ }
+ }
+
+ @Test
+ public void testSimpleTwoFloats() throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ writer.writeFloat(1.0f);
+ writer.writeFloat(2.0f);
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(2, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(1.0f));
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(2.0f));
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testThreeFloatsWithNegative() throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ writer.writeFloat(1.0f);
+ writer.writeFloat(-1.0f);
+ writer.writeFloat(2.0f);
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(3, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(1.0f));
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(-1.0f));
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(2.0f));
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testEncoderDecoderDirectly() {
+ float value = 1.0f;
+ int exponent = 2;
+ int factor = 0;
+
+ AlpCodec.EncodeResult result = new AlpCodec.EncodeResult();
+ AlpCodec.tryEncodeFloat(value, exponent, factor, result);
+ assertThat(result.isException).isFalse();
+ int encoded = AlpCodec.encodeFloat(value, exponent, factor);
+ float decoded = AlpCodec.decodeFloat(encoded, exponent, factor);
+ assertThat(Float.floatToRawIntBits(decoded)).isEqualTo(Float.floatToRawIntBits(value));
+ }
+
+ @Test
+ public void testHeaderFormatValidation() throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ writer.writeFloat(1.0f);
+ writer.writeFloat(2.0f);
+
+ BytesInput input = writer.getBytes();
+ byte[] bytes = input.toByteArray();
+
+ // Parse and validate header (7 bytes: mode + encoding + logVectorSize + count)
+ ByteBuffer buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
+ int compressionMode = buf.get() & 0xFF;
+ int integerEncoding = buf.get() & 0xFF;
+ int logVectorSize = buf.get() & 0xFF;
+ int numElements = buf.getInt();
+
+ assertThat(compressionMode).isEqualTo(AlpConstants.ALP_COMPRESSION_MODE);
+ assertThat(integerEncoding).isEqualTo(AlpConstants.ALP_INTEGER_ENCODING_FOR);
+ assertThat(logVectorSize).isEqualTo(10);
+ assertThat(numElements).isEqualTo(2);
+
+ // Verify offset array follows header (1 vector = 1 offset entry)
+ int offset0 = buf.getInt();
+ // First vector starts right after the offset array (1 vector * 4 bytes = 4)
+ assertThat(offset0).isEqualTo(Integer.BYTES);
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testPartialGroupPacking() throws Exception {
+ // Test with fewer than 8 values to exercise partial group packing/unpacking
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ float[] values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f};
+ for (float v : values) {
+ writer.writeFloat(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (float expected : values) {
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(expected));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testBitWidthZeroPacking() throws Exception {
+ // All identical values should result in bitWidth=0 (no packed data)
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ for (int i = 0; i < 10; i++) {
+ writer.writeFloat(5.0f);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(10, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (int i = 0; i < 10; i++) {
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(5.0f));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testDoublePackingRoundTrip() throws Exception {
+ // Verify double packing round-trip with varying values
+ AlpValuesWriter.DoubleAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.DoubleAlpValuesWriter(512, 512, new DirectByteBufferAllocator());
+ double[] values = {1.0, -1.0, 2.0, 100.5, 0.0, 3.14, 42.0, 99.99, 0.001, 7.77};
+ for (double v : values) {
+ writer.writeDouble(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (double expected : values) {
+ assertThat(Double.doubleToRawLongBits(reader.readDouble()))
+ .isEqualTo(Double.doubleToRawLongBits(expected));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ @Test
+ public void testExactEightValues() throws Exception {
+ // Exactly 8 values = one full packing group, no partial group
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ float[] values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f};
+ for (float v : values) {
+ writer.writeFloat(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (float expected : values) {
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(expected));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpCodecTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpCodecTest.java
new file mode 100644
index 0000000000..4d88c2ba65
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpCodecTest.java
@@ -0,0 +1,460 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.within;
+
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for the core ALP encoder/decoder logic.
+ */
+public class AlpCodecTest {
+
+ // ========== Float Encoding/Decoding Tests ==========
+
+ @Test
+ public void testFloatRoundTrip() {
+ float[] testValues = {0.0f, 1.0f, -1.0f, 3.14159f, 100.5f, 0.001f, 1234567.0f};
+
+ for (float value : testValues) {
+ for (int exponent = 0; exponent <= AlpConstants.FLOAT_MAX_EXPONENT; exponent++) {
+ for (int factor = 0; factor <= exponent; factor++) {
+ if (!isException(value, exponent, factor)) {
+ int encoded = AlpCodec.encodeFloat(value, exponent, factor);
+ float decoded = AlpCodec.decodeFloat(encoded, exponent, factor);
+ assertThat(Float.floatToRawIntBits(decoded))
+ .as("Round-trip failed for value=" + value + ", exponent=" + exponent + ", factor="
+ + factor)
+ .isEqualTo(Float.floatToRawIntBits(value));
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testFloatExceptionDetection() {
+ assertThat(AlpCodec.isIntrinsicFloatException(Float.NaN))
+ .as("NaN should be an exception")
+ .isTrue();
+ assertThat(AlpCodec.isIntrinsicFloatException(Float.POSITIVE_INFINITY))
+ .as("Positive infinity should be an exception")
+ .isTrue();
+ assertThat(AlpCodec.isIntrinsicFloatException(Float.NEGATIVE_INFINITY))
+ .as("Negative infinity should be an exception")
+ .isTrue();
+ assertThat(AlpCodec.isIntrinsicFloatException(-0.0f))
+ .as("Negative zero should be an exception")
+ .isTrue();
+
+ assertThat(AlpCodec.isIntrinsicFloatException(1.0f))
+ .as("1.0f should not be a basic exception")
+ .isFalse();
+ assertThat(AlpCodec.isIntrinsicFloatException(0.0f))
+ .as("0.0f should not be a basic exception")
+ .isFalse();
+ }
+
+ @Test
+ public void testFloatEncoding() {
+ assertThat(AlpCodec.encodeFloat(1.23f, 2, 0)).isEqualTo(123);
+ assertThat(AlpCodec.encodeFloat(12.3f, 2, 1)).isEqualTo(123);
+ assertThat(AlpCodec.encodeFloat(0.0f, 5, 0)).isZero();
+ }
+
+ @Test
+ public void testFloatDecoding() {
+ assertThat(AlpCodec.decodeFloat(123, 2, 0)).isCloseTo(1.23f, within(1e-6f));
+ assertThat(AlpCodec.decodeFloat(123, 2, 1)).isCloseTo(12.3f, within(1e-6f));
+ assertThat(AlpCodec.decodeFloat(0, 5, 0)).isEqualTo(0.0f);
+ }
+
+ @Test
+ public void testFloatEncodeRounding() {
+ // Verify rounding behavior (magic number trick rounds to nearest)
+ assertThat(AlpCodec.encodeFloat(5.4f, 0, 0)).isEqualTo(5);
+ assertThat(AlpCodec.encodeFloat(5.6f, 0, 0)).isEqualTo(6);
+ assertThat(AlpCodec.encodeFloat(-5.4f, 0, 0)).isEqualTo(-5);
+ assertThat(AlpCodec.encodeFloat(-5.6f, 0, 0)).isEqualTo(-6);
+ assertThat(AlpCodec.encodeFloat(0.0f, 0, 0)).isZero();
+ }
+
+ @Test
+ public void testFloatEncodeDecodeWithFactor() {
+ // Verify that encode/decode with non-zero factor works correctly.
+ // The key correctness property: encode uses value * POW10[e] * POW10_NEGATIVE[f],
+ // and decode uses encoded * POW10[f] * POW10_NEGATIVE[e].
+ float value = 12.3f;
+ int encoded = AlpCodec.encodeFloat(value, 2, 1);
+ assertThat(encoded).isEqualTo(123); // 12.3 * 100 * 0.1 = 123
+ float decoded = AlpCodec.decodeFloat(encoded, 2, 1);
+ assertThat(Float.floatToRawIntBits(decoded)).isEqualTo(Float.floatToRawIntBits(value));
+ }
+
+ // ========== Double Encoding/Decoding Tests ==========
+
+ @Test
+ public void testDoubleRoundTrip() {
+ double[] testValues = {0.0, 1.0, -1.0, 3.14159265358979, 100.5, 0.001, 12345678901234.0};
+
+ for (double value : testValues) {
+ for (int exponent = 0; exponent <= Math.min(AlpConstants.DOUBLE_MAX_EXPONENT, 10); exponent++) {
+ for (int factor = 0; factor <= exponent; factor++) {
+ if (!isException(value, exponent, factor)) {
+ long encoded = AlpCodec.encodeDouble(value, exponent, factor);
+ double decoded = AlpCodec.decodeDouble(encoded, exponent, factor);
+ assertThat(Double.doubleToRawLongBits(decoded))
+ .as("Round-trip failed for value=" + value + ", exponent=" + exponent + ", factor="
+ + factor)
+ .isEqualTo(Double.doubleToRawLongBits(value));
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testDoubleExceptionDetection() {
+ assertThat(AlpCodec.isIntrinsicDoubleException(Double.NaN))
+ .as("NaN should be an exception")
+ .isTrue();
+ assertThat(AlpCodec.isIntrinsicDoubleException(Double.POSITIVE_INFINITY))
+ .as("Positive infinity should be an exception")
+ .isTrue();
+ assertThat(AlpCodec.isIntrinsicDoubleException(Double.NEGATIVE_INFINITY))
+ .as("Negative infinity should be an exception")
+ .isTrue();
+ assertThat(AlpCodec.isIntrinsicDoubleException(-0.0))
+ .as("Negative zero should be an exception")
+ .isTrue();
+
+ assertThat(AlpCodec.isIntrinsicDoubleException(1.0))
+ .as("1.0 should not be a basic exception")
+ .isFalse();
+ assertThat(AlpCodec.isIntrinsicDoubleException(0.0))
+ .as("0.0 should not be a basic exception")
+ .isFalse();
+ }
+
+ @Test
+ public void testDoubleEncoding() {
+ assertThat(AlpCodec.encodeDouble(1.23, 2, 0)).isEqualTo(123L);
+ assertThat(AlpCodec.encodeDouble(12.3, 2, 1)).isEqualTo(123L);
+ assertThat(AlpCodec.encodeDouble(0.0, 5, 0)).isZero();
+ }
+
+ @Test
+ public void testDoubleDecoding() {
+ assertThat(AlpCodec.decodeDouble(123, 2, 0)).isCloseTo(1.23, within(1e-10));
+ assertThat(AlpCodec.decodeDouble(123, 2, 1)).isCloseTo(12.3, within(1e-10));
+ assertThat(AlpCodec.decodeDouble(0, 5, 0)).isEqualTo(0.0);
+ }
+
+ @Test
+ public void testDoubleEncodeRounding() {
+ // Verify rounding behavior (magic number trick rounds to nearest)
+ assertThat(AlpCodec.encodeDouble(5.4, 0, 0)).isEqualTo(5L);
+ assertThat(AlpCodec.encodeDouble(5.6, 0, 0)).isEqualTo(6L);
+ assertThat(AlpCodec.encodeDouble(-5.4, 0, 0)).isEqualTo(-5L);
+ assertThat(AlpCodec.encodeDouble(-5.6, 0, 0)).isEqualTo(-6L);
+ assertThat(AlpCodec.encodeDouble(0.0, 0, 0)).isZero();
+ }
+
+ @Test
+ public void testDoubleEncodeDecodeWithFactor() {
+ // Verify that encode/decode with non-zero factor works correctly.
+ // The key correctness property: encode uses value * POW10[e] * POW10_NEGATIVE[f],
+ // and decode uses encoded * POW10[f] * POW10_NEGATIVE[e].
+ double value = 12.3;
+ long encoded = AlpCodec.encodeDouble(value, 2, 1);
+ assertThat(encoded).isEqualTo(123L); // 12.3 * 100 * 0.1 = 123
+ double decoded = AlpCodec.decodeDouble(encoded, 2, 1);
+ assertThat(Double.doubleToRawLongBits(decoded)).isEqualTo(Double.doubleToRawLongBits(value));
+ }
+
+ @Test
+ public void testDoubleEncodeDecodeArithmeticOrder() {
+ // This test verifies that the exact order of operations in encode/decode
+ // is critical for IEEE 754 correctness. The encode uses
+ // fastRound(value * POW10[e] * POW10_NEGATIVE[f]) and decode uses
+ // (encoded * POW10[f] * POW10_NEGATIVE[e]), both as single expressions.
+ // Splitting the multiplies or reordering changes rounding.
+ double[] testValues = {0.123456789, 1.23456789, 12.3456789, 123.456789, 1234.56789};
+ for (double value : testValues) {
+ for (int e = 0; e <= 10; e++) {
+ for (int f = 0; f <= e; f++) {
+ if (!isException(value, e, f)) {
+ long encoded = AlpCodec.encodeDouble(value, e, f);
+ double decoded = AlpCodec.decodeDouble(encoded, e, f);
+ assertThat(Double.doubleToRawLongBits(decoded))
+ .as("Roundtrip failed for " + value + " (e=" + e + ", f=" + f + ")")
+ .isEqualTo(Double.doubleToRawLongBits(value));
+ }
+ }
+ }
+ }
+ }
+
+ // ========== Best Parameters Tests ==========
+
+ @Test
+ public void testFindBestFloatParams() {
+ float[] values = {1.23f, 4.56f, 7.89f, 10.11f, 12.13f};
+ AlpCodec.EncodingParams params =
+ AlpCodec.findBestFloatParams(values, 0, values.length, AlpCodec.ALL_VALID_FLOAT_PAIRS);
+
+ assertThat(params).isNotNull();
+ assertThat(params.exponent).isBetween(0, AlpConstants.FLOAT_MAX_EXPONENT);
+ assertThat(params.factor).isBetween(0, params.exponent);
+
+ for (float v : values) {
+ if (!isException(v, params.exponent, params.factor)) {
+ int encoded = AlpCodec.encodeFloat(v, params.exponent, params.factor);
+ float decoded = AlpCodec.decodeFloat(encoded, params.exponent, params.factor);
+ assertThat(Float.floatToRawIntBits(decoded)).isEqualTo(Float.floatToRawIntBits(v));
+ }
+ }
+ }
+
+ @Test
+ public void testFindBestFloatParamsWithAllExceptions() {
+ float[] values = {Float.NaN, Float.NaN, Float.NaN};
+ AlpCodec.EncodingParams params =
+ AlpCodec.findBestFloatParams(values, 0, values.length, AlpCodec.ALL_VALID_FLOAT_PAIRS);
+
+ assertThat(params).isNotNull();
+ assertThat(params.numExceptions).isEqualTo(values.length);
+ }
+
+ @Test
+ public void testFindBestDoubleParams() {
+ double[] values = {1.23, 4.56, 7.89, 10.11, 12.13};
+ AlpCodec.EncodingParams params =
+ AlpCodec.findBestDoubleParams(values, 0, values.length, AlpCodec.ALL_VALID_DOUBLE_PAIRS);
+
+ assertThat(params).isNotNull();
+ assertThat(params.exponent).isBetween(0, AlpConstants.DOUBLE_MAX_EXPONENT);
+ assertThat(params.factor).isBetween(0, params.exponent);
+
+ for (double v : values) {
+ if (!isException(v, params.exponent, params.factor)) {
+ long encoded = AlpCodec.encodeDouble(v, params.exponent, params.factor);
+ double decoded = AlpCodec.decodeDouble(encoded, params.exponent, params.factor);
+ assertThat(Double.doubleToRawLongBits(decoded)).isEqualTo(Double.doubleToRawLongBits(v));
+ }
+ }
+ }
+
+ @Test
+ public void testFindBestDoubleParamsWithAllExceptions() {
+ double[] values = {Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY};
+ AlpCodec.EncodingParams params =
+ AlpCodec.findBestDoubleParams(values, 0, values.length, AlpCodec.ALL_VALID_DOUBLE_PAIRS);
+
+ assertThat(params).isNotNull();
+ assertThat(params.numExceptions).isEqualTo(values.length);
+ }
+
+ @Test
+ public void testFindBestParamsWithOffset() {
+ float[] values = {Float.NaN, Float.NaN, 1.23f, 4.56f, 7.89f, Float.NaN};
+ AlpCodec.EncodingParams params = AlpCodec.findBestFloatParams(values, 2, 3, AlpCodec.ALL_VALID_FLOAT_PAIRS);
+
+ assertThat(params).isNotNull();
+ assertThat(params.numExceptions).isZero();
+ }
+
+ // ========== Preset-Based Parameter Search Tests ==========
+
+ @Test
+ public void testFindBestFloatParamsOverPresets() {
+ float[] values = {1.23f, 4.56f, 7.89f, 10.11f, 12.13f};
+ int[][] presets = {{2, 0}, {3, 0}, {4, 1}};
+ AlpCodec.EncodingParams params = AlpCodec.findBestFloatParams(values, 0, values.length, presets);
+
+ assertThat(params).isNotNull();
+ // Should select one of the preset combinations
+ boolean foundMatch = false;
+ for (int[] preset : presets) {
+ if (params.exponent == preset[0] && params.factor == preset[1]) {
+ foundMatch = true;
+ break;
+ }
+ }
+ assertThat(foundMatch)
+ .as("Result should be one of the preset combinations")
+ .isTrue();
+ }
+
+ @Test
+ public void testFindBestDoubleParamsOverPresets() {
+ double[] values = {1.23, 4.56, 7.89, 10.11, 12.13};
+ int[][] presets = {{2, 0}, {3, 0}, {4, 1}};
+ AlpCodec.EncodingParams params = AlpCodec.findBestDoubleParams(values, 0, values.length, presets);
+
+ assertThat(params).isNotNull();
+ boolean foundMatch = false;
+ for (int[] preset : presets) {
+ if (params.exponent == preset[0] && params.factor == preset[1]) {
+ foundMatch = true;
+ break;
+ }
+ }
+ assertThat(foundMatch)
+ .as("Result should be one of the preset combinations")
+ .isTrue();
+ }
+
+ @Test
+ public void testPresetsProduceSameResultAsFullSearch() {
+ float[] values = {1.23f, 4.56f, 7.89f};
+ AlpCodec.EncodingParams fullResult =
+ AlpCodec.findBestFloatParams(values, 0, values.length, AlpCodec.ALL_VALID_FLOAT_PAIRS);
+
+ // Include the best params in presets
+ int[][] presets = {{fullResult.exponent, fullResult.factor}, {0, 0}, {1, 0}};
+ AlpCodec.EncodingParams presetResult = AlpCodec.findBestFloatParams(values, 0, values.length, presets);
+
+ assertThat(presetResult.numExceptions)
+ .as("Preset result should be at least as good as full search")
+ .isLessThanOrEqualTo(fullResult.numExceptions);
+ }
+
+ @Test
+ public void findBestFloatParamsMixedSignUsesSignedDeltaRange() {
+ // Mixed-sign, non-integer values are losslessly encodable at some (e, f). A buggy estimator
+ // that computes the FOR range with unsigned subtraction inflates every mixed-sign combo to a
+ // 64-bit width, making it prefer combos that turn values into exceptions. With the correct
+ // signed range (max - min), the estimator finds the lossless encoding with zero exceptions.
+ float[] values = {-3.14f, 2.71f, 100.5f, -50.25f};
+ AlpCodec.EncodingParams params =
+ AlpCodec.findBestFloatParams(values, 0, values.length, AlpCodec.ALL_VALID_FLOAT_PAIRS);
+ assertThat(params.numExceptions).isZero();
+ }
+
+ @Test
+ public void findBestFloatParamsOverPresetsMixedSignUsesSignedDeltaRange() {
+ // Restricting to two presets removes the equivalent-scaling ties, so the chosen exponent is
+ // deterministic: e=0,f=0 has the smallest signed FOR span and wins. A buggy unsigned estimator
+ // would rate both presets at 64 bits, tie, and pick the higher exponent (e=1) instead.
+ float[] values = {-1.0f, 1.0f};
+ int[][] presets = {{0, 0}, {1, 0}}; // e=0,f=0 has the smallest signed FOR span
+ AlpCodec.EncodingParams params = AlpCodec.findBestFloatParams(values, 0, values.length, presets);
+ assertThat(params.exponent).isZero();
+ assertThat(params.factor).isZero();
+ assertThat(params.numExceptions).isZero();
+ }
+
+ /**
+ * tryEncode computes the exception flag and the encoded value in one pass, reusing the scaled
+ * intermediate. It must agree exactly with the older isException-then-encode pair it replaced,
+ * including the raw bits of the encoding, since those bytes go on the wire.
+ */
+ @Test
+ public void testTryEncodeMatchesSeparateExceptionCheckAndEncode() {
+ float[] floats = {
+ 0.0f,
+ -0.0f,
+ 1.0f,
+ -1.0f,
+ 0.01f,
+ 1234.56f,
+ 1e-8f,
+ 1e9f,
+ Float.MIN_VALUE,
+ Float.MAX_VALUE,
+ Float.NaN,
+ Float.POSITIVE_INFINITY,
+ Float.NEGATIVE_INFINITY,
+ 3.14159f,
+ -273.15f
+ };
+ AlpCodec.EncodeResult result = new AlpCodec.EncodeResult();
+ for (float value : floats) {
+ for (int e = 0; e <= AlpConstants.FLOAT_MAX_EXPONENT; e++) {
+ for (int f = 0; f <= e; f++) {
+ AlpCodec.tryEncodeFloat(value, e, f, result);
+ if (!result.isException) {
+ assertThat((int) result.encoded)
+ .as("float encoding for %s at e=%s f=%s", value, e, f)
+ .isEqualTo(AlpCodec.encodeFloat(value, e, f));
+ }
+ }
+ }
+ }
+
+ double[] doubles = {
+ 0.0,
+ -0.0,
+ 1.0,
+ -1.0,
+ 0.01,
+ 1234.56,
+ 1e-15,
+ 1e17,
+ Double.MIN_VALUE,
+ Double.MAX_VALUE,
+ Double.NaN,
+ Double.POSITIVE_INFINITY,
+ Double.NEGATIVE_INFINITY,
+ Math.PI,
+ -273.15
+ };
+ for (double value : doubles) {
+ for (int e = 0; e <= AlpConstants.DOUBLE_MAX_EXPONENT; e++) {
+ for (int f = 0; f <= e; f++) {
+ AlpCodec.tryEncodeDouble(value, e, f, result);
+ if (!result.isException) {
+ assertThat(result.encoded)
+ .as("double encoding for %s at e=%s f=%s", value, e, f)
+ .isEqualTo(AlpCodec.encodeDouble(value, e, f));
+ }
+ }
+ }
+ }
+ }
+
+ /** ±2^63 and ±2^31 round-trip through the cast to long or int, so only the limits reject them. */
+ @Test
+ public void testEncodingLimitsDecideExceptions() {
+ assertThat(isException(Math.scalb(1.0, 62), 0, 0)).isFalse();
+ assertThat(isException(-Math.scalb(1.0, 62), 0, 0)).isFalse();
+ assertThat(isException(Math.scalb(1.0, 63), 0, 0)).isTrue();
+ assertThat(isException(-Math.scalb(1.0, 63), 0, 0)).isTrue();
+
+ assertThat(isException(Math.scalb(1.0f, 30), 0, 0)).isFalse();
+ assertThat(isException(-Math.scalb(1.0f, 30), 0, 0)).isFalse();
+ assertThat(isException(Math.scalb(1.0f, 31), 0, 0)).isTrue();
+ assertThat(isException(-Math.scalb(1.0f, 31), 0, 0)).isTrue();
+ }
+
+ private static boolean isException(float value, int exponent, int factor) {
+ AlpCodec.EncodeResult result = new AlpCodec.EncodeResult();
+ AlpCodec.tryEncodeFloat(value, exponent, factor, result);
+ return result.isException;
+ }
+
+ private static boolean isException(double value, int exponent, int factor) {
+ AlpCodec.EncodeResult result = new AlpCodec.EncodeResult();
+ AlpCodec.tryEncodeDouble(value, exponent, factor, result);
+ return result.isException;
+ }
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpPageBytesTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpPageBytesTest.java
new file mode 100644
index 0000000000..a9017324d6
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpPageBytesTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.alp;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.parquet.bytes.DirectByteBufferAllocator;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Pins the exact bytes the writer produces. Round-trip tests pass for any valid encoding, so they
+ * miss changes that still decode, such as different exceptions, placeholders or preset order.
+ */
+public class AlpPageBytesTest {
+
+ @Test
+ public void testDoublePage() throws Exception {
+ assertThat(doublePage(1.5, 2.25, 3.0))
+ .containsExactly(bytes(
+ "00 00 03 03000000", // header: mode, integer encoding, log vector size 3, 3 values
+ "04000000", // offset array
+ "12 10 0000", // AlpInfo: exponent 18, factor 16, no exceptions
+ "9600000000000000 08", // ForInfo: frame of reference 150, bit width 8
+ "00 4b 96")); // deltas 0, 75, 150
+ }
+
+ @Test
+ public void testDoublePageWithException() throws Exception {
+ assertThat(doublePage(0.5, Double.NaN, 1.25))
+ .containsExactly(bytes(
+ "00 00 03 03000000",
+ "04000000",
+ "12 10 0100", // one exception
+ "3200000000000000 07", // frame of reference 50, bit width 7
+ "00 c0 12", // deltas 0, 0 (the NaN's slot holds the first encoded value), 75
+ "0100", // exception position 1
+ "000000000000f87f")); // NaN
+ }
+
+ @Test
+ public void testFloatPage() throws Exception {
+ assertThat(floatPage(1.5f, 2.25f, 3.0f))
+ .containsExactly(bytes(
+ "00 00 03 03000000",
+ "04000000",
+ "08 06 0000", // exponent 8: at 9, 2.25 and 3.0 would overflow an int
+ "96000000 08", // 4-byte frame of reference
+ "00 4b 96"));
+ }
+
+ @Test
+ public void testPageWithTwoVectors() throws Exception {
+ assertThat(doublePage(1, 2, 3, 4, 5, 6, 7, 8, 9))
+ .containsExactly(bytes(
+ "00 00 03 09000000",
+ "08000000 18000000", // offsets of the two vectors
+ "10 10 0000",
+ "0100000000000000 03",
+ "88 c6 fa", // deltas 0..7 at 3 bits each
+ "00 00 0000", // the second vector holds one value
+ "0900000000000000 00")); // bit width 0, so no packed values follow
+ }
+
+ @Test
+ public void testEmptyPage() throws Exception {
+ assertThat(doublePage()).containsExactly(bytes("00 00 03 00000000"));
+ }
+
+ /** Ties keep HashMap order, which here differs from the C++ order (larger exponent first). */
+ @Test
+ public void testPresetTieOrder() {
+ List Test vector: 3 float values {100.0f, 200.0f, 300.0f} with vectorSize=8.
+ * With exponent=0, factor=0: encoded = {100, 200, 300}.
+ * FOR min = 100, deltas = {0, 100, 200}, maxDelta = 200, bitWidth = 8.
+ * Packed: 3 values in 1 partial group of 8, padded with zeros.
+ * Packed size = ceil(3 * 8 / 8) = 3 bytes.
+ */
+ @Test
+ public void testReaderWithHandCraftedBytes() throws Exception {
+ // Manually build ALP page for {100.0f, 200.0f, 300.0f}, vectorSize=8
+ int vectorSize = 8;
+ int logVectorSize = 3;
+ int numElements = 3;
+ int numVectors = 1;
+
+ // Encoding: e=0, f=0 → multiplier=1.0
+ // encoded = {100, 200, 300}
+ // FOR min = 100
+ // deltas = {0, 100, 200}
+ // maxDelta = 200, bitWidth = 8
+ int exponent = 0;
+ int factor = 0;
+ int numExceptions = 0;
+ int frameOfReference = 100;
+ int bitWidth = 8;
+
+ // Bit-pack deltas using the same BytePacker the reader will use
+ BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(8);
+ int[] deltas = {0, 100, 200, 0, 0, 0, 0, 0}; // padded to 8
+ byte[] packed = new byte[8]; // bitWidth bytes for one group of 8
+ packer.pack8Values(deltas, 0, packed, 0);
+ int packedSize = (3 * 8 + 7) / 8; // = 3 bytes for 3 values at 8 bits
+
+ // Build the page
+ int vectorDataSize = 4 + 5 + packedSize; // AlpInfo + ForInfo + packed
+ int offsetArraySize = numVectors * 4;
+
+ ByteBuffer page =
+ ByteBuffer.allocate(7 + offsetArraySize + vectorDataSize).order(ByteOrder.LITTLE_ENDIAN);
+
+ // Header (7 bytes)
+ page.put((byte) 0); // compression_mode
+ page.put((byte) 0); // integer_encoding
+ page.put((byte) logVectorSize);
+ page.putInt(numElements);
+
+ // Offset array (1 vector)
+ page.putInt(offsetArraySize); // offset₀ = past offset array
+
+ // Vector 0: AlpInfo (4 bytes)
+ page.put((byte) exponent);
+ page.put((byte) factor);
+ page.putShort((short) numExceptions);
+
+ // Vector 0: ForInfo (5 bytes)
+ page.putInt(frameOfReference);
+ page.put((byte) bitWidth);
+
+ // Vector 0: Packed data (3 bytes)
+ page.put(packed, 0, packedSize);
+
+ page.flip();
+
+ // Now read it back
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(numElements, ByteBufferInputStream.wrap(page));
+
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(100.0f));
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(200.0f));
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(300.0f));
+ }
+
+ /**
+ * Hand-craft bytes with exceptions to verify reader handles them independently.
+ * Values: {1.0f, NaN, 3.0f} with vectorSize=8.
+ * e=0, f=0: encoded={1, placeholder=1, 3}, 1 exception at position 1.
+ * FOR min=1, deltas={0, 0, 2}, maxDelta=2, bitWidth=2.
+ */
+ @Test
+ public void testReaderWithHandCraftedExceptions() throws Exception {
+ int vectorSize = 8;
+ int logVectorSize = 3;
+ int numElements = 3;
+
+ int exponent = 0;
+ int factor = 0;
+ int numExceptions = 1;
+ int placeholder = 1; // first non-exception encoded value
+ int frameOfReference = 1; // min of {1, 1, 3}
+ int bitWidth = 2; // ceil(log2(2+1)) = 2
+
+ // deltas = {1-1, 1-1, 3-1} = {0, 0, 2}, padded to 8: {0,0,2,0,0,0,0,0}
+ BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(2);
+ int[] deltas = {0, 0, 2, 0, 0, 0, 0, 0};
+ byte[] packed = new byte[2]; // bitWidth=2 bytes for a group of 8
+ packer.pack8Values(deltas, 0, packed, 0);
+ int packedSize = (3 * 2 + 7) / 8; // = 1 byte
+
+ // Vector data: AlpInfo(4) + ForInfo(5) + packed(1) + excPos(2) + excVal(4) = 16
+ int vectorDataSize = 4 + 5 + packedSize + 2 + 4;
+ int offsetArraySize = 1 * 4;
+
+ ByteBuffer page =
+ ByteBuffer.allocate(7 + offsetArraySize + vectorDataSize).order(ByteOrder.LITTLE_ENDIAN);
+
+ // Header (7 bytes)
+ page.put((byte) 0); // compression_mode
+ page.put((byte) 0); // integer_encoding
+ page.put((byte) logVectorSize);
+ page.putInt(numElements);
+
+ // Offset array
+ page.putInt(offsetArraySize);
+
+ // AlpInfo
+ page.put((byte) exponent);
+ page.put((byte) factor);
+ page.putShort((short) numExceptions);
+
+ // ForInfo
+ page.putInt(frameOfReference);
+ page.put((byte) bitWidth);
+
+ // Packed data
+ page.put(packed, 0, packedSize);
+
+ // Exception positions (uint16 LE)
+ page.putShort((short) 1); // position 1
+
+ // Exception values (float32 LE)
+ page.putFloat(Float.NaN);
+
+ page.flip();
+
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(numElements, ByteBufferInputStream.wrap(page));
+
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(1.0f));
+ float val1 = reader.readFloat();
+ assertThat(Float.isNaN(val1)).as("Expected NaN at position 1").isTrue();
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(3.0f));
+ }
+
+ /**
+ * Hand-craft double ALP bytes and verify reader independence.
+ * Values: {10.0, 20.0, 30.0}, vectorSize=8, e=0, f=0.
+ * encoded={10,20,30}, FOR min=10, deltas={0,10,20}, maxDelta=20, bitWidth=5.
+ */
+ @Test
+ public void testDoubleReaderWithHandCraftedBytes() throws Exception {
+ int numElements = 3;
+ int exponent = 0;
+ int factor = 0;
+ long frameOfReference = 10;
+ int bitWidth = 5; // ceil(log2(20+1)) = 5
+
+ org.apache.parquet.column.values.bitpacking.BytePackerForLong packer =
+ Packer.LITTLE_ENDIAN.newBytePackerForLong(5);
+ long[] deltas = {0, 10, 20, 0, 0, 0, 0, 0};
+ byte[] packed = new byte[5]; // bitWidth bytes for group of 8
+ packer.pack8Values(deltas, 0, packed, 0);
+ int packedSize = (3 * 5 + 7) / 8; // = 2 bytes
+
+ int vectorDataSize = 4 + 9 + packedSize; // AlpInfo + DoubleForInfo + packed
+ int offsetArraySize = 4;
+
+ ByteBuffer page =
+ ByteBuffer.allocate(7 + offsetArraySize + vectorDataSize).order(ByteOrder.LITTLE_ENDIAN);
+
+ // Header (7 bytes)
+ page.put((byte) 0); // compression_mode
+ page.put((byte) 0); // integer_encoding
+ page.put((byte) 3); // log2(8)
+ page.putInt(numElements);
+
+ // Offset array
+ page.putInt(offsetArraySize);
+
+ // AlpInfo
+ page.put((byte) exponent);
+ page.put((byte) factor);
+ page.putShort((short) 0); // no exceptions
+
+ // ForInfo (9 bytes for double)
+ page.putLong(frameOfReference);
+ page.put((byte) bitWidth);
+
+ // Packed data
+ page.put(packed, 0, packedSize);
+
+ page.flip();
+
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(numElements, ByteBufferInputStream.wrap(page));
+
+ assertThat(Double.doubleToRawLongBits(reader.readDouble())).isEqualTo(Double.doubleToRawLongBits(10.0));
+ assertThat(Double.doubleToRawLongBits(reader.readDouble())).isEqualTo(Double.doubleToRawLongBits(20.0));
+ assertThat(Double.doubleToRawLongBits(reader.readDouble())).isEqualTo(Double.doubleToRawLongBits(30.0));
+ }
+
+ // ========== Writer Output → Hand-Verified Bytes ==========
+
+ /**
+ * Write known values, then verify the exact binary output against hand computation.
+ * This verifies the writer independently of the reader.
+ *
+ * Input: {1.0f, 2.0f, 3.0f} with vectorSize=8.
+ * Expected: e=0, f=0, encoded={1,2,3}, FOR min=1, deltas={0,1,2},
+ * maxDelta=2, bitWidth=2. Packed size = ceil(3*2/8) = 1 byte.
+ */
+ @Test
+ public void testWriterOutputExactBytes() throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator(), 8);
+ writer.writeFloat(1.0f);
+ writer.writeFloat(2.0f);
+ writer.writeFloat(3.0f);
+
+ byte[] bytes = writer.getBytes().toByteArray();
+ ByteBuffer buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
+
+ // Header (7 bytes)
+ assertThat(buf.get() & 0xFF).isZero(); // compression_mode
+ assertThat(buf.get() & 0xFF).isZero(); // integer_encoding
+ assertThat(buf.get() & 0xFF).isEqualTo(3); // log2(8) = 3
+ assertThat(buf.getInt()).isEqualTo(3); // num_elements
+
+ // Offset array (1 vector)
+ int offset0 = buf.getInt();
+ assertThat(offset0).isEqualTo(4); // 1 * 4 = past offset array
+
+ // AlpInfo
+ int exp = buf.get() & 0xFF;
+ int fac = buf.get() & 0xFF;
+ int numExc = buf.getShort() & 0xFFFF;
+ assertThat(numExc).isZero();
+
+ // ForInfo
+ int forRef = buf.getInt();
+ int bw = buf.get() & 0xFF;
+
+ // Verify: for e=0, f=0: multiplier=1.0
+ // encoded = {1, 2, 3}, min = 1, deltas = {0, 1, 2}, bitWidth = 2
+ assertThat(forRef).isEqualTo(1); // frame of reference = 1
+ assertThat(bw).isEqualTo(2); // bitWidth for max delta 2
+
+ // Packed data: ceil(3 * 2 / 8) = 1 byte
+ // deltas {0, 1, 2} at bitWidth=2:
+ // Use BytePacker to compute expected packed byte
+ BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(2);
+ int[] expectedDeltas = {0, 1, 2, 0, 0, 0, 0, 0};
+ byte[] expectedPacked = new byte[2]; // bitWidth=2
+ packer.pack8Values(expectedDeltas, 0, expectedPacked, 0);
+
+ byte actualPackedByte = buf.get();
+ assertThat(actualPackedByte).as("packed byte").isEqualTo(expectedPacked[0]);
+
+ // Should have consumed all bytes
+ assertThat(buf.position()).as("all bytes consumed").isEqualTo(bytes.length);
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== NaN Bit Pattern Preservation ==========
+
+ /**
+ * Verify that different NaN bit patterns survive the round-trip exactly.
+ * Java allows multiple NaN representations. Our exception handling must
+ * preserve the exact bit pattern, not normalize to Float.NaN.
+ */
+ @Test
+ public void testNaNBitPatternPreservation() throws Exception {
+ // Standard NaN
+ float standardNaN = Float.NaN; // 0x7FC00000
+ // Signaling NaN (different bit pattern)
+ float signalingNaN = Float.intBitsToFloat(0x7F800001);
+ // Negative NaN
+ float negativeNaN = Float.intBitsToFloat(0xFFC00000);
+ // Another NaN variant
+ float customNaN = Float.intBitsToFloat(0x7FFFFFFF);
+
+ assertThat(Float.isNaN(standardNaN)).isTrue();
+ assertThat(Float.isNaN(signalingNaN)).isTrue();
+ assertThat(Float.isNaN(negativeNaN)).isTrue();
+ assertThat(Float.isNaN(customNaN)).isTrue();
+
+ float[] values = {1.0f, standardNaN, 2.0f, signalingNaN, 3.0f, negativeNaN, 4.0f, customNaN};
+
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ for (float v : values) {
+ writer.writeFloat(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (int i = 0; i < values.length; i++) {
+ float actual = reader.readFloat();
+ assertThat(Float.floatToRawIntBits(actual))
+ .as("Bit pattern mismatch at index " + i + " (0x"
+ + Integer.toHexString(Float.floatToRawIntBits(values[i])) + " vs 0x"
+ + Integer.toHexString(Float.floatToRawIntBits(actual)) + ")")
+ .isEqualTo(Float.floatToRawIntBits(values[i]));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ /**
+ * Same test for double NaN bit patterns.
+ */
+ @Test
+ public void testDoubleNaNBitPatternPreservation() throws Exception {
+ double standardNaN = Double.NaN;
+ double signalingNaN = Double.longBitsToDouble(0x7FF0000000000001L);
+ double negativeNaN = Double.longBitsToDouble(0xFFF8000000000000L);
+ double customNaN = Double.longBitsToDouble(0x7FFFFFFFFFFFFFFFL);
+
+ double[] values = {1.0, standardNaN, 2.0, signalingNaN, 3.0, negativeNaN, 4.0, customNaN};
+
+ AlpValuesWriter.DoubleAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.DoubleAlpValuesWriter(512, 512, new DirectByteBufferAllocator());
+ for (double v : values) {
+ writer.writeDouble(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (int i = 0; i < values.length; i++) {
+ double actual = reader.readDouble();
+ assertThat(Double.doubleToRawLongBits(actual))
+ .as("Bit pattern mismatch at index " + i + " (0x"
+ + Long.toHexString(Double.doubleToRawLongBits(values[i])) + " vs 0x"
+ + Long.toHexString(Double.doubleToRawLongBits(actual)) + ")")
+ .isEqualTo(Double.doubleToRawLongBits(values[i]));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== Negative Zero Bit-Exact Roundtrip ==========
+
+ /**
+ * Verify that -0.0f roundtrips as -0.0f (bit pattern 0x80000000),
+ * not as +0.0f (bit pattern 0x00000000).
+ */
+ @Test
+ public void testNegativeZeroBitExact() throws Exception {
+ float negZero = -0.0f;
+ float posZero = 0.0f;
+
+ // Sanity: they are == but have different bit patterns
+ assertThat(negZero == posZero).isTrue();
+ assertThat(Float.floatToRawIntBits(negZero)).isNotEqualTo(Float.floatToRawIntBits(posZero));
+
+ float[] values = {posZero, negZero, 1.0f, negZero};
+
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator());
+ for (float v : values) {
+ writer.writeFloat(v);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (int i = 0; i < values.length; i++) {
+ float actual = reader.readFloat();
+ assertThat(Float.floatToRawIntBits(actual))
+ .as("Bit pattern at index " + i)
+ .isEqualTo(Float.floatToRawIntBits(values[i]));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== Extreme Float Values ==========
+
+ /**
+ * Test subnormal floats, Float.MIN_VALUE, Float.MAX_VALUE, Float.MIN_NORMAL.
+ * These should all either encode losslessly or be stored as exceptions.
+ */
+ @Test
+ public void testExtremeFloatValues() throws Exception {
+ float[] values = {
+ Float.MIN_VALUE, // smallest positive subnormal: 1.4e-45
+ Float.MIN_NORMAL, // smallest positive normal: 1.17549435e-38
+ Float.MAX_VALUE, // 3.4028235e38
+ -Float.MAX_VALUE, // most negative
+ 1.17549435e-38f, // near MIN_NORMAL
+ 3.4028234e38f, // near MAX_VALUE
+ 1.0e-10f, // very small positive
+ 1.0e10f, // large positive
+ };
+ roundTripFloat(values);
+ }
+
+ /**
+ * Test extreme double values.
+ */
+ @Test
+ public void testExtremeDoubleValues() throws Exception {
+ double[] values = {
+ Double.MIN_VALUE, // smallest positive subnormal
+ Double.MIN_NORMAL, // smallest positive normal
+ Double.MAX_VALUE, // largest positive
+ -Double.MAX_VALUE, // most negative
+ 1.0e-100, // very small
+ 1.0e100, // very large
+ };
+ roundTripDouble(values);
+ }
+
+ // ========== Preset Caching Under Distribution Change ==========
+
+ /**
+ * Write >8 vectors where the optimal (e,f) changes after the sampling phase.
+ * First 8 vectors: monetary data (e=2, f=0 optimal).
+ * Remaining vectors: large integers (e=0, f=0 optimal).
+ * Verify all values survive round-trip despite the distribution change.
+ */
+ @Test
+ public void testPresetCachingWithDistributionChange() throws Exception {
+ int vectorSize = 8; // small vectors to get to 8 quickly
+ int samplingVectors = 8;
+ int postSamplingVectors = 4;
+ int totalValues = vectorSize * (samplingVectors + postSamplingVectors);
+
+ float[] values = new float[totalValues];
+
+ // First 8 vectors: decimal monetary values (best with e=2, f=0)
+ for (int i = 0; i < vectorSize * samplingVectors; i++) {
+ values[i] = 10.00f + (i % 100) * 0.01f; // 10.00, 10.01, 10.02, ...
+ }
+
+ // Last 4 vectors: whole numbers (best with e=0, f=0)
+ for (int i = vectorSize * samplingVectors; i < totalValues; i++) {
+ values[i] = (float) (i * 1000);
+ }
+
+ roundTripFloat(values, vectorSize);
+ }
+
+ /**
+ * Same test for doubles.
+ */
+ @Test
+ public void testDoublePresetCachingWithDistributionChange() throws Exception {
+ int vectorSize = 8;
+ int totalValues = vectorSize * 12; // 8 sampling + 4 post
+
+ double[] values = new double[totalValues];
+
+ for (int i = 0; i < vectorSize * 8; i++) {
+ values[i] = 10.00 + (i % 100) * 0.01;
+ }
+ for (int i = vectorSize * 8; i < totalValues; i++) {
+ values[i] = (double) (i * 1000);
+ }
+
+ roundTripDouble(values, vectorSize);
+ }
+
+ // ========== Multi-Vector Hand-Crafted Binary ==========
+
+ /**
+ * Hand-craft a 2-vector page to verify offset array navigation works.
+ * 10 values with vectorSize=8: vector 0 has values 1-8, vector 1 has 9-10.
+ */
+ @Test
+ public void testMultiVectorHandCrafted() throws Exception {
+ int vectorSize = 8;
+ int numElements = 10;
+ int numVectors = 2;
+
+ // Vector 0: {1,2,3,4,5,6,7,8} → e=0,f=0, encoded={1..8}, FOR min=1, deltas={0..7}
+ // maxDelta=7, bitWidth=3. Packed: 8 values at 3 bits = 3 bytes exactly.
+ int v0BitWidth = 3;
+ BytePacker packer3 = Packer.LITTLE_ENDIAN.newBytePacker(3);
+ int[] v0Deltas = {0, 1, 2, 3, 4, 5, 6, 7};
+ byte[] v0Packed = new byte[3];
+ packer3.pack8Values(v0Deltas, 0, v0Packed, 0);
+ int v0PackedSize = 3; // (8*3+7)/8 = 3
+
+ // Vector 1: {9, 10} → e=0,f=0, encoded={9,10}, FOR min=9, deltas={0,1}
+ // maxDelta=1, bitWidth=1. Packed: ceil(2*1/8) = 1 byte.
+ int v1BitWidth = 1;
+ BytePacker packer1 = Packer.LITTLE_ENDIAN.newBytePacker(1);
+ int[] v1Deltas = {0, 1, 0, 0, 0, 0, 0, 0};
+ byte[] v1Packed = new byte[1];
+ packer1.pack8Values(v1Deltas, 0, v1Packed, 0);
+ int v1PackedSize = 1; // (2*1+7)/8 = 1
+
+ int v0DataSize = 4 + 5 + v0PackedSize; // 12
+ int v1DataSize = 4 + 5 + v1PackedSize; // 10
+ int offsetArraySize = numVectors * 4; // 8
+
+ ByteBuffer page = ByteBuffer.allocate(7 + offsetArraySize + v0DataSize + v1DataSize)
+ .order(ByteOrder.LITTLE_ENDIAN);
+
+ // Header (7 bytes)
+ page.put((byte) 0); // compression_mode
+ page.put((byte) 0); // integer_encoding
+ page.put((byte) 3); // log2(8)
+ page.putInt(numElements);
+
+ // Offset array
+ page.putInt(offsetArraySize); // vector 0 starts at 8
+ page.putInt(offsetArraySize + v0DataSize); // vector 1 starts at 8+12=20
+
+ // Vector 0
+ page.put((byte) 0); // exponent
+ page.put((byte) 0); // factor
+ page.putShort((short) 0); // exceptions
+ page.putInt(1); // FOR = 1
+ page.put((byte) v0BitWidth);
+ page.put(v0Packed, 0, v0PackedSize);
+
+ // Vector 1
+ page.put((byte) 0); // exponent
+ page.put((byte) 0); // factor
+ page.putShort((short) 0); // exceptions
+ page.putInt(9); // FOR = 9
+ page.put((byte) v1BitWidth);
+ page.put(v1Packed, 0, v1PackedSize);
+
+ page.flip();
+
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(numElements, ByteBufferInputStream.wrap(page));
+
+ for (int i = 0; i < numElements; i++) {
+ float expected = (float) (i + 1);
+ float actual = reader.readFloat();
+ assertThat(Float.floatToRawIntBits(actual))
+ .as("Value mismatch at index " + i)
+ .isEqualTo(Float.floatToRawIntBits(expected));
+ }
+ }
+
+ // ========== Skip Over Entire Vector Without Decoding ==========
+
+ /**
+ * Verify that skipping an entire vector and reading the next one works.
+ * Uses hand-crafted bytes to ensure independence from writer.
+ */
+ @Test
+ public void testSkipEntireVectorHandCrafted() throws Exception {
+ // Reuse the multi-vector setup: {1..8} in vector 0, {9, 10} in vector 1
+ int vectorSize = 8;
+ int numElements = 10;
+ int numVectors = 2;
+
+ BytePacker packer3 = Packer.LITTLE_ENDIAN.newBytePacker(3);
+ int[] v0Deltas = {0, 1, 2, 3, 4, 5, 6, 7};
+ byte[] v0Packed = new byte[3];
+ packer3.pack8Values(v0Deltas, 0, v0Packed, 0);
+
+ BytePacker packer1 = Packer.LITTLE_ENDIAN.newBytePacker(1);
+ int[] v1Deltas = {0, 1, 0, 0, 0, 0, 0, 0};
+ byte[] v1Packed = new byte[1];
+ packer1.pack8Values(v1Deltas, 0, v1Packed, 0);
+
+ int v0DataSize = 4 + 5 + 3;
+ int v1DataSize = 4 + 5 + 1;
+ int offsetArraySize = numVectors * 4;
+
+ ByteBuffer page = ByteBuffer.allocate(7 + offsetArraySize + v0DataSize + v1DataSize)
+ .order(ByteOrder.LITTLE_ENDIAN);
+
+ page.put((byte) 0).put((byte) 0).put((byte) 3);
+ page.putInt(numElements);
+ page.putInt(offsetArraySize);
+ page.putInt(offsetArraySize + v0DataSize);
+
+ // Vector 0
+ page.put((byte) 0).put((byte) 0).putShort((short) 0);
+ page.putInt(1).put((byte) 3);
+ page.put(v0Packed, 0, 3);
+
+ // Vector 1
+ page.put((byte) 0).put((byte) 0).putShort((short) 0);
+ page.putInt(9).put((byte) 1);
+ page.put(v1Packed, 0, 1);
+
+ page.flip();
+
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(numElements, ByteBufferInputStream.wrap(page));
+
+ // Skip entire vector 0 (8 values)
+ reader.skip(8);
+
+ // Read from vector 1
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(9.0f));
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(10.0f));
+ }
+
+ // ========== Vector Size Validation ==========
+
+ /**
+ * Verify that vectorSize=65536 is rejected because num_exceptions (uint16)
+ * cannot represent 65536, which would cause silent data corruption if all
+ * values in a vector are exceptions.
+ */
+ @Test
+ public void testVectorSize65536Rejected() throws Exception {
+ assertThatThrownBy(() ->
+ new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator(), 65536))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Vector size log2 must be between");
+ }
+
+ /** Integer.MIN_VALUE has a single bit set, so only the positivity check rejects it. */
+ @Test
+ public void testNonPowerOfTwoVectorSizeRejected() {
+ for (int vectorSize : new int[] {0, 3, 1000, 1025, -1024, Integer.MIN_VALUE}) {
+ assertThatThrownBy(() -> new AlpConfig(vectorSize))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Vector size must be a power of 2");
+ }
+ assertThat(new AlpConfig(1024).getVectorSize()).isEqualTo(1024);
+ }
+
+ /**
+ * Verify that vectorSize=32768 (max allowed) works correctly.
+ */
+ @Test
+ public void testVectorSize32768Works() throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(256, 256, new DirectByteBufferAllocator(), 32768);
+ // Write a small number of values (partial vector)
+ for (int i = 0; i < 10; i++) {
+ writer.writeFloat(i * 1.0f);
+ }
+
+ BytesInput input = writer.getBytes();
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(10, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (int i = 0; i < 10; i++) {
+ assertThat(Float.floatToRawIntBits(reader.readFloat())).isEqualTo(Float.floatToRawIntBits(i * 1.0f));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ // ========== Factor > 0 Tests ==========
+
+ /**
+ * Data where the best (e,f) combo has factor > 0.
+ * Values like 1234.56 have digits on both sides of the decimal point.
+ * With e=4,f=2: 1234.56 * 10000 / 100 = 123456 — exact integer.
+ * This exercises the division path in the encode/decode formula.
+ */
+ @Test
+ public void testDoubleFactorGreaterThanZero() throws Exception {
+ // These values are designed so the best combo has f > 0
+ double[] values = new double[1024];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = 1000.0 + i * 0.01; // 1000.00, 1000.01, ...
+ }
+
+ // Verify the best params actually have f > 0 or f == 0 with low exceptions
+ AlpCodec.EncodingParams params =
+ AlpCodec.findBestDoubleParams(values, 0, values.length, AlpCodec.ALL_VALID_DOUBLE_PAIRS);
+ // Regardless of the chosen f, the roundtrip must be lossless
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testFloatFactorGreaterThanZero() throws Exception {
+ float[] values = new float[1024];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = 1000.0f + i * 0.01f;
+ }
+ roundTripFloat(values);
+ }
+
+ // ========== Overflow Boundary Tests ==========
+
+ /**
+ * Values near the encoding limits that test the overflow check.
+ * The old code used Long.MAX_VALUE (which rounds to 2^63 as double),
+ * the fix uses DOUBLE_ENCODING_UPPER_LIMIT = 9223372036854774784.0.
+ */
+ @Test
+ public void testDoubleNearOverflowBoundary() throws Exception {
+ double[] values = {
+ 9.2e18, // near Long.MAX_VALUE
+ -9.2e18, // near Long.MIN_VALUE
+ 1.7e308, // near Double.MAX_VALUE
+ -1.7e308, // near -Double.MAX_VALUE
+ 4.9e-324, // Double.MIN_VALUE (subnormal)
+ 1.0e18, // large but in range
+ -1.0e18, // large negative but in range
+ 0.0, // normal
+ 1.23, // normal
+ };
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testFloatNearOverflowBoundary() throws Exception {
+ float[] values = {
+ 2.1e9f, // near Integer.MAX_VALUE
+ -2.1e9f, // near Integer.MIN_VALUE
+ 3.4e38f, // near Float.MAX_VALUE
+ -3.4e38f, // near -Float.MAX_VALUE
+ 1.4e-45f, // Float.MIN_VALUE (subnormal)
+ 1.0e9f, // large but in range
+ 0.0f, // normal
+ 1.23f, // normal
+ };
+ roundTripFloat(values);
+ }
+
+ // ========== Large Scale Stress Test ==========
+
+ @Test
+ public void testDoubleLargeScale100K() throws Exception {
+ Random rand = new Random(12345);
+ double[] values = new double[100_000];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = rand.nextDouble() * 10000.0 - 5000.0;
+ }
+ roundTripDouble(values);
+ }
+
+ @Test
+ public void testFloatLargeScale100K() throws Exception {
+ Random rand = new Random(12345);
+ float[] values = new float[100_000];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = rand.nextFloat() * 10000.0f - 5000.0f;
+ }
+ roundTripFloat(values);
+ }
+
+ // ========== All Exceptions in Large Vector ==========
+
+ /**
+ * A full vector where every value is an exception (NaN/Inf/-0.0).
+ * This exercises the case where placeholder search finds no valid value.
+ */
+ @Test
+ public void testFloatEntireVectorAllExceptions() throws Exception {
+ float[] values = new float[DEFAULT_VECTOR_SIZE + 5]; // partial second vector
+ for (int i = 0; i < values.length; i++) {
+ switch (i % 4) {
+ case 0:
+ values[i] = Float.NaN;
+ break;
+ case 1:
+ values[i] = Float.POSITIVE_INFINITY;
+ break;
+ case 2:
+ values[i] = Float.NEGATIVE_INFINITY;
+ break;
+ case 3:
+ values[i] = -0.0f;
+ break;
+ }
+ }
+ roundTripFloat(values);
+ }
+
+ @Test
+ public void testDoubleEntireVectorAllExceptions() throws Exception {
+ double[] values = new double[DEFAULT_VECTOR_SIZE + 5];
+ for (int i = 0; i < values.length; i++) {
+ switch (i % 4) {
+ case 0:
+ values[i] = Double.NaN;
+ break;
+ case 1:
+ values[i] = Double.POSITIVE_INFINITY;
+ break;
+ case 2:
+ values[i] = Double.NEGATIVE_INFINITY;
+ break;
+ case 3:
+ values[i] = -0.0;
+ break;
+ }
+ }
+ roundTripDouble(values);
+ }
+
+ // ========== Verify Encoder Produces Expected Values ==========
+
+ /**
+ * Manually verify that the encoder produces the exact integer values
+ * we expect for known inputs. This is independent of the writer/reader.
+ */
+ @Test
+ public void testEncoderProducesExpectedValues() {
+ // 1.23f * 100 = 123
+ assertThat(AlpCodec.encodeFloat(1.23f, 2, 0)).isEqualTo(123);
+ // 19.99f * 100 = 1999
+ assertThat(AlpCodec.encodeFloat(19.99f, 2, 0)).isEqualTo(1999);
+ // -5.0f * 10 = -50
+ assertThat(AlpCodec.encodeFloat(-5.0f, 1, 0)).isEqualTo(-50);
+ // 0.0f * anything = 0
+ assertThat(AlpCodec.encodeFloat(0.0f, 5, 0)).isZero();
+ // 42.0f * 1 = 42
+ assertThat(AlpCodec.encodeFloat(42.0f, 0, 0)).isEqualTo(42);
+ // 1.5f * 10 = 15
+ assertThat(AlpCodec.encodeFloat(1.5f, 1, 0)).isEqualTo(15);
+
+ // Double path
+ assertThat(AlpCodec.encodeDouble(1.23, 2, 0)).isEqualTo(123L);
+ assertThat(AlpCodec.encodeDouble(19.99, 2, 0)).isEqualTo(1999L);
+ assertThat(AlpCodec.encodeDouble(-5.0, 1, 0)).isEqualTo(-50L);
+ assertThat(AlpCodec.encodeDouble(0.0, 5, 0)).isZero();
+ assertThat(AlpCodec.encodeDouble(42.0, 0, 0)).isEqualTo(42L);
+ }
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java
index 436c13834f..a8a575c82a 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java
@@ -32,6 +32,7 @@
import org.apache.parquet.column.ParquetProperties;
import org.apache.parquet.column.ParquetProperties.WriterVersion;
import org.apache.parquet.column.values.ValuesWriter;
+import org.apache.parquet.column.values.alp.AlpValuesWriter;
import org.apache.parquet.column.values.bytestreamsplit.ByteStreamSplitValuesWriter;
import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriter;
import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForLong;
@@ -460,6 +461,30 @@ public void testDouble_V2_WithByteStreamSplitAndDictionary() {
testFloatingPoint_WithByteStreamSplitAndDictionary(PrimitiveTypeName.DOUBLE, WriterVersion.PARQUET_2_0);
}
+ @Test
+ public void testFloat_V1_WithAlp() {
+ testFloatingPoint_WithAlp(
+ PrimitiveTypeName.FLOAT, WriterVersion.PARQUET_1_0, AlpValuesWriter.FloatAlpValuesWriter.class);
+ }
+
+ @Test
+ public void testDouble_V1_WithAlp() {
+ testFloatingPoint_WithAlp(
+ PrimitiveTypeName.DOUBLE, WriterVersion.PARQUET_1_0, AlpValuesWriter.DoubleAlpValuesWriter.class);
+ }
+
+ @Test
+ public void testFloat_V2_WithAlp() {
+ testFloatingPoint_WithAlp(
+ PrimitiveTypeName.FLOAT, WriterVersion.PARQUET_2_0, AlpValuesWriter.FloatAlpValuesWriter.class);
+ }
+
+ @Test
+ public void testDouble_V2_WithAlp() {
+ testFloatingPoint_WithAlp(
+ PrimitiveTypeName.DOUBLE, WriterVersion.PARQUET_2_0, AlpValuesWriter.DoubleAlpValuesWriter.class);
+ }
+
@Test
public void testColumnWiseDictionaryWithFalseDefault() {
ValuesWriterFactory factory = getDefaultFactory(
@@ -631,6 +656,36 @@ private void testFloatingPoint_WithByteStreamSplitAndDictionary(
PlainValuesWriter.class);
}
+ private void testFloatingPoint_WithAlp(
+ PrimitiveTypeName typeName, WriterVersion writerVersion, Class alpWriterClass) {
+ // Cross-column ALP without a dictionary: the ALP writer is produced directly.
+ doTestValueWriter(
+ createColumnDescriptor(typeName),
+ ParquetProperties.builder()
+ .withWriterVersion(writerVersion)
+ .withDictionaryEncoding(false)
+ .withAlp()
+ .build(),
+ alpWriterClass);
+ // Cross-column ALP with a dictionary: the ALP writer is the dictionary's fallback.
+ doTestValueWriter(
+ createColumnDescriptor(typeName),
+ ParquetProperties.builder()
+ .withWriterVersion(writerVersion)
+ .withAlp()
+ .build(),
+ DictionaryValuesWriter.class,
+ alpWriterClass);
+ // Per-column ALP: only colA opts in, colB keeps the plain fallback.
+ ParquetProperties properties = ParquetProperties.builder()
+ .withWriterVersion(writerVersion)
+ .withDictionaryEncoding(false)
+ .withAlp("colA")
+ .build();
+ doTestValueWriter(createColumnDescriptor(typeName, "colA"), properties, alpWriterClass);
+ doTestValueWriter(createColumnDescriptor(typeName, "colB"), properties, PlainValuesWriter.class);
+ }
+
private void validateFactory(
ValuesWriterFactory factory,
PrimitiveTypeName typeName,
diff --git a/parquet-common/src/main/java/org/apache/parquet/bytes/BytesUtils.java b/parquet-common/src/main/java/org/apache/parquet/bytes/BytesUtils.java
index 07bcc28903..6abbae18eb 100644
--- a/parquet-common/src/main/java/org/apache/parquet/bytes/BytesUtils.java
+++ b/parquet-common/src/main/java/org/apache/parquet/bytes/BytesUtils.java
@@ -50,6 +50,16 @@ public static int getWidthFromMaxInt(int bound) {
return 32 - Integer.numberOfLeadingZeros(bound);
}
+ /**
+ * give the number of bits needed to encode a long given the max value
+ *
+ * @param bound max long that we want to encode
+ * @return the number of bits required
+ */
+ public static int getWidthFromMaxLong(long bound) {
+ return 64 - Long.numberOfLeadingZeros(bound);
+ }
+
/**
* reads an int in little endian at the given position
*
@@ -66,6 +76,29 @@ public static int readIntLittleEndian(ByteBuffer in, int offset) throws IOExcept
return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4 << 0));
}
+ /**
+ * reads an unsigned byte at the given position
+ *
+ * @param in a byte buffer
+ * @param offset an offset into the byte buffer
+ * @return the byte at position offset, from 0 to 255
+ */
+ public static int readUnsignedByte(ByteBuffer in, int offset) {
+ return in.get(offset) & 0xff;
+ }
+
+ /**
+ * reads an unsigned short in little endian at the given position, regardless of the buffer's byte
+ * order
+ *
+ * @param in a byte buffer
+ * @param offset an offset into the byte buffer
+ * @return the short at position offset read using little endian byte order, from 0 to 65535
+ */
+ public static int readUnsignedShortLittleEndian(ByteBuffer in, int offset) {
+ return (in.get(offset) & 0xff) | ((in.get(offset + 1) & 0xff) << 8);
+ }
+
/**
* reads an int in little endian at the given position
*
diff --git a/parquet-common/src/test/java/org/apache/parquet/bytes/TestBytesUtil.java b/parquet-common/src/test/java/org/apache/parquet/bytes/TestBytesUtil.java
index c18d6f5a9c..e8f9bac2c4 100644
--- a/parquet-common/src/test/java/org/apache/parquet/bytes/TestBytesUtil.java
+++ b/parquet-common/src/test/java/org/apache/parquet/bytes/TestBytesUtil.java
@@ -19,8 +19,13 @@
package org.apache.parquet.bytes;
import static org.apache.parquet.bytes.BytesUtils.getWidthFromMaxInt;
+import static org.apache.parquet.bytes.BytesUtils.getWidthFromMaxLong;
+import static org.apache.parquet.bytes.BytesUtils.readUnsignedByte;
+import static org.apache.parquet.bytes.BytesUtils.readUnsignedShortLittleEndian;
import static org.assertj.core.api.Assertions.assertThat;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
import org.junit.jupiter.api.Test;
public class TestBytesUtil {
@@ -46,4 +51,60 @@ public void testWidth() {
assertThat(getWidthFromMaxInt(128)).isEqualTo(8);
assertThat(getWidthFromMaxInt(255)).isEqualTo(8);
}
+
+ @Test
+ public void testWidthLong() {
+ assertThat(getWidthFromMaxLong(0L)).isZero();
+ assertThat(getWidthFromMaxLong(1L)).isEqualTo(1);
+ assertThat(getWidthFromMaxLong(2L)).isEqualTo(2);
+ assertThat(getWidthFromMaxLong(3L)).isEqualTo(2);
+ assertThat(getWidthFromMaxLong(4L)).isEqualTo(3);
+ assertThat(getWidthFromMaxLong(255L)).isEqualTo(8);
+ assertThat(getWidthFromMaxLong(256L)).isEqualTo(9);
+ assertThat(getWidthFromMaxLong(65535L)).isEqualTo(16);
+ assertThat(getWidthFromMaxLong((long) Integer.MAX_VALUE)).isEqualTo(31);
+ // Beyond the int range, where getWidthFromMaxInt cannot be used
+ assertThat(getWidthFromMaxLong(1L << 32)).isEqualTo(33);
+ assertThat(getWidthFromMaxLong(Long.MAX_VALUE)).isEqualTo(63);
+ // A negative bound has the sign bit set, so it occupies the full 64 bits
+ assertThat(getWidthFromMaxLong(-1L)).isEqualTo(64);
+ }
+
+ @Test
+ public void testReadUnsignedByte() {
+ ByteBuffer buf = ByteBuffer.wrap(new byte[] {0, 0x7f, (byte) 0x80, (byte) 0xff});
+ assertThat(readUnsignedByte(buf, 0)).isZero();
+ assertThat(readUnsignedByte(buf, 1)).isEqualTo(127);
+ assertThat(readUnsignedByte(buf, 2)).isEqualTo(128);
+ assertThat(readUnsignedByte(buf, 3)).isEqualTo(255);
+ // The offset is absolute, so the buffer's position doesn't matter
+ buf.position(3);
+ assertThat(readUnsignedByte(buf, 2)).isEqualTo(128);
+ }
+
+ @Test
+ public void testReadUnsignedShortLittleEndian() {
+ byte[] bytes = {0, 0, (byte) 0xff, 0x7f, 0, (byte) 0x80, (byte) 0xff, (byte) 0xff, 0x34, 0x12};
+ for (ByteOrder order : new ByteOrder[] {ByteOrder.LITTLE_ENDIAN, ByteOrder.BIG_ENDIAN}) {
+ ByteBuffer buf = ByteBuffer.wrap(bytes).order(order);
+ assertThat(readUnsignedShortLittleEndian(buf, 0))
+ .as("order=%s", order)
+ .isZero();
+ assertThat(readUnsignedShortLittleEndian(buf, 2))
+ .as("order=%s", order)
+ .isEqualTo(32767);
+ assertThat(readUnsignedShortLittleEndian(buf, 4))
+ .as("order=%s", order)
+ .isEqualTo(32768);
+ assertThat(readUnsignedShortLittleEndian(buf, 6))
+ .as("order=%s", order)
+ .isEqualTo(65535);
+ assertThat(readUnsignedShortLittleEndian(buf, 8))
+ .as("order=%s", order)
+ .isEqualTo(0x1234);
+ assertThat(readUnsignedShortLittleEndian(buf, 3))
+ .as("order=%s", order)
+ .isEqualTo(0x007f);
+ }
+ }
}
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java
index 7dcbb3188c..2df44d49be 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java
@@ -28,6 +28,7 @@
import org.apache.parquet.bytes.ByteBufferAllocator;
import org.apache.parquet.column.ParquetProperties;
import org.apache.parquet.column.ParquetProperties.WriterVersion;
+import org.apache.parquet.column.values.alp.AlpConfig;
import org.apache.parquet.compression.CompressionCodecFactory;
import org.apache.parquet.conf.HadoopParquetConfiguration;
import org.apache.parquet.conf.ParquetConfiguration;
@@ -737,6 +738,36 @@ public SELF withByteStreamSplitEncoding(String columnPath, boolean enableByteStr
return self();
}
+ public SELF withAlp() {
+ encodingPropsBuilder.withAlp();
+ return self();
+ }
+
+ public SELF withAlp(AlpConfig config) {
+ encodingPropsBuilder.withAlp(config);
+ return self();
+ }
+
+ public SELF withAlp(String columnPath) {
+ encodingPropsBuilder.withAlp(columnPath);
+ return self();
+ }
+
+ public SELF withAlp(String columnPath, AlpConfig config) {
+ encodingPropsBuilder.withAlp(columnPath, config);
+ return self();
+ }
+
+ public SELF withoutAlp() {
+ encodingPropsBuilder.withoutAlp();
+ return self();
+ }
+
+ public SELF withoutAlp(String columnPath) {
+ encodingPropsBuilder.withoutAlp(columnPath);
+ return self();
+ }
+
/**
* Enable or disable dictionary encoding of the specified column for the constructed writer.
*
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
index 46ffccf7fa..7ec562c878 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
@@ -589,11 +589,6 @@ public void testEnumEquivalence() {
.isEqualTo(encoding);
}
for (org.apache.parquet.format.Encoding encoding : org.apache.parquet.format.Encoding.values()) {
- // ALP is in the format spec but is not implemented on the Java side yet, so it has no
- // org.apache.parquet.column.Encoding to round trip through. Remove this once it does.
- if (encoding == org.apache.parquet.format.Encoding.ALP) {
- continue;
- }
assertThat(parquetMetadataConverter.getEncoding(parquetMetadataConverter.getEncoding(encoding)))
.isEqualTo(encoding);
}
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInterOpReadAlp.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInterOpReadAlp.java
new file mode 100644
index 0000000000..535037a0af
--- /dev/null
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInterOpReadAlp.java
@@ -0,0 +1,685 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.hadoop;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.column.Encoding;
+import org.apache.parquet.column.ParquetProperties.WriterVersion;
+import org.apache.parquet.column.page.PageReadStore;
+import org.apache.parquet.column.values.alp.AlpConfig;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.simple.SimpleGroup;
+import org.apache.parquet.example.data.simple.convert.GroupRecordConverter;
+import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.io.ColumnIOFactory;
+import org.apache.parquet.io.LocalInputFile;
+import org.apache.parquet.io.LocalOutputFile;
+import org.apache.parquet.io.MessageColumnIO;
+import org.apache.parquet.io.RecordReader;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.MessageTypeParser;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Cross-compatibility tests for ALP (Adaptive Lossless floating-Point) encoding.
+ *
+ * Reads the ALP conformance file from parquet-testing and checks that every ALP column decodes
+ * bit-identically to its PLAIN counterpart, and round-trips Java-written ALP files.
+ */
+public class TestInterOpReadAlp {
+ private static final Logger LOG = LoggerFactory.getLogger(TestInterOpReadAlp.class);
+
+ @TempDir
+ java.nio.file.Path temp;
+
+ /** Mirrors JUnit 4's {@code TemporaryFolder#newFolder()}: a fresh directory per call. */
+ private File newFolder() throws IOException {
+ return Files.createTempDirectory(temp, "junit").toFile();
+ }
+
+ private static final String ALP_EXTENDED_FILE = "alp_extended.zstd.parquet";
+ private static final String CHANGESET = "09f3cdb";
+ private final InterOpTester interop = new InterOpTester();
+
+ private static final int EXTENDED_ROWS = 9032;
+ private static final int EXTENDED_ROW_GROUPS = 5;
+ private static final int EXTENDED_NULLS = 8;
+
+ /** ALP column -> PLAIN column holding the same values. */
+ private static final Map
+ * ┌─────────┬──────────────────────┬──────────────┬──────────────┬─────┐
+ * │ Header │ Offset Array │ Vector 0 │ Vector 1 │ ... │
+ * │ 7 bytes │ 4B × numVectors │ │ │ │
+ * └─────────┴──────────────────────┴──────────────┴──────────────┴─────┘
+ *
+ *
+ *
+ * ┌─────────┬──────────────────────┬──────────────┬──────────────┬─────┐
+ * │ Header │ Offset Array │ Vector 0 │ Vector 1 │ ... │
+ * │ 7 bytes │ 4B × numVectors │ │ │ │
+ * └─────────┴──────────────────────┴──────────────┴──────────────┴─────┘
+ *
+ *
+ *
+ *
+ */
+public class AlpAdversarialTest {
+
+ // ---------------------------------------------------------------------------
+ // Helpers: build a known-good encoded page, then mutate copies of it
+ // ---------------------------------------------------------------------------
+
+ /** Build a valid ALP-encoded double page with N clean values. */
+ private static byte[] validDoublePage(int valueCount, int vectorSize) throws Exception {
+ AlpValuesWriter.DoubleAlpValuesWriter writer = null;
+ try {
+ int cap = Math.max(512, valueCount * 16);
+ writer = new AlpValuesWriter.DoubleAlpValuesWriter(cap, cap, new DirectByteBufferAllocator(), vectorSize);
+ // 2-decimal values — the ALP sweet spot, ensures no exceptions
+ for (int i = 0; i < valueCount; i++) {
+ writer.writeDouble((i % 1000) / 100.0);
+ }
+ BytesInput bi = writer.getBytes();
+ ByteBuffer bb = bi.toByteBuffer();
+ byte[] out = new byte[bb.remaining()];
+ bb.duplicate().get(out);
+ return out;
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ /** Build a valid ALP-encoded float page with N clean values. */
+ private static byte[] validFloatPage(int valueCount, int vectorSize) throws Exception {
+ AlpValuesWriter.FloatAlpValuesWriter writer = null;
+ try {
+ int cap = Math.max(256, valueCount * 8);
+ writer = new AlpValuesWriter.FloatAlpValuesWriter(cap, cap, new DirectByteBufferAllocator(), vectorSize);
+ for (int i = 0; i < valueCount; i++) {
+ writer.writeFloat((i % 1000) / 100.0f);
+ }
+ BytesInput bi = writer.getBytes();
+ ByteBuffer bb = bi.toByteBuffer();
+ byte[] out = new byte[bb.remaining()];
+ bb.duplicate().get(out);
+ return out;
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
+ /** Sanity baseline: the known-good page actually decodes cleanly. */
+ @Test
+ public void sanityBaselineDecodesClean() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ for (int i = 0; i < 32; i++) reader.readDouble();
+ }
+
+ // ---------------------------------------------------------------------------
+ // Header-level validation (already-validated paths)
+ // ---------------------------------------------------------------------------
+
+ @Test
+ public void rejectsBadCompressionMode() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ page[0] = (byte) 0x99; // mode is at byte 0
+ assertThatThrownBy(() -> new AlpValuesReaderForDouble()
+ .initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))))
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("Unsupported ALP compression mode");
+ }
+
+ @Test
+ public void rejectsBadIntegerEncoding() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ page[1] = (byte) 0x99; // integer_encoding is at byte 1
+ assertThatThrownBy(() -> new AlpValuesReaderForDouble()
+ .initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))))
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("Unsupported ALP integer encoding");
+ }
+
+ @Test
+ public void rejectsLogVectorSizeTooLarge() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ page[2] = (byte) 99; // log_vector_size at byte 2
+ assertThatThrownBy(() -> new AlpValuesReaderForDouble()
+ .initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))))
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("Invalid ALP log vector size");
+ }
+
+ @Test
+ public void rejectsLogVectorSizeTooSmall() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ page[2] = (byte) 2; // below MIN_LOG_VECTOR_SIZE=3
+ assertThatThrownBy(() -> new AlpValuesReaderForDouble()
+ .initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))))
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("Invalid ALP log vector size");
+ }
+
+ @Test
+ public void rejectsNegativeNumElements() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ // num_elements is int32 LE at bytes 3..6 — write -1
+ page[3] = (byte) 0xFF;
+ page[4] = (byte) 0xFF;
+ page[5] = (byte) 0xFF;
+ page[6] = (byte) 0xFF;
+ assertThatThrownBy(() -> new AlpValuesReaderForDouble()
+ .initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))))
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("Invalid ALP element count");
+ }
+
+ @Test
+ public void rejectsNumElementsGreaterThanValuesCount() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ // num_elements stays 32; pass valuesCount=10 (smaller than encoded count)
+ assertThatThrownBy(() -> new AlpValuesReaderForDouble()
+ .initFromPage(10, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))))
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("exceeds page valuesCount");
+ }
+
+ // ---------------------------------------------------------------------------
+ // Vector-level validation (already-validated paths, surface lazily on decode)
+ // ---------------------------------------------------------------------------
+
+ /** Helper: find the byte position where the first vector's metadata starts. */
+ private static int firstVectorOffset(byte[] page) {
+ // header (7) + first 4 bytes of offset array = the offset value itself
+ int firstVectorOff =
+ ByteBuffer.wrap(page, 7, 4).order(ByteOrder.LITTLE_ENDIAN).getInt();
+ // offsets are measured from the start of the compression body (after the 7B header)
+ return 7 + firstVectorOff;
+ }
+
+ @Test
+ public void rejectsExponentTooHighDouble() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ int v0 = firstVectorOffset(page);
+ page[v0] = (byte) 99; // exponent byte
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ assertThatThrownBy(reader::readDouble)
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("Invalid ALP double exponent");
+ }
+
+ @Test
+ public void rejectsExponentTooHighFloat() throws Exception {
+ byte[] page = validFloatPage(32, 16);
+ int v0 = firstVectorOffset(page);
+ page[v0] = (byte) 99;
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ assertThatThrownBy(reader::readFloat)
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("Invalid ALP float exponent");
+ }
+
+ @Test
+ public void rejectsFactorGreaterThanExponent() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ int v0 = firstVectorOffset(page);
+ page[v0] = (byte) 2; // exponent
+ page[v0 + 1] = (byte) 5; // factor > exponent
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ assertThatThrownBy(reader::readDouble)
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("Invalid ALP double factor");
+ }
+
+ @Test
+ public void rejectsTooManyExceptions() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ int v0 = firstVectorOffset(page);
+ // num_exceptions at v0+2, uint16 LE — set to 9999, way more than vectorLen=16
+ page[v0 + 2] = (byte) (9999 & 0xFF);
+ page[v0 + 3] = (byte) ((9999 >>> 8) & 0xFF);
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ assertThatThrownBy(reader::readDouble)
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("Invalid ALP numExceptions");
+ }
+
+ @Test
+ public void rejectsBitWidthTooLargeDouble() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ int v0 = firstVectorOffset(page);
+ // Layout: ALP_INFO(4) + frameOfReference(8) then bitWidth byte at v0+12. 99 > 64.
+ page[v0 + 12] = (byte) 99;
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ assertThatThrownBy(reader::readDouble)
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("Invalid ALP double bitWidth");
+ }
+
+ @Test
+ public void rejectsBitWidthTooLargeFloat() throws Exception {
+ byte[] page = validFloatPage(32, 16);
+ int v0 = firstVectorOffset(page);
+ // Layout: ALP_INFO(4) + frameOfReference(4) then bitWidth byte at v0+8. 99 > 32.
+ page[v0 + 8] = (byte) 99;
+ AlpValuesReaderForFloat reader = new AlpValuesReaderForFloat();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ assertThatThrownBy(reader::readFloat)
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("Invalid ALP float bitWidth");
+ }
+
+ /**
+ * A bit width that is legal on its own but larger than the vector actually used makes the packed
+ * values overrun into the next vector. Decoding another vector's bytes would be a silent wrong
+ * answer, so this must be rejected.
+ */
+ @Test
+ public void rejectsPackedValuesRunningPastVectorEnd() throws Exception {
+ // 16 vectors, so the inflated packed body stays inside the page and crosses into the vectors
+ // that follow; the last-vector case, which would run off the end of the page, is below.
+ byte[] page = validDoublePage(256, 16);
+ int v0 = firstVectorOffset(page);
+ // bitWidth lives after ALP_INFO(4) + frameOfReference(8). 64 is the maximum legal width, so it
+ // passes the width check and then claims far more packed bytes than this vector holds.
+ page[v0 + 12] = (byte) 64;
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(256, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ assertThatThrownBy(reader::readDouble)
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("packed values end at");
+ }
+
+ /**
+ * The same forged bit width on the last vector would read past the end of the page. It must be
+ * rejected as a decoding error before unpacking, not surface as an IndexOutOfBoundsException.
+ */
+ @Test
+ public void rejectsPackedValuesRunningPastEndOfLastVector() throws Exception {
+ byte[] page = validDoublePage(20, 8); // 3 vectors, the last holding 4 values
+ int lastOffset = ByteBuffer.wrap(page).order(ByteOrder.LITTLE_ENDIAN).getInt(7 + 2 * Integer.BYTES);
+ page[7 + lastOffset + 12] = (byte) 64;
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(20, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ reader.skip(16);
+ assertThatThrownBy(reader::readDouble)
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("ALP vector 2 packed values end at");
+ }
+
+ /**
+ * An exception count that is legal against the vector size can still declare an exception block
+ * larger than the vector holds. numExceptions == vectorSize passes the AlpInfo check and then
+ * needs 16 * (2 + 8) bytes, far more than this vector's span.
+ */
+ @Test
+ public void rejectsExceptionBlockLargerThanVector() throws Exception {
+ byte[] page = validDoublePage(256, 16);
+ int v0 = firstVectorOffset(page);
+ page[v0 + 2] = (byte) 16; // num_exceptions, uint16 LE
+ page[v0 + 3] = (byte) 0;
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(256, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ assertThatThrownBy(reader::readDouble)
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("exceptions needing");
+ }
+
+ /**
+ * An exception position at or past the end of the vector would write outside the decoded values.
+ * Exercises the static reader directly, which is the whole point of it taking its inputs
+ * explicitly rather than reaching for fields.
+ */
+ @Test
+ public void rejectsExceptionPositionOutOfBounds() {
+ ByteBuffer data = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN);
+ data.putShort(0, (short) 3); // in range
+ data.putShort(2, (short) 16); // == vectorSize, so one past the last valid slot
+ int[] positions = new int[4];
+
+ assertThatThrownBy(() -> AlpValuesReader.readExceptionPositions(data, 0, 2, 16, positions))
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("ALP exception position 16 out of bounds for vectorSize 16");
+ }
+
+ /** Each vector must span at least its fixed headers, the last one included via the end entry. */
+ @Test
+ public void rejectsVectorShorterThanItsHeaders() {
+ // offset array of 2 entries (8 bytes), then vectors at 8 and 30, data ending at 40.
+ assertThatThrownBy(() -> AlpValuesReader.validateVectorOffsets(new int[] {8, 30, 40}, 17, 1000))
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("ALP vector 1 at offset 30 spans 10 bytes, fewer than the 17");
+ }
+
+ /** No vector can be longer than every value packed at full width and also stored as an exception. */
+ @Test
+ public void rejectsVectorLongerThanAnyVectorCanBe() {
+ int max = AlpValuesReader.maxVectorBytes(16, 17, Double.BYTES);
+ assertThat(max).isEqualTo(17 + 16 * 8 + 16 * (2 + 8));
+ AlpValuesReader.validateVectorOffsets(new int[] {8, 8 + max, 8 + max + 17}, 17, max);
+ assertThatThrownBy(() -> AlpValuesReader.validateVectorOffsets(new int[] {8, 9 + max, 9 + max + 17}, 17, max))
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("ALP vector 0 at offset 8 spans " + (max + 1) + " bytes, more than the " + max);
+ }
+
+ /** The end entry makes every in-range index valid, and anything else is a caller error. */
+ @Test
+ public void vectorEndPositionRejectsOutOfRangeIndex() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ assertThat(reader.getVectorEndPosition(1)).isEqualTo(page.length - 7 - 2 * Integer.BYTES);
+ for (int bad : new int[] {-1, 2, Integer.MAX_VALUE}) {
+ assertThatThrownBy(() -> reader.getVectorEndPosition(bad))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("out of range for 2 vectors");
+ }
+ }
+
+ /** The last vector is partial whenever the value count is not a whole number of vectors. */
+ @Test
+ public void vectorSizeAccountsForAPartialLastVector() {
+ // 3 vectors of 16 holding 40 values: two full, one holding the remaining 8.
+ assertThat(AlpValuesReader.getVectorSize(0, 16, 40, 3)).isEqualTo(16);
+ assertThat(AlpValuesReader.getVectorSize(1, 16, 40, 3)).isEqualTo(16);
+ assertThat(AlpValuesReader.getVectorSize(2, 16, 40, 3)).isEqualTo(8);
+ // An exact multiple leaves the last vector full rather than empty.
+ assertThat(AlpValuesReader.getVectorSize(1, 16, 32, 2)).isEqualTo(16);
+ assertThat(AlpValuesReader.isLast(1, 2)).isTrue();
+ assertThat(AlpValuesReader.isLast(0, 2)).isFalse();
+ }
+
+ // ---------------------------------------------------------------------------
+ // Truncation and corrupted offsets
+ // Each of these must fail cleanly: a catchable decoding exception, never an OutOfMemoryError
+ // from an unbounded allocation and never a silent wrong-value decode. See assertFailsCleanly.
+ // ---------------------------------------------------------------------------
+
+ /** Page with only the 7-byte header — nothing else. */
+ @Test
+ public void rejectsHeaderOnlyPage() {
+ byte[] tiny = new byte[] {0x00, 0x00, 0x0A, 0x20, 0x00, 0x00, 0x00}; // 32 elements, log_vec=10
+ assertFailsCleanly(() ->
+ new AlpValuesReaderForDouble().initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(tiny))));
+ }
+
+ @Test
+ public void rejectsPageTruncatedMidOffsetArray() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ // num_vectors = ceil(32/16) = 2, so offset array is 8 bytes. Truncate to chop the 2nd offset.
+ byte[] truncated = new byte[7 + 4]; // header + first offset only
+ System.arraycopy(page, 0, truncated, 0, truncated.length);
+ assertFailsCleanly(() -> new AlpValuesReaderForDouble()
+ .initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(truncated))));
+ }
+
+ @Test
+ public void rejectsPageTruncatedMidVectorData() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ // chop the last 20 bytes — guaranteed to land in the middle of the second vector
+ byte[] truncated = new byte[page.length - 20];
+ System.arraycopy(page, 0, truncated, 0, truncated.length);
+ // The offset array still describes a second vector that the truncated body cannot hold, so
+ // this is now caught up front when the offsets are validated against the body length.
+ assertFailsCleanly(() -> new AlpValuesReaderForDouble()
+ .initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(truncated))));
+ }
+
+ @Test
+ public void rejectsCorruptedOffsetPointingPastEnd() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ // Offset array starts at byte 7. Overwrite the first offset (uint32 LE) with a huge value.
+ page[7] = (byte) 0xFF;
+ page[8] = (byte) 0xFF;
+ page[9] = (byte) 0xFF;
+ page[10] = (byte) 0x7F;
+ // Offsets are validated against the body length in initFromPage, so a bogus offset is
+ // rejected before any vector is decoded rather than reading out of bounds later.
+ assertFailsCleanly(() ->
+ new AlpValuesReaderForDouble().initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page))));
+ }
+
+ // ---------------------------------------------------------------------------
+ // skip() / read() bounds
+ // ---------------------------------------------------------------------------
+
+ @Test
+ public void rejectsSkipPastEnd() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ assertThatThrownBy(() -> reader.skip(33))
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("Cannot skip this many elements");
+ }
+
+ @Test
+ public void rejectsNegativeSkip() throws Exception {
+ byte[] page = validDoublePage(32, 16);
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(32, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ assertThatThrownBy(() -> reader.skip(-1))
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("Cannot skip this many elements");
+ }
+
+ @Test
+ public void rejectsReadPastEnd() throws Exception {
+ byte[] page = validDoublePage(8, 8);
+ AlpValuesReaderForDouble reader = new AlpValuesReaderForDouble();
+ reader.initFromPage(8, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)));
+ for (int i = 0; i < 8; i++) reader.readDouble();
+ assertThatThrownBy(reader::readDouble)
+ .isInstanceOf(ParquetDecodingException.class)
+ .hasMessageContaining("ALP double data was already exhausted");
+ }
+
+ // ---------------------------------------------------------------------------
+ // Utility
+ // ---------------------------------------------------------------------------
+
+ /**
+ * Requires {@code action} to fail, and requires that failure to be a clean one.
+ *
+ *