From 4b9cd0be9b0160cf438dcfc177784b8cd5dbfcc2 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Wed, 18 Nov 2026 16:12:56 +0000 Subject: [PATCH] refactor(bigquery-jdbc): move Arrow batch validation out of the result-set loop --- .../jdbc/BigQueryArrowBatchWrapper.java | 47 +++++- .../bigquery/jdbc/BigQueryArrowResultSet.java | 121 +++++++------- .../bigquery/jdbc/BigQueryStatement.java | 2 +- .../jdbc/BigQueryArrowBatchWrapperTest.java | 96 ++++++++++++ .../jdbc/BigQueryArrowResultSetTest.java | 148 ++++++++++++++++-- 5 files changed, 336 insertions(+), 78 deletions(-) create mode 100644 java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapperTest.java diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java index ad015ce9ba79..90850118fece 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java @@ -16,8 +16,10 @@ package com.google.cloud.bigquery.jdbc; +import com.google.cloud.bigquery.exception.BigQueryJdbcException; import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch; import com.google.cloud.bigquery.storage.v1.ArrowSchema; +import com.google.common.annotations.VisibleForTesting; import org.apache.arrow.vector.util.JsonStringArrayList; /** This class acts as a facade layer and wraps Arrow's VectorSchemaRoot & JsonStringArrayList */ @@ -53,9 +55,9 @@ private BigQueryArrowBatchWrapper( this.exception = exception; } - static BigQueryArrowBatchWrapper of(ArrowRecordBatch currentArrowBatch, boolean... isLast) { - boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0]; - return new BigQueryArrowBatchWrapper(currentArrowBatch, null, null, null, isLastFlag, null); + /** Marks the end of the stream. It carries no data. */ + static BigQueryArrowBatchWrapper ofEndOfStream() { + return new BigQueryArrowBatchWrapper(null, null, null, null, true, null); } /** @@ -70,9 +72,8 @@ static BigQueryArrowBatchWrapper of( } static BigQueryArrowBatchWrapper getNestedFieldValueListWrapper( - JsonStringArrayList nestedRecords, boolean... isLast) { - boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0]; - return new BigQueryArrowBatchWrapper(null, null, null, nestedRecords, isLastFlag, null); + JsonStringArrayList nestedRecords) { + return new BigQueryArrowBatchWrapper(null, null, null, nestedRecords, false, null); } static BigQueryArrowBatchWrapper ofError(Throwable exception) { @@ -91,10 +92,44 @@ ArrowSchema getArrowSchema() { * Rows the source reported for this message, or null if this wrapper was not built from a stream * message. */ + @VisibleForTesting Long getReportedRowCount() { return this.reportedRowCount; } + /** + * True if this stream message carries record batch data. A schema, statistics or throttling + * update carries none. + */ + boolean hasBatchData() { + return this.currentArrowBatch != null + && !this.currentArrowBatch.getSerializedRecordBatch().isEmpty(); + } + + /** + * Checks what this wrapper can check about itself before its batch is decoded: without batch data + * it holds no rows, so it must not report any. + */ + void validate() throws BigQueryJdbcException { + if (!hasBatchData()) { + checkRowCount(0); + } + } + + /** + * Fails unless this message reports exactly {@code actualRows}, the rows its record batch holds. + * A wrapper that is not a stream message reports no count, so there is nothing to check. + */ + void checkRowCount(int actualRows) throws BigQueryJdbcException { + if (this.reportedRowCount == null || this.reportedRowCount == actualRows) { + return; + } + throw new BigQueryJdbcException( + String.format( + "An Arrow stream message reported %d rows but its batch has %d.", + this.reportedRowCount, actualRows)); + } + JsonStringArrayList getNestedRecords() { return this.nestedRecords; } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java index 40b44f189b18..61b8dc3f374e 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java @@ -25,8 +25,8 @@ import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.exception.BigQueryJdbcException; -import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch; import com.google.cloud.bigquery.storage.v1.ArrowSchema; +import com.google.common.annotations.VisibleForTesting; import io.opentelemetry.context.Scope; import java.io.IOException; import java.math.BigDecimal; @@ -46,6 +46,7 @@ import org.apache.arrow.vector.VectorLoader; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.ipc.ReadChannel; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; import org.apache.arrow.vector.ipc.message.MessageSerializer; import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; import org.apache.arrow.vector.util.JsonStringArrayList; @@ -80,11 +81,11 @@ class BigQueryArrowResultSet extends BigQueryBaseResultSet { private boolean afterLast = false; + // Created once the Arrow schema is known: at construction, or from the stream's first message. private ArrowDeserializer arrowDeserializer; - BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); - // Decoder object will be reused to avoid re-allocation and too much garbage collection. + private final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + // What the getters read: the rows of the batch the cursor is in. Null until a batch is decoded. private VectorSchemaRoot vectorSchemaRoot; - private VectorLoader vectorLoader; // producer task's reference private final Future ownedTask; private final boolean enableTimestampPicos; @@ -173,7 +174,6 @@ static BigQueryArrowResultSet of( this.ownedTask = null; this.arrowDeserializer = null; this.vectorSchemaRoot = null; - this.vectorLoader = null; this.enableTimestampPicos = false; } @@ -200,9 +200,15 @@ static BigQueryArrowResultSet getNestedResultSet( enableTimestampPicos); } + /** + * Decodes the record batches of one stream. It owns the vectors it decodes into, and reuses them + * for every batch to avoid re-allocation and too much garbage collection. Their memory comes from + * the result set's allocator. + */ private class ArrowDeserializer implements AutoCloseable { + private final VectorSchemaRoot root; + private final VectorLoader loader; - /* Decoder object will be reused to avoid re-allocation and too much garbage collection. */ private ArrowDeserializer(ArrowSchema arrowSchema) throws IOException { org.apache.arrow.vector.types.pojo.Schema schema = MessageSerializer.deserializeSchema( @@ -214,39 +220,49 @@ private ArrowDeserializer(ArrowSchema arrowSchema) throws IOException { for (org.apache.arrow.vector.types.pojo.Field field : fields) { vectors.add(field.createVector(allocator)); } - vectorSchemaRoot = new VectorSchemaRoot(vectors); - vectorLoader = new VectorLoader(vectorSchemaRoot); + root = new VectorSchemaRoot(vectors); + loader = new VectorLoader(root); } - private void deserializeArrowBatch(ArrowRecordBatch batch) throws SQLException { + /** + * Decodes the message's record batch and returns its rows. The returned vectors hold these rows + * only until the next call. Fails if the batch can't be decoded, or doesn't hold the number of + * rows its message reports. After a failure the vectors hold nothing. + */ + private VectorSchemaRoot deserializeArrowBatch(BigQueryArrowBatchWrapper batchWrapper) + throws SQLException { LOG.finestTrace("deserializeArrowBatch"); try { - if (vectorSchemaRoot != null) { - // Clear vectorSchemaRoot before populating a new batch - vectorSchemaRoot.clear(); - } + // Clear the vectors before populating a new batch + root.clear(); // Read straight from the ByteString instead of copying it into a heap byte[] first. - try (org.apache.arrow.vector.ipc.message.ArrowRecordBatch deserializedBatch = + try (ArrowRecordBatch deserializedBatch = MessageSerializer.deserializeRecordBatch( - new ReadChannel(Channels.newChannel(batch.getSerializedRecordBatch().newInput())), + new ReadChannel( + Channels.newChannel( + batchWrapper.getCurrentArrowBatch().getSerializedRecordBatch().newInput())), allocator)) { + batchWrapper.checkRowCount(deserializedBatch.getLength()); // The vectors keep their own references to the buffers after the batch is closed. - vectorLoader.load(deserializedBatch); + loader.load(deserializedBatch); } } catch (RuntimeException | IOException ex) { + // A batch that fails part-way has already put some of its buffers into the vectors. + root.clear(); throw new BigQueryJdbcException(ex); } + return root; + } + + /** Frees the buffers of the batch decoded last. The vectors stay usable for the next batch. */ + private void releaseBatch() { + root.clear(); } @Override public void close() { - LOG.fineTrace("close", () -> String.format("Closing BigQueryArrowResultSet %s.", this)); - if (vectorSchemaRoot != null) { - vectorSchemaRoot.close(); - } - if (allocator != null) { - allocator.close(); - } + LOG.fineTrace("close", "Closing the Arrow vectors."); + root.close(); } } @@ -268,9 +284,9 @@ public boolean next() throws SQLException { } else { /* Non nested */ if (this.afterLast || this.hasReachedEnd || reachedRowLimit()) { - if (!this.afterLast && this.vectorSchemaRoot != null) { + if (!this.afterLast) { // Free the last batch's Arrow buffers as soon as iteration ends, not at close(). - this.vectorSchemaRoot.clear(); + releaseCurrentBatch(); } this.afterLast = true; return false; @@ -318,6 +334,19 @@ private boolean hasMoreRowsInCurrentBatch() { && this.currentBatchRowIndex < this.vectorSchemaRoot.getRowCount() - 1; } + /** Frees the Arrow buffers of the batch decoded last. Without a schema nothing was decoded. */ + private void releaseCurrentBatch() { + if (this.arrowDeserializer != null) { + this.arrowDeserializer.releaseBatch(); + } + } + + /** Bytes of Arrow memory this result set holds: the buffers of the batch decoded last. */ + @VisibleForTesting + long getAllocatedMemory() { + return this.allocator.getAllocatedMemory(); + } + private void initDeserializerIfNeeded(ArrowSchema arrowSchema) throws SQLException { if (arrowSchema == null || this.arrowDeserializer != null) { return; @@ -332,12 +361,13 @@ private void initDeserializerIfNeeded(ArrowSchema arrowSchema) throws SQLExcepti /** * Loads the next batch that has rows into the vectors. The reader queues every message as-is, so - * this is the single place that decides what to use: + * every message is checked here, when it is consumed: * * * @@ -352,10 +382,8 @@ private boolean loadNextBatch() throws SQLException, InterruptedException { } if (batchWrapper.isLast()) { /* Marks the end of the records */ - if (this.vectorSchemaRoot != null) { - // IMP: To avoid memory leak: clear vectorSchemaRoot as it still holds the last batch - this.vectorSchemaRoot.clear(); - } + // IMP: To avoid memory leak: the vectors still hold the last batch + releaseCurrentBatch(); this.hasReachedEnd = true; if (this.rowCount < this.totalRows) { throw new BigQueryJdbcException( @@ -365,36 +393,20 @@ private boolean loadNextBatch() throws SQLException, InterruptedException { return false; } initDeserializerIfNeeded(batchWrapper.getArrowSchema()); - ArrowRecordBatch arrowBatch = batchWrapper.getCurrentArrowBatch(); - Long reportedRows = batchWrapper.getReportedRowCount(); - if (arrowBatch == null || arrowBatch.getSerializedRecordBatch().isEmpty()) { - if (reportedRows != null && reportedRows > 0) { - throw rowCountMismatch(reportedRows, 0); - } + batchWrapper.validate(); + if (!batchWrapper.hasBatchData()) { continue; // no rows in this message } if (this.arrowDeserializer == null) { throw new BigQueryJdbcException("Received an Arrow record batch before its schema."); } - // Populates vectorSchemaRoot - this.arrowDeserializer.deserializeArrowBatch(arrowBatch); - int decodedRows = this.vectorSchemaRoot.getRowCount(); - if (reportedRows != null && reportedRows != decodedRows) { - throw rowCountMismatch(reportedRows, decodedRows); - } - if (decodedRows > 0) { + this.vectorSchemaRoot = this.arrowDeserializer.deserializeArrowBatch(batchWrapper); + if (this.vectorSchemaRoot.getRowCount() > 0) { return true; } } } - private static SQLException rowCountMismatch(long reportedRows, long decodedRows) { - return new BigQueryJdbcException( - String.format( - "An Arrow stream message reported %d rows but its batch has %d.", - reportedRows, decodedRows)); - } - @Override Object getObjectInternal(int columnIndex) throws SQLException { LOG.finestTrace("getObjectInternal"); @@ -551,13 +563,12 @@ public void close() { if (this.buffer != null) { this.buffer.clear(); // Unblock any consumer thread currently waiting on buffer.take() in next() - this.buffer.offer(BigQueryArrowBatchWrapper.of(null, true)); + this.buffer.offer(BigQueryArrowBatchWrapper.ofEndOfStream()); } if (this.arrowDeserializer != null) { this.arrowDeserializer.close(); - } else if (this.allocator != null) { - this.allocator.close(); } + this.allocator.close(); } finally { super.close(); } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java index 4f3f2d94a3ae..8301a2291ef6 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java @@ -2091,7 +2091,7 @@ private void enqueueError(BlockingQueue queue, Throwa } private void enqueueEndOfStream(BlockingQueue queue) { - enqueueMarker(queue, BigQueryArrowBatchWrapper.of(null, true)); + enqueueMarker(queue, BigQueryArrowBatchWrapper.ofEndOfStream()); } private void enqueueBufferError(BlockingQueue queue, Throwable e) { diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapperTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapperTest.java new file mode 100644 index 000000000000..516419507cf7 --- /dev/null +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapperTest.java @@ -0,0 +1,96 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.jdbc; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.google.cloud.bigquery.exception.BigQueryJdbcException; +import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch; +import com.google.protobuf.ByteString; +import org.apache.arrow.vector.util.JsonStringArrayList; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.function.Executable; + +public class BigQueryArrowBatchWrapperTest { + + // The wrapper only looks at whether the batch has bytes, so they don't have to decode. + private static final ArrowRecordBatch BATCH_WITH_DATA = + ArrowRecordBatch.newBuilder().setSerializedRecordBatch(ByteString.copyFromUtf8("x")).build(); + private static final ArrowRecordBatch BATCH_WITHOUT_DATA = ArrowRecordBatch.getDefaultInstance(); + + @Test + public void testHasBatchData() { + assertThat(BigQueryArrowBatchWrapper.of(null, null, 0).hasBatchData()).isFalse(); + assertThat(BigQueryArrowBatchWrapper.of(BATCH_WITHOUT_DATA, null, 0).hasBatchData()).isFalse(); + assertThat(BigQueryArrowBatchWrapper.of(BATCH_WITH_DATA, null, 1).hasBatchData()).isTrue(); + } + + @Test + public void testValidateRejectsRowsReportedWithoutBatchData() { + assertRowCountMismatch( + () -> BigQueryArrowBatchWrapper.of(null, null, 2).validate(), + "reported 2 rows but its batch has 0"); + assertRowCountMismatch( + () -> BigQueryArrowBatchWrapper.of(BATCH_WITHOUT_DATA, null, 2).validate(), + "reported 2 rows but its batch has 0"); + assertRowCountMismatch( + () -> BigQueryArrowBatchWrapper.of(null, null, -1).validate(), + "reported -1 rows but its batch has 0"); + } + + @Test + public void testValidateAcceptsNoRowsWithoutBatchData() { + assertDoesNotThrow(() -> BigQueryArrowBatchWrapper.of(null, null, 0).validate()); + assertDoesNotThrow(() -> BigQueryArrowBatchWrapper.of(BATCH_WITHOUT_DATA, null, 0).validate()); + } + + @Test + public void testValidateDoesNotCountTheRowsOfABatch() { + // How many rows a batch holds is known only once it is decoded, so the deserializer checks it. + assertDoesNotThrow(() -> BigQueryArrowBatchWrapper.of(BATCH_WITH_DATA, null, 5).validate()); + } + + @Test + public void testCheckRowCount() { + BigQueryArrowBatchWrapper message = BigQueryArrowBatchWrapper.of(BATCH_WITH_DATA, null, 2); + + assertDoesNotThrow(() -> message.checkRowCount(2)); + assertRowCountMismatch(() -> message.checkRowCount(3), "reported 2 rows but its batch has 3"); + assertRowCountMismatch(() -> message.checkRowCount(0), "reported 2 rows but its batch has 0"); + } + + @Test + public void testWrappersThatAreNotStreamMessagesReportNoRowCount() { + assertNothingToCheck(BigQueryArrowBatchWrapper.ofEndOfStream()); + assertNothingToCheck(BigQueryArrowBatchWrapper.ofError(new IllegalStateException("failed"))); + assertNothingToCheck( + BigQueryArrowBatchWrapper.getNestedFieldValueListWrapper(new JsonStringArrayList<>())); + } + + private static void assertNothingToCheck(BigQueryArrowBatchWrapper wrapper) { + assertThat(wrapper.getReportedRowCount()).isNull(); + assertDoesNotThrow(wrapper::validate); + assertDoesNotThrow(() -> wrapper.checkRowCount(3)); + } + + private static void assertRowCountMismatch(Executable check, String expected) { + BigQueryJdbcException e = assertThrows(BigQueryJdbcException.class, check); + assertThat(e.getMessage()).contains(expected); + } +} diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java index 43ffbdb2019e..cb41fe8fd894 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java @@ -21,6 +21,7 @@ import static com.google.common.truth.Truth.assertThat; import static org.apache.arrow.vector.types.Types.MinorType.INT; import static org.apache.arrow.vector.types.Types.MinorType.VARCHAR; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -239,8 +240,8 @@ public void setUp() throws SQLException, IOException { ArrowRecordBatch.newBuilder() .setSerializedRecordBatch(serializeVectorSchemaRoot(vectorSchemaRoot)) .build(); - arrowBatchWrapper = BigQueryArrowBatchWrapper.of(batch); - arrowBatchWrapperLast = BigQueryArrowBatchWrapper.of(null, true); // last flag + arrowBatchWrapper = BigQueryArrowBatchWrapper.of(batch, null, vectorSchemaRoot.getRowCount()); + arrowBatchWrapperLast = BigQueryArrowBatchWrapper.ofEndOfStream(); // last flag buffer.add(arrowBatchWrapper); buffer.add(arrowBatchWrapperLast); bufferWithTwoRows.add(arrowBatchWrapper); @@ -503,8 +504,8 @@ public void testTimestampPicosEnabled() throws SQLException, IOException { .setSerializedRecordBatch(serializeVectorSchemaRoot(root)) .build(); BlockingQueue picosBuffer = new LinkedBlockingDeque<>(); - picosBuffer.add(BigQueryArrowBatchWrapper.of(batch)); - picosBuffer.add(BigQueryArrowBatchWrapper.of(null, true)); + picosBuffer.add(BigQueryArrowBatchWrapper.of(batch, null, root.getRowCount())); + picosBuffer.add(BigQueryArrowBatchWrapper.ofEndOfStream()); BigQueryStatement picosStmt = mock(BigQueryStatement.class); doReturn(true).when(picosStmt).isEnableTimestampPicos(); @@ -542,8 +543,8 @@ public void testTimestampPicosDisabled() throws SQLException, IOException { .setSerializedRecordBatch(serializeVectorSchemaRoot(root)) .build(); BlockingQueue picosBuffer = new LinkedBlockingDeque<>(); - picosBuffer.add(BigQueryArrowBatchWrapper.of(batch)); - picosBuffer.add(BigQueryArrowBatchWrapper.of(null, true)); + picosBuffer.add(BigQueryArrowBatchWrapper.of(batch, null, root.getRowCount())); + picosBuffer.add(BigQueryArrowBatchWrapper.ofEndOfStream()); BigQueryStatement disabledStmt = mock(BigQueryStatement.class); doReturn(false).when(disabledStmt).isEnableTimestampPicos(); @@ -576,8 +577,8 @@ public void testStandardMicrosecondTimestampWithPicosEnabled() throws SQLExcepti .setSerializedRecordBatch(serializeVectorSchemaRoot(root)) .build(); BlockingQueue picosBuffer = new LinkedBlockingDeque<>(); - picosBuffer.add(BigQueryArrowBatchWrapper.of(batch)); - picosBuffer.add(BigQueryArrowBatchWrapper.of(null, true)); + picosBuffer.add(BigQueryArrowBatchWrapper.of(batch, null, root.getRowCount())); + picosBuffer.add(BigQueryArrowBatchWrapper.ofEndOfStream()); BigQueryStatement picosStmt = mock(BigQueryStatement.class); doReturn(true).when(picosStmt).isEnableTimestampPicos(); @@ -624,8 +625,8 @@ private BigQueryArrowResultSet createTemporalArrowResultSet(Instant ref) .setSerializedRecordBatch(serializeVectorSchemaRoot(root)) .build(); BlockingQueue arrowBuffer = new LinkedBlockingDeque<>(); - arrowBuffer.add(BigQueryArrowBatchWrapper.of(batch)); - arrowBuffer.add(BigQueryArrowBatchWrapper.of(null, true)); + arrowBuffer.add(BigQueryArrowBatchWrapper.of(batch, null, root.getRowCount())); + arrowBuffer.add(BigQueryArrowBatchWrapper.ofEndOfStream()); Schema schema = Schema.of( @@ -776,8 +777,8 @@ public void testTimestampPicosAccessorsAcrossJvmZones(String jvmZone) .setSerializedRecordBatch(serializeVectorSchemaRoot(root)) .build(); BlockingQueue picosBuffer = new LinkedBlockingDeque<>(); - picosBuffer.add(BigQueryArrowBatchWrapper.of(batch)); - picosBuffer.add(BigQueryArrowBatchWrapper.of(null, true)); + picosBuffer.add(BigQueryArrowBatchWrapper.of(batch, null, root.getRowCount())); + picosBuffer.add(BigQueryArrowBatchWrapper.ofEndOfStream()); BigQueryStatement picosStmt = mock(BigQueryStatement.class); doReturn(true).when(picosStmt).isEnableTimestampPicos(); @@ -866,8 +867,9 @@ private static ArrowRecordBatch idBatch(int... ids) throws IOException { .build(); } + /** A stream message that reports as many rows as its batch holds. */ private static BigQueryArrowBatchWrapper idWrapper(int... ids) throws IOException { - return BigQueryArrowBatchWrapper.of(idBatch(ids)); + return idMessage(ids.length, ids); } /** A stream message: the ids as an Arrow batch, and the row count the message reports. */ @@ -945,7 +947,121 @@ public void testUndecodableBatchEndsIteration() throws SQLException, IOException .substring(0, whole.getSerializedRecordBatch().size() / 2)) .build(); - assertStreamFailsOn(BigQueryArrowBatchWrapper.of(truncated)); + SQLException e = assertStreamFailsOn(BigQueryArrowBatchWrapper.of(truncated, null, 2)); + // The decoding failed, not the row-count check. + assertThat(e.getCause().getCause()).isNotNull(); + assertThat(e.getCause().getMessage()).doesNotContain("reported"); + } + + @Test + public void testFirstBatchFailureEndsIteration() throws SQLException, IOException { + // No batch was decoded before the failing one. + BigQueryArrowResultSet rs = + idResultSet( + idArrowSchema(), 2, idMessage(3, 1, 2), idMessage(2, 3, 4), arrowBatchWrapperLast); + + SQLException e = assertThrows(SQLException.class, rs::next); + assertThat(e.getCause().getMessage()).contains("reported 3 rows but its batch has 2"); + // The rejected batch was not loaded. + assertThat(rs.getAllocatedMemory()).isEqualTo(0L); + assertThrows(SQLException.class, () -> rs.getLong(1)); + assertThat(rs.next()).isFalse(); + } + + @Test + public void testFailureReleasesTheBatchReadBefore() throws SQLException, IOException { + BigQueryArrowResultSet rs = + idResultSet(idArrowSchema(), 3, idMessage(1, 1), idMessage(3, 1, 2), arrowBatchWrapperLast); + assertThat(rs.next()).isTrue(); + + SQLException e = assertThrows(SQLException.class, rs::next); + assertThat(e.getCause().getMessage()).contains("reported 3 rows but its batch has 2"); + // Neither the rejected batch nor the one read before it is held. + assertThat(rs.getAllocatedMemory()).isEqualTo(0L); + } + + /** Two rows in two columns. The one-column id schema can't load it. */ + private static ArrowRecordBatch twoColumnBatch() throws IOException { + VectorSchemaRoot twoColumns = + new VectorSchemaRoot( + ImmutableList.of(idRoot(1, 2).getVector(0), idRoot(3, 4).getVector(0))); + twoColumns.setRowCount(2); + return ArrowRecordBatch.newBuilder() + .setSerializedRecordBatch(serializeVectorSchemaRoot(twoColumns)) + .build(); + } + + @Test + public void testBatchThatFailsWhileLoadingIsReleased() throws SQLException, IOException { + // The first column is loaded before the batch fails on the second. + BigQueryArrowResultSet rs = + idResultSet( + idArrowSchema(), + 2, + BigQueryArrowBatchWrapper.of(twoColumnBatch(), null, 2), + arrowBatchWrapperLast); + + SQLException e = assertThrows(SQLException.class, rs::next); + assertThat(e.getCause().getMessage()).doesNotContain("reported"); + assertThat(rs.getAllocatedMemory()).isEqualTo(0L); + } + + @Test + public void testRowCountIsCheckedBeforeTheBatchIsLoaded() throws SQLException, IOException { + // The batch can't be loaded either, but its wrong row count is found first. + BigQueryArrowResultSet rs = + idResultSet( + idArrowSchema(), + 3, + BigQueryArrowBatchWrapper.of(twoColumnBatch(), null, 3), + arrowBatchWrapperLast); + + SQLException e = assertThrows(SQLException.class, rs::next); + assertThat(e.getCause().getMessage()).contains("reported 3 rows but its batch has 2"); + } + + @Test + public void testBatchIsReleasedAtTheRowLimit() throws SQLException, IOException { + BigQueryArrowResultSet rs = + idResultSet(idArrowSchema(), 1, idWrapper(1, 2), arrowBatchWrapperLast); + assertThat(rs.next()).isTrue(); + assertThat(rs.getAllocatedMemory()).isGreaterThan(0L); + + assertThat(rs.next()).isFalse(); + assertThat(rs.getAllocatedMemory()).isEqualTo(0L); + } + + @Test + public void testBatchIsReleasedAtTheEndOfTheStream() throws SQLException, IOException { + BigQueryArrowResultSet rs = + idResultSet(idArrowSchema(), 3, idWrapper(1, 2), arrowBatchWrapperLast); + assertThat(rs.next()).isTrue(); + assertThat(rs.next()).isTrue(); + + // The stream ends a row short. The last batch is released before that is reported. + assertThrows(SQLException.class, rs::next); + assertThat(rs.getAllocatedMemory()).isEqualTo(0L); + } + + @Test + public void testCloseReleasesTheBatchUnderTheCursor() throws SQLException, IOException { + BigQueryArrowResultSet rs = + idResultSet(idArrowSchema(), 2, idWrapper(1, 2), arrowBatchWrapperLast); + assertThat(rs.next()).isTrue(); + + // The allocator fails to close while a batch still holds its buffers. + assertDoesNotThrow(rs::close); + assertThat(rs.isClosed()).isTrue(); + } + + @Test + public void testEndAndCloseWithoutASchema() throws SQLException { + // No schema arrived, so there is no deserializer and nothing was decoded. + BigQueryArrowResultSet rs = idResultSet(null, 0, arrowBatchWrapperLast); + + assertThat(rs.next()).isFalse(); + assertDoesNotThrow(rs::close); + assertThat(rs.isClosed()).isTrue(); } @Test @@ -977,9 +1093,9 @@ public void testMatchingReportedRowCountsAreAccepted() throws SQLException, IOEx @Test public void testMessagesWithoutRowsAreSkipped() throws SQLException, IOException { - BigQueryArrowBatchWrapper noBatch = BigQueryArrowBatchWrapper.of(null); + BigQueryArrowBatchWrapper noBatch = BigQueryArrowBatchWrapper.of(null, null, 0); BigQueryArrowBatchWrapper emptyBatch = - BigQueryArrowBatchWrapper.of(ArrowRecordBatch.getDefaultInstance()); + BigQueryArrowBatchWrapper.of(ArrowRecordBatch.getDefaultInstance(), null, 0); BigQueryArrowResultSet rs = idResultSet( idArrowSchema(),