diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java
index ad015ce9ba79..90850118fece 100644
--- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java
+++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java
@@ -16,8 +16,10 @@
package com.google.cloud.bigquery.jdbc;
+import com.google.cloud.bigquery.exception.BigQueryJdbcException;
import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch;
import com.google.cloud.bigquery.storage.v1.ArrowSchema;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.arrow.vector.util.JsonStringArrayList;
/** This class acts as a facade layer and wraps Arrow's VectorSchemaRoot & JsonStringArrayList */
@@ -53,9 +55,9 @@ private BigQueryArrowBatchWrapper(
this.exception = exception;
}
- static BigQueryArrowBatchWrapper of(ArrowRecordBatch currentArrowBatch, boolean... isLast) {
- boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0];
- return new BigQueryArrowBatchWrapper(currentArrowBatch, null, null, null, isLastFlag, null);
+ /** Marks the end of the stream. It carries no data. */
+ static BigQueryArrowBatchWrapper ofEndOfStream() {
+ return new BigQueryArrowBatchWrapper(null, null, null, null, true, null);
}
/**
@@ -70,9 +72,8 @@ static BigQueryArrowBatchWrapper of(
}
static BigQueryArrowBatchWrapper getNestedFieldValueListWrapper(
- JsonStringArrayList nestedRecords, boolean... isLast) {
- boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0];
- return new BigQueryArrowBatchWrapper(null, null, null, nestedRecords, isLastFlag, null);
+ JsonStringArrayList nestedRecords) {
+ return new BigQueryArrowBatchWrapper(null, null, null, nestedRecords, false, null);
}
static BigQueryArrowBatchWrapper ofError(Throwable exception) {
@@ -91,10 +92,44 @@ ArrowSchema getArrowSchema() {
* Rows the source reported for this message, or null if this wrapper was not built from a stream
* message.
*/
+ @VisibleForTesting
Long getReportedRowCount() {
return this.reportedRowCount;
}
+ /**
+ * True if this stream message carries record batch data. A schema, statistics or throttling
+ * update carries none.
+ */
+ boolean hasBatchData() {
+ return this.currentArrowBatch != null
+ && !this.currentArrowBatch.getSerializedRecordBatch().isEmpty();
+ }
+
+ /**
+ * Checks what this wrapper can check about itself before its batch is decoded: without batch data
+ * it holds no rows, so it must not report any.
+ */
+ void validate() throws BigQueryJdbcException {
+ if (!hasBatchData()) {
+ checkRowCount(0);
+ }
+ }
+
+ /**
+ * Fails unless this message reports exactly {@code actualRows}, the rows its record batch holds.
+ * A wrapper that is not a stream message reports no count, so there is nothing to check.
+ */
+ void checkRowCount(int actualRows) throws BigQueryJdbcException {
+ if (this.reportedRowCount == null || this.reportedRowCount == actualRows) {
+ return;
+ }
+ throw new BigQueryJdbcException(
+ String.format(
+ "An Arrow stream message reported %d rows but its batch has %d.",
+ this.reportedRowCount, actualRows));
+ }
+
JsonStringArrayList getNestedRecords() {
return this.nestedRecords;
}
diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java
index 40b44f189b18..61b8dc3f374e 100644
--- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java
+++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java
@@ -25,8 +25,8 @@
import com.google.cloud.bigquery.Schema;
import com.google.cloud.bigquery.StandardSQLTypeName;
import com.google.cloud.bigquery.exception.BigQueryJdbcException;
-import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch;
import com.google.cloud.bigquery.storage.v1.ArrowSchema;
+import com.google.common.annotations.VisibleForTesting;
import io.opentelemetry.context.Scope;
import java.io.IOException;
import java.math.BigDecimal;
@@ -46,6 +46,7 @@
import org.apache.arrow.vector.VectorLoader;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ReadChannel;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel;
import org.apache.arrow.vector.util.JsonStringArrayList;
@@ -80,11 +81,11 @@ class BigQueryArrowResultSet extends BigQueryBaseResultSet {
private boolean afterLast = false;
+ // Created once the Arrow schema is known: at construction, or from the stream's first message.
private ArrowDeserializer arrowDeserializer;
- BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
- // Decoder object will be reused to avoid re-allocation and too much garbage collection.
+ private final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
+ // What the getters read: the rows of the batch the cursor is in. Null until a batch is decoded.
private VectorSchemaRoot vectorSchemaRoot;
- private VectorLoader vectorLoader;
// producer task's reference
private final Future> ownedTask;
private final boolean enableTimestampPicos;
@@ -173,7 +174,6 @@ static BigQueryArrowResultSet of(
this.ownedTask = null;
this.arrowDeserializer = null;
this.vectorSchemaRoot = null;
- this.vectorLoader = null;
this.enableTimestampPicos = false;
}
@@ -200,9 +200,15 @@ static BigQueryArrowResultSet getNestedResultSet(
enableTimestampPicos);
}
+ /**
+ * Decodes the record batches of one stream. It owns the vectors it decodes into, and reuses them
+ * for every batch to avoid re-allocation and too much garbage collection. Their memory comes from
+ * the result set's allocator.
+ */
private class ArrowDeserializer implements AutoCloseable {
+ private final VectorSchemaRoot root;
+ private final VectorLoader loader;
- /* Decoder object will be reused to avoid re-allocation and too much garbage collection. */
private ArrowDeserializer(ArrowSchema arrowSchema) throws IOException {
org.apache.arrow.vector.types.pojo.Schema schema =
MessageSerializer.deserializeSchema(
@@ -214,39 +220,49 @@ private ArrowDeserializer(ArrowSchema arrowSchema) throws IOException {
for (org.apache.arrow.vector.types.pojo.Field field : fields) {
vectors.add(field.createVector(allocator));
}
- vectorSchemaRoot = new VectorSchemaRoot(vectors);
- vectorLoader = new VectorLoader(vectorSchemaRoot);
+ root = new VectorSchemaRoot(vectors);
+ loader = new VectorLoader(root);
}
- private void deserializeArrowBatch(ArrowRecordBatch batch) throws SQLException {
+ /**
+ * Decodes the message's record batch and returns its rows. The returned vectors hold these rows
+ * only until the next call. Fails if the batch can't be decoded, or doesn't hold the number of
+ * rows its message reports. After a failure the vectors hold nothing.
+ */
+ private VectorSchemaRoot deserializeArrowBatch(BigQueryArrowBatchWrapper batchWrapper)
+ throws SQLException {
LOG.finestTrace("deserializeArrowBatch");
try {
- if (vectorSchemaRoot != null) {
- // Clear vectorSchemaRoot before populating a new batch
- vectorSchemaRoot.clear();
- }
+ // Clear the vectors before populating a new batch
+ root.clear();
// Read straight from the ByteString instead of copying it into a heap byte[] first.
- try (org.apache.arrow.vector.ipc.message.ArrowRecordBatch deserializedBatch =
+ try (ArrowRecordBatch deserializedBatch =
MessageSerializer.deserializeRecordBatch(
- new ReadChannel(Channels.newChannel(batch.getSerializedRecordBatch().newInput())),
+ new ReadChannel(
+ Channels.newChannel(
+ batchWrapper.getCurrentArrowBatch().getSerializedRecordBatch().newInput())),
allocator)) {
+ batchWrapper.checkRowCount(deserializedBatch.getLength());
// The vectors keep their own references to the buffers after the batch is closed.
- vectorLoader.load(deserializedBatch);
+ loader.load(deserializedBatch);
}
} catch (RuntimeException | IOException ex) {
+ // A batch that fails part-way has already put some of its buffers into the vectors.
+ root.clear();
throw new BigQueryJdbcException(ex);
}
+ return root;
+ }
+
+ /** Frees the buffers of the batch decoded last. The vectors stay usable for the next batch. */
+ private void releaseBatch() {
+ root.clear();
}
@Override
public void close() {
- LOG.fineTrace("close", () -> String.format("Closing BigQueryArrowResultSet %s.", this));
- if (vectorSchemaRoot != null) {
- vectorSchemaRoot.close();
- }
- if (allocator != null) {
- allocator.close();
- }
+ LOG.fineTrace("close", "Closing the Arrow vectors.");
+ root.close();
}
}
@@ -268,9 +284,9 @@ public boolean next() throws SQLException {
} else {
/* Non nested */
if (this.afterLast || this.hasReachedEnd || reachedRowLimit()) {
- if (!this.afterLast && this.vectorSchemaRoot != null) {
+ if (!this.afterLast) {
// Free the last batch's Arrow buffers as soon as iteration ends, not at close().
- this.vectorSchemaRoot.clear();
+ releaseCurrentBatch();
}
this.afterLast = true;
return false;
@@ -318,6 +334,19 @@ private boolean hasMoreRowsInCurrentBatch() {
&& this.currentBatchRowIndex < this.vectorSchemaRoot.getRowCount() - 1;
}
+ /** Frees the Arrow buffers of the batch decoded last. Without a schema nothing was decoded. */
+ private void releaseCurrentBatch() {
+ if (this.arrowDeserializer != null) {
+ this.arrowDeserializer.releaseBatch();
+ }
+ }
+
+ /** Bytes of Arrow memory this result set holds: the buffers of the batch decoded last. */
+ @VisibleForTesting
+ long getAllocatedMemory() {
+ return this.allocator.getAllocatedMemory();
+ }
+
private void initDeserializerIfNeeded(ArrowSchema arrowSchema) throws SQLException {
if (arrowSchema == null || this.arrowDeserializer != null) {
return;
@@ -332,12 +361,13 @@ private void initDeserializerIfNeeded(ArrowSchema arrowSchema) throws SQLExcepti
/**
* Loads the next batch that has rows into the vectors. The reader queues every message as-is, so
- * this is the single place that decides what to use:
+ * every message is checked here, when it is consumed:
*
*
* - A message without batch data (a schema, statistics or throttling update) is skipped. It
- * must not report rows.
- *
- A message with batch data is decoded, and must hold exactly the rows it reports.
+ * must not report rows, which the message checks itself.
+ *
- A message with batch data is decoded. The deserializer fails unless the batch holds
+ * exactly the rows the message reports.
*
- The stream must deliver all {@code totalRows} rows.
*
*
@@ -352,10 +382,8 @@ private boolean loadNextBatch() throws SQLException, InterruptedException {
}
if (batchWrapper.isLast()) {
/* Marks the end of the records */
- if (this.vectorSchemaRoot != null) {
- // IMP: To avoid memory leak: clear vectorSchemaRoot as it still holds the last batch
- this.vectorSchemaRoot.clear();
- }
+ // IMP: To avoid memory leak: the vectors still hold the last batch
+ releaseCurrentBatch();
this.hasReachedEnd = true;
if (this.rowCount < this.totalRows) {
throw new BigQueryJdbcException(
@@ -365,36 +393,20 @@ private boolean loadNextBatch() throws SQLException, InterruptedException {
return false;
}
initDeserializerIfNeeded(batchWrapper.getArrowSchema());
- ArrowRecordBatch arrowBatch = batchWrapper.getCurrentArrowBatch();
- Long reportedRows = batchWrapper.getReportedRowCount();
- if (arrowBatch == null || arrowBatch.getSerializedRecordBatch().isEmpty()) {
- if (reportedRows != null && reportedRows > 0) {
- throw rowCountMismatch(reportedRows, 0);
- }
+ batchWrapper.validate();
+ if (!batchWrapper.hasBatchData()) {
continue; // no rows in this message
}
if (this.arrowDeserializer == null) {
throw new BigQueryJdbcException("Received an Arrow record batch before its schema.");
}
- // Populates vectorSchemaRoot
- this.arrowDeserializer.deserializeArrowBatch(arrowBatch);
- int decodedRows = this.vectorSchemaRoot.getRowCount();
- if (reportedRows != null && reportedRows != decodedRows) {
- throw rowCountMismatch(reportedRows, decodedRows);
- }
- if (decodedRows > 0) {
+ this.vectorSchemaRoot = this.arrowDeserializer.deserializeArrowBatch(batchWrapper);
+ if (this.vectorSchemaRoot.getRowCount() > 0) {
return true;
}
}
}
- private static SQLException rowCountMismatch(long reportedRows, long decodedRows) {
- return new BigQueryJdbcException(
- String.format(
- "An Arrow stream message reported %d rows but its batch has %d.",
- reportedRows, decodedRows));
- }
-
@Override
Object getObjectInternal(int columnIndex) throws SQLException {
LOG.finestTrace("getObjectInternal");
@@ -551,13 +563,12 @@ public void close() {
if (this.buffer != null) {
this.buffer.clear();
// Unblock any consumer thread currently waiting on buffer.take() in next()
- this.buffer.offer(BigQueryArrowBatchWrapper.of(null, true));
+ this.buffer.offer(BigQueryArrowBatchWrapper.ofEndOfStream());
}
if (this.arrowDeserializer != null) {
this.arrowDeserializer.close();
- } else if (this.allocator != null) {
- this.allocator.close();
}
+ this.allocator.close();
} finally {
super.close();
}
diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java
index 4f3f2d94a3ae..8301a2291ef6 100644
--- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java
+++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java
@@ -2091,7 +2091,7 @@ private void enqueueError(BlockingQueue queue, Throwa
}
private void enqueueEndOfStream(BlockingQueue queue) {
- enqueueMarker(queue, BigQueryArrowBatchWrapper.of(null, true));
+ enqueueMarker(queue, BigQueryArrowBatchWrapper.ofEndOfStream());
}
private void enqueueBufferError(BlockingQueue queue, Throwable e) {
diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapperTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapperTest.java
new file mode 100644
index 000000000000..516419507cf7
--- /dev/null
+++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapperTest.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.bigquery.jdbc;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import com.google.cloud.bigquery.exception.BigQueryJdbcException;
+import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch;
+import com.google.protobuf.ByteString;
+import org.apache.arrow.vector.util.JsonStringArrayList;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.function.Executable;
+
+public class BigQueryArrowBatchWrapperTest {
+
+ // The wrapper only looks at whether the batch has bytes, so they don't have to decode.
+ private static final ArrowRecordBatch BATCH_WITH_DATA =
+ ArrowRecordBatch.newBuilder().setSerializedRecordBatch(ByteString.copyFromUtf8("x")).build();
+ private static final ArrowRecordBatch BATCH_WITHOUT_DATA = ArrowRecordBatch.getDefaultInstance();
+
+ @Test
+ public void testHasBatchData() {
+ assertThat(BigQueryArrowBatchWrapper.of(null, null, 0).hasBatchData()).isFalse();
+ assertThat(BigQueryArrowBatchWrapper.of(BATCH_WITHOUT_DATA, null, 0).hasBatchData()).isFalse();
+ assertThat(BigQueryArrowBatchWrapper.of(BATCH_WITH_DATA, null, 1).hasBatchData()).isTrue();
+ }
+
+ @Test
+ public void testValidateRejectsRowsReportedWithoutBatchData() {
+ assertRowCountMismatch(
+ () -> BigQueryArrowBatchWrapper.of(null, null, 2).validate(),
+ "reported 2 rows but its batch has 0");
+ assertRowCountMismatch(
+ () -> BigQueryArrowBatchWrapper.of(BATCH_WITHOUT_DATA, null, 2).validate(),
+ "reported 2 rows but its batch has 0");
+ assertRowCountMismatch(
+ () -> BigQueryArrowBatchWrapper.of(null, null, -1).validate(),
+ "reported -1 rows but its batch has 0");
+ }
+
+ @Test
+ public void testValidateAcceptsNoRowsWithoutBatchData() {
+ assertDoesNotThrow(() -> BigQueryArrowBatchWrapper.of(null, null, 0).validate());
+ assertDoesNotThrow(() -> BigQueryArrowBatchWrapper.of(BATCH_WITHOUT_DATA, null, 0).validate());
+ }
+
+ @Test
+ public void testValidateDoesNotCountTheRowsOfABatch() {
+ // How many rows a batch holds is known only once it is decoded, so the deserializer checks it.
+ assertDoesNotThrow(() -> BigQueryArrowBatchWrapper.of(BATCH_WITH_DATA, null, 5).validate());
+ }
+
+ @Test
+ public void testCheckRowCount() {
+ BigQueryArrowBatchWrapper message = BigQueryArrowBatchWrapper.of(BATCH_WITH_DATA, null, 2);
+
+ assertDoesNotThrow(() -> message.checkRowCount(2));
+ assertRowCountMismatch(() -> message.checkRowCount(3), "reported 2 rows but its batch has 3");
+ assertRowCountMismatch(() -> message.checkRowCount(0), "reported 2 rows but its batch has 0");
+ }
+
+ @Test
+ public void testWrappersThatAreNotStreamMessagesReportNoRowCount() {
+ assertNothingToCheck(BigQueryArrowBatchWrapper.ofEndOfStream());
+ assertNothingToCheck(BigQueryArrowBatchWrapper.ofError(new IllegalStateException("failed")));
+ assertNothingToCheck(
+ BigQueryArrowBatchWrapper.getNestedFieldValueListWrapper(new JsonStringArrayList<>()));
+ }
+
+ private static void assertNothingToCheck(BigQueryArrowBatchWrapper wrapper) {
+ assertThat(wrapper.getReportedRowCount()).isNull();
+ assertDoesNotThrow(wrapper::validate);
+ assertDoesNotThrow(() -> wrapper.checkRowCount(3));
+ }
+
+ private static void assertRowCountMismatch(Executable check, String expected) {
+ BigQueryJdbcException e = assertThrows(BigQueryJdbcException.class, check);
+ assertThat(e.getMessage()).contains(expected);
+ }
+}
diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java
index 43ffbdb2019e..cb41fe8fd894 100644
--- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java
+++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java
@@ -21,6 +21,7 @@
import static com.google.common.truth.Truth.assertThat;
import static org.apache.arrow.vector.types.Types.MinorType.INT;
import static org.apache.arrow.vector.types.Types.MinorType.VARCHAR;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
@@ -239,8 +240,8 @@ public void setUp() throws SQLException, IOException {
ArrowRecordBatch.newBuilder()
.setSerializedRecordBatch(serializeVectorSchemaRoot(vectorSchemaRoot))
.build();
- arrowBatchWrapper = BigQueryArrowBatchWrapper.of(batch);
- arrowBatchWrapperLast = BigQueryArrowBatchWrapper.of(null, true); // last flag
+ arrowBatchWrapper = BigQueryArrowBatchWrapper.of(batch, null, vectorSchemaRoot.getRowCount());
+ arrowBatchWrapperLast = BigQueryArrowBatchWrapper.ofEndOfStream(); // last flag
buffer.add(arrowBatchWrapper);
buffer.add(arrowBatchWrapperLast);
bufferWithTwoRows.add(arrowBatchWrapper);
@@ -503,8 +504,8 @@ public void testTimestampPicosEnabled() throws SQLException, IOException {
.setSerializedRecordBatch(serializeVectorSchemaRoot(root))
.build();
BlockingQueue picosBuffer = new LinkedBlockingDeque<>();
- picosBuffer.add(BigQueryArrowBatchWrapper.of(batch));
- picosBuffer.add(BigQueryArrowBatchWrapper.of(null, true));
+ picosBuffer.add(BigQueryArrowBatchWrapper.of(batch, null, root.getRowCount()));
+ picosBuffer.add(BigQueryArrowBatchWrapper.ofEndOfStream());
BigQueryStatement picosStmt = mock(BigQueryStatement.class);
doReturn(true).when(picosStmt).isEnableTimestampPicos();
@@ -542,8 +543,8 @@ public void testTimestampPicosDisabled() throws SQLException, IOException {
.setSerializedRecordBatch(serializeVectorSchemaRoot(root))
.build();
BlockingQueue picosBuffer = new LinkedBlockingDeque<>();
- picosBuffer.add(BigQueryArrowBatchWrapper.of(batch));
- picosBuffer.add(BigQueryArrowBatchWrapper.of(null, true));
+ picosBuffer.add(BigQueryArrowBatchWrapper.of(batch, null, root.getRowCount()));
+ picosBuffer.add(BigQueryArrowBatchWrapper.ofEndOfStream());
BigQueryStatement disabledStmt = mock(BigQueryStatement.class);
doReturn(false).when(disabledStmt).isEnableTimestampPicos();
@@ -576,8 +577,8 @@ public void testStandardMicrosecondTimestampWithPicosEnabled() throws SQLExcepti
.setSerializedRecordBatch(serializeVectorSchemaRoot(root))
.build();
BlockingQueue picosBuffer = new LinkedBlockingDeque<>();
- picosBuffer.add(BigQueryArrowBatchWrapper.of(batch));
- picosBuffer.add(BigQueryArrowBatchWrapper.of(null, true));
+ picosBuffer.add(BigQueryArrowBatchWrapper.of(batch, null, root.getRowCount()));
+ picosBuffer.add(BigQueryArrowBatchWrapper.ofEndOfStream());
BigQueryStatement picosStmt = mock(BigQueryStatement.class);
doReturn(true).when(picosStmt).isEnableTimestampPicos();
@@ -624,8 +625,8 @@ private BigQueryArrowResultSet createTemporalArrowResultSet(Instant ref)
.setSerializedRecordBatch(serializeVectorSchemaRoot(root))
.build();
BlockingQueue arrowBuffer = new LinkedBlockingDeque<>();
- arrowBuffer.add(BigQueryArrowBatchWrapper.of(batch));
- arrowBuffer.add(BigQueryArrowBatchWrapper.of(null, true));
+ arrowBuffer.add(BigQueryArrowBatchWrapper.of(batch, null, root.getRowCount()));
+ arrowBuffer.add(BigQueryArrowBatchWrapper.ofEndOfStream());
Schema schema =
Schema.of(
@@ -776,8 +777,8 @@ public void testTimestampPicosAccessorsAcrossJvmZones(String jvmZone)
.setSerializedRecordBatch(serializeVectorSchemaRoot(root))
.build();
BlockingQueue picosBuffer = new LinkedBlockingDeque<>();
- picosBuffer.add(BigQueryArrowBatchWrapper.of(batch));
- picosBuffer.add(BigQueryArrowBatchWrapper.of(null, true));
+ picosBuffer.add(BigQueryArrowBatchWrapper.of(batch, null, root.getRowCount()));
+ picosBuffer.add(BigQueryArrowBatchWrapper.ofEndOfStream());
BigQueryStatement picosStmt = mock(BigQueryStatement.class);
doReturn(true).when(picosStmt).isEnableTimestampPicos();
@@ -866,8 +867,9 @@ private static ArrowRecordBatch idBatch(int... ids) throws IOException {
.build();
}
+ /** A stream message that reports as many rows as its batch holds. */
private static BigQueryArrowBatchWrapper idWrapper(int... ids) throws IOException {
- return BigQueryArrowBatchWrapper.of(idBatch(ids));
+ return idMessage(ids.length, ids);
}
/** A stream message: the ids as an Arrow batch, and the row count the message reports. */
@@ -945,7 +947,121 @@ public void testUndecodableBatchEndsIteration() throws SQLException, IOException
.substring(0, whole.getSerializedRecordBatch().size() / 2))
.build();
- assertStreamFailsOn(BigQueryArrowBatchWrapper.of(truncated));
+ SQLException e = assertStreamFailsOn(BigQueryArrowBatchWrapper.of(truncated, null, 2));
+ // The decoding failed, not the row-count check.
+ assertThat(e.getCause().getCause()).isNotNull();
+ assertThat(e.getCause().getMessage()).doesNotContain("reported");
+ }
+
+ @Test
+ public void testFirstBatchFailureEndsIteration() throws SQLException, IOException {
+ // No batch was decoded before the failing one.
+ BigQueryArrowResultSet rs =
+ idResultSet(
+ idArrowSchema(), 2, idMessage(3, 1, 2), idMessage(2, 3, 4), arrowBatchWrapperLast);
+
+ SQLException e = assertThrows(SQLException.class, rs::next);
+ assertThat(e.getCause().getMessage()).contains("reported 3 rows but its batch has 2");
+ // The rejected batch was not loaded.
+ assertThat(rs.getAllocatedMemory()).isEqualTo(0L);
+ assertThrows(SQLException.class, () -> rs.getLong(1));
+ assertThat(rs.next()).isFalse();
+ }
+
+ @Test
+ public void testFailureReleasesTheBatchReadBefore() throws SQLException, IOException {
+ BigQueryArrowResultSet rs =
+ idResultSet(idArrowSchema(), 3, idMessage(1, 1), idMessage(3, 1, 2), arrowBatchWrapperLast);
+ assertThat(rs.next()).isTrue();
+
+ SQLException e = assertThrows(SQLException.class, rs::next);
+ assertThat(e.getCause().getMessage()).contains("reported 3 rows but its batch has 2");
+ // Neither the rejected batch nor the one read before it is held.
+ assertThat(rs.getAllocatedMemory()).isEqualTo(0L);
+ }
+
+ /** Two rows in two columns. The one-column id schema can't load it. */
+ private static ArrowRecordBatch twoColumnBatch() throws IOException {
+ VectorSchemaRoot twoColumns =
+ new VectorSchemaRoot(
+ ImmutableList.of(idRoot(1, 2).getVector(0), idRoot(3, 4).getVector(0)));
+ twoColumns.setRowCount(2);
+ return ArrowRecordBatch.newBuilder()
+ .setSerializedRecordBatch(serializeVectorSchemaRoot(twoColumns))
+ .build();
+ }
+
+ @Test
+ public void testBatchThatFailsWhileLoadingIsReleased() throws SQLException, IOException {
+ // The first column is loaded before the batch fails on the second.
+ BigQueryArrowResultSet rs =
+ idResultSet(
+ idArrowSchema(),
+ 2,
+ BigQueryArrowBatchWrapper.of(twoColumnBatch(), null, 2),
+ arrowBatchWrapperLast);
+
+ SQLException e = assertThrows(SQLException.class, rs::next);
+ assertThat(e.getCause().getMessage()).doesNotContain("reported");
+ assertThat(rs.getAllocatedMemory()).isEqualTo(0L);
+ }
+
+ @Test
+ public void testRowCountIsCheckedBeforeTheBatchIsLoaded() throws SQLException, IOException {
+ // The batch can't be loaded either, but its wrong row count is found first.
+ BigQueryArrowResultSet rs =
+ idResultSet(
+ idArrowSchema(),
+ 3,
+ BigQueryArrowBatchWrapper.of(twoColumnBatch(), null, 3),
+ arrowBatchWrapperLast);
+
+ SQLException e = assertThrows(SQLException.class, rs::next);
+ assertThat(e.getCause().getMessage()).contains("reported 3 rows but its batch has 2");
+ }
+
+ @Test
+ public void testBatchIsReleasedAtTheRowLimit() throws SQLException, IOException {
+ BigQueryArrowResultSet rs =
+ idResultSet(idArrowSchema(), 1, idWrapper(1, 2), arrowBatchWrapperLast);
+ assertThat(rs.next()).isTrue();
+ assertThat(rs.getAllocatedMemory()).isGreaterThan(0L);
+
+ assertThat(rs.next()).isFalse();
+ assertThat(rs.getAllocatedMemory()).isEqualTo(0L);
+ }
+
+ @Test
+ public void testBatchIsReleasedAtTheEndOfTheStream() throws SQLException, IOException {
+ BigQueryArrowResultSet rs =
+ idResultSet(idArrowSchema(), 3, idWrapper(1, 2), arrowBatchWrapperLast);
+ assertThat(rs.next()).isTrue();
+ assertThat(rs.next()).isTrue();
+
+ // The stream ends a row short. The last batch is released before that is reported.
+ assertThrows(SQLException.class, rs::next);
+ assertThat(rs.getAllocatedMemory()).isEqualTo(0L);
+ }
+
+ @Test
+ public void testCloseReleasesTheBatchUnderTheCursor() throws SQLException, IOException {
+ BigQueryArrowResultSet rs =
+ idResultSet(idArrowSchema(), 2, idWrapper(1, 2), arrowBatchWrapperLast);
+ assertThat(rs.next()).isTrue();
+
+ // The allocator fails to close while a batch still holds its buffers.
+ assertDoesNotThrow(rs::close);
+ assertThat(rs.isClosed()).isTrue();
+ }
+
+ @Test
+ public void testEndAndCloseWithoutASchema() throws SQLException {
+ // No schema arrived, so there is no deserializer and nothing was decoded.
+ BigQueryArrowResultSet rs = idResultSet(null, 0, arrowBatchWrapperLast);
+
+ assertThat(rs.next()).isFalse();
+ assertDoesNotThrow(rs::close);
+ assertThat(rs.isClosed()).isTrue();
}
@Test
@@ -977,9 +1093,9 @@ public void testMatchingReportedRowCountsAreAccepted() throws SQLException, IOEx
@Test
public void testMessagesWithoutRowsAreSkipped() throws SQLException, IOException {
- BigQueryArrowBatchWrapper noBatch = BigQueryArrowBatchWrapper.of(null);
+ BigQueryArrowBatchWrapper noBatch = BigQueryArrowBatchWrapper.of(null, null, 0);
BigQueryArrowBatchWrapper emptyBatch =
- BigQueryArrowBatchWrapper.of(ArrowRecordBatch.getDefaultInstance());
+ BigQueryArrowBatchWrapper.of(ArrowRecordBatch.getDefaultInstance(), null, 0);
BigQueryArrowResultSet rs =
idResultSet(
idArrowSchema(),