diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java
index 62588c58ab42..ad015ce9ba79 100644
--- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java
+++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java
@@ -17,12 +17,19 @@
package com.google.cloud.bigquery.jdbc;
import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch;
+import com.google.cloud.bigquery.storage.v1.ArrowSchema;
import org.apache.arrow.vector.util.JsonStringArrayList;
/** This class acts as a facade layer and wraps Arrow's VectorSchemaRoot & JsonStringArrayList */
class BigQueryArrowBatchWrapper {
// Reference to the current arrowBatch
private final ArrowRecordBatch currentArrowBatch;
+ // Arrow schema carried with this batch, set only when the source sent one (e.g. the first
+ // ReadRowsResponse of a stream)
+ private final ArrowSchema arrowSchema;
+ // Rows the source reported for this message (ReadRowsResponse.row_count), or null when the
+ // wrapper was not built from a stream message (markers, nested records)
+ private final Long reportedRowCount;
// Reference to the nested Records, set as null otherwise (Arrays)
private final JsonStringArrayList nestedRecords;
@@ -33,10 +40,14 @@ class BigQueryArrowBatchWrapper {
private BigQueryArrowBatchWrapper(
ArrowRecordBatch currentArrowBatch,
+ ArrowSchema arrowSchema,
+ Long reportedRowCount,
JsonStringArrayList nestedRecords,
boolean isLast,
Throwable exception) {
this.currentArrowBatch = currentArrowBatch;
+ this.arrowSchema = arrowSchema;
+ this.reportedRowCount = reportedRowCount;
this.nestedRecords = nestedRecords;
this.isLast = isLast;
this.exception = exception;
@@ -44,23 +55,46 @@ private BigQueryArrowBatchWrapper(
static BigQueryArrowBatchWrapper of(ArrowRecordBatch currentArrowBatch, boolean... isLast) {
boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0];
- return new BigQueryArrowBatchWrapper(currentArrowBatch, null, isLastFlag, null);
+ return new BigQueryArrowBatchWrapper(currentArrowBatch, null, null, null, isLastFlag, null);
+ }
+
+ /**
+ * Wraps one stream message: its record batch, the schema sent with it, and the row count it
+ * reports. The batch and the schema may each be null (a schema-only message carries no batch;
+ * later messages carry no schema).
+ */
+ static BigQueryArrowBatchWrapper of(
+ ArrowRecordBatch currentArrowBatch, ArrowSchema arrowSchema, long reportedRowCount) {
+ return new BigQueryArrowBatchWrapper(
+ currentArrowBatch, arrowSchema, reportedRowCount, null, false, null);
}
static BigQueryArrowBatchWrapper getNestedFieldValueListWrapper(
JsonStringArrayList nestedRecords, boolean... isLast) {
boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0];
- return new BigQueryArrowBatchWrapper(null, nestedRecords, isLastFlag, null);
+ return new BigQueryArrowBatchWrapper(null, null, null, nestedRecords, isLastFlag, null);
}
static BigQueryArrowBatchWrapper ofError(Throwable exception) {
- return new BigQueryArrowBatchWrapper(null, null, true, exception);
+ return new BigQueryArrowBatchWrapper(null, null, null, null, true, exception);
}
ArrowRecordBatch getCurrentArrowBatch() {
return this.currentArrowBatch;
}
+ ArrowSchema getArrowSchema() {
+ return this.arrowSchema;
+ }
+
+ /**
+ * Rows the source reported for this message, or null if this wrapper was not built from a stream
+ * message.
+ */
+ Long getReportedRowCount() {
+ return this.reportedRowCount;
+ }
+
JsonStringArrayList getNestedRecords() {
return this.nestedRecords;
}
diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java
index 620f91641933..40b44f189b18 100644
--- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java
+++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java
@@ -54,6 +54,8 @@
/** {@link ResultSet} Implementation for Arrow datasource (Using Storage Read APIs) */
class BigQueryArrowResultSet extends BigQueryBaseResultSet {
+ // Number of rows to return. Iteration stops once this many rows have been returned (this is
+ // also how Statement.setMaxRows is applied).
private final long totalRows;
// count of rows read by the current instance of ResultSet
private long rowCount = 0;
@@ -112,12 +114,8 @@ private BigQueryArrowResultSet(
this.nestedRowIndex = fromIndex - 1;
this.ownedTask = ownedTask;
this.enableTimestampPicos = enableTimestampPicos;
- if (!isNested && arrowSchema != null) {
- try {
- this.arrowDeserializer = new ArrowDeserializer(arrowSchema);
- } catch (IOException ex) {
- throw new BigQueryJdbcException("IOException during ArrowDeserializer creation", ex);
- }
+ if (!isNested) {
+ initDeserializerIfNeeded(arrowSchema);
}
}
@@ -269,49 +267,34 @@ public boolean next() throws SQLException {
return false;
} else {
/* Non nested */
- if (this.hasReachedEnd || this.isLast()) {
+ if (this.afterLast || this.hasReachedEnd || reachedRowLimit()) {
+ if (!this.afterLast && this.vectorSchemaRoot != null) {
+ // Free the last batch's Arrow buffers as soon as iteration ends, not at close().
+ this.vectorSchemaRoot.clear();
+ }
this.afterLast = true;
return false;
}
- try {
- if (this.currentBatchRowIndex == -1
- || this.currentBatchRowIndex == (this.vectorSchemaRoot.getRowCount() - 1)) {
- /* Start of iteration or we have exhausted the current batch */
- // Advance the cursor. Potentially blocking operation.
- try (Scope scope = makeOriginalContextCurrent()) {
- BigQueryArrowBatchWrapper batchWrapper = this.buffer.take();
- checkClosed();
- if (batchWrapper.getException() != null) {
- throw backgroundFetchError(batchWrapper.getException());
- }
- if (batchWrapper.isLast()) {
- /* Marks the end of the records */
- if (this.vectorSchemaRoot != null) {
- // IMP: To avoid memory leak: clear vectorSchemaRoot as it still holds
- // the last batch
- this.vectorSchemaRoot.clear();
- }
- this.hasReachedEnd = true;
- this.rowCount++;
- return false;
- }
- // Valid batch, process it
- ArrowRecordBatch arrowBatch = batchWrapper.getCurrentArrowBatch();
- // Populates vectorSchemaRoot
- this.arrowDeserializer.deserializeArrowBatch(arrowBatch);
- // Pointing to the first row in this fresh batch
- this.currentBatchRowIndex = 0;
- this.rowCount++;
- return true;
- }
- }
- // There are rows left in the current batch.
- else if (this.currentBatchRowIndex < this.vectorSchemaRoot.getRowCount()) {
- this.currentBatchRowIndex++;
- this.rowCount++;
- return true;
+ if (hasMoreRowsInCurrentBatch()) {
+ this.currentBatchRowIndex++;
+ this.rowCount++;
+ return true;
+ }
+ // Start of iteration or the current batch is exhausted. Potentially blocking.
+ try (Scope scope = makeOriginalContextCurrent()) {
+ if (!loadNextBatch()) {
+ this.afterLast = true;
+ return false;
}
+ // Pointing to the first row in this fresh batch
+ this.currentBatchRowIndex = 0;
+ this.rowCount++;
+ return true;
} catch (SQLException ex) {
+ // A stream error ends iteration: the next call returns false and no row stays readable.
+ // The failed message is never skipped, even if the caller ignores this exception.
+ this.hasReachedEnd = true;
+ this.currentBatchRowIndex = -1;
throw new BigQueryJdbcException(
"Error occurred while advancing the cursor. This could happen when connection is closed while the next method is being called.",
ex);
@@ -322,7 +305,94 @@ else if (this.currentBatchRowIndex < this.vectorSchemaRoot.getRowCount()) {
ex);
}
}
- return false;
+ }
+
+ /** True once totalRows rows (or the row limit) have been returned. */
+ private boolean reachedRowLimit() {
+ return this.rowCount >= this.totalRows;
+ }
+
+ private boolean hasMoreRowsInCurrentBatch() {
+ return this.currentBatchRowIndex >= 0
+ && this.vectorSchemaRoot != null
+ && this.currentBatchRowIndex < this.vectorSchemaRoot.getRowCount() - 1;
+ }
+
+ private void initDeserializerIfNeeded(ArrowSchema arrowSchema) throws SQLException {
+ if (arrowSchema == null || this.arrowDeserializer != null) {
+ return;
+ }
+ try {
+ this.arrowDeserializer = new ArrowDeserializer(arrowSchema);
+ } catch (RuntimeException | IOException ex) {
+ // Arrow throws unchecked exceptions on malformed schema bytes.
+ throw new BigQueryJdbcException("Failed to create the Arrow deserializer", ex);
+ }
+ }
+
+ /**
+ * Loads the next batch that has rows into the vectors. The reader queues every message as-is, so
+ * this is the single place that decides what to use:
+ *
+ *
+ * - A message without batch data (a schema, statistics or throttling update) is skipped. It
+ * must not report rows.
+ *
- A message with batch data is decoded, and must hold exactly the rows it reports.
+ *
- The stream must deliver all {@code totalRows} rows.
+ *
+ *
+ * Returns false at the end of the stream. A failure ends iteration, see {@link #next()}.
+ */
+ private boolean loadNextBatch() throws SQLException, InterruptedException {
+ while (true) {
+ BigQueryArrowBatchWrapper batchWrapper = this.buffer.take();
+ checkClosed();
+ if (batchWrapper.getException() != null) {
+ throw backgroundFetchError(batchWrapper.getException());
+ }
+ if (batchWrapper.isLast()) {
+ /* Marks the end of the records */
+ if (this.vectorSchemaRoot != null) {
+ // IMP: To avoid memory leak: clear vectorSchemaRoot as it still holds the last batch
+ this.vectorSchemaRoot.clear();
+ }
+ this.hasReachedEnd = true;
+ if (this.rowCount < this.totalRows) {
+ throw new BigQueryJdbcException(
+ String.format(
+ "The Arrow stream ended after %d of %d rows.", this.rowCount, this.totalRows));
+ }
+ return false;
+ }
+ initDeserializerIfNeeded(batchWrapper.getArrowSchema());
+ ArrowRecordBatch arrowBatch = batchWrapper.getCurrentArrowBatch();
+ Long reportedRows = batchWrapper.getReportedRowCount();
+ if (arrowBatch == null || arrowBatch.getSerializedRecordBatch().isEmpty()) {
+ if (reportedRows != null && reportedRows > 0) {
+ throw rowCountMismatch(reportedRows, 0);
+ }
+ continue; // no rows in this message
+ }
+ if (this.arrowDeserializer == null) {
+ throw new BigQueryJdbcException("Received an Arrow record batch before its schema.");
+ }
+ // Populates vectorSchemaRoot
+ this.arrowDeserializer.deserializeArrowBatch(arrowBatch);
+ int decodedRows = this.vectorSchemaRoot.getRowCount();
+ if (reportedRows != null && reportedRows != decodedRows) {
+ throw rowCountMismatch(reportedRows, decodedRows);
+ }
+ if (decodedRows > 0) {
+ return true;
+ }
+ }
+ }
+
+ private static SQLException rowCountMismatch(long reportedRows, long decodedRows) {
+ return new BigQueryJdbcException(
+ String.format(
+ "An Arrow stream message reported %d rows but its batch has %d.",
+ reportedRows, decodedRows));
}
@Override
@@ -500,7 +570,8 @@ public boolean isBeforeFirst() throws SQLException {
if (this.isNested) {
return this.nestedRowIndex < this.fromIndex;
} else {
- return this.rowCount == 0;
+ // Per JDBC, false when the result set contains no rows.
+ return !this.afterLast && this.rowCount == 0 && this.totalRows > 0;
}
}
@@ -518,7 +589,8 @@ public boolean isFirst() throws SQLException {
if (this.isNested) {
return !this.afterLast && this.nestedRowIndex == this.fromIndex;
} else {
- return !this.afterLast && this.rowCount == 1;
+ // The index is negative after a stream error: the cursor is on no row then.
+ return !this.afterLast && this.rowCount == 1 && this.currentBatchRowIndex >= 0;
}
}
@@ -529,7 +601,7 @@ public boolean isLast() throws SQLException {
if (this.isNested) {
return !this.afterLast && this.nestedRowIndex == this.toIndexExclusive - 1;
} else {
- return !this.afterLast && this.rowCount == this.totalRows;
+ return !this.afterLast && this.rowCount > 0 && this.rowCount == this.totalRows;
}
}
}
diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java
index f26751102db7..5716aad64b00 100644
--- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java
+++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java
@@ -109,6 +109,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
BigQueryJdbcUrlUtility.DEFAULT_DATASET_PROPERTY_NAME,
BigQueryJdbcUrlUtility.LOCATION_PROPERTY_NAME,
BigQueryJdbcUrlUtility.ENABLE_HTAPI_PROPERTY_NAME,
+ BigQueryJdbcUrlUtility.ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME,
BigQueryJdbcUrlUtility.UNSUPPORTED_HTAPI_FALLBACK_PROPERTY_NAME,
BigQueryJdbcUrlUtility.HTAPI_MIN_TABLE_SIZE_PROPERTY_NAME,
BigQueryJdbcUrlUtility.HTAPI_ACTIVATION_RATIO_PROPERTY_NAME,
@@ -196,6 +197,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
DatasetId defaultDataset;
String location;
boolean enableHighThroughputAPI;
+ boolean enableQueryResultsArrowFormat;
int highThroughputMinTableSize;
int highThroughputActivationRatio;
boolean enableSession;
@@ -359,6 +361,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
this.defaultDataset = BigQueryJdbcUrlUtility.parseDefaultDataset(ds.getDefaultDataset());
this.location = ds.getLocation();
this.enableHighThroughputAPI = ds.getEnableHighThroughputAPI();
+ this.enableQueryResultsArrowFormat = ds.getEnableQueryResultsArrowFormat();
this.highThroughputMinTableSize = ds.getHighThroughputMinTableSize();
this.highThroughputActivationRatio = ds.getHighThroughputActivationRatio();
this.useQueryCache = ds.getUseQueryCache();
@@ -793,6 +796,10 @@ boolean isEnableHighThroughputAPI() {
return this.enableHighThroughputAPI;
}
+ boolean isEnableQueryResultsArrowFormat() {
+ return this.enableQueryResultsArrowFormat;
+ }
+
boolean isUseQueryCache() {
return useQueryCache;
}
diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java
index d22846470081..3ce1aaaeb1d3 100644
--- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java
+++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java
@@ -110,6 +110,8 @@ protected boolean removeEldestEntry(Map.Entry> eldes
static final String DEFAULT_OAUTH_CLIENT_ID = "977385342095.apps.googleusercontent.com";
static final String DEFAULT_OAUTH_CLIENT_SECRET = "wbER7576mc_1YOII0dGk7jEE";
static final String ENABLE_HTAPI_PROPERTY_NAME = "EnableHighThroughputAPI";
+ static final String ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME =
+ "EnableQueryResultsArrowFormat";
static final String PROXY_HOST_PROPERTY_NAME = "ProxyHost";
static final String PROXY_PORT_PROPERTY_NAME = "ProxyPort";
static final String PROXY_USER_ID_PROPERTY_NAME = "ProxyUid";
@@ -117,6 +119,7 @@ protected boolean removeEldestEntry(Map.Entry> eldes
static final String HTTP_CONNECT_TIMEOUT_PROPERTY_NAME = "HttpConnectTimeout";
static final String HTTP_READ_TIMEOUT_PROPERTY_NAME = "HttpReadTimeout";
static final boolean DEFAULT_ENABLE_HTAPI_VALUE = false;
+ static final boolean DEFAULT_ENABLE_QUERY_RESULTS_ARROW_FORMAT_VALUE = false;
static final boolean DEFAULT_ENABLE_SESSION_VALUE = false;
static final int DEFAULT_LOG_LEVEL = 0;
static final String LOG_LEVEL_PROPERTY_NAME = "LogLevel";
@@ -445,6 +448,15 @@ protected boolean removeEldestEntry(Map.Entry> eldes
+ " to the BigQuery Storage API for faster processing.")
.setDefaultValue(String.valueOf(DEFAULT_HTAPI_MIN_TABLE_SIZE_VALUE))
.build(),
+ BigQueryConnectionProperty.newBuilder()
+ .setName(ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME)
+ .setDescription(
+ "Reads query results in the Apache Arrow format: the first page of"
+ + " jobs.query and the query job's Storage Read API stream (Preview)."
+ + " Disabled by default.")
+ .setDefaultValue(
+ String.valueOf(DEFAULT_ENABLE_QUERY_RESULTS_ARROW_FORMAT_VALUE))
+ .build(),
BigQueryConnectionProperty.newBuilder()
.setName(ENABLE_SESSION_PROPERTY_NAME)
.setDescription(
diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQuerySettings.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQuerySettings.java
index bc1b932022f5..bb5767842fe5 100644
--- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQuerySettings.java
+++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQuerySettings.java
@@ -44,6 +44,7 @@ class BigQuerySettings {
private final boolean enableSession;
private final boolean enableTimestampPicos;
+ private final boolean enableQueryResultsArrowFormat;
private final ConnectionProperty sessionInfoConnectionProperty;
@@ -122,6 +123,7 @@ private BigQuerySettings(Builder builder) {
this.defaultDataset = builder.defaultDataset;
this.enableSession = builder.enableSession;
this.enableTimestampPicos = builder.enableTimestampPicos;
+ this.enableQueryResultsArrowFormat = builder.enableQueryResultsArrowFormat;
this.unsupportedHTAPIFallback = builder.unsupportedHTAPIFallback;
this.sessionInfoConnectionProperty = builder.sessionInfoConnectionProperty;
this.useWriteAPI = builder.useWriteAPI;
@@ -182,6 +184,15 @@ boolean isEnableTimestampPicos() {
return enableTimestampPicos;
}
+ /**
+ * Returns whether query results should be requested in Arrow format.
+ *
+ * @return true if Arrow format is enabled, false otherwise.
+ */
+ boolean isEnableQueryResultsArrowFormat() {
+ return enableQueryResultsArrowFormat;
+ }
+
/**
* When the connector uses fetch workflows not supported on the High-Throughput API, this option
* specifies whether the connector falls back to the REST API or returns an error. By default it
@@ -387,6 +398,9 @@ public String toString() {
+ "useReadAPI="
+ useReadAPI
+ ", "
+ + "enableQueryResultsArrowFormat="
+ + enableQueryResultsArrowFormat
+ + ", "
+ "kmsKeyName="
+ kmsKeyName
+ ", "
@@ -488,6 +502,7 @@ static final class Builder {
private int highThroughputActivationRatio;
private boolean enableSession;
private boolean enableTimestampPicos;
+ private boolean enableQueryResultsArrowFormat;
private boolean unsupportedHTAPIFallback;
private ConnectionProperty sessionInfoConnectionProperty;
private boolean useQueryCache;
@@ -529,6 +544,7 @@ private Builder(BigQuerySettings querySettings) {
this.highThroughputActivationRatio = querySettings.getHighThroughputActivationRatio();
this.enableSession = querySettings.isEnableSession();
this.enableTimestampPicos = querySettings.isEnableTimestampPicos();
+ this.enableQueryResultsArrowFormat = querySettings.isEnableQueryResultsArrowFormat();
this.unsupportedHTAPIFallback = querySettings.isUnsupportedHTAPIFallback();
this.sessionInfoConnectionProperty = querySettings.getSessionInfoConnectionProperty();
this.useQueryCache = querySettings.getUseQueryCache();
@@ -630,6 +646,15 @@ Builder setEnableTimestampPicos(boolean enableTimestampPicos) {
return this;
}
+ /**
+ * Setting true requests query results in Arrow format when the driver supports it for the
+ * query. Disabled by default.
+ */
+ Builder setEnableQueryResultsArrowFormat(boolean enableQueryResultsArrowFormat) {
+ this.enableQueryResultsArrowFormat = enableQueryResultsArrowFormat;
+ return this;
+ }
+
/**
* When the connector uses fetch workflows not supported on the High-Throughput API, this option
* specifies whether the connector falls back to the REST API or returns an error. By default it
diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java
index 13181dfc4806..4f3f2d94a3ae 100644
--- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java
+++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java
@@ -52,7 +52,6 @@
import com.google.cloud.bigquery.jdbc.telemetry.v1.QueryApiType;
import com.google.cloud.bigquery.jdbc.telemetry.v1.StatementExecution;
import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager;
-import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch;
import com.google.cloud.bigquery.storage.v1.ArrowSchema;
import com.google.cloud.bigquery.storage.v1.ArrowSerializationOptions;
import com.google.cloud.bigquery.storage.v1.BigQueryReadClient;
@@ -63,6 +62,7 @@
import com.google.cloud.bigquery.storage.v1.ReadSession;
import com.google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
import io.grpc.Status;
import io.grpc.StatusRuntimeException;
import io.opentelemetry.api.common.AttributeKey;
@@ -91,6 +91,7 @@
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadFactory;
+import java.util.function.Supplier;
import java.util.logging.Level;
/**
@@ -233,6 +234,8 @@ private BigQuerySettings generateBigQuerySettings() throws SQLException {
querySettings.setWriteAPIActivationRowCount(this.connection.getWriteAPIActivationRowCount());
querySettings.setWriteAPIAppendRowCount(this.connection.getWriteAPIAppendRowCount());
querySettings.setEnableTimestampPicos(this.connection.isEnableTimestampPicos());
+ querySettings.setEnableQueryResultsArrowFormat(
+ this.connection.isEnableQueryResultsArrowFormat());
return querySettings.build();
}
@@ -1002,12 +1005,84 @@ private TableReadOptions buildTableReadOptions() {
.build();
}
+ /**
+ * Arrow options for a job stream. They are sent with every request instead of left to the
+ * stream's defaults, because they change the schema the stream returns: all requests for one
+ * result must use the same values. Compression stays unset, which means none.
+ */
+ @VisibleForTesting
+ ArrowSerializationOptions buildArrowSerializationOptions() {
+ return ArrowSerializationOptions.newBuilder()
+ .setPicosTimestampPrecision(
+ isEnableTimestampPicos()
+ ? ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_PICOS
+ : ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_MICROS)
+ .build();
+ }
+
+ /**
+ * Name of the stream that carries a query job's results. The API does not return it, so it is
+ * built from the job reference. The location must be the job's own: the stream does not exist in
+ * any other location.
+ */
+ @VisibleForTesting
+ static String jobStreamName(JobId jobId) throws BigQueryJdbcException {
+ if (jobId == null
+ || Strings.isNullOrEmpty(jobId.getProject())
+ || Strings.isNullOrEmpty(jobId.getLocation())
+ || Strings.isNullOrEmpty(jobId.getJob())) {
+ throw new BigQueryJdbcException(
+ "A job stream needs the job's project, location and job ID, got: " + jobId);
+ }
+ return String.format(
+ "projects/%s/locations/%s/jobs/%s/streams/_default",
+ jobId.getProject(), jobId.getLocation(), jobId.getJob());
+ }
+
/** Asynchronously reads results and populates an arrow record queue */
@InternalApi
Future> populateArrowBufferedQueue(
ReadSession readSession,
BlockingQueue arrowBatchWrapperBlockingQueue,
BigQueryReadClient bqReadClient) {
+ // Use the first stream to perform reading. The session already carries the read options:
+ // ReadRows accepts Arrow options for job streams only.
+ return readRowsIntoArrowQueue(
+ () ->
+ ReadRowsRequest.newBuilder().setReadStream(readSession.getStreams(0).getName()).build(),
+ arrowBatchWrapperBlockingQueue,
+ bqReadClient);
+ }
+
+ /**
+ * Asynchronously reads a query job's results from its {@code _default} stream, starting at row
+ * {@code startOffset}, and populates an arrow record queue. No read session is created.
+ */
+ @InternalApi
+ Future> populateArrowBufferedQueueFromJobStream(
+ JobId jobId,
+ long startOffset,
+ BlockingQueue arrowBatchWrapperBlockingQueue,
+ BigQueryReadClient bqReadClient)
+ throws BigQueryJdbcException {
+ ReadRowsRequest firstRequest =
+ ReadRowsRequest.newBuilder()
+ .setReadStream(jobStreamName(jobId))
+ .setOffset(startOffset)
+ .setArrowSerializationOptions(buildArrowSerializationOptions())
+ .build();
+ return readRowsIntoArrowQueue(() -> firstRequest, arrowBatchWrapperBlockingQueue, bqReadClient);
+ }
+
+ /**
+ * Reads one stream into the queue. Every attempt sends the first request moved forward by the
+ * rows already read, so a resumed request keeps the stream name and the Arrow options. The first
+ * request is resolved on the worker, where a failure reaches the queue as an error.
+ */
+ private Future> readRowsIntoArrowQueue(
+ Supplier firstRequestSupplier,
+ BlockingQueue arrowBatchWrapperBlockingQueue,
+ BigQueryReadClient bqReadClient) {
LOG.finer("++enter++");
ExecutorService executor = connection.getExecutorService();
@@ -1019,17 +1094,15 @@ Future> populateArrowBufferedQueue(
long rowsRead = 0;
int retryCount = 0;
try {
- // Use the first stream to perform reading.
- String streamName = readSession.getStreams(0).getName();
+ ReadRowsRequest firstRequest = firstRequestSupplier.get();
while (true) {
ServerStream stream = null;
boolean streamCompleted = false;
try {
ReadRowsRequest readRowsRequest =
- ReadRowsRequest.newBuilder()
- .setReadStream(streamName)
- .setOffset(rowsRead)
+ firstRequest.toBuilder()
+ .setOffset(firstRequest.getOffset() + rowsRead)
.build();
// Process each block of rows as they arrive and decode using our simple row
@@ -1041,9 +1114,14 @@ Future> populateArrowBufferedQueue(
"Arrow stream reading was interrupted or shut down before completion.");
}
- ArrowRecordBatch currentBatch = response.getArrowRecordBatch();
+ // Queue every response as-is; the result set decides what to use.
arrowBatchWrapperBlockingQueue.put(
- BigQueryArrowBatchWrapper.of(currentBatch));
+ BigQueryArrowBatchWrapper.of(
+ response.hasArrowRecordBatch()
+ ? response.getArrowRecordBatch()
+ : null,
+ response.hasArrowSchema() ? response.getArrowSchema() : null,
+ response.getRowCount()));
rowsRead += response.getRowCount();
}
if (isWorkerShuttingDown()) {
@@ -1054,7 +1132,7 @@ Future> populateArrowBufferedQueue(
break;
} catch (ApiException e) {
if (e.getStatusCode().getCode() == StatusCode.Code.NOT_FOUND) {
- LOG.warning("Read session expired or not found: %s", e.getMessage());
+ LOG.warning("Read stream expired or not found: %s", e.getMessage());
enqueueError(arrowBatchWrapperBlockingQueue, e);
break;
}
@@ -1072,7 +1150,7 @@ Future> populateArrowBufferedQueue(
LOG.warning(
"Connection interrupted during arrow stream read, retrying. attempt: %d",
retryCount);
- Thread.sleep(RETRY_DELAY_MS);
+ Thread.sleep(getArrowRetryDelayMs());
} finally {
if (stream != null && !streamCompleted) {
stream.cancel();
@@ -1567,6 +1645,12 @@ int getArrowBufferSize() {
return DEFAULT_ARROW_BUFFER_SIZE;
}
+ /** Wait between two attempts to read an Arrow stream. */
+ @VisibleForTesting
+ long getArrowRetryDelayMs() {
+ return RETRY_DELAY_MS;
+ }
+
/** Returns the destinationTable from jobId by calling `jobs.get` API */
TableId getDestinationTable(JobId jobId) {
Job job = this.bigQuery.getJob(jobId);
diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java
index 7d91c128d9a7..cb5f0ead55f7 100644
--- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java
+++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java
@@ -53,6 +53,7 @@ public class DataSource implements javax.sql.DataSource {
private String location;
private String userAgent;
private Boolean enableHighThroughputAPI;
+ private Boolean enableQueryResultsArrowFormat;
private Integer highThroughputMinTableSize;
private Integer highThroughputActivationRatio;
private Boolean unsupportedHTAPIFallback;
@@ -160,6 +161,13 @@ public class DataSource implements javax.sql.DataSource {
ds.setEnableHighThroughputAPI(
BigQueryJdbcUrlUtility.convertToBoolean(
val, BigQueryJdbcUrlUtility.ENABLE_HTAPI_PROPERTY_NAME)))
+ .put(
+ BigQueryJdbcUrlUtility.ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME,
+ (ds, val) ->
+ ds.setEnableQueryResultsArrowFormat(
+ BigQueryJdbcUrlUtility.convertToBoolean(
+ val,
+ BigQueryJdbcUrlUtility.ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME)))
.put(
BigQueryJdbcUrlUtility.UNSUPPORTED_HTAPI_FALLBACK_PROPERTY_NAME,
(ds, val) ->
@@ -486,6 +494,11 @@ Properties createProperties() {
BigQueryJdbcUrlUtility.ENABLE_HTAPI_PROPERTY_NAME,
String.valueOf(this.enableHighThroughputAPI));
}
+ if (this.enableQueryResultsArrowFormat != null) {
+ connectionProperties.setProperty(
+ BigQueryJdbcUrlUtility.ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME,
+ String.valueOf(this.enableQueryResultsArrowFormat));
+ }
if (this.unsupportedHTAPIFallback != null) {
connectionProperties.setProperty(
BigQueryJdbcUrlUtility.UNSUPPORTED_HTAPI_FALLBACK_PROPERTY_NAME,
@@ -846,6 +859,16 @@ public void setEnableHighThroughputAPI(Boolean enableHighThroughputAPI) {
this.enableHighThroughputAPI = enableHighThroughputAPI;
}
+ public Boolean getEnableQueryResultsArrowFormat() {
+ return enableQueryResultsArrowFormat != null
+ ? enableQueryResultsArrowFormat
+ : BigQueryJdbcUrlUtility.DEFAULT_ENABLE_QUERY_RESULTS_ARROW_FORMAT_VALUE;
+ }
+
+ public void setEnableQueryResultsArrowFormat(Boolean enableQueryResultsArrowFormat) {
+ this.enableQueryResultsArrowFormat = enableQueryResultsArrowFormat;
+ }
+
public Integer getHighThroughputMinTableSize() {
return highThroughputMinTableSize != null
? highThroughputMinTableSize
diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java
index 72a91dc42145..43ffbdb2019e 100644
--- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java
+++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java
@@ -36,6 +36,7 @@
import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch;
import com.google.cloud.bigquery.storage.v1.ArrowSchema;
import com.google.common.collect.ImmutableList;
+import com.google.protobuf.ByteString;
import java.io.IOException;
import java.math.BigDecimal;
import java.sql.Array;
@@ -302,6 +303,11 @@ public void testRowCount() throws SQLException, IOException {
QUERY_SCHEMA, arrowSchema, 1, statement, bufferWithTwoRows, workerTask, null);
assertThat(resultSetRowCount(bigQueryArrowResultSet2)).isEqualTo(1);
+ // ResultSet with 3 rows buffer and 2 total rows (e.g. Statement.setMaxRows(2)).
+ assertThat(
+ resultSetRowCount(
+ idResultSet(idArrowSchema(), 2, idWrapper(1, 2, 3), arrowBatchWrapperLast)))
+ .isEqualTo(2);
}
@Test
@@ -833,4 +839,236 @@ public void testGetterWhenNotOnRowThrows() throws SQLException {
assertThat(bigQueryArrowResultSetNested.isLast()).isFalse();
assertThrows(SQLException.class, () -> bigQueryArrowResultSetNested.getObject(2));
}
+
+ private static final Schema ID_SCHEMA = Schema.of(Field.of("id", StandardSQLTypeName.INT64));
+
+ private static VectorSchemaRoot idRoot(int... ids) {
+ IntVector idVector = new IntVector("id", new RootAllocator());
+ idVector.allocateNew(Math.max(ids.length, 1));
+ for (int i = 0; i < ids.length; i++) {
+ idVector.set(i, ids[i]);
+ }
+ idVector.setValueCount(ids.length);
+ VectorSchemaRoot root = new VectorSchemaRoot(ImmutableList.of(idVector));
+ root.setRowCount(ids.length);
+ return root;
+ }
+
+ private static ArrowSchema idArrowSchema() throws IOException {
+ return ArrowSchema.newBuilder()
+ .setSerializedSchema(serializeSchema(idRoot().getSchema()))
+ .build();
+ }
+
+ private static ArrowRecordBatch idBatch(int... ids) throws IOException {
+ return ArrowRecordBatch.newBuilder()
+ .setSerializedRecordBatch(serializeVectorSchemaRoot(idRoot(ids)))
+ .build();
+ }
+
+ private static BigQueryArrowBatchWrapper idWrapper(int... ids) throws IOException {
+ return BigQueryArrowBatchWrapper.of(idBatch(ids));
+ }
+
+ /** A stream message: the ids as an Arrow batch, and the row count the message reports. */
+ private static BigQueryArrowBatchWrapper idMessage(long reportedRows, int... ids)
+ throws IOException {
+ return BigQueryArrowBatchWrapper.of(idBatch(ids), null, reportedRows);
+ }
+
+ private BigQueryArrowResultSet idResultSet(
+ ArrowSchema arrowSchema, long totalRows, BigQueryArrowBatchWrapper... items)
+ throws SQLException {
+ BlockingQueue queue =
+ new LinkedBlockingDeque<>(Arrays.asList(items));
+ return BigQueryArrowResultSet.of(
+ ID_SCHEMA, arrowSchema, totalRows, statement, queue, mock(Future.class), null);
+ }
+
+ /**
+ * Reads a stream of one good row, then {@code failing}, then two more good rows. next() must fail
+ * on {@code failing}, leave no row readable, and not move on to the rows behind it. The row limit
+ * (3) is below the stream's total, so if {@code failing} were skipped, the rows behind it would
+ * fill the limit and nothing would report the gap.
+ */
+ private SQLException assertStreamFailsOn(BigQueryArrowBatchWrapper failing)
+ throws SQLException, IOException {
+ BigQueryArrowResultSet rs =
+ idResultSet(
+ idArrowSchema(),
+ 3,
+ idMessage(1, 1),
+ failing,
+ idMessage(2, 3, 4),
+ arrowBatchWrapperLast);
+
+ assertThat(rs.next()).isTrue();
+ SQLException e = assertThrows(SQLException.class, rs::next);
+ assertThat(rs.isFirst()).isFalse();
+ assertThrows(SQLException.class, () -> rs.getLong(1));
+ assertThat(rs.next()).isFalse();
+ return e;
+ }
+
+ private void assertRowCountMismatch(BigQueryArrowBatchWrapper message, String expected)
+ throws SQLException, IOException {
+ assertThat(assertStreamFailsOn(message).getCause().getMessage()).contains(expected);
+ }
+
+ @Test
+ public void testRowsReportedWithoutDataThrows() throws SQLException, IOException {
+ assertRowCountMismatch(
+ BigQueryArrowBatchWrapper.of(null, null, 2), "reported 2 rows but its batch has 0");
+ assertRowCountMismatch(
+ BigQueryArrowBatchWrapper.of(ArrowRecordBatch.getDefaultInstance(), null, 2),
+ "reported 2 rows but its batch has 0");
+ }
+
+ @Test
+ public void testRowsReportedBeforeSchemaThrows() throws SQLException, IOException {
+ BigQueryArrowResultSet rs =
+ idResultSet(null, 2, BigQueryArrowBatchWrapper.of(null, null, 2), arrowBatchWrapperLast);
+
+ SQLException e = assertThrows(SQLException.class, rs::next);
+ assertThat(e.getCause().getMessage()).contains("reported 2 rows but its batch has 0");
+ assertThat(rs.next()).isFalse();
+ }
+
+ @Test
+ public void testUndecodableBatchEndsIteration() throws SQLException, IOException {
+ ArrowRecordBatch whole = idBatch(8, 9);
+ ArrowRecordBatch truncated =
+ ArrowRecordBatch.newBuilder()
+ .setSerializedRecordBatch(
+ whole
+ .getSerializedRecordBatch()
+ .substring(0, whole.getSerializedRecordBatch().size() / 2))
+ .build();
+
+ assertStreamFailsOn(BigQueryArrowBatchWrapper.of(truncated));
+ }
+
+ @Test
+ public void testBatchRowCountMismatchThrows() throws SQLException, IOException {
+ assertRowCountMismatch(idMessage(3, 1, 2), "reported 3 rows but its batch has 2");
+ assertRowCountMismatch(idMessage(0, 1, 2), "reported 0 rows but its batch has 2");
+ assertRowCountMismatch(idMessage(2), "reported 2 rows but its batch has 0");
+ }
+
+ @Test
+ public void testMatchingReportedRowCountsAreAccepted() throws SQLException, IOException {
+ BigQueryArrowResultSet rs =
+ idResultSet(
+ null,
+ 3,
+ BigQueryArrowBatchWrapper.of(null, idArrowSchema(), 0),
+ idMessage(2, 1, 2),
+ BigQueryArrowBatchWrapper.of(ArrowRecordBatch.getDefaultInstance(), null, 0),
+ idMessage(0),
+ idMessage(1, 3),
+ arrowBatchWrapperLast);
+
+ assertThat(rs.next()).isTrue();
+ assertThat(rs.next()).isTrue();
+ assertThat(rs.next()).isTrue();
+ assertThat(rs.getLong(1)).isEqualTo(3L);
+ assertThat(rs.next()).isFalse();
+ }
+
+ @Test
+ public void testMessagesWithoutRowsAreSkipped() throws SQLException, IOException {
+ BigQueryArrowBatchWrapper noBatch = BigQueryArrowBatchWrapper.of(null);
+ BigQueryArrowBatchWrapper emptyBatch =
+ BigQueryArrowBatchWrapper.of(ArrowRecordBatch.getDefaultInstance());
+ BigQueryArrowResultSet rs =
+ idResultSet(
+ idArrowSchema(),
+ 3,
+ idWrapper(1, 2),
+ noBatch,
+ idWrapper(),
+ emptyBatch,
+ idWrapper(3),
+ arrowBatchWrapperLast);
+
+ assertThat(rs.next()).isTrue();
+ assertThat(rs.next()).isTrue();
+ assertThat(rs.next()).isTrue();
+ assertThat(rs.getLong(1)).isEqualTo(3L);
+ assertThat(rs.next()).isFalse();
+ }
+
+ @Test
+ public void testStreamEndingBeforeTotalRowsThrows() throws SQLException, IOException {
+ BigQueryArrowResultSet rs =
+ idResultSet(idArrowSchema(), 3, idWrapper(1, 2), arrowBatchWrapperLast);
+
+ assertThat(rs.next()).isTrue();
+ assertThat(rs.next()).isTrue();
+ SQLException e = assertThrows(SQLException.class, rs::next);
+ assertThat(e.getCause().getMessage()).contains("ended after 2 of 3 rows");
+ }
+
+ @Test
+ public void testEmptyResultIsNotBeforeFirst() throws SQLException, IOException {
+ BigQueryArrowResultSet rs = idResultSet(idArrowSchema(), 0, arrowBatchWrapperLast);
+
+ assertThat(rs.isBeforeFirst()).isFalse();
+ assertThat(rs.isLast()).isFalse();
+ assertThat(rs.next()).isFalse();
+ }
+
+ @Test
+ public void testSchemaFromStreamMessage() throws SQLException, IOException {
+ BigQueryArrowResultSet rs =
+ idResultSet(
+ null,
+ 1,
+ BigQueryArrowBatchWrapper.of(null, idArrowSchema(), 0),
+ idWrapper(7),
+ arrowBatchWrapperLast);
+
+ assertThat(rs.next()).isTrue();
+ assertThat(rs.getLong(1)).isEqualTo(7L);
+ assertThat(rs.next()).isFalse();
+ }
+
+ @Test
+ public void testBatchBeforeSchemaThrows() throws SQLException, IOException {
+ BigQueryArrowResultSet rs =
+ idResultSet(
+ null,
+ 1,
+ idWrapper(1),
+ BigQueryArrowBatchWrapper.of(idBatch(2), idArrowSchema(), 1),
+ arrowBatchWrapperLast);
+
+ assertThrows(SQLException.class, rs::next);
+ // The message behind the failure has a schema and a row. It must not be served.
+ assertThat(rs.next()).isFalse();
+ }
+
+ @Test
+ public void testUndecodableSchemaEndsIteration() throws SQLException, IOException {
+ // A negative length prefix: Arrow fails on it with an unchecked exception.
+ ArrowSchema undecodable =
+ ArrowSchema.newBuilder()
+ .setSerializedSchema(
+ ByteString.copyFrom(
+ new byte[] {(byte) 0xFE, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF}))
+ .build();
+ BigQueryArrowResultSet rs =
+ idResultSet(
+ null,
+ 1,
+ BigQueryArrowBatchWrapper.of(null, undecodable, 0),
+ BigQueryArrowBatchWrapper.of(idBatch(2), idArrowSchema(), 1),
+ arrowBatchWrapperLast);
+
+ SQLException e = assertThrows(SQLException.class, rs::next);
+ assertThat(e.getCause().getMessage()).contains("Failed to create the Arrow deserializer");
+ assertThat(e.getCause().getCause()).isInstanceOf(RuntimeException.class);
+ // The message behind the failure has a schema and a row. It must not be served.
+ assertThat(rs.next()).isFalse();
+ }
}
diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java
index fd82c6babb62..54488ca8ef1b 100644
--- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java
+++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java
@@ -945,6 +945,21 @@ public void testEnableTimestampPicosConfigured() throws Exception {
}
}
+ @Test
+ public void testEnableQueryResultsArrowFormatDefault() throws Exception {
+ try (BigQueryConnection connection = new BigQueryConnection(BASE_URL)) {
+ assertFalse(connection.isEnableQueryResultsArrowFormat());
+ }
+ }
+
+ @Test
+ public void testEnableQueryResultsArrowFormatConfigured() throws Exception {
+ String url = BASE_URL + "EnableQueryResultsArrowFormat=1;";
+ try (BigQueryConnection connection = new BigQueryConnection(url)) {
+ assertTrue(connection.isEnableQueryResultsArrowFormat());
+ }
+ }
+
@Test
public void testSessionIdIsMatchedRegardlessOfCase() throws Exception {
String url = BASE_URL + ";QueryProperties=Session_Id=abc123";
diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java
index 74b948a75135..63ba3381cd09 100644
--- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java
+++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java
@@ -162,6 +162,12 @@ public class BigQueryStatementTest {
private static final TableId TABLE_ID = TableId.of(DEFAULT_TEST_DATASET, PROJECT);
+ private static final JobId STREAM_JOB_ID =
+ JobId.newBuilder().setProject("p").setLocation("us-central1").setJob("j").build();
+
+ private static final String JOB_STREAM_NAME =
+ "projects/p/locations/us-central1/jobs/j/streams/_default";
+
private static ArrowSchema arrowSchema;
private final Map LABELS =
@@ -1303,6 +1309,13 @@ public void testEnableTimestampPicosPropagation() throws SQLException {
assertTrue(statement.isEnableTimestampPicos());
}
+ @Test
+ public void testEnableQueryResultsArrowFormatPropagation() throws SQLException {
+ doReturn(true).when(bigQueryConnection).isEnableQueryResultsArrowFormat();
+ BigQueryStatement statement = new BigQueryStatement(bigQueryConnection);
+ assertTrue(statement.querySettings.isEnableQueryResultsArrowFormat());
+ }
+
@Test
public void testJoblessQueryPopulatesQueryStatistics() throws Exception {
doReturn(true).when(bigQueryConnection).getUseStatelessQueryMode();
@@ -1582,6 +1595,251 @@ public ReadRowsResponse next() {
"Stopping early due to worker shutdown must enqueue an error marker rather than normal end-of-stream");
}
+ private static ReadRowsResponse rowsResponse(long rowCount) {
+ return ReadRowsResponse.newBuilder()
+ .setRowCount(rowCount)
+ .setArrowRecordBatch(ArrowRecordBatch.getDefaultInstance())
+ .build();
+ }
+
+ /** One ReadRows attempt: delivers {@code responses}, then throws {@code failure} unless null. */
+ private static ServerStream readRowsAttempt(
+ RuntimeException failure, ReadRowsResponse... responses) {
+ Iterator delivered = Arrays.asList(responses).iterator();
+ Iterator attempt =
+ new Iterator() {
+ @Override
+ public boolean hasNext() {
+ if (!delivered.hasNext() && failure != null) {
+ throw failure;
+ }
+ return delivered.hasNext();
+ }
+
+ @Override
+ public ReadRowsResponse next() {
+ return delivered.next();
+ }
+ };
+ @SuppressWarnings("unchecked")
+ ServerStream stream = mock(ServerStream.class);
+ doReturn(attempt).when(stream).iterator();
+ return stream;
+ }
+
+ /** A read client that serves {@code attempts} in order and records each request it is sent. */
+ @SafeVarargs
+ private static BigQueryReadClient readClientServing(
+ List requests, ServerStream... attempts) {
+ Iterator> remaining = Arrays.asList(attempts).iterator();
+ @SuppressWarnings("unchecked")
+ ServerStreamingCallable callable =
+ mock(ServerStreamingCallable.class);
+ Mockito.doAnswer(
+ invocation -> {
+ requests.add(invocation.getArgument(0));
+ return remaining.next();
+ })
+ .when(callable)
+ .call(any(ReadRowsRequest.class));
+ BigQueryReadClient readClient = mock(BigQueryReadClient.class);
+ doReturn(callable).when(readClient).readRowsCallable();
+ return readClient;
+ }
+
+ /** A statement with picosecond timestamps on, whose reader retries without waiting. */
+ private BigQueryStatement picosStatementWithoutRetryDelay() throws SQLException {
+ doReturn(true).when(bigQueryConnection).isEnableTimestampPicos();
+ BigQueryStatement statement = Mockito.spy(new BigQueryStatement(bigQueryConnection));
+ doReturn(0L).when(statement).getArrowRetryDelayMs();
+ return statement;
+ }
+
+ @Test
+ public void testJobStreamNameUsesJobReference() throws SQLException {
+ assertEquals(JOB_STREAM_NAME, BigQueryStatement.jobStreamName(STREAM_JOB_ID));
+ }
+
+ @Test
+ public void testJobStreamNameRejectsMissingParts() {
+ for (JobId incomplete :
+ Arrays.asList(
+ null,
+ STREAM_JOB_ID.toBuilder().setProject(null).build(),
+ STREAM_JOB_ID.toBuilder().setLocation(null).build(),
+ STREAM_JOB_ID.toBuilder().setLocation("").build(),
+ STREAM_JOB_ID.toBuilder().setJob(null).build())) {
+ assertThrows(
+ BigQueryJdbcException.class,
+ () -> BigQueryStatement.jobStreamName(incomplete),
+ "Expected no stream name for " + incomplete);
+ }
+ }
+
+ @Test
+ public void testArrowSerializationOptionsDefaultToMicros() {
+ ArrowSerializationOptions options = bigQueryStatement.buildArrowSerializationOptions();
+
+ assertEquals(
+ ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_MICROS,
+ options.getPicosTimestampPrecision());
+ assertEquals(
+ ArrowSerializationOptions.CompressionCodec.COMPRESSION_UNSPECIFIED,
+ options.getBufferCompression());
+ }
+
+ @Test
+ public void testArrowSerializationOptionsUsePicosWhenEnabled() throws SQLException {
+ doReturn(true).when(bigQueryConnection).isEnableTimestampPicos();
+ BigQueryStatement statement = new BigQueryStatement(bigQueryConnection);
+
+ assertEquals(
+ ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_PICOS,
+ statement.buildArrowSerializationOptions().getPicosTimestampPrecision());
+ }
+
+ @Test
+ public void testJobStreamReaderSendsNameOffsetAndOptions() throws Exception {
+ BlockingQueue queue = new LinkedBlockingDeque<>();
+ List requests = new ArrayList<>();
+ BigQueryReadClient readClient =
+ readClientServing(requests, readRowsAttempt(null, rowsResponse(3)));
+
+ bigQueryStatement
+ .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 1000L, queue, readClient)
+ .get(5, TimeUnit.SECONDS);
+
+ assertEquals(1, requests.size());
+ ReadRowsRequest request = requests.get(0);
+ assertEquals(JOB_STREAM_NAME, request.getReadStream());
+ assertEquals(1000L, request.getOffset());
+ assertTrue(request.hasArrowSerializationOptions());
+ assertEquals(
+ ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_MICROS,
+ request.getArrowSerializationOptions().getPicosTimestampPrecision());
+ }
+
+ @Test
+ public void testJobStreamReaderResumeKeepsOptionsAndOffset() throws Exception {
+ BigQueryStatement statement = picosStatementWithoutRetryDelay();
+ BlockingQueue queue = new LinkedBlockingDeque<>();
+ List requests = new ArrayList<>();
+ BigQueryReadClient readClient =
+ readClientServing(
+ requests,
+ readRowsAttempt(
+ mockApiException(StatusCode.Code.UNAVAILABLE), rowsResponse(2), rowsResponse(1)),
+ readRowsAttempt(null, rowsResponse(4)));
+
+ statement
+ .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 1000L, queue, readClient)
+ .get(5, TimeUnit.SECONDS);
+
+ assertEquals(2, requests.size());
+ ReadRowsRequest first = requests.get(0);
+ ReadRowsRequest resumed = requests.get(1);
+ assertEquals(1000L, first.getOffset());
+ assertEquals(1003L, resumed.getOffset());
+ assertEquals(
+ ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_PICOS,
+ resumed.getArrowSerializationOptions().getPicosTimestampPrecision());
+ // The resumed request differs from the first one in its offset only.
+ assertEquals(first, resumed.toBuilder().setOffset(first.getOffset()).build());
+
+ // Three messages, then a normal end of stream.
+ List queued = new ArrayList<>(queue);
+ assertEquals(4, queued.size());
+ assertTrue(queued.get(3).isLast());
+ assertNull(queued.get(3).getException());
+ }
+
+ @Test
+ public void testReadSessionReaderSendsNoArrowOptions() throws Exception {
+ // Picoseconds are on, so the session carries the Arrow options. ReadRows on a session stream
+ // must not: the request accepts them for job streams only.
+ BigQueryStatement statement = picosStatementWithoutRetryDelay();
+ String sessionStream = "projects/p/locations/l/sessions/s/streams/0";
+ ReadSession readSession =
+ ReadSession.newBuilder().addStreams(ReadStream.newBuilder().setName(sessionStream)).build();
+ BlockingQueue queue = new LinkedBlockingDeque<>();
+ List requests = new ArrayList<>();
+ BigQueryReadClient readClient =
+ readClientServing(
+ requests,
+ readRowsAttempt(mockApiException(StatusCode.Code.UNAVAILABLE), rowsResponse(2)),
+ readRowsAttempt(null, rowsResponse(4)));
+
+ statement.populateArrowBufferedQueue(readSession, queue, readClient).get(5, TimeUnit.SECONDS);
+
+ assertEquals(
+ Arrays.asList(
+ ReadRowsRequest.newBuilder().setReadStream(sessionStream).build(),
+ ReadRowsRequest.newBuilder().setReadStream(sessionStream).setOffset(2).build()),
+ requests);
+ }
+
+ @Test
+ public void testJobStreamReaderQueuesMessagesAsReceived() throws Exception {
+ ReadRowsResponse schemaAndRows =
+ rowsResponse(3).toBuilder().setArrowSchema(arrowSchema).build();
+ ReadRowsResponse noRows = ReadRowsResponse.getDefaultInstance();
+ BlockingQueue queue = new LinkedBlockingDeque<>();
+ BigQueryReadClient readClient =
+ readClientServing(new ArrayList<>(), readRowsAttempt(null, schemaAndRows, noRows));
+
+ bigQueryStatement
+ .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 0L, queue, readClient)
+ .get(5, TimeUnit.SECONDS);
+
+ BigQueryArrowBatchWrapper first = queue.poll();
+ assertEquals(arrowSchema, first.getArrowSchema());
+ assertNotNull(first.getCurrentArrowBatch());
+ assertEquals(3L, first.getReportedRowCount());
+ BigQueryArrowBatchWrapper second = queue.poll();
+ assertNull(second.getArrowSchema());
+ assertNull(second.getCurrentArrowBatch());
+ assertEquals(0L, second.getReportedRowCount());
+ BigQueryArrowBatchWrapper end = queue.poll();
+ assertTrue(end.isLast());
+ assertNull(end.getException());
+ assertNull(end.getReportedRowCount());
+ assertTrue(queue.isEmpty());
+ }
+
+ @Test
+ public void testJobStreamReaderDoesNotRetryNotFound() throws Exception {
+ ApiException notFound = mockApiException(StatusCode.Code.NOT_FOUND);
+ BlockingQueue queue = new LinkedBlockingDeque<>();
+ List requests = new ArrayList<>();
+ BigQueryReadClient readClient = readClientServing(requests, readRowsAttempt(notFound));
+
+ bigQueryStatement
+ .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 0L, queue, readClient)
+ .get(5, TimeUnit.SECONDS);
+
+ assertEquals(1, requests.size());
+ assertEquals(notFound, queue.poll().getException());
+ BigQueryArrowBatchWrapper end = queue.poll();
+ assertTrue(end.isLast());
+ assertNull(end.getException());
+ assertTrue(queue.isEmpty());
+ }
+
+ @Test
+ public void testJobStreamReaderRejectsIncompleteJobReferenceBeforeStartingAWorker() {
+ BlockingQueue queue = new LinkedBlockingDeque<>();
+ JobId withoutProjectAndLocation = JobId.of("j");
+
+ assertThrows(
+ BigQueryJdbcException.class,
+ () ->
+ bigQueryStatement.populateArrowBufferedQueueFromJobStream(
+ withoutProjectAndLocation, 0L, queue, storageReadClient));
+
+ verify(bigQueryConnection, Mockito.never()).getExecutorService();
+ assertTrue(queue.isEmpty());
+ }
+
private static FieldValueList createTestRow(boolean first, long second) {
return FieldValueList.of(
Arrays.asList(