diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java index 62588c58ab42..ad015ce9ba79 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowBatchWrapper.java @@ -17,12 +17,19 @@ package com.google.cloud.bigquery.jdbc; import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch; +import com.google.cloud.bigquery.storage.v1.ArrowSchema; import org.apache.arrow.vector.util.JsonStringArrayList; /** This class acts as a facade layer and wraps Arrow's VectorSchemaRoot & JsonStringArrayList */ class BigQueryArrowBatchWrapper { // Reference to the current arrowBatch private final ArrowRecordBatch currentArrowBatch; + // Arrow schema carried with this batch, set only when the source sent one (e.g. the first + // ReadRowsResponse of a stream) + private final ArrowSchema arrowSchema; + // Rows the source reported for this message (ReadRowsResponse.row_count), or null when the + // wrapper was not built from a stream message (markers, nested records) + private final Long reportedRowCount; // Reference to the nested Records, set as null otherwise (Arrays) private final JsonStringArrayList nestedRecords; @@ -33,10 +40,14 @@ class BigQueryArrowBatchWrapper { private BigQueryArrowBatchWrapper( ArrowRecordBatch currentArrowBatch, + ArrowSchema arrowSchema, + Long reportedRowCount, JsonStringArrayList nestedRecords, boolean isLast, Throwable exception) { this.currentArrowBatch = currentArrowBatch; + this.arrowSchema = arrowSchema; + this.reportedRowCount = reportedRowCount; this.nestedRecords = nestedRecords; this.isLast = isLast; this.exception = exception; @@ -44,23 +55,46 @@ private BigQueryArrowBatchWrapper( static BigQueryArrowBatchWrapper of(ArrowRecordBatch currentArrowBatch, boolean... isLast) { boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0]; - return new BigQueryArrowBatchWrapper(currentArrowBatch, null, isLastFlag, null); + return new BigQueryArrowBatchWrapper(currentArrowBatch, null, null, null, isLastFlag, null); + } + + /** + * Wraps one stream message: its record batch, the schema sent with it, and the row count it + * reports. The batch and the schema may each be null (a schema-only message carries no batch; + * later messages carry no schema). + */ + static BigQueryArrowBatchWrapper of( + ArrowRecordBatch currentArrowBatch, ArrowSchema arrowSchema, long reportedRowCount) { + return new BigQueryArrowBatchWrapper( + currentArrowBatch, arrowSchema, reportedRowCount, null, false, null); } static BigQueryArrowBatchWrapper getNestedFieldValueListWrapper( JsonStringArrayList nestedRecords, boolean... isLast) { boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0]; - return new BigQueryArrowBatchWrapper(null, nestedRecords, isLastFlag, null); + return new BigQueryArrowBatchWrapper(null, null, null, nestedRecords, isLastFlag, null); } static BigQueryArrowBatchWrapper ofError(Throwable exception) { - return new BigQueryArrowBatchWrapper(null, null, true, exception); + return new BigQueryArrowBatchWrapper(null, null, null, null, true, exception); } ArrowRecordBatch getCurrentArrowBatch() { return this.currentArrowBatch; } + ArrowSchema getArrowSchema() { + return this.arrowSchema; + } + + /** + * Rows the source reported for this message, or null if this wrapper was not built from a stream + * message. + */ + Long getReportedRowCount() { + return this.reportedRowCount; + } + JsonStringArrayList getNestedRecords() { return this.nestedRecords; } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java index 620f91641933..40b44f189b18 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSet.java @@ -54,6 +54,8 @@ /** {@link ResultSet} Implementation for Arrow datasource (Using Storage Read APIs) */ class BigQueryArrowResultSet extends BigQueryBaseResultSet { + // Number of rows to return. Iteration stops once this many rows have been returned (this is + // also how Statement.setMaxRows is applied). private final long totalRows; // count of rows read by the current instance of ResultSet private long rowCount = 0; @@ -112,12 +114,8 @@ private BigQueryArrowResultSet( this.nestedRowIndex = fromIndex - 1; this.ownedTask = ownedTask; this.enableTimestampPicos = enableTimestampPicos; - if (!isNested && arrowSchema != null) { - try { - this.arrowDeserializer = new ArrowDeserializer(arrowSchema); - } catch (IOException ex) { - throw new BigQueryJdbcException("IOException during ArrowDeserializer creation", ex); - } + if (!isNested) { + initDeserializerIfNeeded(arrowSchema); } } @@ -269,49 +267,34 @@ public boolean next() throws SQLException { return false; } else { /* Non nested */ - if (this.hasReachedEnd || this.isLast()) { + if (this.afterLast || this.hasReachedEnd || reachedRowLimit()) { + if (!this.afterLast && this.vectorSchemaRoot != null) { + // Free the last batch's Arrow buffers as soon as iteration ends, not at close(). + this.vectorSchemaRoot.clear(); + } this.afterLast = true; return false; } - try { - if (this.currentBatchRowIndex == -1 - || this.currentBatchRowIndex == (this.vectorSchemaRoot.getRowCount() - 1)) { - /* Start of iteration or we have exhausted the current batch */ - // Advance the cursor. Potentially blocking operation. - try (Scope scope = makeOriginalContextCurrent()) { - BigQueryArrowBatchWrapper batchWrapper = this.buffer.take(); - checkClosed(); - if (batchWrapper.getException() != null) { - throw backgroundFetchError(batchWrapper.getException()); - } - if (batchWrapper.isLast()) { - /* Marks the end of the records */ - if (this.vectorSchemaRoot != null) { - // IMP: To avoid memory leak: clear vectorSchemaRoot as it still holds - // the last batch - this.vectorSchemaRoot.clear(); - } - this.hasReachedEnd = true; - this.rowCount++; - return false; - } - // Valid batch, process it - ArrowRecordBatch arrowBatch = batchWrapper.getCurrentArrowBatch(); - // Populates vectorSchemaRoot - this.arrowDeserializer.deserializeArrowBatch(arrowBatch); - // Pointing to the first row in this fresh batch - this.currentBatchRowIndex = 0; - this.rowCount++; - return true; - } - } - // There are rows left in the current batch. - else if (this.currentBatchRowIndex < this.vectorSchemaRoot.getRowCount()) { - this.currentBatchRowIndex++; - this.rowCount++; - return true; + if (hasMoreRowsInCurrentBatch()) { + this.currentBatchRowIndex++; + this.rowCount++; + return true; + } + // Start of iteration or the current batch is exhausted. Potentially blocking. + try (Scope scope = makeOriginalContextCurrent()) { + if (!loadNextBatch()) { + this.afterLast = true; + return false; } + // Pointing to the first row in this fresh batch + this.currentBatchRowIndex = 0; + this.rowCount++; + return true; } catch (SQLException ex) { + // A stream error ends iteration: the next call returns false and no row stays readable. + // The failed message is never skipped, even if the caller ignores this exception. + this.hasReachedEnd = true; + this.currentBatchRowIndex = -1; throw new BigQueryJdbcException( "Error occurred while advancing the cursor. This could happen when connection is closed while the next method is being called.", ex); @@ -322,7 +305,94 @@ else if (this.currentBatchRowIndex < this.vectorSchemaRoot.getRowCount()) { ex); } } - return false; + } + + /** True once totalRows rows (or the row limit) have been returned. */ + private boolean reachedRowLimit() { + return this.rowCount >= this.totalRows; + } + + private boolean hasMoreRowsInCurrentBatch() { + return this.currentBatchRowIndex >= 0 + && this.vectorSchemaRoot != null + && this.currentBatchRowIndex < this.vectorSchemaRoot.getRowCount() - 1; + } + + private void initDeserializerIfNeeded(ArrowSchema arrowSchema) throws SQLException { + if (arrowSchema == null || this.arrowDeserializer != null) { + return; + } + try { + this.arrowDeserializer = new ArrowDeserializer(arrowSchema); + } catch (RuntimeException | IOException ex) { + // Arrow throws unchecked exceptions on malformed schema bytes. + throw new BigQueryJdbcException("Failed to create the Arrow deserializer", ex); + } + } + + /** + * Loads the next batch that has rows into the vectors. The reader queues every message as-is, so + * this is the single place that decides what to use: + * + * + * + * Returns false at the end of the stream. A failure ends iteration, see {@link #next()}. + */ + private boolean loadNextBatch() throws SQLException, InterruptedException { + while (true) { + BigQueryArrowBatchWrapper batchWrapper = this.buffer.take(); + checkClosed(); + if (batchWrapper.getException() != null) { + throw backgroundFetchError(batchWrapper.getException()); + } + if (batchWrapper.isLast()) { + /* Marks the end of the records */ + if (this.vectorSchemaRoot != null) { + // IMP: To avoid memory leak: clear vectorSchemaRoot as it still holds the last batch + this.vectorSchemaRoot.clear(); + } + this.hasReachedEnd = true; + if (this.rowCount < this.totalRows) { + throw new BigQueryJdbcException( + String.format( + "The Arrow stream ended after %d of %d rows.", this.rowCount, this.totalRows)); + } + return false; + } + initDeserializerIfNeeded(batchWrapper.getArrowSchema()); + ArrowRecordBatch arrowBatch = batchWrapper.getCurrentArrowBatch(); + Long reportedRows = batchWrapper.getReportedRowCount(); + if (arrowBatch == null || arrowBatch.getSerializedRecordBatch().isEmpty()) { + if (reportedRows != null && reportedRows > 0) { + throw rowCountMismatch(reportedRows, 0); + } + continue; // no rows in this message + } + if (this.arrowDeserializer == null) { + throw new BigQueryJdbcException("Received an Arrow record batch before its schema."); + } + // Populates vectorSchemaRoot + this.arrowDeserializer.deserializeArrowBatch(arrowBatch); + int decodedRows = this.vectorSchemaRoot.getRowCount(); + if (reportedRows != null && reportedRows != decodedRows) { + throw rowCountMismatch(reportedRows, decodedRows); + } + if (decodedRows > 0) { + return true; + } + } + } + + private static SQLException rowCountMismatch(long reportedRows, long decodedRows) { + return new BigQueryJdbcException( + String.format( + "An Arrow stream message reported %d rows but its batch has %d.", + reportedRows, decodedRows)); } @Override @@ -500,7 +570,8 @@ public boolean isBeforeFirst() throws SQLException { if (this.isNested) { return this.nestedRowIndex < this.fromIndex; } else { - return this.rowCount == 0; + // Per JDBC, false when the result set contains no rows. + return !this.afterLast && this.rowCount == 0 && this.totalRows > 0; } } @@ -518,7 +589,8 @@ public boolean isFirst() throws SQLException { if (this.isNested) { return !this.afterLast && this.nestedRowIndex == this.fromIndex; } else { - return !this.afterLast && this.rowCount == 1; + // The index is negative after a stream error: the cursor is on no row then. + return !this.afterLast && this.rowCount == 1 && this.currentBatchRowIndex >= 0; } } @@ -529,7 +601,7 @@ public boolean isLast() throws SQLException { if (this.isNested) { return !this.afterLast && this.nestedRowIndex == this.toIndexExclusive - 1; } else { - return !this.afterLast && this.rowCount == this.totalRows; + return !this.afterLast && this.rowCount > 0 && this.rowCount == this.totalRows; } } } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index f26751102db7..5716aad64b00 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -109,6 +109,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { BigQueryJdbcUrlUtility.DEFAULT_DATASET_PROPERTY_NAME, BigQueryJdbcUrlUtility.LOCATION_PROPERTY_NAME, BigQueryJdbcUrlUtility.ENABLE_HTAPI_PROPERTY_NAME, + BigQueryJdbcUrlUtility.ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME, BigQueryJdbcUrlUtility.UNSUPPORTED_HTAPI_FALLBACK_PROPERTY_NAME, BigQueryJdbcUrlUtility.HTAPI_MIN_TABLE_SIZE_PROPERTY_NAME, BigQueryJdbcUrlUtility.HTAPI_ACTIVATION_RATIO_PROPERTY_NAME, @@ -196,6 +197,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { DatasetId defaultDataset; String location; boolean enableHighThroughputAPI; + boolean enableQueryResultsArrowFormat; int highThroughputMinTableSize; int highThroughputActivationRatio; boolean enableSession; @@ -359,6 +361,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { this.defaultDataset = BigQueryJdbcUrlUtility.parseDefaultDataset(ds.getDefaultDataset()); this.location = ds.getLocation(); this.enableHighThroughputAPI = ds.getEnableHighThroughputAPI(); + this.enableQueryResultsArrowFormat = ds.getEnableQueryResultsArrowFormat(); this.highThroughputMinTableSize = ds.getHighThroughputMinTableSize(); this.highThroughputActivationRatio = ds.getHighThroughputActivationRatio(); this.useQueryCache = ds.getUseQueryCache(); @@ -793,6 +796,10 @@ boolean isEnableHighThroughputAPI() { return this.enableHighThroughputAPI; } + boolean isEnableQueryResultsArrowFormat() { + return this.enableQueryResultsArrowFormat; + } + boolean isUseQueryCache() { return useQueryCache; } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java index d22846470081..3ce1aaaeb1d3 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java @@ -110,6 +110,8 @@ protected boolean removeEldestEntry(Map.Entry> eldes static final String DEFAULT_OAUTH_CLIENT_ID = "977385342095.apps.googleusercontent.com"; static final String DEFAULT_OAUTH_CLIENT_SECRET = "wbER7576mc_1YOII0dGk7jEE"; static final String ENABLE_HTAPI_PROPERTY_NAME = "EnableHighThroughputAPI"; + static final String ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME = + "EnableQueryResultsArrowFormat"; static final String PROXY_HOST_PROPERTY_NAME = "ProxyHost"; static final String PROXY_PORT_PROPERTY_NAME = "ProxyPort"; static final String PROXY_USER_ID_PROPERTY_NAME = "ProxyUid"; @@ -117,6 +119,7 @@ protected boolean removeEldestEntry(Map.Entry> eldes static final String HTTP_CONNECT_TIMEOUT_PROPERTY_NAME = "HttpConnectTimeout"; static final String HTTP_READ_TIMEOUT_PROPERTY_NAME = "HttpReadTimeout"; static final boolean DEFAULT_ENABLE_HTAPI_VALUE = false; + static final boolean DEFAULT_ENABLE_QUERY_RESULTS_ARROW_FORMAT_VALUE = false; static final boolean DEFAULT_ENABLE_SESSION_VALUE = false; static final int DEFAULT_LOG_LEVEL = 0; static final String LOG_LEVEL_PROPERTY_NAME = "LogLevel"; @@ -445,6 +448,15 @@ protected boolean removeEldestEntry(Map.Entry> eldes + " to the BigQuery Storage API for faster processing.") .setDefaultValue(String.valueOf(DEFAULT_HTAPI_MIN_TABLE_SIZE_VALUE)) .build(), + BigQueryConnectionProperty.newBuilder() + .setName(ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME) + .setDescription( + "Reads query results in the Apache Arrow format: the first page of" + + " jobs.query and the query job's Storage Read API stream (Preview)." + + " Disabled by default.") + .setDefaultValue( + String.valueOf(DEFAULT_ENABLE_QUERY_RESULTS_ARROW_FORMAT_VALUE)) + .build(), BigQueryConnectionProperty.newBuilder() .setName(ENABLE_SESSION_PROPERTY_NAME) .setDescription( diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQuerySettings.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQuerySettings.java index bc1b932022f5..bb5767842fe5 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQuerySettings.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQuerySettings.java @@ -44,6 +44,7 @@ class BigQuerySettings { private final boolean enableSession; private final boolean enableTimestampPicos; + private final boolean enableQueryResultsArrowFormat; private final ConnectionProperty sessionInfoConnectionProperty; @@ -122,6 +123,7 @@ private BigQuerySettings(Builder builder) { this.defaultDataset = builder.defaultDataset; this.enableSession = builder.enableSession; this.enableTimestampPicos = builder.enableTimestampPicos; + this.enableQueryResultsArrowFormat = builder.enableQueryResultsArrowFormat; this.unsupportedHTAPIFallback = builder.unsupportedHTAPIFallback; this.sessionInfoConnectionProperty = builder.sessionInfoConnectionProperty; this.useWriteAPI = builder.useWriteAPI; @@ -182,6 +184,15 @@ boolean isEnableTimestampPicos() { return enableTimestampPicos; } + /** + * Returns whether query results should be requested in Arrow format. + * + * @return true if Arrow format is enabled, false otherwise. + */ + boolean isEnableQueryResultsArrowFormat() { + return enableQueryResultsArrowFormat; + } + /** * When the connector uses fetch workflows not supported on the High-Throughput API, this option * specifies whether the connector falls back to the REST API or returns an error. By default it @@ -387,6 +398,9 @@ public String toString() { + "useReadAPI=" + useReadAPI + ", " + + "enableQueryResultsArrowFormat=" + + enableQueryResultsArrowFormat + + ", " + "kmsKeyName=" + kmsKeyName + ", " @@ -488,6 +502,7 @@ static final class Builder { private int highThroughputActivationRatio; private boolean enableSession; private boolean enableTimestampPicos; + private boolean enableQueryResultsArrowFormat; private boolean unsupportedHTAPIFallback; private ConnectionProperty sessionInfoConnectionProperty; private boolean useQueryCache; @@ -529,6 +544,7 @@ private Builder(BigQuerySettings querySettings) { this.highThroughputActivationRatio = querySettings.getHighThroughputActivationRatio(); this.enableSession = querySettings.isEnableSession(); this.enableTimestampPicos = querySettings.isEnableTimestampPicos(); + this.enableQueryResultsArrowFormat = querySettings.isEnableQueryResultsArrowFormat(); this.unsupportedHTAPIFallback = querySettings.isUnsupportedHTAPIFallback(); this.sessionInfoConnectionProperty = querySettings.getSessionInfoConnectionProperty(); this.useQueryCache = querySettings.getUseQueryCache(); @@ -630,6 +646,15 @@ Builder setEnableTimestampPicos(boolean enableTimestampPicos) { return this; } + /** + * Setting true requests query results in Arrow format when the driver supports it for the + * query. Disabled by default. + */ + Builder setEnableQueryResultsArrowFormat(boolean enableQueryResultsArrowFormat) { + this.enableQueryResultsArrowFormat = enableQueryResultsArrowFormat; + return this; + } + /** * When the connector uses fetch workflows not supported on the High-Throughput API, this option * specifies whether the connector falls back to the REST API or returns an error. By default it diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java index 13181dfc4806..4f3f2d94a3ae 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java @@ -52,7 +52,6 @@ import com.google.cloud.bigquery.jdbc.telemetry.v1.QueryApiType; import com.google.cloud.bigquery.jdbc.telemetry.v1.StatementExecution; import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; -import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch; import com.google.cloud.bigquery.storage.v1.ArrowSchema; import com.google.cloud.bigquery.storage.v1.ArrowSerializationOptions; import com.google.cloud.bigquery.storage.v1.BigQueryReadClient; @@ -63,6 +62,7 @@ import com.google.cloud.bigquery.storage.v1.ReadSession; import com.google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.opentelemetry.api.common.AttributeKey; @@ -91,6 +91,7 @@ import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadFactory; +import java.util.function.Supplier; import java.util.logging.Level; /** @@ -233,6 +234,8 @@ private BigQuerySettings generateBigQuerySettings() throws SQLException { querySettings.setWriteAPIActivationRowCount(this.connection.getWriteAPIActivationRowCount()); querySettings.setWriteAPIAppendRowCount(this.connection.getWriteAPIAppendRowCount()); querySettings.setEnableTimestampPicos(this.connection.isEnableTimestampPicos()); + querySettings.setEnableQueryResultsArrowFormat( + this.connection.isEnableQueryResultsArrowFormat()); return querySettings.build(); } @@ -1002,12 +1005,84 @@ private TableReadOptions buildTableReadOptions() { .build(); } + /** + * Arrow options for a job stream. They are sent with every request instead of left to the + * stream's defaults, because they change the schema the stream returns: all requests for one + * result must use the same values. Compression stays unset, which means none. + */ + @VisibleForTesting + ArrowSerializationOptions buildArrowSerializationOptions() { + return ArrowSerializationOptions.newBuilder() + .setPicosTimestampPrecision( + isEnableTimestampPicos() + ? ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_PICOS + : ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_MICROS) + .build(); + } + + /** + * Name of the stream that carries a query job's results. The API does not return it, so it is + * built from the job reference. The location must be the job's own: the stream does not exist in + * any other location. + */ + @VisibleForTesting + static String jobStreamName(JobId jobId) throws BigQueryJdbcException { + if (jobId == null + || Strings.isNullOrEmpty(jobId.getProject()) + || Strings.isNullOrEmpty(jobId.getLocation()) + || Strings.isNullOrEmpty(jobId.getJob())) { + throw new BigQueryJdbcException( + "A job stream needs the job's project, location and job ID, got: " + jobId); + } + return String.format( + "projects/%s/locations/%s/jobs/%s/streams/_default", + jobId.getProject(), jobId.getLocation(), jobId.getJob()); + } + /** Asynchronously reads results and populates an arrow record queue */ @InternalApi Future populateArrowBufferedQueue( ReadSession readSession, BlockingQueue arrowBatchWrapperBlockingQueue, BigQueryReadClient bqReadClient) { + // Use the first stream to perform reading. The session already carries the read options: + // ReadRows accepts Arrow options for job streams only. + return readRowsIntoArrowQueue( + () -> + ReadRowsRequest.newBuilder().setReadStream(readSession.getStreams(0).getName()).build(), + arrowBatchWrapperBlockingQueue, + bqReadClient); + } + + /** + * Asynchronously reads a query job's results from its {@code _default} stream, starting at row + * {@code startOffset}, and populates an arrow record queue. No read session is created. + */ + @InternalApi + Future populateArrowBufferedQueueFromJobStream( + JobId jobId, + long startOffset, + BlockingQueue arrowBatchWrapperBlockingQueue, + BigQueryReadClient bqReadClient) + throws BigQueryJdbcException { + ReadRowsRequest firstRequest = + ReadRowsRequest.newBuilder() + .setReadStream(jobStreamName(jobId)) + .setOffset(startOffset) + .setArrowSerializationOptions(buildArrowSerializationOptions()) + .build(); + return readRowsIntoArrowQueue(() -> firstRequest, arrowBatchWrapperBlockingQueue, bqReadClient); + } + + /** + * Reads one stream into the queue. Every attempt sends the first request moved forward by the + * rows already read, so a resumed request keeps the stream name and the Arrow options. The first + * request is resolved on the worker, where a failure reaches the queue as an error. + */ + private Future readRowsIntoArrowQueue( + Supplier firstRequestSupplier, + BlockingQueue arrowBatchWrapperBlockingQueue, + BigQueryReadClient bqReadClient) { LOG.finer("++enter++"); ExecutorService executor = connection.getExecutorService(); @@ -1019,17 +1094,15 @@ Future populateArrowBufferedQueue( long rowsRead = 0; int retryCount = 0; try { - // Use the first stream to perform reading. - String streamName = readSession.getStreams(0).getName(); + ReadRowsRequest firstRequest = firstRequestSupplier.get(); while (true) { ServerStream stream = null; boolean streamCompleted = false; try { ReadRowsRequest readRowsRequest = - ReadRowsRequest.newBuilder() - .setReadStream(streamName) - .setOffset(rowsRead) + firstRequest.toBuilder() + .setOffset(firstRequest.getOffset() + rowsRead) .build(); // Process each block of rows as they arrive and decode using our simple row @@ -1041,9 +1114,14 @@ Future populateArrowBufferedQueue( "Arrow stream reading was interrupted or shut down before completion."); } - ArrowRecordBatch currentBatch = response.getArrowRecordBatch(); + // Queue every response as-is; the result set decides what to use. arrowBatchWrapperBlockingQueue.put( - BigQueryArrowBatchWrapper.of(currentBatch)); + BigQueryArrowBatchWrapper.of( + response.hasArrowRecordBatch() + ? response.getArrowRecordBatch() + : null, + response.hasArrowSchema() ? response.getArrowSchema() : null, + response.getRowCount())); rowsRead += response.getRowCount(); } if (isWorkerShuttingDown()) { @@ -1054,7 +1132,7 @@ Future populateArrowBufferedQueue( break; } catch (ApiException e) { if (e.getStatusCode().getCode() == StatusCode.Code.NOT_FOUND) { - LOG.warning("Read session expired or not found: %s", e.getMessage()); + LOG.warning("Read stream expired or not found: %s", e.getMessage()); enqueueError(arrowBatchWrapperBlockingQueue, e); break; } @@ -1072,7 +1150,7 @@ Future populateArrowBufferedQueue( LOG.warning( "Connection interrupted during arrow stream read, retrying. attempt: %d", retryCount); - Thread.sleep(RETRY_DELAY_MS); + Thread.sleep(getArrowRetryDelayMs()); } finally { if (stream != null && !streamCompleted) { stream.cancel(); @@ -1567,6 +1645,12 @@ int getArrowBufferSize() { return DEFAULT_ARROW_BUFFER_SIZE; } + /** Wait between two attempts to read an Arrow stream. */ + @VisibleForTesting + long getArrowRetryDelayMs() { + return RETRY_DELAY_MS; + } + /** Returns the destinationTable from jobId by calling `jobs.get` API */ TableId getDestinationTable(JobId jobId) { Job job = this.bigQuery.getJob(jobId); diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java index 7d91c128d9a7..cb5f0ead55f7 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java @@ -53,6 +53,7 @@ public class DataSource implements javax.sql.DataSource { private String location; private String userAgent; private Boolean enableHighThroughputAPI; + private Boolean enableQueryResultsArrowFormat; private Integer highThroughputMinTableSize; private Integer highThroughputActivationRatio; private Boolean unsupportedHTAPIFallback; @@ -160,6 +161,13 @@ public class DataSource implements javax.sql.DataSource { ds.setEnableHighThroughputAPI( BigQueryJdbcUrlUtility.convertToBoolean( val, BigQueryJdbcUrlUtility.ENABLE_HTAPI_PROPERTY_NAME))) + .put( + BigQueryJdbcUrlUtility.ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME, + (ds, val) -> + ds.setEnableQueryResultsArrowFormat( + BigQueryJdbcUrlUtility.convertToBoolean( + val, + BigQueryJdbcUrlUtility.ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME))) .put( BigQueryJdbcUrlUtility.UNSUPPORTED_HTAPI_FALLBACK_PROPERTY_NAME, (ds, val) -> @@ -486,6 +494,11 @@ Properties createProperties() { BigQueryJdbcUrlUtility.ENABLE_HTAPI_PROPERTY_NAME, String.valueOf(this.enableHighThroughputAPI)); } + if (this.enableQueryResultsArrowFormat != null) { + connectionProperties.setProperty( + BigQueryJdbcUrlUtility.ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME, + String.valueOf(this.enableQueryResultsArrowFormat)); + } if (this.unsupportedHTAPIFallback != null) { connectionProperties.setProperty( BigQueryJdbcUrlUtility.UNSUPPORTED_HTAPI_FALLBACK_PROPERTY_NAME, @@ -846,6 +859,16 @@ public void setEnableHighThroughputAPI(Boolean enableHighThroughputAPI) { this.enableHighThroughputAPI = enableHighThroughputAPI; } + public Boolean getEnableQueryResultsArrowFormat() { + return enableQueryResultsArrowFormat != null + ? enableQueryResultsArrowFormat + : BigQueryJdbcUrlUtility.DEFAULT_ENABLE_QUERY_RESULTS_ARROW_FORMAT_VALUE; + } + + public void setEnableQueryResultsArrowFormat(Boolean enableQueryResultsArrowFormat) { + this.enableQueryResultsArrowFormat = enableQueryResultsArrowFormat; + } + public Integer getHighThroughputMinTableSize() { return highThroughputMinTableSize != null ? highThroughputMinTableSize diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java index 72a91dc42145..43ffbdb2019e 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java @@ -36,6 +36,7 @@ import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch; import com.google.cloud.bigquery.storage.v1.ArrowSchema; import com.google.common.collect.ImmutableList; +import com.google.protobuf.ByteString; import java.io.IOException; import java.math.BigDecimal; import java.sql.Array; @@ -302,6 +303,11 @@ public void testRowCount() throws SQLException, IOException { QUERY_SCHEMA, arrowSchema, 1, statement, bufferWithTwoRows, workerTask, null); assertThat(resultSetRowCount(bigQueryArrowResultSet2)).isEqualTo(1); + // ResultSet with 3 rows buffer and 2 total rows (e.g. Statement.setMaxRows(2)). + assertThat( + resultSetRowCount( + idResultSet(idArrowSchema(), 2, idWrapper(1, 2, 3), arrowBatchWrapperLast))) + .isEqualTo(2); } @Test @@ -833,4 +839,236 @@ public void testGetterWhenNotOnRowThrows() throws SQLException { assertThat(bigQueryArrowResultSetNested.isLast()).isFalse(); assertThrows(SQLException.class, () -> bigQueryArrowResultSetNested.getObject(2)); } + + private static final Schema ID_SCHEMA = Schema.of(Field.of("id", StandardSQLTypeName.INT64)); + + private static VectorSchemaRoot idRoot(int... ids) { + IntVector idVector = new IntVector("id", new RootAllocator()); + idVector.allocateNew(Math.max(ids.length, 1)); + for (int i = 0; i < ids.length; i++) { + idVector.set(i, ids[i]); + } + idVector.setValueCount(ids.length); + VectorSchemaRoot root = new VectorSchemaRoot(ImmutableList.of(idVector)); + root.setRowCount(ids.length); + return root; + } + + private static ArrowSchema idArrowSchema() throws IOException { + return ArrowSchema.newBuilder() + .setSerializedSchema(serializeSchema(idRoot().getSchema())) + .build(); + } + + private static ArrowRecordBatch idBatch(int... ids) throws IOException { + return ArrowRecordBatch.newBuilder() + .setSerializedRecordBatch(serializeVectorSchemaRoot(idRoot(ids))) + .build(); + } + + private static BigQueryArrowBatchWrapper idWrapper(int... ids) throws IOException { + return BigQueryArrowBatchWrapper.of(idBatch(ids)); + } + + /** A stream message: the ids as an Arrow batch, and the row count the message reports. */ + private static BigQueryArrowBatchWrapper idMessage(long reportedRows, int... ids) + throws IOException { + return BigQueryArrowBatchWrapper.of(idBatch(ids), null, reportedRows); + } + + private BigQueryArrowResultSet idResultSet( + ArrowSchema arrowSchema, long totalRows, BigQueryArrowBatchWrapper... items) + throws SQLException { + BlockingQueue queue = + new LinkedBlockingDeque<>(Arrays.asList(items)); + return BigQueryArrowResultSet.of( + ID_SCHEMA, arrowSchema, totalRows, statement, queue, mock(Future.class), null); + } + + /** + * Reads a stream of one good row, then {@code failing}, then two more good rows. next() must fail + * on {@code failing}, leave no row readable, and not move on to the rows behind it. The row limit + * (3) is below the stream's total, so if {@code failing} were skipped, the rows behind it would + * fill the limit and nothing would report the gap. + */ + private SQLException assertStreamFailsOn(BigQueryArrowBatchWrapper failing) + throws SQLException, IOException { + BigQueryArrowResultSet rs = + idResultSet( + idArrowSchema(), + 3, + idMessage(1, 1), + failing, + idMessage(2, 3, 4), + arrowBatchWrapperLast); + + assertThat(rs.next()).isTrue(); + SQLException e = assertThrows(SQLException.class, rs::next); + assertThat(rs.isFirst()).isFalse(); + assertThrows(SQLException.class, () -> rs.getLong(1)); + assertThat(rs.next()).isFalse(); + return e; + } + + private void assertRowCountMismatch(BigQueryArrowBatchWrapper message, String expected) + throws SQLException, IOException { + assertThat(assertStreamFailsOn(message).getCause().getMessage()).contains(expected); + } + + @Test + public void testRowsReportedWithoutDataThrows() throws SQLException, IOException { + assertRowCountMismatch( + BigQueryArrowBatchWrapper.of(null, null, 2), "reported 2 rows but its batch has 0"); + assertRowCountMismatch( + BigQueryArrowBatchWrapper.of(ArrowRecordBatch.getDefaultInstance(), null, 2), + "reported 2 rows but its batch has 0"); + } + + @Test + public void testRowsReportedBeforeSchemaThrows() throws SQLException, IOException { + BigQueryArrowResultSet rs = + idResultSet(null, 2, BigQueryArrowBatchWrapper.of(null, null, 2), arrowBatchWrapperLast); + + SQLException e = assertThrows(SQLException.class, rs::next); + assertThat(e.getCause().getMessage()).contains("reported 2 rows but its batch has 0"); + assertThat(rs.next()).isFalse(); + } + + @Test + public void testUndecodableBatchEndsIteration() throws SQLException, IOException { + ArrowRecordBatch whole = idBatch(8, 9); + ArrowRecordBatch truncated = + ArrowRecordBatch.newBuilder() + .setSerializedRecordBatch( + whole + .getSerializedRecordBatch() + .substring(0, whole.getSerializedRecordBatch().size() / 2)) + .build(); + + assertStreamFailsOn(BigQueryArrowBatchWrapper.of(truncated)); + } + + @Test + public void testBatchRowCountMismatchThrows() throws SQLException, IOException { + assertRowCountMismatch(idMessage(3, 1, 2), "reported 3 rows but its batch has 2"); + assertRowCountMismatch(idMessage(0, 1, 2), "reported 0 rows but its batch has 2"); + assertRowCountMismatch(idMessage(2), "reported 2 rows but its batch has 0"); + } + + @Test + public void testMatchingReportedRowCountsAreAccepted() throws SQLException, IOException { + BigQueryArrowResultSet rs = + idResultSet( + null, + 3, + BigQueryArrowBatchWrapper.of(null, idArrowSchema(), 0), + idMessage(2, 1, 2), + BigQueryArrowBatchWrapper.of(ArrowRecordBatch.getDefaultInstance(), null, 0), + idMessage(0), + idMessage(1, 3), + arrowBatchWrapperLast); + + assertThat(rs.next()).isTrue(); + assertThat(rs.next()).isTrue(); + assertThat(rs.next()).isTrue(); + assertThat(rs.getLong(1)).isEqualTo(3L); + assertThat(rs.next()).isFalse(); + } + + @Test + public void testMessagesWithoutRowsAreSkipped() throws SQLException, IOException { + BigQueryArrowBatchWrapper noBatch = BigQueryArrowBatchWrapper.of(null); + BigQueryArrowBatchWrapper emptyBatch = + BigQueryArrowBatchWrapper.of(ArrowRecordBatch.getDefaultInstance()); + BigQueryArrowResultSet rs = + idResultSet( + idArrowSchema(), + 3, + idWrapper(1, 2), + noBatch, + idWrapper(), + emptyBatch, + idWrapper(3), + arrowBatchWrapperLast); + + assertThat(rs.next()).isTrue(); + assertThat(rs.next()).isTrue(); + assertThat(rs.next()).isTrue(); + assertThat(rs.getLong(1)).isEqualTo(3L); + assertThat(rs.next()).isFalse(); + } + + @Test + public void testStreamEndingBeforeTotalRowsThrows() throws SQLException, IOException { + BigQueryArrowResultSet rs = + idResultSet(idArrowSchema(), 3, idWrapper(1, 2), arrowBatchWrapperLast); + + assertThat(rs.next()).isTrue(); + assertThat(rs.next()).isTrue(); + SQLException e = assertThrows(SQLException.class, rs::next); + assertThat(e.getCause().getMessage()).contains("ended after 2 of 3 rows"); + } + + @Test + public void testEmptyResultIsNotBeforeFirst() throws SQLException, IOException { + BigQueryArrowResultSet rs = idResultSet(idArrowSchema(), 0, arrowBatchWrapperLast); + + assertThat(rs.isBeforeFirst()).isFalse(); + assertThat(rs.isLast()).isFalse(); + assertThat(rs.next()).isFalse(); + } + + @Test + public void testSchemaFromStreamMessage() throws SQLException, IOException { + BigQueryArrowResultSet rs = + idResultSet( + null, + 1, + BigQueryArrowBatchWrapper.of(null, idArrowSchema(), 0), + idWrapper(7), + arrowBatchWrapperLast); + + assertThat(rs.next()).isTrue(); + assertThat(rs.getLong(1)).isEqualTo(7L); + assertThat(rs.next()).isFalse(); + } + + @Test + public void testBatchBeforeSchemaThrows() throws SQLException, IOException { + BigQueryArrowResultSet rs = + idResultSet( + null, + 1, + idWrapper(1), + BigQueryArrowBatchWrapper.of(idBatch(2), idArrowSchema(), 1), + arrowBatchWrapperLast); + + assertThrows(SQLException.class, rs::next); + // The message behind the failure has a schema and a row. It must not be served. + assertThat(rs.next()).isFalse(); + } + + @Test + public void testUndecodableSchemaEndsIteration() throws SQLException, IOException { + // A negative length prefix: Arrow fails on it with an unchecked exception. + ArrowSchema undecodable = + ArrowSchema.newBuilder() + .setSerializedSchema( + ByteString.copyFrom( + new byte[] {(byte) 0xFE, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF})) + .build(); + BigQueryArrowResultSet rs = + idResultSet( + null, + 1, + BigQueryArrowBatchWrapper.of(null, undecodable, 0), + BigQueryArrowBatchWrapper.of(idBatch(2), idArrowSchema(), 1), + arrowBatchWrapperLast); + + SQLException e = assertThrows(SQLException.class, rs::next); + assertThat(e.getCause().getMessage()).contains("Failed to create the Arrow deserializer"); + assertThat(e.getCause().getCause()).isInstanceOf(RuntimeException.class); + // The message behind the failure has a schema and a row. It must not be served. + assertThat(rs.next()).isFalse(); + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java index fd82c6babb62..54488ca8ef1b 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java @@ -945,6 +945,21 @@ public void testEnableTimestampPicosConfigured() throws Exception { } } + @Test + public void testEnableQueryResultsArrowFormatDefault() throws Exception { + try (BigQueryConnection connection = new BigQueryConnection(BASE_URL)) { + assertFalse(connection.isEnableQueryResultsArrowFormat()); + } + } + + @Test + public void testEnableQueryResultsArrowFormatConfigured() throws Exception { + String url = BASE_URL + "EnableQueryResultsArrowFormat=1;"; + try (BigQueryConnection connection = new BigQueryConnection(url)) { + assertTrue(connection.isEnableQueryResultsArrowFormat()); + } + } + @Test public void testSessionIdIsMatchedRegardlessOfCase() throws Exception { String url = BASE_URL + ";QueryProperties=Session_Id=abc123"; diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java index 74b948a75135..63ba3381cd09 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java @@ -162,6 +162,12 @@ public class BigQueryStatementTest { private static final TableId TABLE_ID = TableId.of(DEFAULT_TEST_DATASET, PROJECT); + private static final JobId STREAM_JOB_ID = + JobId.newBuilder().setProject("p").setLocation("us-central1").setJob("j").build(); + + private static final String JOB_STREAM_NAME = + "projects/p/locations/us-central1/jobs/j/streams/_default"; + private static ArrowSchema arrowSchema; private final Map LABELS = @@ -1303,6 +1309,13 @@ public void testEnableTimestampPicosPropagation() throws SQLException { assertTrue(statement.isEnableTimestampPicos()); } + @Test + public void testEnableQueryResultsArrowFormatPropagation() throws SQLException { + doReturn(true).when(bigQueryConnection).isEnableQueryResultsArrowFormat(); + BigQueryStatement statement = new BigQueryStatement(bigQueryConnection); + assertTrue(statement.querySettings.isEnableQueryResultsArrowFormat()); + } + @Test public void testJoblessQueryPopulatesQueryStatistics() throws Exception { doReturn(true).when(bigQueryConnection).getUseStatelessQueryMode(); @@ -1582,6 +1595,251 @@ public ReadRowsResponse next() { "Stopping early due to worker shutdown must enqueue an error marker rather than normal end-of-stream"); } + private static ReadRowsResponse rowsResponse(long rowCount) { + return ReadRowsResponse.newBuilder() + .setRowCount(rowCount) + .setArrowRecordBatch(ArrowRecordBatch.getDefaultInstance()) + .build(); + } + + /** One ReadRows attempt: delivers {@code responses}, then throws {@code failure} unless null. */ + private static ServerStream readRowsAttempt( + RuntimeException failure, ReadRowsResponse... responses) { + Iterator delivered = Arrays.asList(responses).iterator(); + Iterator attempt = + new Iterator() { + @Override + public boolean hasNext() { + if (!delivered.hasNext() && failure != null) { + throw failure; + } + return delivered.hasNext(); + } + + @Override + public ReadRowsResponse next() { + return delivered.next(); + } + }; + @SuppressWarnings("unchecked") + ServerStream stream = mock(ServerStream.class); + doReturn(attempt).when(stream).iterator(); + return stream; + } + + /** A read client that serves {@code attempts} in order and records each request it is sent. */ + @SafeVarargs + private static BigQueryReadClient readClientServing( + List requests, ServerStream... attempts) { + Iterator> remaining = Arrays.asList(attempts).iterator(); + @SuppressWarnings("unchecked") + ServerStreamingCallable callable = + mock(ServerStreamingCallable.class); + Mockito.doAnswer( + invocation -> { + requests.add(invocation.getArgument(0)); + return remaining.next(); + }) + .when(callable) + .call(any(ReadRowsRequest.class)); + BigQueryReadClient readClient = mock(BigQueryReadClient.class); + doReturn(callable).when(readClient).readRowsCallable(); + return readClient; + } + + /** A statement with picosecond timestamps on, whose reader retries without waiting. */ + private BigQueryStatement picosStatementWithoutRetryDelay() throws SQLException { + doReturn(true).when(bigQueryConnection).isEnableTimestampPicos(); + BigQueryStatement statement = Mockito.spy(new BigQueryStatement(bigQueryConnection)); + doReturn(0L).when(statement).getArrowRetryDelayMs(); + return statement; + } + + @Test + public void testJobStreamNameUsesJobReference() throws SQLException { + assertEquals(JOB_STREAM_NAME, BigQueryStatement.jobStreamName(STREAM_JOB_ID)); + } + + @Test + public void testJobStreamNameRejectsMissingParts() { + for (JobId incomplete : + Arrays.asList( + null, + STREAM_JOB_ID.toBuilder().setProject(null).build(), + STREAM_JOB_ID.toBuilder().setLocation(null).build(), + STREAM_JOB_ID.toBuilder().setLocation("").build(), + STREAM_JOB_ID.toBuilder().setJob(null).build())) { + assertThrows( + BigQueryJdbcException.class, + () -> BigQueryStatement.jobStreamName(incomplete), + "Expected no stream name for " + incomplete); + } + } + + @Test + public void testArrowSerializationOptionsDefaultToMicros() { + ArrowSerializationOptions options = bigQueryStatement.buildArrowSerializationOptions(); + + assertEquals( + ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_MICROS, + options.getPicosTimestampPrecision()); + assertEquals( + ArrowSerializationOptions.CompressionCodec.COMPRESSION_UNSPECIFIED, + options.getBufferCompression()); + } + + @Test + public void testArrowSerializationOptionsUsePicosWhenEnabled() throws SQLException { + doReturn(true).when(bigQueryConnection).isEnableTimestampPicos(); + BigQueryStatement statement = new BigQueryStatement(bigQueryConnection); + + assertEquals( + ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_PICOS, + statement.buildArrowSerializationOptions().getPicosTimestampPrecision()); + } + + @Test + public void testJobStreamReaderSendsNameOffsetAndOptions() throws Exception { + BlockingQueue queue = new LinkedBlockingDeque<>(); + List requests = new ArrayList<>(); + BigQueryReadClient readClient = + readClientServing(requests, readRowsAttempt(null, rowsResponse(3))); + + bigQueryStatement + .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 1000L, queue, readClient) + .get(5, TimeUnit.SECONDS); + + assertEquals(1, requests.size()); + ReadRowsRequest request = requests.get(0); + assertEquals(JOB_STREAM_NAME, request.getReadStream()); + assertEquals(1000L, request.getOffset()); + assertTrue(request.hasArrowSerializationOptions()); + assertEquals( + ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_MICROS, + request.getArrowSerializationOptions().getPicosTimestampPrecision()); + } + + @Test + public void testJobStreamReaderResumeKeepsOptionsAndOffset() throws Exception { + BigQueryStatement statement = picosStatementWithoutRetryDelay(); + BlockingQueue queue = new LinkedBlockingDeque<>(); + List requests = new ArrayList<>(); + BigQueryReadClient readClient = + readClientServing( + requests, + readRowsAttempt( + mockApiException(StatusCode.Code.UNAVAILABLE), rowsResponse(2), rowsResponse(1)), + readRowsAttempt(null, rowsResponse(4))); + + statement + .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 1000L, queue, readClient) + .get(5, TimeUnit.SECONDS); + + assertEquals(2, requests.size()); + ReadRowsRequest first = requests.get(0); + ReadRowsRequest resumed = requests.get(1); + assertEquals(1000L, first.getOffset()); + assertEquals(1003L, resumed.getOffset()); + assertEquals( + ArrowSerializationOptions.PicosTimestampPrecision.TIMESTAMP_PRECISION_PICOS, + resumed.getArrowSerializationOptions().getPicosTimestampPrecision()); + // The resumed request differs from the first one in its offset only. + assertEquals(first, resumed.toBuilder().setOffset(first.getOffset()).build()); + + // Three messages, then a normal end of stream. + List queued = new ArrayList<>(queue); + assertEquals(4, queued.size()); + assertTrue(queued.get(3).isLast()); + assertNull(queued.get(3).getException()); + } + + @Test + public void testReadSessionReaderSendsNoArrowOptions() throws Exception { + // Picoseconds are on, so the session carries the Arrow options. ReadRows on a session stream + // must not: the request accepts them for job streams only. + BigQueryStatement statement = picosStatementWithoutRetryDelay(); + String sessionStream = "projects/p/locations/l/sessions/s/streams/0"; + ReadSession readSession = + ReadSession.newBuilder().addStreams(ReadStream.newBuilder().setName(sessionStream)).build(); + BlockingQueue queue = new LinkedBlockingDeque<>(); + List requests = new ArrayList<>(); + BigQueryReadClient readClient = + readClientServing( + requests, + readRowsAttempt(mockApiException(StatusCode.Code.UNAVAILABLE), rowsResponse(2)), + readRowsAttempt(null, rowsResponse(4))); + + statement.populateArrowBufferedQueue(readSession, queue, readClient).get(5, TimeUnit.SECONDS); + + assertEquals( + Arrays.asList( + ReadRowsRequest.newBuilder().setReadStream(sessionStream).build(), + ReadRowsRequest.newBuilder().setReadStream(sessionStream).setOffset(2).build()), + requests); + } + + @Test + public void testJobStreamReaderQueuesMessagesAsReceived() throws Exception { + ReadRowsResponse schemaAndRows = + rowsResponse(3).toBuilder().setArrowSchema(arrowSchema).build(); + ReadRowsResponse noRows = ReadRowsResponse.getDefaultInstance(); + BlockingQueue queue = new LinkedBlockingDeque<>(); + BigQueryReadClient readClient = + readClientServing(new ArrayList<>(), readRowsAttempt(null, schemaAndRows, noRows)); + + bigQueryStatement + .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 0L, queue, readClient) + .get(5, TimeUnit.SECONDS); + + BigQueryArrowBatchWrapper first = queue.poll(); + assertEquals(arrowSchema, first.getArrowSchema()); + assertNotNull(first.getCurrentArrowBatch()); + assertEquals(3L, first.getReportedRowCount()); + BigQueryArrowBatchWrapper second = queue.poll(); + assertNull(second.getArrowSchema()); + assertNull(second.getCurrentArrowBatch()); + assertEquals(0L, second.getReportedRowCount()); + BigQueryArrowBatchWrapper end = queue.poll(); + assertTrue(end.isLast()); + assertNull(end.getException()); + assertNull(end.getReportedRowCount()); + assertTrue(queue.isEmpty()); + } + + @Test + public void testJobStreamReaderDoesNotRetryNotFound() throws Exception { + ApiException notFound = mockApiException(StatusCode.Code.NOT_FOUND); + BlockingQueue queue = new LinkedBlockingDeque<>(); + List requests = new ArrayList<>(); + BigQueryReadClient readClient = readClientServing(requests, readRowsAttempt(notFound)); + + bigQueryStatement + .populateArrowBufferedQueueFromJobStream(STREAM_JOB_ID, 0L, queue, readClient) + .get(5, TimeUnit.SECONDS); + + assertEquals(1, requests.size()); + assertEquals(notFound, queue.poll().getException()); + BigQueryArrowBatchWrapper end = queue.poll(); + assertTrue(end.isLast()); + assertNull(end.getException()); + assertTrue(queue.isEmpty()); + } + + @Test + public void testJobStreamReaderRejectsIncompleteJobReferenceBeforeStartingAWorker() { + BlockingQueue queue = new LinkedBlockingDeque<>(); + JobId withoutProjectAndLocation = JobId.of("j"); + + assertThrows( + BigQueryJdbcException.class, + () -> + bigQueryStatement.populateArrowBufferedQueueFromJobStream( + withoutProjectAndLocation, 0L, queue, storageReadClient)); + + verify(bigQueryConnection, Mockito.never()).getExecutorService(); + assertTrue(queue.isEmpty()); + } + private static FieldValueList createTestRow(boolean first, long second) { return FieldValueList.of( Arrays.asList(