From fd56f252ac0a4e02c17acbf17a7aa25ada4fa1e4 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Thu, 19 Nov 2026 14:58:07 +0000 Subject: [PATCH 1/3] feat(bigquery-jdbc): read large results from the job stream --- .../bigquery/jdbc/BigQueryJdbcUrlUtility.java | 11 +- .../bigquery/jdbc/BigQueryStatement.java | 172 ++++++-- .../bigquery/jdbc/BigQueryStatementTest.java | 403 +++++++++++++++++- .../google/cloud/bigquery/jdbc/it/ITBase.java | 10 +- .../bigquery/jdbc/it/ITBigQueryJDBCTest.java | 128 ++++++ 5 files changed, 679 insertions(+), 45 deletions(-) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java index 3ce1aaaeb1d3..7e0b12fdd94c 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java @@ -413,7 +413,9 @@ protected boolean removeEldestEntry(Map.Entry> eldes BigQueryConnectionProperty.newBuilder() .setName(ENABLE_HTAPI_PROPERTY_NAME) .setDescription( - "Enables or disables Read API usage in the Driver. Disabled by default.") + "Enables or disables Read API usage in the Driver. Disabled by default." + + " With EnableQueryResultsArrowFormat enabled, this applies only" + + " when LargeResultTable or LargeResultDataset is set.") .setDefaultValue(String.valueOf(DEFAULT_ENABLE_HTAPI_VALUE)) .build(), BigQueryConnectionProperty.newBuilder() @@ -453,7 +455,12 @@ protected boolean removeEldestEntry(Map.Entry> eldes .setDescription( "Reads query results in the Apache Arrow format: the first page of" + " jobs.query and the query job's Storage Read API stream (Preview)." - + " Disabled by default.") + + " A result with more than one page and at least" + + " HighThroughputMinTableSize rows is read from the job stream," + + " whatever EnableHighThroughputAPI and" + + " HighThroughputActivationRatio are set to, unless" + + " LargeResultTable or LargeResultDataset is set. Disabled by" + + " default.") .setDefaultValue( String.valueOf(DEFAULT_ENABLE_QUERY_RESULTS_ARROW_FORMAT_VALUE)) .build(), diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java index 4f3f2d94a3ae..8b0a21e9dfcb 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java @@ -147,8 +147,6 @@ public class BigQueryStatement extends BigQueryNoOpsStatement { private BlockingQueue bigQueryFieldValueListWrapperBlockingQueue; - private BlockingQueue arrowBatchWrapperBlockingQueue; - // Variables Required for the ReferenceQueue implementation static ReferenceQueue referenceQueueArrowRs = new ReferenceQueue<>(); static ReferenceQueue referenceQueueJsonRs = new ReferenceQueue<>(); @@ -549,7 +547,6 @@ private void closeStatementResources() throws SQLException { tmp.close(); } this.batchQueries.clear(); - this.arrowBatchWrapperBlockingQueue = null; this.bigQueryFieldValueListWrapperBlockingQueue = null; this.currentUpdateCount = -1; this.currentJobIdIndex = -1; @@ -923,10 +920,8 @@ ResultSet processArrowResultSet(TableResult results, Job job) throws SQLExceptio LOG.finer("++enter++"); // set the resultset - long totalRows = getEffectiveTotalRows(results); JobId currentJobId = results.getJobId(); TableId destinationTable = getDestinationTable(currentJobId); - Schema schema = results.getSchema(); Future populateBufferWorker = null; try { String parent = String.format("projects/%s", destinationTable.getProject()); @@ -953,45 +948,91 @@ ResultSet processArrowResultSet(TableResult results, Job job) throws SQLExceptio .setMaxStreamCount(1); ReadSession readSession = getReadSession(builder.build()); - this.arrowBatchWrapperBlockingQueue = new LinkedBlockingDeque<>(getArrowBufferSize()); + BlockingQueue queue = + new LinkedBlockingDeque<>(getArrowBufferSize()); // deserialize and populate the buffer async, so that the client isn't blocked populateBufferWorker = - populateArrowBufferedQueue( - readSession, this.arrowBatchWrapperBlockingQueue, getBigQueryReadClient()); - - BigQueryArrowResultSet arrowResultSet = - BigQueryArrowResultSet.of( - schema, - getArrowSchema(readSession), - totalRows, - this, - this.arrowBatchWrapperBlockingQueue, - populateBufferWorker, - this.bigQuery, - job); - arrowResultSetFinalizers.add( - new BigQueryResultSetFinalizers.ArrowResultSetFinalizer( - arrowResultSet, referenceQueueArrowRs, populateBufferWorker)); - arrowResultSet.setJobId(currentJobId); - arrowResultSet.setQueryId(results.getQueryId()); - if (job == null && currentJobId == null) { - arrowResultSet.setQueryStatistics(results.extractQueryStatistics()); - } - return arrowResultSet; + populateArrowBufferedQueue(readSession, queue, getBigQueryReadClient()); + + return newArrowResultSet( + results, job, getArrowSchema(readSession), queue, populateBufferWorker); } catch (Exception | OutOfMemoryError ex) { if (populateBufferWorker != null) { populateBufferWorker.cancel(true); } - if (ex instanceof OutOfMemoryError || ex instanceof RejectedExecutionException) { - throw new BigQueryJdbcException( - "Failed to execute query: Unable to allocate background threads to process the query results. Connection-scoped thread pool limit of 100 threads was reached or system is out of memory.", - ex); + throw arrowResultSetException(ex); + } + } + + /** + * Reads a query job's results from its {@code _default} stream, starting at row 0. No read + * session is created and the job is not looked up. The first page in {@code results} is not + * returned: it supplies the column types and the row count. The Arrow schema arrives in the + * stream's first message. + */ + private ResultSet processJobStreamResultSet(TableResult results, Job job) throws SQLException { + LOG.finer("++enter++"); + Future populateBufferWorker = null; + try { + BlockingQueue queue = + new LinkedBlockingDeque<>(getArrowBufferSize()); + populateBufferWorker = + populateArrowBufferedQueueFromJobStream( + results.getJobId(), 0, queue, getBigQueryReadClient()); + + return newArrowResultSet(results, job, null, queue, populateBufferWorker); + } catch (Exception | OutOfMemoryError ex) { + if (populateBufferWorker != null) { + populateBufferWorker.cancel(true); } - throw new BigQueryJdbcException(ex.getMessage(), ex); + throw arrowResultSetException(ex); } } + /** + * Builds the result set over {@code queue}, which {@code reader} fills. Without an {@code + * arrowSchema}, the result set takes it from the first message in the queue. + */ + private BigQueryArrowResultSet newArrowResultSet( + TableResult results, + Job job, + ArrowSchema arrowSchema, + BlockingQueue queue, + Future reader) + throws SQLException { + JobId currentJobId = results.getJobId(); + BigQueryArrowResultSet arrowResultSet = + BigQueryArrowResultSet.of( + results.getSchema(), + arrowSchema, + getEffectiveTotalRows(results), + this, + queue, + reader, + this.bigQuery, + job); + arrowResultSetFinalizers.add( + new BigQueryResultSetFinalizers.ArrowResultSetFinalizer( + arrowResultSet, referenceQueueArrowRs, reader)); + arrowResultSet.setJobId(currentJobId); + arrowResultSet.setQueryId(results.getQueryId()); + if (job == null && currentJobId == null) { + arrowResultSet.setQueryStatistics(results.extractQueryStatistics()); + } + return arrowResultSet; + } + + /** The exception a statement throws when it can't set up an Arrow result set. */ + private static BigQueryJdbcException arrowResultSetException(Throwable ex) { + if (ex instanceof OutOfMemoryError || ex instanceof RejectedExecutionException) { + return new BigQueryJdbcException( + "Failed to execute query: Unable to allocate background threads to process the query results. Connection-scoped thread pool limit of 100 threads was reached or system is out of memory.", + ex); + } + return new BigQueryJdbcException(ex.getMessage(), ex); + } + private TableReadOptions buildTableReadOptions() { if (!isEnableTimestampPicos()) { return null; @@ -1027,10 +1068,7 @@ ArrowSerializationOptions buildArrowSerializationOptions() { */ @VisibleForTesting static String jobStreamName(JobId jobId) throws BigQueryJdbcException { - if (jobId == null - || Strings.isNullOrEmpty(jobId.getProject()) - || Strings.isNullOrEmpty(jobId.getLocation()) - || Strings.isNullOrEmpty(jobId.getJob())) { + if (!isCompleteJobReference(jobId)) { throw new BigQueryJdbcException( "A job stream needs the job's project, location and job ID, got: " + jobId); } @@ -1039,6 +1077,14 @@ static String jobStreamName(JobId jobId) throws BigQueryJdbcException { jobId.getProject(), jobId.getLocation(), jobId.getJob()); } + /** A job stream can be named only from a reference that has a project, location and job ID. */ + private static boolean isCompleteJobReference(JobId jobId) { + return jobId != null + && !Strings.isNullOrEmpty(jobId.getProject()) + && !Strings.isNullOrEmpty(jobId.getLocation()) + && !Strings.isNullOrEmpty(jobId.getJob()); + } + /** Asynchronously reads results and populates an arrow record queue */ @InternalApi Future populateArrowBufferedQueue( @@ -1212,7 +1258,14 @@ void processQueryResponse(String query, TableResult results, Job job) throws SQL LOG.fine("Processing query response. Query: %s", query); ResultSet resultSet = null; - if (jobId != null && useReadAPI(results)) { + if (isJobStreamAllowed()) { + // The job stream or JSON. A read session is never used where the job stream is allowed. + if (useJobStream(results)) { + LOG.info("Using the job stream to read the data."); + resultSet = processJobStreamResultSet(results, job); + this.currentExecutionBuilder.setQueryApiType(QueryApiType.QUERY_API_TYPE_READ_API); + } + } else if (jobId != null && useReadAPI(results)) { try { LOG.info("Using ReadAPI to read the data."); resultSet = processArrowResultSet(results, job); @@ -1253,6 +1306,49 @@ private boolean isPermissionDeniedException(Throwable t) { return false; } + /** + * True when a result may be read from its job's stream: {@code EnableQueryResultsArrowFormat} is + * on and the user set no destination table. A job with a user-set destination table has no + * stream, so it keeps the High-Throughput API rules. + */ + private boolean isJobStreamAllowed() { + return querySettings.isEnableQueryResultsArrowFormat() && !isLargeResultsEnabled(); + } + + /** + * Whether a SELECT result is read from the query job's stream. The result must have more pages, + * and return at least {@code HighThroughputMinTableSize} rows and more rows than its first page + * holds. Rows are counted after {@link #setMaxRows(int)}. Any other result is read as JSON, + * starting with the page already fetched. + */ + @VisibleForTesting + boolean useJobStream(TableResult results) { + if (!isJobStreamAllowed()) { + return false; + } + long rows = getEffectiveTotalRows(results); + Long rowsInPage = results.getRowsInPage(); + int threshold = querySettings.getHighThroughputMinTableSize(); + LOG.fine( + "Job stream rule. More pages: %s, rows to return: %d, rows in the first page: %s," + + " threshold: %d.", + results.hasNextPage(), rows, rowsInPage, threshold); + if (!results.hasNextPage() || rows < threshold) { + return false; + } + if (rowsInPage != null && rows <= rowsInPage) { + // setMaxRows stops within the first page, and that page is already here. + return false; + } + if (!isCompleteJobReference(results.getJobId())) { + LOG.warning( + "Reading the results as JSON: the job reference %s can't name a job stream.", + results.getJobId()); + return false; + } + return true; + } + // The read Ratio should be met // AND the User must not have disabled the Read API @VisibleForTesting diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java index 63ba3381cd09..a0d7644cf89a 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java @@ -17,6 +17,7 @@ package com.google.cloud.bigquery.jdbc; import static com.google.cloud.bigquery.jdbc.utils.ArrowUtilities.serializeSchema; +import static com.google.cloud.bigquery.jdbc.utils.ArrowUtilities.serializeVectorSchemaRoot; import static com.google.common.truth.Truth.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -26,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -66,8 +68,10 @@ import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableResult; import com.google.cloud.bigquery.exception.BigQueryJdbcException; +import com.google.cloud.bigquery.exception.BigQueryJdbcRuntimeException; import com.google.cloud.bigquery.jdbc.BigQueryStatement.JobIdWrapper; import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.QueryApiType; import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import com.google.cloud.bigquery.spi.BigQueryRpcFactory; import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch; @@ -107,6 +111,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Stream; @@ -1840,6 +1845,387 @@ public void testJobStreamReaderRejectsIncompleteJobReferenceBeforeStartingAWorke assertTrue(queue.isEmpty()); } + /** A statement with EnableQueryResultsArrowFormat on and a small-result threshold of 100 rows. */ + private BigQueryStatement arrowFormatStatement() throws SQLException { + return arrowFormatStatement(100); + } + + private BigQueryStatement arrowFormatStatement(int smallResultThreshold) throws SQLException { + doReturn(true).when(bigQueryConnection).isEnableQueryResultsArrowFormat(); + doReturn(smallResultThreshold).when(bigQueryConnection).getHighThroughputMinTableSize(); + return new BigQueryStatement(bigQueryConnection); + } + + private static TableResult selectResult(JobId jobId, long totalRows, boolean hasNextPage) { + TableResult result = mock(TableResult.class); + doReturn(jobId).when(result).getJobId(); + doReturn(Schema.of(fieldList)).when(result).getSchema(); + doReturn(totalRows).when(result).getTotalRows(); + doReturn(hasNextPage).when(result).hasNextPage(); + return result; + } + + /** A result with more pages and a complete job reference: streamed if it has enough rows. */ + private static TableResult multiPageResult(long totalRows) { + return selectResult(STREAM_JOB_ID, totalRows, true); + } + + @Test + public void testUseJobStream_LargeResultWithNextPage() throws SQLException { + // 100 rows is the threshold itself. + assertTrue(arrowFormatStatement().useJobStream(multiPageResult(100L))); + } + + @Test + public void testUseJobStream_PropertyOff() throws SQLException { + doReturn(100).when(bigQueryConnection).getHighThroughputMinTableSize(); + BigQueryStatement statement = new BigQueryStatement(bigQueryConnection); + + assertFalse(statement.useJobStream(multiPageResult(5000L))); + } + + @Test + public void testUseJobStream_DestinationTableSet() throws SQLException { + doReturn("destination_table").when(bigQueryConnection).getDestinationTable(); + + assertFalse(arrowFormatStatement().useJobStream(multiPageResult(5000L))); + } + + @Test + public void testUseJobStream_DestinationDatasetSet() throws SQLException { + doReturn("destination_dataset").when(bigQueryConnection).getDestinationDataset(); + + assertFalse(arrowFormatStatement().useJobStream(multiPageResult(5000L))); + } + + @Test + public void testUseJobStream_NoNextPage() throws SQLException { + assertFalse(arrowFormatStatement().useJobStream(selectResult(STREAM_JOB_ID, 5000L, false))); + } + + @Test + public void testUseJobStream_BelowThreshold() throws SQLException { + assertFalse(arrowFormatStatement().useJobStream(multiPageResult(99L))); + } + + @Test + public void testUseJobStream_CountsRowsAfterMaxRows() throws SQLException { + BigQueryStatement statement = arrowFormatStatement(); + TableResult result = multiPageResult(1_000_000L); + + statement.setMaxRows(99); + assertFalse(statement.useJobStream(result)); + + statement.setMaxRows(100); + assertTrue(statement.useJobStream(result)); + } + + @Test + public void testUseJobStream_MaxRowsWithinFirstPage() throws SQLException { + BigQueryStatement statement = arrowFormatStatement(); + TableResult result = multiPageResult(1_000_000L); + doReturn(1000L).when(result).getRowsInPage(); + + // The first page already holds every row that setMaxRows lets through. + statement.setMaxRows(1000); + assertFalse(statement.useJobStream(result)); + + statement.setMaxRows(1001); + assertTrue(statement.useJobStream(result)); + } + + @Test + public void testUseJobStream_FirstPageSizeUnknown() throws SQLException { + TableResult result = multiPageResult(5000L); + // The count is nullable in TableResult. A mock returns 0 for it unless told otherwise. + doReturn(null).when(result).getRowsInPage(); + + assertTrue(arrowFormatStatement().useJobStream(result)); + } + + @Test + public void testUseJobStream_IncompleteJobReference() throws SQLException { + BigQueryStatement statement = arrowFormatStatement(); + + for (JobId incomplete : + Arrays.asList( + null, + JobId.of("j"), + STREAM_JOB_ID.toBuilder().setProject(null).build(), + STREAM_JOB_ID.toBuilder().setLocation(null).build(), + STREAM_JOB_ID.toBuilder().setLocation("").build(), + STREAM_JOB_ID.toBuilder().setJob(null).build())) { + assertFalse( + statement.useJobStream(selectResult(incomplete, 5000L, true)), + "Expected no job stream for " + incomplete); + } + } + + @Test + public void testUseJobStream_IgnoresActivationRatio() throws SQLException { + // The High-Throughput API would need more than 1000 pages here. The job stream has no such + // rule. + doReturn(1000).when(bigQueryConnection).getHighThroughputActivationRatio(); + + assertTrue(arrowFormatStatement().useJobStream(multiPageResult(5000L))); + } + + /** {@link #arrowFormatStatement()} as a spy that starts no real job-stream reader. */ + private BigQueryStatement jobStreamStatementSpy() throws SQLException { + BigQueryStatement stmtSpy = Mockito.spy(arrowFormatStatement()); + doReturn(storageReadClient).when(stmtSpy).getBigQueryReadClient(); + doReturn(mock(Future.class)) + .when(stmtSpy) + .populateArrowBufferedQueueFromJobStream( + any(JobId.class), anyLong(), any(BlockingQueue.class), any(BigQueryReadClient.class)); + return stmtSpy; + } + + private static void verifyNoJobStreamReader(BigQueryStatement stmtSpy) throws SQLException { + verify(stmtSpy, Mockito.never()) + .populateArrowBufferedQueueFromJobStream( + any(), anyLong(), any(BlockingQueue.class), any(BigQueryReadClient.class)); + } + + @Test + public void testProcessQueryResponseReadsLargeResultFromJobStream() throws SQLException { + BigQueryStatement stmtSpy = jobStreamStatementSpy(); + + stmtSpy.processQueryResponse("SELECT 1", multiPageResult(5000L)); + + assertThat(stmtSpy.currentResultSet).isInstanceOf(BigQueryArrowResultSet.class); + assertEquals(-1, stmtSpy.getUpdateCount()); + assertEquals( + QueryApiType.QUERY_API_TYPE_READ_API, stmtSpy.currentExecutionBuilder.getQueryApiType()); + // The stream is the result's own job's, read from row 0. + verify(stmtSpy) + .populateArrowBufferedQueueFromJobStream( + eq(STREAM_JOB_ID), eq(0L), any(BlockingQueue.class), eq(storageReadClient)); + // Neither a read session nor the jobs.get call that finds its table. + verify(stmtSpy, Mockito.never()).getReadSession(any()); + verify(bigquery, Mockito.never()).getJob(any(JobId.class)); + verify(stmtSpy, Mockito.never()).processJsonResultSet(any(), any()); + } + + @Test + public void testProcessQueryResponseUsesNoReadSessionInArrowFormat() throws SQLException { + // The High-Throughput API is on too, and its own rule is met: 5 pages of 1,000 rows. + doReturn(true).when(bigQueryConnection).isEnableHighThroughputAPI(); + doReturn(2).when(bigQueryConnection).getHighThroughputActivationRatio(); + BigQueryStatement stmtSpy = jobStreamStatementSpy(); + TableResult result = multiPageResult(5000L); + assertTrue(stmtSpy.useReadAPI(result)); + // Fewer rows than the job stream needs. + stmtSpy.setMaxRows(50); + BigQueryJsonResultSet jsonResultSet = mock(BigQueryJsonResultSet.class); + doReturn(jsonResultSet).when(stmtSpy).processJsonResultSet(eq(result), any()); + + stmtSpy.processQueryResponse("SELECT 1", result); + + assertThat(stmtSpy.currentResultSet).isEqualTo(jsonResultSet); + assertEquals( + QueryApiType.QUERY_API_TYPE_STANDARD_REST_API, + stmtSpy.currentExecutionBuilder.getQueryApiType()); + verify(stmtSpy, Mockito.never()).processArrowResultSet(any(), any()); + verifyNoJobStreamReader(stmtSpy); + } + + @Test + public void testProcessQueryResponseKeepsReadSessionForDestinationTable() throws SQLException { + doReturn(true).when(bigQueryConnection).isEnableHighThroughputAPI(); + doReturn(2).when(bigQueryConnection).getHighThroughputActivationRatio(); + doReturn("destination_table").when(bigQueryConnection).getDestinationTable(); + BigQueryStatement stmtSpy = jobStreamStatementSpy(); + TableResult result = multiPageResult(5000L); + ResultSet readSessionResultSet = mock(ResultSet.class); + doReturn(readSessionResultSet).when(stmtSpy).processArrowResultSet(eq(result), any()); + + stmtSpy.processQueryResponse("SELECT 1", result); + + assertThat(stmtSpy.currentResultSet).isEqualTo(readSessionResultSet); + verifyNoJobStreamReader(stmtSpy); + } + + @Test + public void testProcessQueryResponseFailsWhenJobStreamReaderCannotStart() throws SQLException { + BigQueryStatement stmtSpy = jobStreamStatementSpy(); + Mockito.doThrow(new RejectedExecutionException("The pool is full")) + .when(stmtSpy) + .populateArrowBufferedQueueFromJobStream( + any(JobId.class), anyLong(), any(BlockingQueue.class), any(BigQueryReadClient.class)); + + SQLException thrown = + assertThrows( + SQLException.class, + () -> stmtSpy.processQueryResponse("SELECT 1", multiPageResult(5000L))); + + assertThat(thrown.getMessage()).contains("Unable to allocate background threads"); + // The fallback to JSON is not part of this change. + verify(stmtSpy, Mockito.never()).processJsonResultSet(any(), any()); + } + + @Test + public void testProcessQueryResponseFailsWhenReadClientCannotBeCreated() throws SQLException { + BigQueryStatement stmtSpy = jobStreamStatementSpy(); + BigQueryJdbcRuntimeException noClient = + new BigQueryJdbcRuntimeException("Failed to initialize BigQueryReadClient"); + Mockito.doThrow(noClient).when(stmtSpy).getBigQueryReadClient(); + + SQLException thrown = + assertThrows( + SQLException.class, + () -> stmtSpy.processQueryResponse("SELECT 1", multiPageResult(5000L))); + + assertThat(thrown.getMessage()).contains("Failed to initialize BigQueryReadClient"); + assertEquals(noClient, thrown.getCause()); + verifyNoJobStreamReader(stmtSpy); + verify(stmtSpy, Mockito.never()).processJsonResultSet(any(), any()); + } + + @Test + public void testProcessQueryResponseCancelsReaderWhenResultSetCannotBeBuilt() + throws SQLException { + BigQueryStatement stmtSpy = jobStreamStatementSpy(); + Future reader = mock(Future.class); + doReturn(reader) + .when(stmtSpy) + .populateArrowBufferedQueueFromJobStream( + any(JobId.class), anyLong(), any(BlockingQueue.class), any(BigQueryReadClient.class)); + TableResult result = multiPageResult(5000L); + Mockito.doThrow(new IllegalStateException("No schema")).when(result).getSchema(); + + SQLException thrown = + assertThrows(SQLException.class, () -> stmtSpy.processQueryResponse("SELECT 1", result)); + + assertThat(thrown.getMessage()).contains("No schema"); + // The reader had started, so it must not be left running. + verify(reader).cancel(true); + } + + @Test + public void testCancelDuringJobStreamSetupDoesNotBreakTheResultSet() throws Exception { + BigQueryStatement stmtSpy = jobStreamStatementSpy(); + // cancel() from another thread lands right after the reader starts, and clears the statement's + // state. The stub then queues an end of stream; a real reader would keep reading. + // Known gap, not what this test is for: nothing checks for a cancel between here and the + // result set being stored, so the cancel is lost and the statement keeps the result set. When + // that is fixed, this test must change. + Mockito.doAnswer( + invocation -> { + BlockingQueue queue = invocation.getArgument(2); + stmtSpy.cancel(); + queue.put(BigQueryArrowBatchWrapper.of(null, true)); + return mock(Future.class); + }) + .when(stmtSpy) + .populateArrowBufferedQueueFromJobStream( + any(JobId.class), anyLong(), any(BlockingQueue.class), any(BigQueryReadClient.class)); + + stmtSpy.processQueryResponse("SELECT 1", multiPageResult(5000L)); + + // The result set reads the queue its reader was given: it sees the stream end early and + // reports that. A queue taken from statement state would be null by now, and next() would + // throw NullPointerException. + try (ResultSet resultSet = stmtSpy.getResultSet()) { + SQLException thrown = assertThrows(SQLException.class, resultSet::next); + assertThat(thrown.getCause().getMessage()).contains("ended after 0 of 5000 rows"); + } + } + + @Test + public void testLongRunningJobIsReadFromJobStream() throws Exception { + BigQueryStatement stmtSpy = jobStreamStatementSpy(); + // jobs.query returned before the job finished, so the driver waits on the job. + Job job = + getJobMock( + multiPageResult(5000L), + QueryJobConfiguration.newBuilder(query).build(), + StatementType.SELECT); + doReturn(job).when(bigquery).queryWithTimeout(any(QueryJobConfiguration.class), any(), any()); + + assertTrue(stmtSpy.execute(query)); + + verify(job).getQueryResults(QueryResultsOption.pageSize(1000L)); + assertThat(stmtSpy.getResultSet()).isInstanceOf(BigQueryArrowResultSet.class); + verify(stmtSpy) + .populateArrowBufferedQueueFromJobStream( + eq(STREAM_JOB_ID), eq(0L), any(BlockingQueue.class), eq(storageReadClient)); + } + + @Test + public void testScriptUsesJsonForSmallSelectAndJobStreamForLargeSelect() throws Exception { + BigQueryStatement stmtSpy = jobStreamStatementSpy(); + + TableResult parentResult = mockTableResultWithJob("parent-script-job"); + doReturn(StatementType.SCRIPT).when(parentResult).getStatementType(); + doReturn(parentResult) + .when(bigquery) + .queryWithTimeout(any(QueryJobConfiguration.class), any(), any()); + + JobId largeChildId = STREAM_JOB_ID.toBuilder().setJob("child-large-select").build(); + Job smallChildJob = + createScriptChildJobMock("child-small-select", StatementType.SELECT, 10L, 10L, false, null); + Job largeChildJob = + createScriptChildJobMock(largeChildId, StatementType.SELECT, 5000L, 1000L, true, null); + + @SuppressWarnings("unchecked") + Page childJobsPage = mock(Page.class); + doReturn(Arrays.asList(largeChildJob, smallChildJob)).when(childJobsPage).iterateAll(); + doReturn(childJobsPage) + .when(bigquery) + .listJobs(eq(JobListOption.parentJobId("parent-script-job"))); + + assertTrue(stmtSpy.execute("SELECT 1; SELECT * FROM UNNEST(GENERATE_ARRAY(1, 5000));")); + assertThat(stmtSpy.getResultSet()).isInstanceOf(BigQueryJsonResultSet.class); + verifyNoJobStreamReader(stmtSpy); + + assertTrue(stmtSpy.getMoreResults()); + assertThat(stmtSpy.getResultSet()).isInstanceOf(BigQueryArrowResultSet.class); + // Each child job has its own stream. + verify(stmtSpy) + .populateArrowBufferedQueueFromJobStream( + eq(largeChildId), eq(0L), any(BlockingQueue.class), eq(storageReadClient)); + + assertFalse(stmtSpy.getMoreResults()); + verify(stmtSpy, Mockito.never()).getReadSession(any()); + } + + @Test + public void testJobStreamResultSetReturnsStreamedRows() throws Exception { + BigQueryStatement stmtSpy = Mockito.spy(arrowFormatStatement(2)); + // The test root holds one row: (false, 1). + ReadRowsResponse schemaAndRow = + ReadRowsResponse.newBuilder() + .setArrowSchema(arrowSchema) + .setArrowRecordBatch( + ArrowRecordBatch.newBuilder() + .setSerializedRecordBatch(serializeVectorSchemaRoot(getTestVectorSchemaRoot()))) + .setRowCount(1) + .build(); + ReadRowsResponse row = schemaAndRow.toBuilder().clearArrowSchema().build(); + List requests = new ArrayList<>(); + doReturn(readClientServing(requests, readRowsAttempt(null, schemaAndRow, row))) + .when(stmtSpy) + .getBigQueryReadClient(); + + stmtSpy.processQueryResponse("SELECT 1", multiPageResult(2L)); + + int rows = 0; + try (ResultSet resultSet = stmtSpy.getResultSet()) { + while (resultSet.next()) { + assertFalse(resultSet.getBoolean(1)); + assertEquals(1L, resultSet.getLong(2)); + rows++; + } + } + assertEquals(2, rows); + // One request: the job's stream from row 0, with the Arrow options. + assertEquals(1, requests.size()); + assertEquals(JOB_STREAM_NAME, requests.get(0).getReadStream()); + assertEquals(0L, requests.get(0).getOffset()); + assertTrue(requests.get(0).hasArrowSerializationOptions()); + } + private static FieldValueList createTestRow(boolean first, long second) { return FieldValueList.of( Arrays.asList( @@ -2133,11 +2519,20 @@ private Job createScriptChildJobMock( boolean hasNextPage, Long dmlAffectedRows) throws InterruptedException { - TableResult childResult = mockTableResultWithJob(childId); - doReturn(Schema.of(fieldList)).when(childResult).getSchema(); - doReturn(totalRows).when(childResult).getTotalRows(); + return createScriptChildJobMock( + JobId.of(childId), statementType, totalRows, rowsInPage, hasNextPage, dmlAffectedRows); + } + + private Job createScriptChildJobMock( + JobId childId, + StatementType statementType, + long totalRows, + long rowsInPage, + boolean hasNextPage, + Long dmlAffectedRows) + throws InterruptedException { + TableResult childResult = selectResult(childId, totalRows, hasNextPage); doReturn(rowsInPage).when(childResult).getRowsInPage(); - doReturn(hasNextPage).when(childResult).hasNextPage(); if (dmlAffectedRows != null) { doReturn(dmlAffectedRows).when(childResult).getNumDmlAffectedRows(); } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITBase.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITBase.java index 4ad678312fbb..2952fce3c966 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITBase.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITBase.java @@ -443,11 +443,19 @@ protected static GoogleCredentials getCredentials() throws IOException { new ByteArrayInputStream(authJson.toString().getBytes(StandardCharsets.UTF_8))); } - /** Asserts that the result set was served by the Storage Read (High-Throughput) API. */ + /** + * Asserts that the result set was read in Arrow from the Storage Read API: a read session or a + * job stream. + */ protected static void assertArrowResultSet(ResultSet resultSet) { assertTrue(resultSet.getClass().getName().contains("BigQueryArrowResultSet")); } + /** Asserts that the result set was read as JSON, not from the Storage Read API. */ + protected static void assertJsonResultSet(ResultSet resultSet) { + assertTrue(resultSet.getClass().getName().contains("BigQueryJsonResultSet")); + } + protected int resultSetRowCount(ResultSet resultSet) throws SQLException { int rowCount = 0; while (resultSet.next()) { diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITBigQueryJDBCTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITBigQueryJDBCTest.java index 21906893c7c3..9c6fcb26ce09 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITBigQueryJDBCTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITBigQueryJDBCTest.java @@ -37,6 +37,7 @@ import com.google.cloud.bigquery.exception.BigQueryJdbcSqlSyntaxErrorException; import com.google.cloud.bigquery.jdbc.BigQueryConnection; import com.google.cloud.bigquery.jdbc.BigQueryDriver; +import com.google.cloud.bigquery.jdbc.BigQueryResultSet; import com.google.cloud.bigquery.jdbc.DataSource; import com.google.cloud.bigquery.jdbc.OpenTelemetryJulHandler; import com.google.common.collect.ImmutableMap; @@ -84,6 +85,11 @@ public class ITBigQueryJDBCTest extends ITBase { private static final String BASE_QUERY = "SELECT * FROM bigquery-public-data.new_york_taxi_trips.tlc_yellow_trips_2017 order by" + " trip_distance asc LIMIT %s"; + + /** Rows 1..N in order; use with {@code String.format(ORDERED_ARRAY_QUERY, n)}. */ + private static final String ORDERED_ARRAY_QUERY = + "SELECT n FROM UNNEST(GENERATE_ARRAY(1, %s)) AS n ORDER BY n;"; + private static final Random random = new Random(); private static final int randomNumber = random.nextInt(9999); private static String DATASET; @@ -284,6 +290,128 @@ public void testReadAPIPathLargeWithThresholdNotMet() throws SQLException { connection.close(); } + /** + * Properties that read a result of 1,000 rows or more from the query job's stream. The + * High-Throughput API is off, so an Arrow result set can only come from the job stream. The query + * cache is off, so every run reads a job that ran the query. + */ + private static Properties jobStreamProperties() { + Properties properties = new Properties(); + properties.setProperty("EnableQueryResultsArrowFormat", "1"); + properties.setProperty("EnableHighThroughputAPI", "0"); + properties.setProperty("HighThroughputMinTableSize", "1000"); + properties.setProperty("MaxResults", "300"); + properties.setProperty("UseQueryCache", "0"); + return properties; + } + + /** Asserts that the result set returns 1, 2, ... {@code expectedCnt} in that order, then ends. */ + private static void assertRowsInOrder(ResultSet resultSet, int expectedCnt) throws SQLException { + for (int expected = 1; expected <= expectedCnt; expected++) { + assertTrue(resultSet.next(), "The result ended before row " + expected); + assertEquals(expected, resultSet.getInt(1)); + } + assertFalse(resultSet.next()); + } + + @Test + @Tag("disable_tpc") + public void testJobStreamPathLarge() throws SQLException { + int expectedCnt = 5000; + try (Connection connection = + DriverManager.getConnection(connection_uri, jobStreamProperties()); + Statement statement = connection.createStatement(); + ResultSet resultSet = + statement.executeQuery(String.format(ORDERED_ARRAY_QUERY, expectedCnt))) { + assertArrowResultSet(resultSet); + assertRowsInOrder(resultSet, expectedCnt); + } + } + + @Test + @Tag("disable_tpc") + public void testJobStreamPathWithMaxRows() throws SQLException { + // 2,000 rows are more than the threshold and more than the first page, so the result is + // streamed. The stream holds 5,000 rows and the result set stops at 2,000. + int maxRows = 2000; + try (Connection connection = + DriverManager.getConnection(connection_uri, jobStreamProperties()); + Statement statement = connection.createStatement()) { + statement.setMaxRows(maxRows); + try (ResultSet resultSet = statement.executeQuery(String.format(ORDERED_ARRAY_QUERY, 5000))) { + assertArrowResultSet(resultSet); + assertRowsInOrder(resultSet, maxRows); + } + } + } + + @Test + @Tag("disable_tpc") + public void testJobStreamReadsCachedResult() throws SQLException { + // The second run of a query takes its result from the query cache. That job's stream must + // return the rows too. No other test sends this text, so none shares its cache entry. + int expectedCnt = 3000; + String query = String.format(ORDERED_ARRAY_QUERY, expectedCnt); + Properties properties = jobStreamProperties(); + properties.setProperty("UseQueryCache", "1"); + try (Connection connection = DriverManager.getConnection(connection_uri, properties); + Statement statement = connection.createStatement()) { + try (ResultSet first = statement.executeQuery(query)) { + assertArrowResultSet(first); + assertRowsInOrder(first, expectedCnt); + } + + try (ResultSet cached = statement.executeQuery(query)) { + assertEquals( + Boolean.TRUE, + cached.unwrap(BigQueryResultSet.class).getQueryStatistics().getCacheHit(), + "The second run was not served from the query cache, so no cached result was read"); + assertArrowResultSet(cached); + assertRowsInOrder(cached, expectedCnt); + } + } + } + + @Test + @Tag("disable_tpc") + public void testJobStreamSmallResultUsesJson() throws SQLException { + // 500 rows span two pages of 300, but are fewer than the 1,000 a job stream needs. + int expectedCnt = 500; + try (Connection connection = + DriverManager.getConnection(connection_uri, jobStreamProperties()); + Statement statement = connection.createStatement(); + ResultSet resultSet = + statement.executeQuery(String.format(GENERATE_ARRAY_QUERY, expectedCnt))) { + assertJsonResultSet(resultSet); + assertEquals(expectedCnt, resultSetRowCount(resultSet)); + } + } + + @Test + @Tag("disable_tpc") + public void testJobStreamScriptChildJobs() throws SQLException { + String script = + String.format(GENERATE_ARRAY_QUERY, 10) + String.format(GENERATE_ARRAY_QUERY, 5000); + try (Connection connection = + DriverManager.getConnection(connection_uri, jobStreamProperties()); + Statement statement = connection.createStatement()) { + assertTrue(statement.execute(script)); + try (ResultSet small = statement.getResultSet()) { + assertJsonResultSet(small); + assertEquals(10, resultSetRowCount(small)); + } + + // Each child job of a script has its own stream. + assertTrue(statement.getMoreResults()); + try (ResultSet large = statement.getResultSet()) { + assertArrowResultSet(large); + assertEquals(5000, resultSetRowCount(large)); + } + + assertFalse(statement.getMoreResults()); + } + } + @Test public void testStatelessQueryPathSmall() throws SQLException { Properties jobCreationMode = new Properties(); From ee2fe3f072ec289b1e5853f89eb541aee736c8a1 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Thu, 19 Nov 2026 15:55:46 +0000 Subject: [PATCH 2/3] fix throwing exception --- .../bigquery/jdbc/BigQueryStatement.java | 2 +- .../bigquery/jdbc/BigQueryStatementTest.java | 45 ++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java index 8b0a21e9dfcb..e6ee7fe2abf0 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java @@ -1030,7 +1030,7 @@ private static BigQueryJdbcException arrowResultSetException(Throwable ex) { "Failed to execute query: Unable to allocate background threads to process the query results. Connection-scoped thread pool limit of 100 threads was reached or system is out of memory.", ex); } - return new BigQueryJdbcException(ex.getMessage(), ex); + return new BigQueryJdbcException(ex); } private TableReadOptions buildTableReadOptions() { diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java index a0d7644cf89a..22bab374a95b 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java @@ -45,6 +45,7 @@ import com.google.cloud.bigquery.BigQuery.JobListOption; import com.google.cloud.bigquery.BigQuery.QueryResultsOption; import com.google.cloud.bigquery.BigQuery.TableDataListOption; +import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.DatasetId; @@ -1398,6 +1399,21 @@ public void testProcessArrowResultSetUsesBoundedQueueCapacity() throws SQLExcept defaultQueueCaptor.getValue().remainingCapacity()); } + @Test + public void testProcessArrowResultSetFailureReportsItsCauseOnce() throws SQLException { + BigQueryStatement stmtSpy = Mockito.spy(bigQueryStatement); + doReturn(TABLE_ID).when(stmtSpy).getDestinationTable(any()); + IllegalStateException noSession = new IllegalStateException("No read session"); + Mockito.doThrow(noSession).when(stmtSpy).getReadSession(any(CreateReadSessionRequest.class)); + + SQLException thrown = + assertThrows( + SQLException.class, () -> stmtSpy.processArrowResultSet(mock(TableResult.class), null)); + + assertThat(thrown.getMessage()).isEqualTo("No read session"); + assertThat(thrown.getCause()).isSameInstanceAs(noSession); + } + @Test public void testPopulateArrowBufferedQueueStopsAndCancelsStreamWhenResultSetClosedWhileQueueFull() throws Exception { @@ -2076,12 +2092,39 @@ public void testProcessQueryResponseFailsWhenReadClientCannotBeCreated() throws SQLException.class, () -> stmtSpy.processQueryResponse("SELECT 1", multiPageResult(5000L))); - assertThat(thrown.getMessage()).contains("Failed to initialize BigQueryReadClient"); + // The cause's message is reported once, not repeated. + assertThat(thrown.getMessage()).isEqualTo("Failed to initialize BigQueryReadClient"); assertEquals(noClient, thrown.getCause()); verifyNoJobStreamReader(stmtSpy); verify(stmtSpy, Mockito.never()).processJsonResultSet(any(), any()); } + @Test + public void testProcessQueryResponsePassesOnJdbcExceptionFromJobStreamSetup() + throws SQLException { + BigQueryStatement stmtSpy = jobStreamStatementSpy(); + BigQueryException accessDenied = + new BigQueryException( + 403, "Access Denied", new BigQueryError("accessDenied", null, "Access Denied")); + BigQueryJdbcException failure = new BigQueryJdbcException("Query failed", accessDenied); + Mockito.doThrow(failure) + .when(stmtSpy) + .populateArrowBufferedQueueFromJobStream( + any(JobId.class), anyLong(), any(BlockingQueue.class), any(BigQueryReadClient.class)); + + SQLException thrown = + assertThrows( + SQLException.class, + () -> stmtSpy.processQueryResponse("SELECT 1", multiPageResult(5000L))); + + // The driver's own exception keeps its details and is not nested in a second one. + assertThat(thrown.getMessage()).isEqualTo(failure.getMessage()); + assertThat(thrown.getSQLState()).isEqualTo("42501"); // insufficient privilege + assertThat(thrown.getCause()).isSameInstanceAs(accessDenied); + assertThat(((BigQueryJdbcException) thrown).getBigQueryException()) + .isSameInstanceAs(accessDenied); + } + @Test public void testProcessQueryResponseCancelsReaderWhenResultSetCannotBeBuilt() throws SQLException { From 0c80bb09e11a18437d7af9382c6a65bfdd4ff001 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Thu, 19 Nov 2026 18:50:10 +0000 Subject: [PATCH 3/3] test(bigquery-jdbc): run the permission-denied fallback through processArrowResultSet --- .../bigquery/jdbc/BigQueryStatementTest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java index 22bab374a95b..2eb57d1d4177 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java @@ -914,6 +914,24 @@ public void testProcessQueryResponseNoFallbackOnNonPermissionFailure() throws SQ verify(statementSpy, Mockito.never()).processJsonResultSet(eq(tableResultMock), any()); } + @Test + public void testProcessQueryResponseFallsBackToJsonWhenReadSessionIsDenied() throws SQLException { + BigQueryStatement statementSpy = Mockito.spy(bigQueryStatement); + TableResult tableResultMock = mockTableResultWithJob("job-id"); + doReturn(true).when(statementSpy).useReadAPI(tableResultMock); + doReturn(TABLE_ID).when(statementSpy).getDestinationTable(any()); + // processArrowResultSet is not stubbed: the fallback inspects the exception it really throws. + Mockito.doThrow(mockApiException(StatusCode.Code.PERMISSION_DENIED)) + .when(statementSpy) + .getReadSession(any(CreateReadSessionRequest.class)); + BigQueryJsonResultSet jsonResultSetMock = mock(BigQueryJsonResultSet.class); + doReturn(jsonResultSetMock).when(statementSpy).processJsonResultSet(eq(tableResultMock), any()); + + statementSpy.processQueryResponse("SELECT 1", tableResultMock); + + assertThat(statementSpy.currentResultSet).isEqualTo(jsonResultSetMock); + } + private TableResult mockTableResultWithJob(String jobId) { TableResult tableResult = mock(TableResult.class); doReturn(JobId.of(jobId)).when(tableResult).getJobId();