From dda0cd51f1ac88b3b45c757537d97fe534d0718d Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Thu, 19 Nov 2026 16:46:51 -0500 Subject: [PATCH] fix(bigquery-jdbc): harden Storage Write API writer concurrency and batch lifecycle Stack: PR 1 of 3 (base: `main`) ### Summary - Fix a `Phaser` registration race in `BigQueryJdbcBulkInsertWriter.append()` by registering before submitting `append(...)` and idempotently deregistering on synchronous or callback failure. - Make `BigQueryJdbcBulkInsertWriter.cleanup()` interruptible (`awaitAdvanceInterruptibly`) and add idempotent `closeQuietly()` cleanup on failure. - Keep the connection-cached `BigQueryWriteClient` open across `BigQueryPreparedStatement.executeBatch()` calls, wrap Storage Write API failures in `BigQueryJdbcException`, reject `null` SQL in `BigQueryPreparedStatement`, and clear `batchParameters` on `clearBatch()`, `close()`, and `executeBatch()` completion. ### Testing - `make lint`, `mvn -q -Denforcer.skip=true -Dclirr.skip=true test-compile`, and `make unittest` (148 targeted tests passing; full suite 1305/1306 non-skipped passing, 1 pre-existing `BigQueryJdbcOAuthUtilityTest` P12 env failure; ITs require GCP credentials). --- .../jdbc/BigQueryJdbcBulkInsertWriter.java | 97 ++++-- .../jdbc/BigQueryPreparedStatement.java | 167 ++++++--- .../BigQueryJdbcBulkInsertWriterTest.java | 165 ++++++++- ...BigQueryPreparedStatementWriteApiTest.java | 322 ++++++++++++++++++ 4 files changed, 651 insertions(+), 100 deletions(-) create mode 100644 java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriter.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriter.java index 8bf9c75a6a49..87bbd5a372f1 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriter.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriter.java @@ -34,17 +34,20 @@ import com.google.protobuf.Descriptors.DescriptorValidationException; import java.io.IOException; import java.util.concurrent.Phaser; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import javax.annotation.concurrent.GuardedBy; class BigQueryJdbcBulkInsertWriter { private static final BigQueryJdbcCustomLogger LOG = new BigQueryJdbcCustomLogger(BigQueryJdbcBulkInsertWriter.class.getName()); - private JsonStreamWriter jsonStreamWriter; + JsonStreamWriter jsonStreamWriter; + private volatile String streamName; private final Phaser openRequestCount = new Phaser(1); + private final AtomicBoolean writerClosed = new AtomicBoolean(false); private final Object streamLock = new Object(); - @GuardedBy("lock") + @GuardedBy("streamLock") private RuntimeException error = null; void initialize(TableName parentTable, BigQueryWriteClient client, RetrySettings retrySettings) @@ -59,15 +62,21 @@ void initialize(TableName parentTable, BigQueryWriteClient client, RetrySettings .setWriteStream(stream) .build(); WriteStream writeStream = client.createWriteStream(createWriteStreamRequest); + this.streamName = writeStream.getName(); JsonStreamWriter.Builder jsonStreamWriterBuilder = - JsonStreamWriter.newBuilder(writeStream.getName(), writeStream.getTableSchema(), client); + JsonStreamWriter.newBuilder(this.streamName, writeStream.getTableSchema(), client); if (retrySettings != null) { jsonStreamWriterBuilder.setRetrySettings(retrySettings); } - this.jsonStreamWriter = jsonStreamWriterBuilder.build(); + this.jsonStreamWriter = buildJsonStreamWriter(jsonStreamWriterBuilder); + } + + JsonStreamWriter buildJsonStreamWriter(JsonStreamWriter.Builder builder) + throws DescriptorValidationException, IOException, InterruptedException { + return builder.build(); } void append(JsonArray data, long offset) throws DescriptorValidationException, IOException { @@ -80,17 +89,28 @@ void append(JsonArray data, long offset) throws DescriptorValidationException, I } } - ApiFuture future = jsonStreamWriter.append(data, offset); - ApiFutures.addCallback( - future, new AppendCompleteCallback(this), MoreExecutors.directExecutor()); + // Register before submitting so a synchronously completed future cannot drop the party count + // to zero (which permanently terminates the Phaser) before registration occurs. openRequestCount.register(); + AppendCompleteCallback callback = new AppendCompleteCallback(this); + try { + ApiFuture future = jsonStreamWriter.append(data, offset); + ApiFutures.addCallback(future, callback, MoreExecutors.directExecutor()); + } catch (Throwable t) { + callback.done(); + throw t; + } } - long cleanup(BigQueryWriteClient client) { + long cleanup(BigQueryWriteClient client) throws InterruptedException { LOG.finer("++enter++"); - LOG.fine("Cleaning up bulk insert writer for stream: %s", jsonStreamWriter.getStreamName()); - openRequestCount.arriveAndAwaitAdvance(); - jsonStreamWriter.close(); + String currentStreamName = getStreamName(); + LOG.fine("Cleaning up bulk insert writer for stream: %s", currentStreamName); + try { + openRequestCount.awaitAdvanceInterruptibly(openRequestCount.arrive()); + } finally { + closeStreamWriter(); + } synchronized (this.streamLock) { if (this.error != null) { @@ -98,44 +118,69 @@ long cleanup(BigQueryWriteClient client) { } } - // Finalize the stream. - FinalizeWriteStreamResponse finalizeResponse = - client.finalizeWriteStream(jsonStreamWriter.getStreamName()); - LOG.finer("Rows written: " + finalizeResponse.getRowCount()); + FinalizeWriteStreamResponse finalizeResponse = client.finalizeWriteStream(currentStreamName); + LOG.finer("Rows written: %d", finalizeResponse.getRowCount()); return finalizeResponse.getRowCount(); } + void closeQuietly() { + if (this.jsonStreamWriter != null && this.writerClosed.compareAndSet(false, true)) { + try { + this.jsonStreamWriter.close(); + } catch (RuntimeException ex) { + LOG.warning(ex, "Failed to close JsonStreamWriter during error cleanup."); + } + } + } + + private void closeStreamWriter() { + if (this.jsonStreamWriter != null && this.writerClosed.compareAndSet(false, true)) { + this.jsonStreamWriter.close(); + } + } + String getStreamName() { - return jsonStreamWriter.getStreamName(); + if (this.streamName != null) { + return this.streamName; + } + return this.jsonStreamWriter != null ? this.jsonStreamWriter.getStreamName() : null; } static class AppendCompleteCallback implements ApiFutureCallback { private final BigQueryJdbcBulkInsertWriter parent; + private final AtomicBoolean deregistered = new AtomicBoolean(false); AppendCompleteCallback(BigQueryJdbcBulkInsertWriter parent) { this.parent = parent; } + @Override public void onSuccess(AppendRowsResponse response) { done(); } + @Override public void onFailure(Throwable throwable) { - parent.LOG.warning( - "Append failed for stream %s: %s", parent.getStreamName(), throwable.getMessage()); - synchronized (this.parent.streamLock) { - if (this.parent.error == null) { - StorageException storageException = Exceptions.toStorageException(throwable); - this.parent.error = - (storageException != null) ? storageException : new RuntimeException(throwable); + try { + synchronized (this.parent.streamLock) { + if (this.parent.error == null) { + StorageException storageException = Exceptions.toStorageException(throwable); + this.parent.error = + (storageException != null) ? storageException : new RuntimeException(throwable); + } } + LOG.warning( + "Append failed for stream %s: %s", parent.getStreamName(), throwable.getMessage()); + } finally { + done(); } - done(); } - private void done() { - this.parent.openRequestCount.arriveAndDeregister(); + void done() { + if (this.deregistered.compareAndSet(false, true)) { + this.parent.openRequestCount.arriveAndDeregister(); + } } } } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java index b035dd3f9333..8d079ba3ce2b 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java @@ -21,7 +21,6 @@ import com.google.cloud.Tuple; import com.google.cloud.bigquery.FieldList; import com.google.cloud.bigquery.JobStatistics.QueryStatistics; -import com.google.cloud.bigquery.JobStatistics.QueryStatistics.StatementType; import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.QueryParameterValue; import com.google.cloud.bigquery.Schema; @@ -30,7 +29,10 @@ import com.google.cloud.bigquery.exception.BigQueryJdbcException; import com.google.cloud.bigquery.exception.BigQueryJdbcSqlFeatureNotSupportedException; import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.QueryApiType; import com.google.cloud.bigquery.jdbc.telemetry.v1.StatementExecution; +import com.google.cloud.bigquery.jdbc.telemetry.v1.StatementType; +import com.google.cloud.bigquery.jdbc.telemetry.v1.Status; import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import com.google.cloud.bigquery.storage.v1.BatchCommitWriteStreamsRequest; import com.google.cloud.bigquery.storage.v1.BatchCommitWriteStreamsResponse; @@ -75,7 +77,7 @@ class BigQueryPreparedStatement extends BigQueryStatement implements PreparedSta private static final char POSITIONAL_PARAMETER_CHAR = '?'; // parameterHandler is inherited from BigQueryStatement protected final int parameterCount; - protected String currentQuery; + protected final String currentQuery; private Queue> batchParameters = new LinkedList<>(); // Both hold statistics.getSchema(), which reports different things per statement type: the rows // a SELECT returns, versus the columns an INSERT writes. @@ -85,6 +87,9 @@ class BigQueryPreparedStatement extends BigQueryStatement implements PreparedSta BigQueryPreparedStatement(BigQueryConnection connection, String query) throws SQLException { super(connection); + if (query == null) { + throw new BigQueryJdbcException("SQL query cannot be null."); + } this.currentQuery = query; QueryStatistics queryStatistics = describeQueryQuietly(); this.parameterCount = resolveParameterCount(query, queryStatistics); @@ -159,7 +164,7 @@ private static StandardSQLTypeName toStandardSqlType(String typeName) { // Remembers the table and schema of single-table INSERT targets, for the Storage Write API. // Failure is delegated to executeBatch instead of the constructor. private void captureInsertMetadata(QueryStatistics statistics) { - if (!StatementType.INSERT.equals(statistics.getStatementType()) + if (!QueryStatistics.StatementType.INSERT.equals(statistics.getStatementType()) || statistics.getSchema() == null || statistics.getReferencedTables() == null || statistics.getReferencedTables().stream().distinct().count() != 1) { @@ -176,7 +181,7 @@ private void captureInsertMetadata(QueryStatistics statistics) { // The column names are as they are returned from the query dryRun and may not match the actual // table column names. private void captureResultSchema(QueryStatistics statistics) { - if (StatementType.SELECT.equals(statistics.getStatementType())) { + if (QueryStatistics.StatementType.SELECT.equals(statistics.getStatementType())) { this.resultSchema = statistics.getSchema(); } } @@ -378,16 +383,48 @@ public void addBatch() { this.batchParameters.add(currentParameterList); } + @Override + public void clearBatch() { + super.clearBatch(); + this.batchParameters.clear(); + } + + @Override + public void close() throws SQLException { + try { + super.close(); + } finally { + this.batchParameters.clear(); + } + } + private ArrayList deepCopyParameterList( ArrayList parametersList) { - ArrayList copiedParameterList = new ArrayList<>(); + ArrayList copiedParameterList = new ArrayList<>(parametersList.size()); for (BigQueryJdbcParameter parameter : parametersList) { - BigQueryJdbcParameter newParameter = new BigQueryJdbcParameter(parameter); - copiedParameterList.add(newParameter); + copiedParameterList.add(parameter == null ? null : new BigQueryJdbcParameter(parameter)); } return copiedParameterList; } + private void validateBatchParameters() throws BigQueryJdbcException { + for (ArrayList parameterList : this.batchParameters) { + for (int i = 1; i <= this.parameterCount; i++) { + int arrayIndex = i - 1; + if (parameterList == null + || parameterList.size() <= arrayIndex + || parameterList.get(arrayIndex) == null + || !parameterList.get(arrayIndex).isBound()) { + throw new BigQueryJdbcException( + String.format( + "One or more parameters missing in Prepared statement. No value was set for" + + " parameter %d of %d.", + i, this.parameterCount)); + } + } + } + } + @Override public int[] executeBatch() throws SQLException { validateExecution(); @@ -395,7 +432,15 @@ public int[] executeBatch() throws SQLException { if (this.batchParameters.isEmpty()) { return result; } + try { + validateBatchParameters(); + return executeBatchInternal(result); + } finally { + this.batchParameters.clear(); + } + } + private int[] executeBatchInternal(int[] result) throws SQLException { // The constructor's dry run may have happened before the target table existed. if (this.insertSchema == null && this.querySettings.isUseWriteAPI() @@ -410,37 +455,41 @@ public int[] executeBatch() throws SQLException { if (useWriteAPI()) { long startTime = System.currentTimeMillis(); + int fallbackErrorCode = 1000; StatementExecution.Builder writeApiExecutionBuilder = StatementExecution.newBuilder() - .setStatementType( - com.google.cloud.bigquery.jdbc.telemetry.v1.StatementType.STATEMENT_TYPE_INSERT) - .setQueryApiType( - com.google.cloud.bigquery.jdbc.telemetry.v1.QueryApiType - .QUERY_API_TYPE_WRITE_API); - try (BigQueryWriteClient writeClient = this.connection.getBigQueryWriteClient()) { + .setStatementType(StatementType.STATEMENT_TYPE_INSERT) + .setQueryApiType(QueryApiType.QUERY_API_TYPE_WRITE_API); + try { + // Lifecycle of the cached BigQueryWriteClient belongs to BigQueryConnection.close(). + BigQueryWriteClient writeClient = this.connection.getBigQueryWriteClient(); LOG.info("Using Write API for bulk INSERT operation."); long rowCount = bulkInsertWithWriteAPI(writeClient); int[] insertArray = new int[Math.toIntExact(rowCount)]; Arrays.fill(insertArray, 1); - writeApiExecutionBuilder.setStatus( - com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_SUCCESS); + writeApiExecutionBuilder.setStatus(Status.STATUS_SUCCESS); return insertArray; } catch (InterruptedException e) { Thread.currentThread().interrupt(); - writeApiExecutionBuilder - .setStatus(com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_ERROR) - .setErrorCode(TelemetryManager.extractErrorCode(e)); + fallbackErrorCode = TelemetryManager.extractErrorCode(e); + writeApiExecutionBuilder.setStatus(Status.STATUS_ERROR).setErrorCode(fallbackErrorCode); throw new BigQueryJdbcException("Interrupted during Write API batch", e); - } catch (DescriptorValidationException | IOException e) { - writeApiExecutionBuilder - .setStatus(com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_ERROR) - .setErrorCode(TelemetryManager.extractErrorCode(e)); + } catch (BigQueryJdbcException e) { + fallbackErrorCode = TelemetryManager.extractErrorCode(e); + writeApiExecutionBuilder.setStatus(Status.STATUS_ERROR).setErrorCode(fallbackErrorCode); + throw e; + } catch (DescriptorValidationException | IOException | RuntimeException e) { + fallbackErrorCode = TelemetryManager.extractErrorCode(e); + writeApiExecutionBuilder.setStatus(Status.STATUS_ERROR).setErrorCode(fallbackErrorCode); throw new BigQueryJdbcException("Failed to execute batch with Write API", e); } finally { + if (writeApiExecutionBuilder.getStatus() == Status.STATUS_UNSPECIFIED) { + writeApiExecutionBuilder.setStatus(Status.STATUS_ERROR).setErrorCode(fallbackErrorCode); + } long durationMs = System.currentTimeMillis() - startTime; TelemetryManager.recordStatementExecution(writeApiExecutionBuilder, durationMs); TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); @@ -478,6 +527,10 @@ public int[] executeBatch() throws SQLException { } } + BigQueryJdbcBulkInsertWriter createBulkInsertWriter() { + return new BigQueryJdbcBulkInsertWriter(); + } + private long bulkInsertWithWriteAPI(BigQueryWriteClient bigQueryWriteClient) throws DescriptorValidationException, IOException, @@ -486,52 +539,54 @@ private long bulkInsertWithWriteAPI(BigQueryWriteClient bigQueryWriteClient) LOG.finer("++enter++"); RetrySettings retrySettings = this.connection.getRetrySettings(); - BigQueryJdbcBulkInsertWriter bulkInsertWriter = new BigQueryJdbcBulkInsertWriter(); - bulkInsertWriter.initialize(this.insertTableName, bigQueryWriteClient, retrySettings); - + BigQueryJdbcBulkInsertWriter bulkInsertWriter = createBulkInsertWriter(); + boolean cleanedUp = false; try { + bulkInsertWriter.initialize(this.insertTableName, bigQueryWriteClient, retrySettings); + long offset = 0; JsonArray jsonArray = new JsonArray(); Gson gson = new Gson(); + FieldList fieldLists = this.insertSchema.getFields(); int count = this.batchParameters.size(); for (int i = 0; i < count; i++) { - ArrayList parameterList = this.batchParameters.poll(); - FieldList fieldLists = this.insertSchema.getFields(); - if (fieldLists.size() == parameterList.size()) { - - JsonObject rowObject = createJsonRow(fieldLists, parameterList, gson); - jsonArray.add(rowObject); - - if (jsonArray.size() == this.querySettings.getWriteAPIAppendRowCount() - || this.batchParameters.size() == 0) { - bulkInsertWriter.append(jsonArray, offset); - LOG.finer("Append called "); - offset += jsonArray.size(); - jsonArray = new JsonArray(); - } - } else { + if (fieldLists.size() != parameterList.size()) { throw new BigQueryJdbcException("Mismatch between field count and parameter count."); } + + JsonObject rowObject = createJsonRow(fieldLists, parameterList, gson); + jsonArray.add(rowObject); + + if (jsonArray.size() == this.querySettings.getWriteAPIAppendRowCount() + || this.batchParameters.isEmpty()) { + bulkInsertWriter.append(jsonArray, offset); + LOG.finer("Append called "); + offset += jsonArray.size(); + jsonArray = new JsonArray(); + } } - } catch (BigQueryJdbcException e) { - throw new BigQueryJdbcException("BigQueryJdbcException during bulkInsertWithWriteAPI", e); - } - long rowCount = bulkInsertWriter.cleanup(bigQueryWriteClient); - - BatchCommitWriteStreamsRequest commitRequest = - BatchCommitWriteStreamsRequest.newBuilder() - .setParent(this.insertTableName.toString()) - .addWriteStreams(bulkInsertWriter.getStreamName()) - .build(); - BatchCommitWriteStreamsResponse commitResponse = - bigQueryWriteClient.batchCommitWriteStreams(commitRequest); - if (commitResponse.hasCommitTime() == false) { - throw new BigQueryJdbcException("Error committing the streams"); + long rowCount = bulkInsertWriter.cleanup(bigQueryWriteClient); + cleanedUp = true; + + BatchCommitWriteStreamsRequest commitRequest = + BatchCommitWriteStreamsRequest.newBuilder() + .setParent(this.insertTableName.toString()) + .addWriteStreams(bulkInsertWriter.getStreamName()) + .build(); + BatchCommitWriteStreamsResponse commitResponse = + bigQueryWriteClient.batchCommitWriteStreams(commitRequest); + if (!commitResponse.hasCommitTime()) { + throw new BigQueryJdbcException("Error committing the streams"); + } + LOG.finer("Commit called."); + return rowCount; + } finally { + if (!cleanedUp) { + bulkInsertWriter.closeQuietly(); + } } - LOG.finer("Commit called."); - return rowCount; } static JsonObject createJsonRow( diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriterTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriterTest.java index 98b4bd1392a9..581fb0ebe9b9 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriterTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriterTest.java @@ -16,45 +16,174 @@ package com.google.cloud.bigquery.jdbc; -import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.api.core.ApiFutures; +import com.google.api.core.SettableApiFuture; import com.google.api.gax.core.NoCredentialsProvider; +import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; import com.google.cloud.bigquery.storage.v1.BigQueryWriteSettings; import com.google.cloud.bigquery.storage.v1.CreateWriteStreamRequest; +import com.google.cloud.bigquery.storage.v1.FinalizeWriteStreamResponse; +import com.google.cloud.bigquery.storage.v1.JsonStreamWriter; import com.google.cloud.bigquery.storage.v1.TableName; import com.google.cloud.bigquery.storage.v1.TableSchema; import com.google.cloud.bigquery.storage.v1.WriteStream; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.grpc.Status; +import java.time.Duration; import org.junit.jupiter.api.Test; public class BigQueryJdbcBulkInsertWriterTest { + private static final String STREAM_NAME = + "projects/test-project/datasets/test_dataset/tables/test_table/streams/_pending1"; + + private static JsonArray singleRowBatch() { + JsonArray rows = new JsonArray(); + rows.add(new JsonObject()); + return rows; + } + + private static BigQueryJdbcBulkInsertWriter writerWithMockStream(JsonStreamWriter streamWriter) { + when(streamWriter.getStreamName()).thenReturn(STREAM_NAME); + BigQueryJdbcBulkInsertWriter writer = new BigQueryJdbcBulkInsertWriter(); + writer.jsonStreamWriter = streamWriter; + return writer; + } + @Test public void testInitializePassesClientToJsonStreamWriter() throws Exception { BigQueryWriteClient mockClient = mock(BigQueryWriteClient.class); - BigQueryWriteSettings mockSettings = - BigQueryWriteSettings.newBuilder() - .setCredentialsProvider(NoCredentialsProvider.create()) - .build(); - when(mockClient.getSettings()).thenReturn(mockSettings); - - WriteStream stream = - WriteStream.newBuilder() - .setName( - "projects/test-project/datasets/test_dataset/tables/test_table/streams/_default") - .setTableSchema(TableSchema.newBuilder().build()) - .build(); - when(mockClient.createWriteStream(any(CreateWriteStreamRequest.class))).thenReturn(stream); - - TableName tableName = TableName.of("test-project", "test_dataset", "test_table"); + when(mockClient.getSettings()) + .thenReturn( + BigQueryWriteSettings.newBuilder() + .setCredentialsProvider(NoCredentialsProvider.create()) + .build()); + when(mockClient.createWriteStream(any(CreateWriteStreamRequest.class))) + .thenReturn( + WriteStream.newBuilder() + .setName(STREAM_NAME) + .setTableSchema(TableSchema.getDefaultInstance()) + .build()); + BigQueryJdbcBulkInsertWriter writer = new BigQueryJdbcBulkInsertWriter(); - writer.initialize(tableName, mockClient, null); + writer.initialize(TableName.of("test-project", "test_dataset", "test_table"), mockClient, null); - assertNotNull(writer.getStreamName()); + assertEquals(STREAM_NAME, writer.getStreamName()); verify(mockClient).getSettings(); + writer.closeQuietly(); + } + + @Test + public void testCloseQuietlyIsIdempotentAndSwallowsErrors() { + BigQueryJdbcBulkInsertWriter uninitialized = new BigQueryJdbcBulkInsertWriter(); + assertNull(uninitialized.getStreamName()); + assertDoesNotThrow(uninitialized::closeQuietly); + + JsonStreamWriter mockStreamWriter = mock(JsonStreamWriter.class); + doThrow(new RuntimeException("close boom")).when(mockStreamWriter).close(); + BigQueryJdbcBulkInsertWriter writer = writerWithMockStream(mockStreamWriter); + + assertDoesNotThrow(writer::closeQuietly); + assertDoesNotThrow(writer::closeQuietly); + verify(mockStreamWriter, times(1)).close(); + } + + @Test + public void testAppendImmediateAndAsyncFuturesCoordinatePhaserAndCloseOnce() throws Exception { + JsonStreamWriter mockStreamWriter = mock(JsonStreamWriter.class); + SettableApiFuture delayedSecondBatch = SettableApiFuture.create(); + when(mockStreamWriter.append(any(JsonArray.class), anyLong())) + .thenReturn(ApiFutures.immediateFuture(AppendRowsResponse.getDefaultInstance())) + .thenReturn(delayedSecondBatch); + BigQueryJdbcBulkInsertWriter writer = writerWithMockStream(mockStreamWriter); + + writer.append(singleRowBatch(), 0L); + writer.append(singleRowBatch(), 1L); + Thread completer = + new Thread(() -> delayedSecondBatch.set(AppendRowsResponse.getDefaultInstance())); + completer.start(); + + BigQueryWriteClient mockClient = mock(BigQueryWriteClient.class); + when(mockClient.finalizeWriteStream(STREAM_NAME)) + .thenReturn(FinalizeWriteStreamResponse.newBuilder().setRowCount(2L).build()); + + assertEquals(2L, writer.cleanup(mockClient)); + completer.join(); + writer.closeQuietly(); + verify(mockStreamWriter, times(1)).close(); + } + + @Test + public void testSyncAppendThrowOrNullFutureDeregistersPhaserWithoutHangingCleanup() + throws Exception { + JsonStreamWriter mockStreamWriter = mock(JsonStreamWriter.class); + RuntimeException syncFailure = new IllegalArgumentException("bad serialization"); + when(mockStreamWriter.append(any(JsonArray.class), anyLong())) + .thenThrow(syncFailure) + .thenReturn(null); + BigQueryJdbcBulkInsertWriter writer = writerWithMockStream(mockStreamWriter); + + assertSame( + syncFailure, + assertThrows(RuntimeException.class, () -> writer.append(singleRowBatch(), 0L))); + assertThrows(NullPointerException.class, () -> writer.append(singleRowBatch(), 1L)); + + BigQueryWriteClient mockClient = mock(BigQueryWriteClient.class); + when(mockClient.finalizeWriteStream(STREAM_NAME)) + .thenReturn(FinalizeWriteStreamResponse.newBuilder().setRowCount(0L).build()); + assertEquals( + 0L, assertTimeoutPreemptively(Duration.ofSeconds(2), () -> writer.cleanup(mockClient))); + verify(mockStreamWriter, times(1)).close(); + } + + @Test + public void testCleanupInterruptibleAndAsyncCallbackFailurePropagates() throws Exception { + JsonStreamWriter stalledWriter = mock(JsonStreamWriter.class); + when(stalledWriter.append(any(JsonArray.class), anyLong())) + .thenReturn(SettableApiFuture.create()); + BigQueryJdbcBulkInsertWriter writer = writerWithMockStream(stalledWriter); + writer.append(singleRowBatch(), 0L); + + BigQueryWriteClient mockClient = mock(BigQueryWriteClient.class); + assertTimeoutPreemptively( + Duration.ofSeconds(2), + () -> { + Thread.currentThread().interrupt(); + try { + assertThrows(InterruptedException.class, () -> writer.cleanup(mockClient)); + } finally { + Thread.interrupted(); + } + }); + verify(stalledWriter, times(1)).close(); + verify(mockClient, never()).finalizeWriteStream(anyString()); + + JsonStreamWriter failedWriter = mock(JsonStreamWriter.class); + when(failedWriter.append(any(JsonArray.class), anyLong())) + .thenReturn( + ApiFutures.immediateFailedFuture( + Status.INVALID_ARGUMENT.withDescription("append rejected").asRuntimeException())); + BigQueryJdbcBulkInsertWriter failingBulkWriter = writerWithMockStream(failedWriter); + failingBulkWriter.append(singleRowBatch(), 0L); + assertThrows(RuntimeException.class, () -> failingBulkWriter.cleanup(mockClient)); + verify(failedWriter, times(1)).close(); } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java new file mode 100644 index 000000000000..6ee46e027880 --- /dev/null +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java @@ -0,0 +1,322 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.jdbc; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.core.ApiFutures; +import com.google.api.core.SettableApiFuture; +import com.google.api.services.bigquery.model.QueryParameter; +import com.google.api.services.bigquery.model.QueryParameterType; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.JobStatistics.QueryStatistics; +import com.google.cloud.bigquery.JobStatistics.QueryStatistics.StatementType; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.exception.BigQueryJdbcException; +import com.google.cloud.bigquery.jdbc.telemetry.v1.QueryApiType; +import com.google.cloud.bigquery.jdbc.telemetry.v1.StatementExecution; +import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; +import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1.BatchCommitWriteStreamsRequest; +import com.google.cloud.bigquery.storage.v1.BatchCommitWriteStreamsResponse; +import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; +import com.google.cloud.bigquery.storage.v1.CreateWriteStreamRequest; +import com.google.cloud.bigquery.storage.v1.FinalizeWriteStreamResponse; +import com.google.cloud.bigquery.storage.v1.JsonStreamWriter; +import com.google.cloud.bigquery.storage.v1.TableSchema; +import com.google.cloud.bigquery.storage.v1.WriteStream; +import com.google.gson.JsonArray; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import java.io.IOException; +import java.sql.SQLException; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; + +public class BigQueryPreparedStatementWriteApiTest { + + private static final String STREAM_NAME = "projects/proj/datasets/ds/tables/TAB/streams/_p1"; + + private BigQueryConnection connection; + private BigQuery bigQuery; + + @BeforeEach + public void setUp() { + connection = mock(BigQueryConnection.class); + bigQuery = mock(BigQuery.class); + doReturn(bigQuery).when(connection).getBigQuery(); + doReturn("SQL").when(connection).getQueryDialect(); + doReturn("test-conn-id").when(connection).getConnectionId(); + doReturn(BigQueryConnection.SessionState.empty()).when(connection).getSessionStateSnapshot(); + doReturn(true).when(connection).isEnableWriteAPI(); + doReturn(3).when(connection).getWriteAPIActivationRowCount(); + doReturn(1000).when(connection).getWriteAPIAppendRowCount(); + } + + private static QueryParameter queryParam(String typeName) { + return new QueryParameter().setParameterType(new QueryParameterType().setType(typeName)); + } + + private Job mockInsertDryRunJob(TableId tableId, Schema schema, List parameters) { + Job dryRunJob = mock(Job.class); + QueryStatistics stats = mock(QueryStatistics.class); + doReturn(StatementType.INSERT).when(stats).getStatementType(); + doReturn(schema).when(stats).getSchema(); + doReturn(Collections.singletonList(tableId)).when(stats).getReferencedTables(); + doReturn(parameters).when(stats).getQueryParameters(); + doReturn(stats).when(dryRunJob).getStatistics(); + return dryRunJob; + } + + private BigQueryWriteClient stubWriteClient(long finalizedRows, boolean commitHasTimestamp) + throws SQLException { + BigQueryWriteClient writeClient = mock(BigQueryWriteClient.class); + when(writeClient.createWriteStream(any(CreateWriteStreamRequest.class))) + .thenReturn( + WriteStream.newBuilder() + .setName(STREAM_NAME) + .setTableSchema(TableSchema.getDefaultInstance()) + .build()); + if (finalizedRows >= 0) { + when(writeClient.finalizeWriteStream(STREAM_NAME)) + .thenReturn(FinalizeWriteStreamResponse.newBuilder().setRowCount(finalizedRows).build()); + BatchCommitWriteStreamsResponse.Builder commitResp = + BatchCommitWriteStreamsResponse.newBuilder(); + if (commitHasTimestamp) { + commitResp.getCommitTimeBuilder().setSeconds(1700000000L); + } + when(writeClient.batchCommitWriteStreams(any(BatchCommitWriteStreamsRequest.class))) + .thenReturn(commitResp.build()); + } + doReturn(writeClient).when(connection).getBigQueryWriteClient(); + return writeClient; + } + + private BigQueryPreparedStatement newWriteApiPreparedStatement( + String sql, JsonStreamWriter mockJsonStreamWriter) throws SQLException { + return new BigQueryPreparedStatement(connection, sql) { + @Override + BigQueryJdbcBulkInsertWriter createBulkInsertWriter() { + return new BigQueryJdbcBulkInsertWriter() { + @Override + JsonStreamWriter buildJsonStreamWriter(JsonStreamWriter.Builder builder) { + return mockJsonStreamWriter; + } + }; + } + }; + } + + @Test + public void testConstructorRejectsNullQueryAndBatchLifecycleClearsQueue() throws Exception { + assertThrows( + BigQueryJdbcException.class, () -> new BigQueryPreparedStatement(connection, null)); + + when(bigQuery.create(any(JobInfo.class))).thenThrow(new BigQueryException(400, "dry run off")); + BigQueryPreparedStatement ps = + new BigQueryPreparedStatement(connection, "INSERT INTO TAB (a, b) VALUES (?, ?)"); + ps.setInt(2, 42); + ps.addBatch(); + + BigQueryJdbcException missingEx = assertThrows(BigQueryJdbcException.class, ps::executeBatch); + assertTrue(missingEx.getMessage().contains("parameter 1 of 2")); + assertArrayEquals(new int[0], ps.executeBatch()); + + ps.setString(1, "x"); + ps.setInt(2, 42); + ps.addBatch(); + ps.clearBatch(); + assertArrayEquals(new int[0], ps.executeBatch()); + + ps.addBatch(); + ps.close(); + assertTrue(ps.isClosed()); + } + + @Test + public void testExecuteBatchWriteApiMultiChunkSuccessOffsetsCommitAndTelemetry() + throws Exception { + Schema schema = + Schema.of( + Field.of("id", StandardSQLTypeName.INT64), + Field.of("name", StandardSQLTypeName.STRING)); + doReturn( + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + schema, + Arrays.asList(queryParam("INT64"), queryParam("STRING")))) + .when(bigQuery) + .create(any(JobInfo.class)); + doReturn(2).when(connection).getWriteAPIAppendRowCount(); + + BigQueryWriteClient mockWriteClient = stubWriteClient(5L, true); + JsonStreamWriter mockStreamWriter = mock(JsonStreamWriter.class); + when(mockStreamWriter.append(any(JsonArray.class), anyLong())) + .thenReturn(ApiFutures.immediateFuture(AppendRowsResponse.getDefaultInstance())); + + BigQueryPreparedStatement ps = + newWriteApiPreparedStatement("INSERT INTO TAB (id, name) VALUES (?, ?)", mockStreamWriter); + for (int i = 0; i < 5; i++) { + ps.setInt(1, i + 10); + ps.setString(2, "user-" + i); + ps.addBatch(); + } + + try (MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { + assertArrayEquals(new int[] {1, 1, 1, 1, 1}, ps.executeBatch()); + ArgumentCaptor execCaptor = + ArgumentCaptor.forClass(StatementExecution.Builder.class); + telemetry.verify( + () -> TelemetryManager.recordStatementExecution(execCaptor.capture(), anyLong())); + assertEquals(QueryApiType.QUERY_API_TYPE_WRITE_API, execCaptor.getValue().getQueryApiType()); + assertEquals("STATUS_SUCCESS", execCaptor.getValue().getStatus().name()); + } + + ArgumentCaptor batchCaptor = ArgumentCaptor.forClass(JsonArray.class); + ArgumentCaptor offsetCaptor = ArgumentCaptor.forClass(Long.class); + verify(mockStreamWriter, times(3)).append(batchCaptor.capture(), offsetCaptor.capture()); + assertEquals(Arrays.asList(0L, 2L, 4L), offsetCaptor.getAllValues()); + assertEquals( + "user-0", + batchCaptor.getAllValues().get(0).get(0).getAsJsonObject().get("name").getAsString()); + verify(mockStreamWriter, times(1)).close(); + verify(mockWriteClient, never()).close(); + assertArrayEquals(new int[0], ps.executeBatch()); + } + + @Test + public void testExecuteBatchWriteApiFailurePathsPreserveClientCloseWriterAndRecordError() + throws Exception { + doReturn( + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + Schema.of(Field.of("col1", StandardSQLTypeName.STRING)), + Collections.singletonList(queryParam("STRING")))) + .when(bigQuery) + .create(any(JobInfo.class)); + + // 1. Stream creation RuntimeException wraps in BigQueryJdbcException without closing client. + BigQueryWriteClient initFailClient = mock(BigQueryWriteClient.class); + StatusRuntimeException streamError = + Status.INVALID_ARGUMENT.withDescription("simulated SWA failure").asRuntimeException(); + when(initFailClient.createWriteStream(any(CreateWriteStreamRequest.class))) + .thenThrow(streamError); + doReturn(initFailClient).when(connection).getBigQueryWriteClient(); + BigQueryPreparedStatement psInitFail = + new BigQueryPreparedStatement(connection, "INSERT INTO TAB VALUES (?)"); + for (int i = 0; i < 3; i++) { + psInitFail.setString(1, "r" + i); + psInitFail.addBatch(); + } + assertSame( + streamError, + assertThrows(BigQueryJdbcException.class, psInitFail::executeBatch).getCause()); + verify(initFailClient, never()).close(); + assertArrayEquals(new int[0], psInitFail.executeBatch()); + + // 2. Append IOException closes writer once via closeQuietly without finalizing or closing + // client. + BigQueryWriteClient appendFailClient = stubWriteClient(-1L, false); + JsonStreamWriter ioFailWriter = mock(JsonStreamWriter.class); + IOException brokenPipe = new IOException("broken pipe"); + when(ioFailWriter.append(any(JsonArray.class), anyLong())).thenThrow(brokenPipe); + BigQueryPreparedStatement psIoFail = + newWriteApiPreparedStatement("INSERT INTO TAB VALUES (?)", ioFailWriter); + for (int i = 0; i < 3; i++) { + psIoFail.setString(1, "r" + i); + psIoFail.addBatch(); + } + assertSame( + brokenPipe, assertThrows(BigQueryJdbcException.class, psIoFail::executeBatch).getCause()); + verify(ioFailWriter, times(1)).close(); + verify(appendFailClient, never()).finalizeWriteStream(anyString()); + verify(appendFailClient, never()).close(); + + // 3. Missing commit timestamp throws BigQueryJdbcException and records STATUS_ERROR. + BigQueryWriteClient commitFailClient = stubWriteClient(3L, false); + JsonStreamWriter okAppendWriter = mock(JsonStreamWriter.class); + when(okAppendWriter.append(any(JsonArray.class), anyLong())) + .thenReturn(ApiFutures.immediateFuture(AppendRowsResponse.getDefaultInstance())); + BigQueryPreparedStatement psCommitFail = + newWriteApiPreparedStatement("INSERT INTO TAB VALUES (?)", okAppendWriter); + for (int i = 0; i < 3; i++) { + psCommitFail.setString(1, "r" + i); + psCommitFail.addBatch(); + } + assertTrue( + assertThrows(BigQueryJdbcException.class, psCommitFail::executeBatch) + .getMessage() + .contains("Error committing the streams")); + verify(okAppendWriter, times(1)).close(); + verify(commitFailClient, never()).close(); + + // 4. Interrupted cleanup restores thread interrupt flag and closes writer once. + stubWriteClient(-1L, false); + JsonStreamWriter stalledWriter = mock(JsonStreamWriter.class); + when(stalledWriter.append(any(JsonArray.class), anyLong())) + .thenReturn(SettableApiFuture.create()); + BigQueryPreparedStatement psInterrupt = + newWriteApiPreparedStatement("INSERT INTO TAB VALUES (?)", stalledWriter); + for (int i = 0; i < 3; i++) { + psInterrupt.setString(1, "r" + i); + psInterrupt.addBatch(); + } + assertTimeoutPreemptively( + Duration.ofSeconds(2), + () -> { + Thread.currentThread().interrupt(); + try { + BigQueryJdbcException ex = + assertThrows(BigQueryJdbcException.class, psInterrupt::executeBatch); + assertTrue(ex.getCause() instanceof InterruptedException); + assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + }); + verify(stalledWriter, times(1)).close(); + } +}