From a65725fcfeffe5380d1251e4cc67d4f9d556d571 Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Thu, 19 Nov 2026 09:58:54 -0500 Subject: [PATCH] fix(bigquery-jdbc): gate Storage Write API on direct VALUES inserts and unify write-path data conversion --- .../jdbc/BigQueryJdbcBulkInsertWriter.java | 70 +- .../jdbc/BigQueryParameterHandler.java | 325 +++- .../jdbc/BigQueryPreparedStatement.java | 755 +++++++-- .../bigquery/jdbc/BigQueryStatement.java | 8 +- .../bigquery/jdbc/BigQueryTypeRegistry.java | 126 +- .../BigQueryJdbcBulkInsertWriterTest.java | 158 +- .../jdbc/BigQueryParameterHandlerTest.java | 103 ++ ...BigQueryPreparedStatementWriteApiTest.java | 1430 +++++++++++++++++ .../jdbc/BigQueryTypeRegistryTest.java | 49 + 9 files changed, 2788 insertions(+), 236 deletions(-) create mode 100644 java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriter.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriter.java index 8bf9c75a6a49..46c711869b6e 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriter.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriter.java @@ -34,17 +34,21 @@ import com.google.protobuf.Descriptors.DescriptorValidationException; import java.io.IOException; import java.util.concurrent.Phaser; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import javax.annotation.concurrent.GuardedBy; class BigQueryJdbcBulkInsertWriter { private static final BigQueryJdbcCustomLogger LOG = new BigQueryJdbcCustomLogger(BigQueryJdbcBulkInsertWriter.class.getName()); - private JsonStreamWriter jsonStreamWriter; + JsonStreamWriter jsonStreamWriter; + // Written once during initialize() and read by background gRPC completion callbacks. + private volatile String streamName; private final Phaser openRequestCount = new Phaser(1); + private final AtomicBoolean writerClosed = new AtomicBoolean(false); private final Object streamLock = new Object(); - @GuardedBy("lock") + @GuardedBy("streamLock") private RuntimeException error = null; void initialize(TableName parentTable, BigQueryWriteClient client, RetrySettings retrySettings) @@ -59,15 +63,21 @@ void initialize(TableName parentTable, BigQueryWriteClient client, RetrySettings .setWriteStream(stream) .build(); WriteStream writeStream = client.createWriteStream(createWriteStreamRequest); + this.streamName = writeStream.getName(); JsonStreamWriter.Builder jsonStreamWriterBuilder = - JsonStreamWriter.newBuilder(writeStream.getName(), writeStream.getTableSchema(), client); + JsonStreamWriter.newBuilder(this.streamName, writeStream.getTableSchema(), client); if (retrySettings != null) { jsonStreamWriterBuilder.setRetrySettings(retrySettings); } - this.jsonStreamWriter = jsonStreamWriterBuilder.build(); + this.jsonStreamWriter = buildJsonStreamWriter(jsonStreamWriterBuilder); + } + + JsonStreamWriter buildJsonStreamWriter(JsonStreamWriter.Builder builder) + throws DescriptorValidationException, IOException, InterruptedException { + return builder.build(); } void append(JsonArray data, long offset) throws DescriptorValidationException, IOException { @@ -80,17 +90,29 @@ void append(JsonArray data, long offset) throws DescriptorValidationException, I } } - ApiFuture future = jsonStreamWriter.append(data, offset); + // Register before submitting so a synchronously completed future cannot drop the party count + // to zero (which permanently terminates the Phaser) before registration occurs. + openRequestCount.register(); + ApiFuture future; + try { + future = jsonStreamWriter.append(data, offset); + } catch (Throwable t) { + openRequestCount.arriveAndDeregister(); + throw t; + } ApiFutures.addCallback( future, new AppendCompleteCallback(this), MoreExecutors.directExecutor()); - openRequestCount.register(); } - long cleanup(BigQueryWriteClient client) { + long cleanup(BigQueryWriteClient client) throws InterruptedException { LOG.finer("++enter++"); - LOG.fine("Cleaning up bulk insert writer for stream: %s", jsonStreamWriter.getStreamName()); - openRequestCount.arriveAndAwaitAdvance(); - jsonStreamWriter.close(); + String currentStreamName = getStreamName(); + LOG.fine("Cleaning up bulk insert writer for stream: %s", currentStreamName); + try { + openRequestCount.awaitAdvanceInterruptibly(openRequestCount.arrive()); + } finally { + closeStreamWriter(); + } synchronized (this.streamLock) { if (this.error != null) { @@ -99,14 +121,32 @@ long cleanup(BigQueryWriteClient client) { } // Finalize the stream. - FinalizeWriteStreamResponse finalizeResponse = - client.finalizeWriteStream(jsonStreamWriter.getStreamName()); - LOG.finer("Rows written: " + finalizeResponse.getRowCount()); + FinalizeWriteStreamResponse finalizeResponse = client.finalizeWriteStream(currentStreamName); + LOG.finer("Rows written: %d", finalizeResponse.getRowCount()); return finalizeResponse.getRowCount(); } + void closeQuietly() { + if (this.jsonStreamWriter != null && this.writerClosed.compareAndSet(false, true)) { + try { + this.jsonStreamWriter.close(); + } catch (RuntimeException ex) { + LOG.warning(ex, "Failed to close JsonStreamWriter during error cleanup."); + } + } + } + + private void closeStreamWriter() { + if (this.jsonStreamWriter != null && this.writerClosed.compareAndSet(false, true)) { + this.jsonStreamWriter.close(); + } + } + String getStreamName() { - return jsonStreamWriter.getStreamName(); + if (this.streamName != null) { + return this.streamName; + } + return this.jsonStreamWriter != null ? this.jsonStreamWriter.getStreamName() : null; } static class AppendCompleteCallback implements ApiFutureCallback { @@ -122,7 +162,7 @@ public void onSuccess(AppendRowsResponse response) { } public void onFailure(Throwable throwable) { - parent.LOG.warning( + LOG.warning( "Append failed for stream %s: %s", parent.getStreamName(), throwable.getMessage()); synchronized (this.parent.streamLock) { if (this.parent.error == null) { diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java index 2fc89afbf173..7dcd6af0c13b 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java @@ -17,6 +17,8 @@ package com.google.cloud.bigquery.jdbc; import com.google.cloud.Tuple; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FieldList; import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.QueryParameterValue; import com.google.cloud.bigquery.StandardSQLTypeName; @@ -24,6 +26,10 @@ import com.google.cloud.bigquery.exception.BigQueryJdbcSqlFeatureNotSupportedException; import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; +import com.google.gson.JsonArray; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Date; import java.sql.SQLException; @@ -33,12 +39,14 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; -import java.time.OffsetDateTime; import java.time.ZoneOffset; -import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.format.ResolverStyle; import java.util.ArrayList; +import java.util.List; import java.util.Locale; +import java.util.Queue; class BigQueryParameterHandler { private static final BigQueryJdbcCustomLogger LOG = @@ -54,13 +62,28 @@ class BigQueryParameterHandler { DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSSSSSXXX", Locale.ROOT); private static final DateTimeFormatter TIMESTAMP_NANOS_UTC = DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSSSSSSSSXXX", Locale.ROOT); + // Storage Write API TIMESTAMP(12) proto string fields require >9 fractional digits so + // JsonToProtoMessage matches ISO8601_TIMESTAMP_HIGH_PRECISION_PATTERN without truncating nanos; + // using space + 'Z' matches QueryParameterValue.timestamp() as well. + private static final DateTimeFormatter TIMESTAMP_PICOS_UTC = + DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSSSSSSSS'000'XXX", Locale.ROOT); // DATETIME is a zone-less wall clock; the validator requires six fraction digits and no offset. private static final DateTimeFormatter DATETIME_MICROS = - DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSSSSS", Locale.ROOT); + DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSSSSS", Locale.ROOT) + .withResolverStyle(ResolverStyle.STRICT); private static final DateTimeFormatter DATE_DIGITS = - DateTimeFormatter.ofPattern("uuuu-MM-dd", Locale.ROOT); + DateTimeFormatter.ofPattern("uuuu-MM-dd", Locale.ROOT) + .withResolverStyle(ResolverStyle.STRICT); private static final DateTimeFormatter TIME_MICROS = DateTimeFormatter.ofPattern("HH:mm:ss.SSSSSS", Locale.ROOT); + private static final BigDecimal NUMERIC_MAX = + new BigDecimal("99999999999999999999999999999.999999999"); + private static final BigDecimal BIGNUMERIC_MAX = + new BigDecimal( + "578960446186580977117854925043439539266.34992332820282019728792003956564819967"); + private static final Instant MIN_WRITE_API_INSTANT = Instant.parse("0001-01-01T00:00:00Z"); + private static final Instant MAX_WRITE_API_INSTANT = + Instant.parse("9999-12-31T23:59:59.999999999Z"); /** Wall-clock digits for DATETIME (read as UTC if sent to TIMESTAMP). Nanos are truncated. */ static String formatDateTimeDigits(LocalDateTime value) { @@ -83,14 +106,9 @@ static Tuple> bindingForJavaTime(Object value) { if (value instanceof LocalDateTime) { return Tuple.of(formatDateTimeDigits((LocalDateTime) value), String.class); } - if (value instanceof OffsetDateTime) { - return Tuple.of(Timestamp.from(((OffsetDateTime) value).toInstant()), Timestamp.class); - } - if (value instanceof ZonedDateTime) { - return Tuple.of(Timestamp.from(((ZonedDateTime) value).toInstant()), Timestamp.class); - } - if (value instanceof Instant) { - return Tuple.of(Timestamp.from((Instant) value), Timestamp.class); + Instant instant = BigQueryTypeRegistry.extractInstant(value); + if (instant != null && !(value instanceof Timestamp)) { + return Tuple.of(Timestamp.from(instant), Timestamp.class); } return null; } @@ -128,9 +146,9 @@ QueryJobConfiguration.Builder configureParameters( for (int i = 1; i <= this.parametersArraySize; i++) { int arrayIndex = i - 1; - if (this.parametersList.size() <= arrayIndex - || this.parametersList.get(arrayIndex) == null - || !this.parametersList.get(arrayIndex).isBound()) { + BigQueryJdbcParameter parameter = + arrayIndex < this.parametersList.size() ? this.parametersList.get(arrayIndex) : null; + if (parameter == null || !parameter.isBound()) { throw new BigQueryJdbcException( String.format( "One or more parameters missing in Prepared statement. No value was set for" @@ -138,19 +156,24 @@ QueryJobConfiguration.Builder configureParameters( i, this.parametersArraySize)); } - Object parameterValue = getParameter(i); - StandardSQLTypeName sqlType = getSqlType(i); - parameterValue = - formatValueForQueryParameter(parameterValue, sqlType, this.enableTimestampPicos); + QueryParameterValue queryParameter = + toQueryParameterValue(parameter, this.enableTimestampPicos); LOG.finest( "Parameter %s of type %s at index %s added to QueryJobConfiguration", - parameterValue, sqlType, i); - jobConfigurationBuilder.addPositionalParameter( - QueryParameterValue.of(parameterValue, sqlType)); + queryParameter.getValue(), queryParameter.getType(), i); + jobConfigurationBuilder.addPositionalParameter(queryParameter); } return jobConfigurationBuilder; } + static QueryParameterValue toQueryParameterValue( + BigQueryJdbcParameter parameter, boolean enableTimestampPicos) { + StandardSQLTypeName sqlType = parameter.getSqlType(); + Object parameterValue = + formatValueForQueryParameter(parameter.getValue(), sqlType, enableTimestampPicos); + return QueryParameterValue.of(parameterValue, sqlType); + } + static Object formatValueForQueryParameter( Object parameterValue, StandardSQLTypeName sqlType, boolean enableTimestampPicos) { if (parameterValue == null) { @@ -167,18 +190,9 @@ static Object formatValueForQueryParameter( } if (sqlType == StandardSQLTypeName.TIMESTAMP) { // Reduce to an Instant and render in UTC so digits and 'Z' can never disagree. - if (parameterValue instanceof Timestamp) { - return formatInstantUtc(((Timestamp) parameterValue).toInstant(), enableTimestampPicos); - } - if (parameterValue instanceof OffsetDateTime) { - return formatInstantUtc( - ((OffsetDateTime) parameterValue).toInstant(), enableTimestampPicos); - } - if (parameterValue instanceof ZonedDateTime) { - return formatInstantUtc(((ZonedDateTime) parameterValue).toInstant(), enableTimestampPicos); - } - if (parameterValue instanceof Instant) { - return formatInstantUtc((Instant) parameterValue, enableTimestampPicos); + Instant instant = BigQueryTypeRegistry.extractInstant(parameterValue); + if (instant != null) { + return formatInstantUtc(instant, enableTimestampPicos); } if (parameterValue instanceof String) { String str = ((String) parameterValue).trim(); @@ -487,4 +501,247 @@ StandardSQLTypeName getSqlType(String name) { int getParametersArraySize() { return this.parametersArraySize; } + + private static Object formatParameterValueForField( + Object value, + Field field, + StandardSQLTypeName fallbackSqlType, + boolean enableTimestampPicos) { + if (value == null) { + return null; + } + StandardSQLTypeName targetType = + field != null && field.getType() != null && field.getType().getStandardType() != null + ? field.getType().getStandardType() + : fallbackSqlType; + if (targetType == StandardSQLTypeName.TIMESTAMP) { + if (field != null + && enableTimestampPicos + && BigQueryTemporalUtility.isPicosecondTimestamp(field)) { + Instant instant = BigQueryTypeRegistry.extractInstant(value); + return instant != null + ? TIMESTAMP_PICOS_UTC.format(instant.atOffset(ZoneOffset.UTC)) + : formatValueForQueryParameter(value, targetType, true); + } + return formatValueForQueryParameter(value, targetType, field == null && enableTimestampPicos); + } + return formatValueForQueryParameter(value, targetType, enableTimestampPicos); + } + + static JsonObject createWriteApiJsonRow( + FieldList fieldLists, + List parameterList, + boolean enableTimestampPicos) { + JsonObject rowObject = new JsonObject(); + for (int j = 0; j < parameterList.size(); j++) { + Field field = fieldLists.get(j); + String fieldName = field.getName(); + BigQueryJdbcParameter parameter = parameterList.get(j); + Object formatted = + formatParameterValueForField( + parameter.getValue(), field, parameter.getSqlType(), enableTimestampPicos); + if (formatted == null) { + rowObject.add(fieldName, JsonNull.INSTANCE); + } else if (formatted instanceof byte[]) { + byte[] bytes = (byte[]) formatted; + JsonArray byteArray = new JsonArray(); + for (byte b : bytes) { + byteArray.add(b); + } + rowObject.add(fieldName, byteArray); + } else if (formatted instanceof Boolean) { + rowObject.addProperty(fieldName, (Boolean) formatted); + } else if (formatted instanceof BigDecimal) { + BigDecimal bd = (BigDecimal) formatted; + rowObject.addProperty( + fieldName, bd.signum() == 0 ? "0" : bd.stripTrailingZeros().toPlainString()); + } else { + rowObject.addProperty(fieldName, formatted.toString()); + } + } + return rowObject; + } + + static boolean areBatchParametersCompatibleWithWriteApi( + FieldList fields, Queue> batchParameters) { + for (ArrayList parameterList : batchParameters) { + if (parameterList == null || parameterList.size() != fields.size()) { + return false; + } + for (int i = 0; i < fields.size(); i++) { + Field field = fields.get(i); + BigQueryJdbcParameter parameter = parameterList.get(i); + if (parameter == null || !parameter.isBound()) { + return false; + } + StandardSQLTypeName columnType = field.getType().getStandardType(); + Object value = parameter.getValue(); + StandardSQLTypeName paramSqlType = parameter.getSqlType(); + if (value == null) { + if (field.getMode() == Field.Mode.REQUIRED + || !isSqlTypeAssignableForWriteApi(paramSqlType, columnType)) { + return false; + } + } else if (!isValueAssignableForWriteApi(value, paramSqlType, field, columnType)) { + return false; + } + } + } + return true; + } + + private static boolean isSqlTypeAssignableForWriteApi( + StandardSQLTypeName paramType, StandardSQLTypeName columnType) { + if (paramType == null || columnType == null) { + return false; + } + if (paramType == columnType) { + return true; + } + switch (columnType) { + case NUMERIC: + return paramType == StandardSQLTypeName.INT64; + case BIGNUMERIC: + return paramType == StandardSQLTypeName.INT64 || paramType == StandardSQLTypeName.NUMERIC; + case FLOAT64: + return paramType == StandardSQLTypeName.INT64 + || paramType == StandardSQLTypeName.NUMERIC + || paramType == StandardSQLTypeName.BIGNUMERIC; + case DATETIME: + return paramType == StandardSQLTypeName.STRING; + default: + return false; + } + } + + private static boolean isValueAssignableForWriteApi( + Object value, StandardSQLTypeName paramSqlType, Field field, StandardSQLTypeName columnType) { + if (!isSqlTypeAssignableForWriteApi(paramSqlType, columnType)) { + return false; + } + switch (columnType) { + case STRING: + case BYTES: + case BOOL: + case INT64: + case TIME: + return BigQueryTypeRegistry.isSupportedJavaValue(columnType, value); + case NUMERIC: + case BIGNUMERIC: + StandardSQLTypeName effectiveNumericType = + paramSqlType == StandardSQLTypeName.NUMERIC + || (paramSqlType == StandardSQLTypeName.INT64 + && columnType == StandardSQLTypeName.NUMERIC) + ? StandardSQLTypeName.NUMERIC + : StandardSQLTypeName.BIGNUMERIC; + BigDecimal numericMax = + effectiveNumericType == StandardSQLTypeName.NUMERIC ? NUMERIC_MAX : BIGNUMERIC_MAX; + int maxScale = BigQueryTypeRegistry.getColumnTypeInfo(effectiveNumericType).decimalDigits; + if (paramSqlType == StandardSQLTypeName.INT64) { + return BigQueryTypeRegistry.isSupportedJavaValue(StandardSQLTypeName.INT64, value) + && isValidWriteApiBigDecimal( + BigDecimal.valueOf(((Number) value).longValue()), + maxScale, + numericMax, + field.getPrecision(), + field.getScale()); + } + return BigQueryTypeRegistry.isSupportedJavaValue(columnType, value) + && isValidWriteApiBigDecimal( + (BigDecimal) value, maxScale, numericMax, field.getPrecision(), field.getScale()); + case FLOAT64: + if (paramSqlType == StandardSQLTypeName.FLOAT64) { + return BigQueryTypeRegistry.isSupportedJavaValue(StandardSQLTypeName.FLOAT64, value) + && Double.isFinite(((Number) value).doubleValue()); + } + if (paramSqlType == StandardSQLTypeName.INT64) { + return BigQueryTypeRegistry.isSupportedJavaValue(StandardSQLTypeName.INT64, value); + } + StandardSQLTypeName floatDecimalType = + paramSqlType == StandardSQLTypeName.NUMERIC + ? StandardSQLTypeName.NUMERIC + : StandardSQLTypeName.BIGNUMERIC; + return BigQueryTypeRegistry.isSupportedJavaValue(paramSqlType, value) + && isValidWriteApiBigDecimal( + (BigDecimal) value, + BigQueryTypeRegistry.getColumnTypeInfo(floatDecimalType).decimalDigits, + paramSqlType == StandardSQLTypeName.NUMERIC ? NUMERIC_MAX : BIGNUMERIC_MAX, + null, + null); + case DATE: + if (!BigQueryTypeRegistry.isSupportedJavaValue(StandardSQLTypeName.DATE, value)) { + return false; + } + if (value instanceof LocalDate) { + int year = ((LocalDate) value).getYear(); + return year >= 1 && year <= 9999; + } + String formattedDate = + formatValueForQueryParameter(value, StandardSQLTypeName.DATE, false).toString(); + if (formattedDate.length() != 10) { + return false; + } + try { + int year = LocalDate.parse(formattedDate, DATE_DIGITS).getYear(); + return year >= 1 && year <= 9999; + } catch (DateTimeParseException ex) { + return false; + } + case DATETIME: + if (paramSqlType == StandardSQLTypeName.DATETIME) { + if (!(value instanceof LocalDateTime)) { + return false; + } + int year = ((LocalDateTime) value).getYear(); + return year >= 1 && year <= 9999; + } + if (!(value instanceof String) || ((String) value).length() != 26) { + return false; + } + try { + int year = LocalDateTime.parse((String) value, DATETIME_MICROS).getYear(); + return year >= 1 && year <= 9999; + } catch (DateTimeParseException ex) { + return false; + } + case TIMESTAMP: + if (!BigQueryTypeRegistry.isSupportedJavaValue(StandardSQLTypeName.TIMESTAMP, value)) { + return false; + } + Instant instant = BigQueryTypeRegistry.extractInstant(value); + return instant != null + && !instant.isBefore(MIN_WRITE_API_INSTANT) + && !instant.isAfter(MAX_WRITE_API_INSTANT); + default: + return false; + } + } + + private static boolean isValidWriteApiBigDecimal( + BigDecimal value, + int maxScale, + BigDecimal maxMagnitude, + Long columnPrecision, + Long columnScale) { + BigDecimal normalized = value.signum() == 0 ? BigDecimal.ZERO : value.stripTrailingZeros(); + if (normalized.scale() > maxScale) { + return false; + } + if (columnScale != null && normalized.scale() > columnScale) { + return false; + } + if (columnPrecision != null) { + long effectiveScale = columnScale != null ? columnScale : 0L; + if (normalized.scale() > effectiveScale) { + return false; + } + long maxIntDigits = columnPrecision - effectiveScale; + int intDigits = + normalized.signum() == 0 ? 0 : Math.max(0, normalized.precision() - normalized.scale()); + if (intDigits > maxIntDigits) { + return false; + } + } + return normalized.abs().compareTo(maxMagnitude) <= 0; + } } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java index b035dd3f9333..b62e8fc05359 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java @@ -19,9 +19,10 @@ import com.google.api.gax.retrying.RetrySettings; import com.google.api.services.bigquery.model.QueryParameter; import com.google.cloud.Tuple; +import com.google.cloud.bigquery.DatasetId; +import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.FieldList; import com.google.cloud.bigquery.JobStatistics.QueryStatistics; -import com.google.cloud.bigquery.JobStatistics.QueryStatistics.StatementType; import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.QueryParameterValue; import com.google.cloud.bigquery.Schema; @@ -30,7 +31,10 @@ import com.google.cloud.bigquery.exception.BigQueryJdbcException; import com.google.cloud.bigquery.exception.BigQueryJdbcSqlFeatureNotSupportedException; import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature; +import com.google.cloud.bigquery.jdbc.telemetry.v1.QueryApiType; import com.google.cloud.bigquery.jdbc.telemetry.v1.StatementExecution; +import com.google.cloud.bigquery.jdbc.telemetry.v1.StatementType; +import com.google.cloud.bigquery.jdbc.telemetry.v1.Status; import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; import com.google.cloud.bigquery.storage.v1.BatchCommitWriteStreamsRequest; import com.google.cloud.bigquery.storage.v1.BatchCommitWriteStreamsResponse; @@ -38,7 +42,6 @@ import com.google.cloud.bigquery.storage.v1.TableName; import com.google.gson.Gson; import com.google.gson.JsonArray; -import com.google.gson.JsonNull; import com.google.gson.JsonObject; import com.google.protobuf.Descriptors.DescriptorValidationException; import java.io.IOException; @@ -65,9 +68,15 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Queue; +import java.util.Set; class BigQueryPreparedStatement extends BigQueryStatement implements PreparedStatement { private static final BigQueryJdbcCustomLogger LOG = @@ -75,17 +84,23 @@ class BigQueryPreparedStatement extends BigQueryStatement implements PreparedSta private static final char POSITIONAL_PARAMETER_CHAR = '?'; // parameterHandler is inherited from BigQueryStatement protected final int parameterCount; - protected String currentQuery; + protected final String currentQuery; + private final DirectInsertShape directInsertShape; + private boolean insertMetadataEvaluated = false; private Queue> batchParameters = new LinkedList<>(); - // Both hold statistics.getSchema(), which reports different things per statement type: the rows - // a SELECT returns, versus the columns an INSERT writes. + // resultSchema feeds getMetaData() for SELECT statements; insertSchema holds the target columns + // in placeholder order for single-table direct-VALUES INSERT statements on the Storage Write API. Schema resultSchema = null; // Feeds getMetaData(). Schema insertSchema = null; // Feeds the Storage Write API. private TableName insertTableName = null; BigQueryPreparedStatement(BigQueryConnection connection, String query) throws SQLException { super(connection); + if (query == null) { + throw new BigQueryJdbcException("SQL query cannot be null."); + } this.currentQuery = query; + this.directInsertShape = parseDirectInsertValues(query); QueryStatistics queryStatistics = describeQueryQuietly(); this.parameterCount = resolveParameterCount(query, queryStatistics); this.parameterHandler = @@ -112,7 +127,7 @@ private int resolveParameterCount(String query, QueryStatistics queryStatistics) * of the query text, so a statement stays usable against a service that refuses the dry run. */ private QueryStatistics describeQueryQuietly() { - if (this.currentQuery.indexOf(POSITIONAL_PARAMETER_CHAR) == -1) { + if (this.currentQuery.indexOf(POSITIONAL_PARAMETER_CHAR) == -1 || getUseLegacySql()) { return null; } try { @@ -156,27 +171,397 @@ private static StandardSQLTypeName toStandardSqlType(String typeName) { } } - // Remembers the table and schema of single-table INSERT targets, for the Storage Write API. - // Failure is delegated to executeBatch instead of the constructor. + // Remembers the table and target column schema of direct single-table INSERT ... VALUES (?, ...) + // statements for the Storage Write API. Statements with SQL expressions, partial column lists, or + // unsupported column types leave insertSchema null so executeBatch uses standard SQL execution. private void captureInsertMetadata(QueryStatistics statistics) { - if (!StatementType.INSERT.equals(statistics.getStatementType()) + if (statistics == null) { + return; + } + this.insertMetadataEvaluated = true; + if (this.directInsertShape == null + || !QueryStatistics.StatementType.INSERT.equals(statistics.getStatementType()) || statistics.getSchema() == null || statistics.getReferencedTables() == null || statistics.getReferencedTables().stream().distinct().count() != 1) { return; } - this.insertSchema = statistics.getSchema(); TableId tableId = statistics.getReferencedTables().get(0); - this.insertTableName = - TableName.of(tableId.getProject(), tableId.getDataset(), tableId.getTable()); + String project = resolveTargetProject(tableId); + if (project == null || tableId.getDataset() == null || tableId.getTable() == null) { + return; + } + Schema resolvedSchema = + resolveInsertSchema(statistics.getSchema(), this.directInsertShape, this.parameterCount); + if (resolvedSchema == null) { + LOG.fine( + "INSERT target schema is incompatible with direct Storage Write API streaming;" + + " falling back to standard SQL batch execution."); + return; + } + this.insertSchema = resolvedSchema; + this.insertTableName = TableName.of(project, tableId.getDataset(), tableId.getTable()); LOG.finer("insertTableName: %s, insertSchema: %s", this.insertTableName, this.insertSchema); } + private String resolveTargetProject(TableId tableId) { + if (tableId.getProject() != null) { + return tableId.getProject(); + } + DatasetId defaultDataset = this.querySettings.getDefaultDataset(); + String catalog = this.connection != null ? this.connection.getCatalog() : null; + if (defaultDataset != null + && defaultDataset.getProject() != null + && (catalog == null + || (tableId.getDataset() != null + && tableId.getDataset().equals(defaultDataset.getDataset())))) { + return defaultDataset.getProject(); + } + return catalog; + } + + static Schema resolveInsertSchema( + Schema tableSchema, DirectInsertShape shape, int parameterCount) { + if (tableSchema == null || shape == null) { + return null; + } + FieldList tableFields = tableSchema.getFields(); + if (tableFields == null + || shape.placeholderCount == 0 + || shape.placeholderCount != parameterCount + || tableFields.size() != shape.placeholderCount) { + return null; + } + for (Field field : tableFields) { + if (!BigQueryTypeRegistry.isSupportedWriteApiField(field)) { + return null; + } + } + if (shape.columnNames == null) { + return tableSchema; + } + if (shape.columnNames.size() != tableFields.size()) { + return null; + } + Map fieldsByLowerName = new HashMap<>(tableFields.size() * 2); + for (Field field : tableFields) { + fieldsByLowerName.put(field.getName().toLowerCase(Locale.ROOT), field); + } + List orderedFields = new ArrayList<>(shape.columnNames.size()); + Set seenColumns = new HashSet<>(shape.columnNames.size() * 2); + for (String columnName : shape.columnNames) { + String normalized = columnName.toLowerCase(Locale.ROOT); + if (!seenColumns.add(normalized)) { + return null; + } + Field matched = fieldsByLowerName.get(normalized); + if (matched == null) { + return null; + } + orderedFields.add(matched); + } + return Schema.of(orderedFields); + } + + static final class DirectInsertShape { + final int placeholderCount; + final List columnNames; + + DirectInsertShape(int placeholderCount, List columnNames) { + this.placeholderCount = placeholderCount; + this.columnNames = + columnNames == null ? null : Collections.unmodifiableList(new ArrayList<>(columnNames)); + } + } + + /** + * Recognizes single-row {@code INSERT [INTO] [(, ...)] VALUES (?, ...) [;]} + * statements whose {@code VALUES} tuple consists strictly of bare positional {@code ?} + * placeholders. Returns {@code null} for any SQL expression, function/UDF call, cast, literal, + * {@code DEFAULT}, {@code SELECT}, multi-row tuple list, or {@code _SESSION}-qualified temporary + * table so the statement executes on BigQuery's SQL engine instead of the Storage Write API. + */ + static DirectInsertShape parseDirectInsertValues(String sql) { + if (sql == null) { + return null; + } + int pos = skipTrivia(sql, 0); + if (pos < 0) { + return null; + } + pos = matchKeyword(sql, pos, "INSERT"); + if (pos < 0) { + return null; + } + pos = skipTrivia(sql, pos); + if (pos < 0) { + return null; + } + int afterInto = matchKeyword(sql, pos, "INTO"); + if (afterInto >= 0) { + pos = skipTrivia(sql, afterInto); + if (pos < 0) { + return null; + } + } + pos = consumeTablePath(sql, pos); + if (pos < 0 || pos >= sql.length()) { + return null; + } + + List columnNames = null; + if (sql.charAt(pos) == '(') { + columnNames = new ArrayList<>(); + pos++; + while (true) { + pos = skipTrivia(sql, pos); + if (pos < 0 || pos >= sql.length()) { + return null; + } + Tuple column = readColumnIdentifier(sql, pos); + if (column == null) { + return null; + } + columnNames.add(column.x()); + pos = skipTrivia(sql, column.y()); + if (pos < 0 || pos >= sql.length()) { + return null; + } + char sep = sql.charAt(pos); + if (sep == ',') { + pos++; + } else if (sep == ')') { + pos++; + break; + } else { + return null; + } + } + } + + pos = skipTrivia(sql, pos); + if (pos < 0) { + return null; + } + pos = matchKeyword(sql, pos, "VALUES"); + if (pos < 0) { + return null; + } + pos = skipTrivia(sql, pos); + if (pos < 0 || pos >= sql.length() || sql.charAt(pos) != '(') { + return null; + } + pos++; + + int placeholderCount = 0; + while (true) { + pos = skipTrivia(sql, pos); + if (pos < 0 || pos >= sql.length() || sql.charAt(pos) != POSITIONAL_PARAMETER_CHAR) { + return null; + } + placeholderCount++; + pos = skipTrivia(sql, pos + 1); + if (pos < 0 || pos >= sql.length()) { + return null; + } + char sep = sql.charAt(pos); + if (sep == ',') { + pos++; + } else if (sep == ')') { + pos++; + break; + } else { + return null; + } + } + + pos = skipTrivia(sql, pos); + if (pos < 0) { + return null; + } + if (pos < sql.length() && sql.charAt(pos) == ';') { + pos = skipTrivia(sql, pos + 1); + if (pos < 0) { + return null; + } + } + if (pos != sql.length()) { + return null; + } + return new DirectInsertShape(placeholderCount, columnNames); + } + + private static int consumeTablePath(String sql, int pos) { + Tuple firstSegment = readTableSegment(sql, pos); + if (firstSegment == null) { + return -1; + } + String firstName = firstSegment.x(); + pos = firstSegment.y(); + int segmentCount = 1; + while (true) { + int next = skipTrivia(sql, pos); + if (next < 0) { + return -1; + } + if (next < sql.length() && (sql.charAt(next) == '.' || sql.charAt(next) == ':')) { + pos = skipTrivia(sql, next + 1); + if (pos < 0) { + return -1; + } + Tuple segment = readTableSegment(sql, pos); + if (segment == null) { + return -1; + } + pos = segment.y(); + segmentCount++; + } else { + pos = next; + break; + } + } + if (segmentCount == 2 && "_session".equalsIgnoreCase(firstName)) { + return -1; + } + if (segmentCount == 1 + && firstName.length() > 9 + && firstName.regionMatches(true, 0, "_session.", 0, 9)) { + return -1; + } + return pos; + } + + private static Tuple readTableSegment(String sql, int pos) { + if (pos >= sql.length()) { + return null; + } + char first = sql.charAt(pos); + if (first == '`') { + return readBacktickIdentifier(sql, pos); + } + if (!isUnquotedTableStart(first)) { + return null; + } + int i = pos + 1; + while (i < sql.length() && isUnquotedTablePart(sql.charAt(i))) { + i++; + } + return Tuple.of(sql.substring(pos, i), i); + } + + private static Tuple readColumnIdentifier(String sql, int pos) { + if (pos >= sql.length()) { + return null; + } + char first = sql.charAt(pos); + if (first == '`') { + return readBacktickIdentifier(sql, pos); + } + if (!isIdentifierStart(first)) { + return null; + } + int i = pos + 1; + while (i < sql.length() && isIdentifierPart(sql.charAt(i))) { + i++; + } + return Tuple.of(sql.substring(pos, i), i); + } + + private static Tuple readBacktickIdentifier(String sql, int pos) { + StringBuilder sb = new StringBuilder(); + int i = pos + 1; + while (i < sql.length()) { + char c = sql.charAt(i); + if (c == '\\') { + if (i + 1 >= sql.length()) { + return null; + } + char escaped = sql.charAt(i + 1); + if (escaped != '\\' && escaped != '`') { + return null; + } + sb.append(escaped); + i += 2; + } else if (c == '`') { + if (sb.length() == 0) { + return null; + } + return Tuple.of(sb.toString(), i + 1); + } else { + sb.append(c); + i++; + } + } + return null; + } + + private static int matchKeyword(String sql, int pos, String keyword) { + int end = pos + keyword.length(); + if (end > sql.length() || !sql.regionMatches(true, pos, keyword, 0, keyword.length())) { + return -1; + } + if (end < sql.length() && isIdentifierPart(sql.charAt(end))) { + return -1; + } + return end; + } + + private static int skipTrivia(String sql, int pos) { + int len = sql.length(); + while (pos < len) { + char ch = sql.charAt(pos); + if (Character.isWhitespace(ch)) { + pos++; + } else if (ch == '#') { + pos = skipToEndOfLine(sql, pos + 1); + } else if (ch == '-' && pos + 1 < len && sql.charAt(pos + 1) == '-') { + pos = skipToEndOfLine(sql, pos + 2); + } else if (ch == '/' && pos + 1 < len && sql.charAt(pos + 1) == '*') { + int closeIdx = sql.indexOf("*/", pos + 2); + if (closeIdx < 0) { + return -1; + } + pos = closeIdx + 2; + } else { + break; + } + } + return pos; + } + + private static int skipToEndOfLine(String sql, int pos) { + int len = sql.length(); + while (pos < len) { + char ch = sql.charAt(pos); + if (ch == '\n' || ch == '\r') { + return pos + 1; + } + pos++; + } + return len; + } + + private static boolean isIdentifierStart(char ch) { + return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || ch == '_'; + } + + private static boolean isIdentifierPart(char ch) { + return isIdentifierStart(ch) || (ch >= '0' && ch <= '9'); + } + + private static boolean isUnquotedTableStart(char ch) { + return isIdentifierPart(ch); + } + + private static boolean isUnquotedTablePart(char ch) { + return isIdentifierPart(ch) || ch == '-'; + } + // This populates the schema of the ResultSet that is being returned by the query. // The column names are as they are returned from the query dryRun and may not match the actual // table column names. private void captureResultSchema(QueryStatistics statistics) { - if (StatementType.SELECT.equals(statistics.getStatementType())) { + if (QueryStatistics.StatementType.SELECT.equals(statistics.getStatementType())) { this.resultSchema = statistics.getSchema(); } } @@ -378,16 +763,48 @@ public void addBatch() { this.batchParameters.add(currentParameterList); } + @Override + public void clearBatch() { + super.clearBatch(); + this.batchParameters.clear(); + } + + @Override + public void close() throws SQLException { + try { + super.close(); + } finally { + this.batchParameters.clear(); + } + } + private ArrayList deepCopyParameterList( ArrayList parametersList) { - ArrayList copiedParameterList = new ArrayList<>(); + ArrayList copiedParameterList = new ArrayList<>(parametersList.size()); for (BigQueryJdbcParameter parameter : parametersList) { - BigQueryJdbcParameter newParameter = new BigQueryJdbcParameter(parameter); - copiedParameterList.add(newParameter); + copiedParameterList.add(parameter == null ? null : new BigQueryJdbcParameter(parameter)); } return copiedParameterList; } + private void validateBatchParameters() throws BigQueryJdbcException { + for (ArrayList parameterList : this.batchParameters) { + for (int i = 1; i <= this.parameterCount; i++) { + int arrayIndex = i - 1; + if (parameterList == null + || parameterList.size() <= arrayIndex + || parameterList.get(arrayIndex) == null + || !parameterList.get(arrayIndex).isBound()) { + throw new BigQueryJdbcException( + String.format( + "One or more parameters missing in Prepared statement. No value was set for" + + " parameter %d of %d.", + i, this.parameterCount)); + } + } + } + } + @Override public int[] executeBatch() throws SQLException { validateExecution(); @@ -396,88 +813,114 @@ public int[] executeBatch() throws SQLException { return result; } - // The constructor's dry run may have happened before the target table existed. - if (this.insertSchema == null - && this.querySettings.isUseWriteAPI() - && this.batchParameters.size() >= this.querySettings.getWriteAPIActivationRowCount()) { - try { - captureInsertMetadata( - getQueryStatistics(getWriteBatchJobConfiguration(this.batchParameters.peek()))); - } catch (SQLException | RuntimeException ex) { - LOG.warning(ex, "Could not describe INSERT target; using the standard batch path."); - } - } - - if (useWriteAPI()) { - long startTime = System.currentTimeMillis(); - StatementExecution.Builder writeApiExecutionBuilder = - StatementExecution.newBuilder() - .setStatementType( - com.google.cloud.bigquery.jdbc.telemetry.v1.StatementType.STATEMENT_TYPE_INSERT) - .setQueryApiType( - com.google.cloud.bigquery.jdbc.telemetry.v1.QueryApiType - .QUERY_API_TYPE_WRITE_API); - try (BigQueryWriteClient writeClient = this.connection.getBigQueryWriteClient()) { - LOG.info("Using Write API for bulk INSERT operation."); - - long rowCount = bulkInsertWithWriteAPI(writeClient); - int[] insertArray = new int[Math.toIntExact(rowCount)]; - Arrays.fill(insertArray, 1); - - writeApiExecutionBuilder.setStatus( - com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_SUCCESS); - - return insertArray; - - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - writeApiExecutionBuilder - .setStatus(com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_ERROR) - .setErrorCode(TelemetryManager.extractErrorCode(e)); - throw new BigQueryJdbcException("Interrupted during Write API batch", e); - } catch (DescriptorValidationException | IOException e) { - writeApiExecutionBuilder - .setStatus(com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_ERROR) - .setErrorCode(TelemetryManager.extractErrorCode(e)); - throw new BigQueryJdbcException("Failed to execute batch with Write API", e); - } finally { - long durationMs = System.currentTimeMillis() - startTime; - TelemetryManager.recordStatementExecution(writeApiExecutionBuilder, durationMs); - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); + try { + validateBatchParameters(); + + // The constructor's dry run may have happened before the target table existed. + if (!this.insertMetadataEvaluated + && this.directInsertShape != null + && this.querySettings.isUseWriteAPI() + && !getUseLegacySql() + && (this.connection == null || !this.connection.isTransactionStarted()) + && this.batchParameters.size() >= this.querySettings.getWriteAPIActivationRowCount()) { + try { + QueryStatistics statistics = describePositionalParameterQuery(this.currentQuery); + this.insertMetadataEvaluated = true; + if (statistics != null) { + applyInferredParameterTypes(statistics); + captureInsertMetadata(statistics); + } + } catch (SQLException | RuntimeException ex) { + LOG.warning(ex, "Could not describe INSERT target; using the standard batch path."); + } } - } else { - try { - LOG.info("Using individual INSERT query runs."); - int count = this.batchParameters.size(); - StringBuilder combinedQuery = new StringBuilder(); - for (int i = 0; i < count; i++) { - - if (this.currentQuery.trim().endsWith(";")) { - combinedQuery.append(this.currentQuery); - } else { - combinedQuery.append(this.currentQuery).append(";"); + if (useWriteAPI()) { + long startTime = System.currentTimeMillis(); + int fallbackErrorCode = 1000; + StatementExecution.Builder writeApiExecutionBuilder = + StatementExecution.newBuilder() + .setStatementType(StatementType.STATEMENT_TYPE_INSERT) + .setQueryApiType(QueryApiType.QUERY_API_TYPE_WRITE_API); + try { + // Lifecycle of the cached BigQueryWriteClient belongs to BigQueryConnection.close(). + BigQueryWriteClient writeClient = this.connection.getBigQueryWriteClient(); + LOG.info("Using Write API for bulk INSERT operation."); + + long rowCount = bulkInsertWithWriteAPI(writeClient); + int[] insertArray = new int[Math.toIntExact(rowCount)]; + Arrays.fill(insertArray, 1); + + writeApiExecutionBuilder.setStatus(Status.STATUS_SUCCESS); + + return insertArray; + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fallbackErrorCode = TelemetryManager.extractErrorCode(e); + writeApiExecutionBuilder.setStatus(Status.STATUS_ERROR).setErrorCode(fallbackErrorCode); + throw new BigQueryJdbcException("Interrupted during Write API batch", e); + } catch (BigQueryJdbcException e) { + fallbackErrorCode = TelemetryManager.extractErrorCode(e); + writeApiExecutionBuilder.setStatus(Status.STATUS_ERROR).setErrorCode(fallbackErrorCode); + throw e; + } catch (DescriptorValidationException | IOException | RuntimeException e) { + fallbackErrorCode = TelemetryManager.extractErrorCode(e); + writeApiExecutionBuilder.setStatus(Status.STATUS_ERROR).setErrorCode(fallbackErrorCode); + throw new BigQueryJdbcException("Failed to execute batch with Write API", e); + } finally { + if (writeApiExecutionBuilder.getStatus() == Status.STATUS_UNSPECIFIED) { + writeApiExecutionBuilder.setStatus(Status.STATUS_ERROR).setErrorCode(fallbackErrorCode); } + long durationMs = System.currentTimeMillis() - startTime; + TelemetryManager.recordStatementExecution(writeApiExecutionBuilder, durationMs); + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); } - // executeBatch in PreparedStatement is used for BulkInsert/DML. - // If not correct Type, fails later. - runQuery( - combinedQuery.toString(), getStandardBatchJobConfiguration(combinedQuery.toString())); - int i = 0; - while (getUpdateCount() != -1 && i < count) { - result[i] = getUpdateCount(); - getMoreResults(); - i++; + + } else { + if (this.querySettings.isUseWriteAPI() + && this.batchParameters.size() >= this.querySettings.getWriteAPIActivationRowCount()) { + LOG.fine( + "Batch query is not eligible for Storage Write API; falling back to standard SQL" + + " batch execution."); + } + try { + LOG.info("Using individual INSERT query runs."); + int count = this.batchParameters.size(); + StringBuilder combinedQuery = new StringBuilder(); + for (int i = 0; i < count; i++) { + if (this.currentQuery.trim().endsWith(";")) { + combinedQuery.append(this.currentQuery); + } else { + combinedQuery.append(this.currentQuery).append(";"); + } + } + // executeBatch in PreparedStatement is used for BulkInsert/DML. + // If not correct Type, fails later. + runQuery( + combinedQuery.toString(), getStandardBatchJobConfiguration(combinedQuery.toString())); + int i = 0; + while (getUpdateCount() != -1 && i < count) { + result[i] = getUpdateCount(); + getMoreResults(); + i++; + } + return result; + } catch (SQLException e) { + throw new BigQueryJdbcException("SQL error during individual INSERT batch", e); + } finally { + TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); } - return result; - } catch (SQLException e) { - throw new BigQueryJdbcException("SQL error during individual INSERT batch", e); - } finally { - TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS); } + } finally { + this.batchParameters.clear(); } } + BigQueryJdbcBulkInsertWriter createBulkInsertWriter() { + return new BigQueryJdbcBulkInsertWriter(); + } + private long bulkInsertWithWriteAPI(BigQueryWriteClient bigQueryWriteClient) throws DescriptorValidationException, IOException, @@ -486,68 +929,67 @@ private long bulkInsertWithWriteAPI(BigQueryWriteClient bigQueryWriteClient) LOG.finer("++enter++"); RetrySettings retrySettings = this.connection.getRetrySettings(); - BigQueryJdbcBulkInsertWriter bulkInsertWriter = new BigQueryJdbcBulkInsertWriter(); - bulkInsertWriter.initialize(this.insertTableName, bigQueryWriteClient, retrySettings); - + BigQueryJdbcBulkInsertWriter bulkInsertWriter = createBulkInsertWriter(); + boolean cleanedUp = false; try { + bulkInsertWriter.initialize(this.insertTableName, bigQueryWriteClient, retrySettings); + long offset = 0; JsonArray jsonArray = new JsonArray(); - Gson gson = new Gson(); + FieldList fieldLists = this.insertSchema.getFields(); int count = this.batchParameters.size(); + boolean timestampPicos = this.isEnableTimestampPicos(); for (int i = 0; i < count; i++) { - ArrayList parameterList = this.batchParameters.poll(); - FieldList fieldLists = this.insertSchema.getFields(); - if (fieldLists.size() == parameterList.size()) { - - JsonObject rowObject = createJsonRow(fieldLists, parameterList, gson); - jsonArray.add(rowObject); - - if (jsonArray.size() == this.querySettings.getWriteAPIAppendRowCount() - || this.batchParameters.size() == 0) { - bulkInsertWriter.append(jsonArray, offset); - LOG.finer("Append called "); - offset += jsonArray.size(); - jsonArray = new JsonArray(); - } - } else { + if (fieldLists.size() != parameterList.size()) { throw new BigQueryJdbcException("Mismatch between field count and parameter count."); } - } - } catch (BigQueryJdbcException e) { - throw new BigQueryJdbcException("BigQueryJdbcException during bulkInsertWithWriteAPI", e); - } - long rowCount = bulkInsertWriter.cleanup(bigQueryWriteClient); + JsonObject rowObject = createJsonRow(fieldLists, parameterList, timestampPicos); + jsonArray.add(rowObject); + + if (jsonArray.size() == this.querySettings.getWriteAPIAppendRowCount() + || this.batchParameters.isEmpty()) { + bulkInsertWriter.append(jsonArray, offset); + LOG.finer("Append called "); + offset += jsonArray.size(); + jsonArray = new JsonArray(); + } + } - BatchCommitWriteStreamsRequest commitRequest = - BatchCommitWriteStreamsRequest.newBuilder() - .setParent(this.insertTableName.toString()) - .addWriteStreams(bulkInsertWriter.getStreamName()) - .build(); - BatchCommitWriteStreamsResponse commitResponse = - bigQueryWriteClient.batchCommitWriteStreams(commitRequest); - if (commitResponse.hasCommitTime() == false) { - throw new BigQueryJdbcException("Error committing the streams"); + long rowCount = bulkInsertWriter.cleanup(bigQueryWriteClient); + cleanedUp = true; + + BatchCommitWriteStreamsRequest commitRequest = + BatchCommitWriteStreamsRequest.newBuilder() + .setParent(this.insertTableName.toString()) + .addWriteStreams(bulkInsertWriter.getStreamName()) + .build(); + BatchCommitWriteStreamsResponse commitResponse = + bigQueryWriteClient.batchCommitWriteStreams(commitRequest); + if (!commitResponse.hasCommitTime()) { + throw new BigQueryJdbcException("Error committing the streams"); + } + LOG.finer("Commit called."); + return rowCount; + } finally { + if (!cleanedUp) { + bulkInsertWriter.closeQuietly(); + } } - LOG.finer("Commit called."); - return rowCount; } static JsonObject createJsonRow( FieldList fieldLists, List parameterList, Gson gson) { - JsonObject rowObject = new JsonObject(); - for (int j = 0; j < parameterList.size(); j++) { - BigQueryJdbcParameter parameter = parameterList.get(j); - if (parameter.getValue() == null) { - rowObject.add(fieldLists.get(j).getName(), JsonNull.INSTANCE); - } else if (parameter.getSqlType() == StandardSQLTypeName.STRING) { - rowObject.addProperty(fieldLists.get(j).getName(), parameter.getValue().toString()); - } else { - rowObject.addProperty(fieldLists.get(j).getName(), gson.toJson(parameter.getValue())); - } - } - return rowObject; + return createJsonRow(fieldLists, parameterList, false); + } + + static JsonObject createJsonRow( + FieldList fieldLists, + List parameterList, + boolean enableTimestampPicos) { + return BigQueryParameterHandler.createWriteApiJsonRow( + fieldLists, parameterList, enableTimestampPicos); } QueryJobConfiguration getWriteBatchJobConfiguration( @@ -567,19 +1009,19 @@ QueryJobConfiguration getStandardBatchJobConfiguration(String query) throws SQLE QueryJobConfiguration.Builder jobConfiguration = getJobConfig(query); jobConfiguration.setParameterMode("POSITIONAL"); jobConfiguration.setPriority(QueryJobConfiguration.Priority.BATCH); + boolean enableTimestampPicos = this.isEnableTimestampPicos(); int index = 0; while (!this.batchParameters.isEmpty()) { ArrayList parameterList = this.batchParameters.poll(); - for (BigQueryJdbcParameter parameter : parameterList) { - Object parameterValue = - BigQueryParameterHandler.formatValueForQueryParameter( - parameter.getValue(), parameter.getSqlType(), this.isEnableTimestampPicos()); - StandardSQLTypeName sqlType = parameter.getSqlType(); + for (int i = 0; i < this.parameterCount; i++) { + BigQueryJdbcParameter parameter = parameterList.get(i); + QueryParameterValue queryParameter = + BigQueryParameterHandler.toQueryParameterValue(parameter, enableTimestampPicos); LOG.finer( "Parameter %s of type %s at index %s added to QueryJobConfiguration", - parameterValue, sqlType, index++); - jobConfiguration.addPositionalParameter(QueryParameterValue.of(parameterValue, sqlType)); + queryParameter.getValue(), queryParameter.getType(), index++); + jobConfiguration.addPositionalParameter(queryParameter); } } return jobConfiguration.build(); @@ -588,9 +1030,32 @@ QueryJobConfiguration getStandardBatchJobConfiguration(String query) throws SQLE Boolean useWriteAPI() { LOG.finer("++enter++"); return this.querySettings.isUseWriteAPI() - // the constructor dry run already confirmed a single-table INSERT && this.insertSchema != null - && this.batchParameters.size() >= this.querySettings.getWriteAPIActivationRowCount(); + && this.batchParameters.size() >= this.querySettings.getWriteAPIActivationRowCount() + && isWriteApiAllowedBySessionAndTransaction() + && BigQueryParameterHandler.areBatchParametersCompatibleWithWriteApi( + this.insertSchema.getFields(), this.batchParameters); + } + + private boolean isWriteApiAllowedBySessionAndTransaction() { + if (this.connection == null) { + return true; + } + if (this.connection.isTransactionStarted()) { + return false; + } + if (this.insertTableName != null && this.insertTableName.getDataset().startsWith("_")) { + BigQueryConnection.SessionState snapshot = this.connection.getSessionStateSnapshot(); + boolean sessionActive = + this.connection.isSessionEnabled() || (snapshot != null && snapshot.sessionInfo != null); + DatasetId defaultDataset = this.querySettings.getDefaultDataset(); + if (sessionActive + && (defaultDataset == null + || !this.insertTableName.getDataset().equals(defaultDataset.getDataset()))) { + return false; + } + } + return true; } @Override diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java index 13181dfc4806..a7f26a58891d 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java @@ -192,7 +192,7 @@ private BigQuerySettings generateBigQuerySettings() throws SQLException { BigQuerySettings.Builder querySettings = BigQuerySettings.newBuilder(); DatasetId defaultDataset = this.connection.getDefaultDataset(); if (defaultDataset != null) { - querySettings.setDefaultDataset(this.connection.defaultDataset); + querySettings.setDefaultDataset(defaultDataset); } Long maxBytesBilled = this.connection.getMaxBytesBilled(); if (maxBytesBilled > 0) { @@ -1634,9 +1634,9 @@ QueryJobConfiguration.Builder getJobConfig(String query) { return queryConfigBuilder; } - private boolean getUseLegacySql() { - return QueryDialectType.BIG_QUERY.equals( - QueryDialectType.valueOf(this.querySettings.getQueryDialect())); + boolean getUseLegacySql() { + String dialect = this.querySettings.getQueryDialect(); + return dialect != null && QueryDialectType.BIG_QUERY.equals(QueryDialectType.valueOf(dialect)); } boolean isEnableTimestampPicos() { diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistry.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistry.java index 45cb14f1d83f..5f3120a2f24c 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistry.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistry.java @@ -16,6 +16,7 @@ package com.google.cloud.bigquery.jdbc; +import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.FieldValue; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.exception.BigQueryJdbcException; @@ -39,9 +40,13 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Base64; +import java.util.Collections; +import java.util.EnumSet; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import org.apache.arrow.vector.PeriodDuration; @@ -54,6 +59,23 @@ final class BigQueryTypeRegistry { private static final TypeDescriptor[] DESCRIPTORS_BY_ORDINAL; private static final Map, TypeDescriptor> DESCRIPTORS_BY_CLASS; private static final Map> DESCRIPTORS_BY_JDBC_TYPE; + private static final Set SUPPORTED_WRITE_API_TYPES = + Collections.unmodifiableSet( + EnumSet.of( + StandardSQLTypeName.STRING, + StandardSQLTypeName.BYTES, + StandardSQLTypeName.INT64, + StandardSQLTypeName.FLOAT64, + StandardSQLTypeName.NUMERIC, + StandardSQLTypeName.BIGNUMERIC, + StandardSQLTypeName.BOOL, + StandardSQLTypeName.DATE, + StandardSQLTypeName.TIME, + StandardSQLTypeName.DATETIME, + StandardSQLTypeName.TIMESTAMP, + StandardSQLTypeName.JSON, + StandardSQLTypeName.GEOGRAPHY, + StandardSQLTypeName.INTERVAL)); private static final DateTimeFormatter TIMESTAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS"); @@ -366,25 +388,19 @@ static TypeDescriptor createTimestampDescriptor() { || targetClass == LocalTime.class || targetClass == Date.class || targetClass == Time.class) { - Instant instant; - if (val instanceof Instant) { - instant = (Instant) val; - } else if (val instanceof OffsetDateTime) { - instant = ((OffsetDateTime) val).toInstant(); - } else if (val instanceof ZonedDateTime) { - instant = ((ZonedDateTime) val).toInstant(); - } else if (val instanceof Timestamp) { - instant = ((Timestamp) val).toInstant(); - } else if (val instanceof java.util.Date) { - instant = Instant.ofEpochMilli(((java.util.Date) val).getTime()); - } else if (val instanceof LocalDateTime) { - instant = ((LocalDateTime) val).toInstant(ZoneOffset.UTC); - } else if (val instanceof Long) { - instant = Instant.EPOCH.plus((Long) val, java.time.temporal.ChronoUnit.MICROS); - } else if (val instanceof String) { - instant = BigQueryTemporalUtility.boxTimestamp((String) val).toInstant(); - } else { - throw new BigQueryJdbcException("Cannot convert to TIMESTAMP: " + val); + Instant instant = extractInstant(val); + if (instant == null) { + if (val instanceof java.util.Date) { + instant = Instant.ofEpochMilli(((java.util.Date) val).getTime()); + } else if (val instanceof LocalDateTime) { + instant = ((LocalDateTime) val).toInstant(ZoneOffset.UTC); + } else if (val instanceof Long) { + instant = Instant.EPOCH.plus((Long) val, ChronoUnit.MICROS); + } else if (val instanceof String) { + instant = BigQueryTemporalUtility.boxTimestamp((String) val).toInstant(); + } else { + throw new BigQueryJdbcException("Cannot convert to TIMESTAMP: " + val); + } } if (isInstantView(targetClass)) { @@ -414,24 +430,21 @@ static TypeDescriptor createTimestampDescriptor() { Timestamp ts; if (val instanceof Timestamp) { ts = (Timestamp) val; - } else if (val instanceof java.util.Date) { - ts = new Timestamp(((java.util.Date) val).getTime()); - } else if (val instanceof Instant) { - ts = Timestamp.from((Instant) val); - } else if (val instanceof OffsetDateTime) { - ts = Timestamp.from(((OffsetDateTime) val).toInstant()); - } else if (val instanceof ZonedDateTime) { - ts = Timestamp.from(((ZonedDateTime) val).toInstant()); - } else if (val instanceof LocalDateTime) { - ts = Timestamp.from(((LocalDateTime) val).toInstant(ZoneOffset.UTC)); - } else if (val instanceof Long) { - ts = - Timestamp.from( - Instant.EPOCH.plus((Long) val, java.time.temporal.ChronoUnit.MICROS)); - } else if (val instanceof String) { - ts = BigQueryTemporalUtility.boxTimestamp((String) val); } else { - throw new BigQueryJdbcException("Cannot convert to TIMESTAMP: " + val); + Instant instant = extractInstant(val); + if (instant != null) { + ts = Timestamp.from(instant); + } else if (val instanceof java.util.Date) { + ts = new Timestamp(((java.util.Date) val).getTime()); + } else if (val instanceof LocalDateTime) { + ts = Timestamp.from(((LocalDateTime) val).toInstant(ZoneOffset.UTC)); + } else if (val instanceof Long) { + ts = Timestamp.from(Instant.EPOCH.plus((Long) val, ChronoUnit.MICROS)); + } else if (val instanceof String) { + ts = BigQueryTemporalUtility.boxTimestamp((String) val); + } else { + throw new BigQueryJdbcException("Cannot convert to TIMESTAMP: " + val); + } } return ts; @@ -998,4 +1011,45 @@ StandardSQLTypeName.ARRAY, new ColumnTypeInfo(Types.ARRAY, "ARRAY", null, null, public static ColumnTypeInfo getColumnTypeInfo(StandardSQLTypeName bqType) { return STANDARD_TYPE_INFO.get(bqType); } + + static boolean isSupportedWriteApiField(Field field) { + return field != null + && field.getMode() != Field.Mode.REPEATED + && (field.getSubFields() == null || field.getSubFields().isEmpty()) + && field.getType() != null + && field.getType().getStandardType() != null + && SUPPORTED_WRITE_API_TYPES.contains(field.getType().getStandardType()); + } + + static boolean isSupportedJavaValue(StandardSQLTypeName bqType, Object value) { + if (bqType == null || value == null) { + return false; + } + int ordinal = bqType.ordinal(); + if (ordinal >= DESCRIPTORS_BY_ORDINAL.length || DESCRIPTORS_BY_ORDINAL[ordinal] == null) { + return false; + } + for (Class supported : DESCRIPTORS_BY_ORDINAL[ordinal].getSupportedJavaTypes()) { + if (supported.isInstance(value)) { + return true; + } + } + return false; + } + + static Instant extractInstant(Object value) { + if (value instanceof Instant) { + return (Instant) value; + } + if (value instanceof OffsetDateTime) { + return ((OffsetDateTime) value).toInstant(); + } + if (value instanceof ZonedDateTime) { + return ((ZonedDateTime) value).toInstant(); + } + if (value instanceof Timestamp) { + return ((Timestamp) value).toInstant(); + } + return null; + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriterTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriterTest.java index 98b4bd1392a9..6fe11d927590 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriterTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcBulkInsertWriterTest.java @@ -16,23 +16,47 @@ package com.google.cloud.bigquery.jdbc; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.api.core.ApiFutures; +import com.google.api.core.SettableApiFuture; import com.google.api.gax.core.NoCredentialsProvider; +import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; import com.google.cloud.bigquery.storage.v1.BigQueryWriteSettings; import com.google.cloud.bigquery.storage.v1.CreateWriteStreamRequest; +import com.google.cloud.bigquery.storage.v1.FinalizeWriteStreamResponse; +import com.google.cloud.bigquery.storage.v1.JsonStreamWriter; import com.google.cloud.bigquery.storage.v1.TableName; import com.google.cloud.bigquery.storage.v1.TableSchema; import com.google.cloud.bigquery.storage.v1.WriteStream; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import java.time.Duration; import org.junit.jupiter.api.Test; public class BigQueryJdbcBulkInsertWriterTest { + private static final String STREAM_NAME = + "projects/test-project/datasets/test_dataset/tables/test_table/streams/_pending1"; + @Test public void testInitializePassesClientToJsonStreamWriter() throws Exception { BigQueryWriteClient mockClient = mock(BigQueryWriteClient.class); @@ -44,8 +68,7 @@ public void testInitializePassesClientToJsonStreamWriter() throws Exception { WriteStream stream = WriteStream.newBuilder() - .setName( - "projects/test-project/datasets/test_dataset/tables/test_table/streams/_default") + .setName(STREAM_NAME) .setTableSchema(TableSchema.newBuilder().build()) .build(); when(mockClient.createWriteStream(any(CreateWriteStreamRequest.class))).thenReturn(stream); @@ -55,6 +78,137 @@ public void testInitializePassesClientToJsonStreamWriter() throws Exception { writer.initialize(tableName, mockClient, null); assertNotNull(writer.getStreamName()); + assertEquals(STREAM_NAME, writer.getStreamName()); verify(mockClient).getSettings(); + writer.closeQuietly(); + } + + @Test + public void testCloseQuietlyBeforeInitializeIsNoOp() { + BigQueryJdbcBulkInsertWriter writer = new BigQueryJdbcBulkInsertWriter(); + assertNull(writer.getStreamName()); + assertDoesNotThrow(writer::closeQuietly); + } + + @Test + public void testCloseQuietlyClosesOnceAndSwallowsRuntimeException() { + BigQueryJdbcBulkInsertWriter writer = new BigQueryJdbcBulkInsertWriter(); + JsonStreamWriter mockStreamWriter = mock(JsonStreamWriter.class); + doThrow(new RuntimeException("close boom")).when(mockStreamWriter).close(); + writer.jsonStreamWriter = mockStreamWriter; + + assertDoesNotThrow(writer::closeQuietly); + assertDoesNotThrow(writer::closeQuietly); + verify(mockStreamWriter, times(1)).close(); + } + + @Test + public void testAppendWithImmediateFuturesDoesNotPrematurelyTerminatePhaser() throws Exception { + BigQueryJdbcBulkInsertWriter writer = new BigQueryJdbcBulkInsertWriter(); + JsonStreamWriter mockStreamWriter = mock(JsonStreamWriter.class); + when(mockStreamWriter.getStreamName()).thenReturn(STREAM_NAME); + writer.jsonStreamWriter = mockStreamWriter; + + SettableApiFuture delayedSecondBatch = SettableApiFuture.create(); + when(mockStreamWriter.append(any(JsonArray.class), anyLong())) + .thenReturn(ApiFutures.immediateFuture(AppendRowsResponse.getDefaultInstance())) + .thenReturn(delayedSecondBatch); + + JsonArray rows = new JsonArray(); + rows.add(new JsonObject()); + // First batch completes synchronously inside addCallback; second batch stays in flight. + writer.append(rows, 0L); + writer.append(rows, 1L); + + // Complete second batch on another thread while cleanup awaits openRequestCount. + Thread completer = + new Thread(() -> delayedSecondBatch.set(AppendRowsResponse.getDefaultInstance())); + completer.start(); + + BigQueryWriteClient mockClient = mock(BigQueryWriteClient.class); + when(mockClient.finalizeWriteStream(STREAM_NAME)) + .thenReturn(FinalizeWriteStreamResponse.newBuilder().setRowCount(2L).build()); + + long rowCount = writer.cleanup(mockClient); + completer.join(); + + assertEquals(2L, rowCount); + verify(mockStreamWriter, times(1)).close(); + // Subsequent closeQuietly after cleanup must not double-close. + writer.closeQuietly(); + verify(mockStreamWriter, times(1)).close(); + } + + @Test + public void testSynchronousAppendFailureDeregistersPhaserAndAllowsCleanCleanup() + throws Exception { + BigQueryJdbcBulkInsertWriter writer = new BigQueryJdbcBulkInsertWriter(); + JsonStreamWriter mockStreamWriter = mock(JsonStreamWriter.class); + when(mockStreamWriter.getStreamName()).thenReturn(STREAM_NAME); + RuntimeException syncFailure = new IllegalArgumentException("bad serialization"); + when(mockStreamWriter.append(any(JsonArray.class), anyLong())).thenThrow(syncFailure); + writer.jsonStreamWriter = mockStreamWriter; + + JsonArray rows = new JsonArray(); + rows.add(new JsonObject()); + + RuntimeException thrown = assertThrows(RuntimeException.class, () -> writer.append(rows, 0L)); + assertSame(syncFailure, thrown); + + BigQueryWriteClient mockClient = mock(BigQueryWriteClient.class); + when(mockClient.finalizeWriteStream(STREAM_NAME)) + .thenReturn(FinalizeWriteStreamResponse.newBuilder().setRowCount(0L).build()); + long rowCount = + assertTimeoutPreemptively(Duration.ofSeconds(2), () -> writer.cleanup(mockClient)); + assertEquals(0L, rowCount); + verify(mockStreamWriter, times(1)).close(); + } + + @Test + public void testCleanupInterruptibleWhenAppendStalls() throws Exception { + BigQueryJdbcBulkInsertWriter writer = new BigQueryJdbcBulkInsertWriter(); + JsonStreamWriter mockStreamWriter = mock(JsonStreamWriter.class); + when(mockStreamWriter.getStreamName()).thenReturn(STREAM_NAME); + SettableApiFuture stalledBatch = SettableApiFuture.create(); + when(mockStreamWriter.append(any(JsonArray.class), anyLong())).thenReturn(stalledBatch); + writer.jsonStreamWriter = mockStreamWriter; + + JsonArray rows = new JsonArray(); + rows.add(new JsonObject()); + writer.append(rows, 0L); + + BigQueryWriteClient mockClient = mock(BigQueryWriteClient.class); + assertTimeoutPreemptively( + Duration.ofSeconds(2), + () -> { + Thread.currentThread().interrupt(); + try { + assertThrows(InterruptedException.class, () -> writer.cleanup(mockClient)); + } finally { + Thread.interrupted(); + } + }); + verify(mockStreamWriter, times(1)).close(); + verify(mockClient, never()).finalizeWriteStream(anyString()); + } + + @Test + public void testAsyncCallbackFailurePropagatesOnCleanupAndClosesStreamOnce() throws Exception { + BigQueryJdbcBulkInsertWriter writer = new BigQueryJdbcBulkInsertWriter(); + JsonStreamWriter mockStreamWriter = mock(JsonStreamWriter.class); + when(mockStreamWriter.getStreamName()).thenReturn(STREAM_NAME); + StatusRuntimeException grpcFailure = + Status.INVALID_ARGUMENT.withDescription("append rejected").asRuntimeException(); + when(mockStreamWriter.append(any(JsonArray.class), anyLong())) + .thenReturn(ApiFutures.immediateFailedFuture(grpcFailure)); + writer.jsonStreamWriter = mockStreamWriter; + + JsonArray rows = new JsonArray(); + rows.add(new JsonObject()); + writer.append(rows, 0L); + + BigQueryWriteClient mockClient = mock(BigQueryWriteClient.class); + assertThrows(RuntimeException.class, () -> writer.cleanup(mockClient)); + verify(mockStreamWriter, times(1)).close(); } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandlerTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandlerTest.java index 496982f42683..a8526439207f 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandlerTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandlerTest.java @@ -17,20 +17,34 @@ package com.google.cloud.bigquery.jdbc; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.QueryParameterValue; +import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.exception.BigQueryJdbcException; import com.google.cloud.bigquery.jdbc.BigQueryParameterHandler.BigQueryStatementParameterType; +import com.google.gson.JsonObject; +import java.sql.Date; +import java.sql.Time; import java.sql.Timestamp; import java.time.Instant; +import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.LocalTime; import java.time.OffsetDateTime; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.Queue; import java.util.TimeZone; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -373,4 +387,93 @@ public void testFormatDateTimeDigits_isZoneFreeAndFixedWidth(String zone) { BigQueryParameterHandler.formatDateTimeDigits( LocalDateTime.of(2025, 1, 1, 12, 0, 0, 123_456_789))); } + + @Test + public void testDoor1AndDoor2ShareConversionAndStrictCalendarValidation() { + Schema schema = + Schema.of( + Field.of("s_int", StandardSQLTypeName.INT64), + Field.of("b_int", StandardSQLTypeName.INT64), + Field.of("f_col", StandardSQLTypeName.FLOAT64), + Field.of("d_col", StandardSQLTypeName.DATE), + Field.of("t_col", StandardSQLTypeName.TIME), + Field.of("dt_col", StandardSQLTypeName.DATETIME), + Field.of("ts6_col", StandardSQLTypeName.TIMESTAMP), + Field.newBuilder("ts12_col", StandardSQLTypeName.TIMESTAMP) + .setTimestampPrecision(12L) + .build()); + + BigQueryParameterHandler handler = new BigQueryParameterHandler(8, true); + handler.setParameter(1, (short) 7, Short.class); + handler.setParameter(2, (byte) 9, Byte.class); + handler.setParameter(3, 0.25f, Float.class); + handler.setParameter(4, LocalDate.of(2026, 11, 18), Date.class); + handler.setParameter(5, LocalTime.of(9, 8, 7, 123_456_000), Time.class); + handler.setParameter( + 6, + BigQueryParameterHandler.formatDateTimeDigits(LocalDateTime.of(2026, 11, 18, 9, 8, 7)), + String.class); + Instant tsInstant = Instant.parse("2026-11-18T19:39:36.123456789Z"); + handler.setParameter(7, Timestamp.from(tsInstant), Timestamp.class); + handler.setParameter(8, Timestamp.from(tsInstant), Timestamp.class); + + Queue> batches = new LinkedList<>(); + batches.add(handler.parametersList); + assertTrue( + BigQueryParameterHandler.areBatchParametersCompatibleWithWriteApi( + schema.getFields(), batches)); + + JsonObject jsonRow = + BigQueryParameterHandler.createWriteApiJsonRow( + schema.getFields(), handler.parametersList, true); + for (int i = 0; i < 6; i++) { + String fieldName = schema.getFields().get(i).getName(); + QueryParameterValue qpv = + BigQueryParameterHandler.toQueryParameterValue(handler.parametersList.get(i), true); + assertEquals(qpv.getValue(), jsonRow.get(fieldName).getAsString()); + } + assertEquals("2026-11-18 19:39:36.123456Z", jsonRow.get("ts6_col").getAsString()); + assertEquals("2026-11-18 19:39:36.123456789000Z", jsonRow.get("ts12_col").getAsString()); + + // ResolverStyle.STRICT rejects non-existent calendar dates like 1500-02-29 or 2026-02-31. + Schema dtSchema = Schema.of(Field.of("dt_col", StandardSQLTypeName.DATETIME)); + for (String badDateTime : + new String[] {"1500-02-29 10:20:30.000000", "2026-02-31 10:20:30.000000"}) { + BigQueryParameterHandler invalidDtHandler = new BigQueryParameterHandler(1); + invalidDtHandler.setParameter(1, badDateTime, String.class); + Queue> invalidBatch = new LinkedList<>(); + invalidBatch.add(invalidDtHandler.parametersList); + assertFalse( + BigQueryParameterHandler.areBatchParametersCompatibleWithWriteApi( + dtSchema.getFields(), invalidBatch)); + } + + // ResolverStyle.STRICT on DATE_DIGITS rejects Julian leap date 1500-02-29 for DATE columns. + Schema dateSchema = Schema.of(Field.of("d_col", StandardSQLTypeName.DATE)); + BigQueryParameterHandler invalidDateHandler = new BigQueryParameterHandler(1); + invalidDateHandler.setParameter(1, Date.valueOf("1500-02-29"), Date.class); + Queue> invalidDateBatch = new LinkedList<>(); + invalidDateBatch.add(invalidDateHandler.parametersList); + assertFalse( + BigQueryParameterHandler.areBatchParametersCompatibleWithWriteApi( + dateSchema.getFields(), invalidDateBatch)); + } + + @Test + public void testConfigureParametersThrowsBigQueryJdbcExceptionWhenParametersUnset() { + BigQueryParameterHandler unsetAllHandler = new BigQueryParameterHandler(2); + QueryJobConfiguration.Builder builder1 = QueryJobConfiguration.newBuilder("SELECT ?, ?"); + BigQueryJdbcException missingFirst = + assertThrows( + BigQueryJdbcException.class, () -> unsetAllHandler.configureParameters(builder1)); + assertTrue(missingFirst.getMessage().contains("parameter 1 of 2")); + + BigQueryParameterHandler partialHandler = new BigQueryParameterHandler(2); + partialHandler.setParameter(1, "only-first", String.class); + QueryJobConfiguration.Builder builder2 = QueryJobConfiguration.newBuilder("SELECT ?, ?"); + BigQueryJdbcException missingSecond = + assertThrows( + BigQueryJdbcException.class, () -> partialHandler.configureParameters(builder2)); + assertTrue(missingSecond.getMessage().contains("parameter 2 of 2")); + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java new file mode 100644 index 000000000000..7d5f092b7df7 --- /dev/null +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java @@ -0,0 +1,1430 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.jdbc; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.core.ApiFutures; +import com.google.api.core.SettableApiFuture; +import com.google.api.services.bigquery.model.QueryParameter; +import com.google.api.services.bigquery.model.QueryParameterType; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.ConnectionProperty; +import com.google.cloud.bigquery.DatasetId; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FieldElementType; +import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.JobStatistics.QueryStatistics; +import com.google.cloud.bigquery.JobStatistics.QueryStatistics.StatementType; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.QueryParameterValue; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableResult; +import com.google.cloud.bigquery.exception.BigQueryJdbcException; +import com.google.cloud.bigquery.jdbc.telemetry.v1.QueryApiType; +import com.google.cloud.bigquery.jdbc.telemetry.v1.StatementExecution; +import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager; +import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1.BQTableSchemaToProtoDescriptor; +import com.google.cloud.bigquery.storage.v1.BatchCommitWriteStreamsRequest; +import com.google.cloud.bigquery.storage.v1.BatchCommitWriteStreamsResponse; +import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; +import com.google.cloud.bigquery.storage.v1.CreateWriteStreamRequest; +import com.google.cloud.bigquery.storage.v1.FinalizeWriteStreamResponse; +import com.google.cloud.bigquery.storage.v1.JsonStreamWriter; +import com.google.cloud.bigquery.storage.v1.JsonToProtoMessage; +import com.google.cloud.bigquery.storage.v1.TableFieldSchema; +import com.google.cloud.bigquery.storage.v1.TableSchema; +import com.google.cloud.bigquery.storage.v1.WriteStream; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.Int64Value; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Date; +import java.sql.SQLException; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.TimeZone; +import java.util.stream.Stream; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; + +public class BigQueryPreparedStatementWriteApiTest { + + private BigQueryConnection connection; + private BigQuery bigQuery; + + @BeforeEach + public void setUp() { + connection = mock(BigQueryConnection.class); + bigQuery = mock(BigQuery.class); + doReturn(bigQuery).when(connection).getBigQuery(); + doReturn("SQL").when(connection).getQueryDialect(); + doReturn("test-conn-id").when(connection).getConnectionId(); + doReturn(BigQueryConnection.SessionState.empty()).when(connection).getSessionStateSnapshot(); + doReturn(true).when(connection).isEnableWriteAPI(); + doReturn(3).when(connection).getWriteAPIActivationRowCount(); + doReturn(1000).when(connection).getWriteAPIAppendRowCount(); + } + + private static QueryParameter queryParam(String typeName) { + return new QueryParameter().setParameterType(new QueryParameterType().setType(typeName)); + } + + private Job mockInsertDryRunJob(TableId tableId, Schema schema, List parameters) { + Job dryRunJob = mock(Job.class); + QueryStatistics stats = mock(QueryStatistics.class); + doReturn(StatementType.INSERT).when(stats).getStatementType(); + doReturn(schema).when(stats).getSchema(); + doReturn(Collections.singletonList(tableId)).when(stats).getReferencedTables(); + doReturn(parameters).when(stats).getQueryParameters(); + doReturn(stats).when(dryRunJob).getStatistics(); + return dryRunJob; + } + + @FunctionalInterface + private interface BatchRowBinder { + void bindRow(int rowIndex) throws SQLException; + } + + private static void populateBatchAndAssertWriteApi( + BigQueryPreparedStatement ps, int rows, BatchRowBinder binder, boolean expectedUseWriteApi) + throws SQLException { + ps.clearBatch(); + for (int i = 0; i < rows; i++) { + binder.bindRow(i); + ps.addBatch(); + } + assertEquals(expectedUseWriteApi, ps.useWriteAPI()); + } + + private BigQueryPreparedStatement newWriteApiPreparedStatement( + String sql, JsonStreamWriter mockJsonStreamWriter) throws SQLException { + return new BigQueryPreparedStatement(connection, sql) { + @Override + BigQueryJdbcBulkInsertWriter createBulkInsertWriter() { + return new BigQueryJdbcBulkInsertWriter() { + @Override + JsonStreamWriter buildJsonStreamWriter(JsonStreamWriter.Builder builder) { + return mockJsonStreamWriter; + } + }; + } + }; + } + + static Stream validDirectInsertQueries() { + return Stream.of( + Arguments.of("INSERT INTO TAB VALUES (?)", 1, null), + Arguments.of( + "insert tab (TARG_FIELD) values (?);", 1, Collections.singletonList("TARG_FIELD")), + Arguments.of( + "/* header */ INSERT INTO `my-proj.my_ds.TAB` (`col_b`, col_a) -- comment\n" + + "VALUES ( ? , ? ) ; # trailing", + 2, + Arrays.asList("col_b", "col_a")), + Arguments.of( + "INSERT INTO proj:ds.tab (c1) -- mac line ending\rVALUES (?)", + 1, + Collections.singletonList("c1")), + Arguments.of( + "INSERT INTO `my\\`tab` (`col\\\\1`, `col\\`2`) VALUES (?, ?)", + 2, + Arrays.asList("col\\1", "col`2"))); + } + + @ParameterizedTest + @MethodSource("validDirectInsertQueries") + public void testParseDirectInsertValuesAcceptsValidForms( + String sql, int expectedPlaceholders, List expectedColumns) { + BigQueryPreparedStatement.DirectInsertShape shape = + BigQueryPreparedStatement.parseDirectInsertValues(sql); + assertNotNull(shape); + assertEquals(expectedPlaceholders, shape.placeholderCount); + assertEquals(expectedColumns, shape.columnNames); + } + + @ParameterizedTest + @ValueSource( + strings = { + "", + " ", + "SELECT ?", + "UPDATE TAB SET a = ? WHERE b = ?", + "MERGE INTO TAB USING (SELECT ? AS v) ON FALSE WHEN NOT MATCHED THEN INSERT (a) VALUES (v)", + "INSERT INTO TAB (TARG_FIELD) VALUES (UDF(CAST(? AS STRING), 'XXXXXX'));", + "INSERT INTO TAB (a) VALUES (CAST(? AS BYTES))", + "INSERT INTO TAB (a) VALUES (UPPER(?))", + "INSERT INTO TAB (a) VALUES (? + 1)", + "INSERT INTO TAB (a) VALUES (-?)", + "INSERT INTO TAB (a) VALUES ((?))", + "INSERT INTO TAB (a, b) VALUES (?, 'constant')", + "INSERT INTO TAB (a, b) VALUES (?, DEFAULT)", + "INSERT INTO TAB (a, b) VALUES (?, NULL)", + "INSERT INTO TAB (a) VALUES (?), (?)", + "INSERT INTO TAB (a) SELECT a FROM TAB WHERE a = ?", + "INSERT INTO _SESSION.temp_tab (a) VALUES (?)", + "INSERT INTO `_SESSION.temp_tab` (a) VALUES (?)", + "INSERT INTO `\\u005fSESSION.temp_tab` (a) VALUES (?)", + "INSERT INTO TAB () VALUES (?)", + "INSERT INTO TAB (s.a) VALUES (?)", + "INSERT INTO TAB (`bad\\nescape`) VALUES (?)", + "INSERT INTO TAB (a) VALUES (?); INSERT INTO TAB (a) VALUES (?)", + "/* unclosed comment INSERT INTO TAB VALUES (?)" + }) + public void testParseDirectInsertValuesRejectsIneligibleSql(String sql) { + assertNull(BigQueryPreparedStatement.parseDirectInsertValues(sql)); + } + + @Test + public void testParseDirectInsertValuesAndConstructorRejectNullQuery() { + assertNull(BigQueryPreparedStatement.parseDirectInsertValues(null)); + assertThrows( + BigQueryJdbcException.class, () -> new BigQueryPreparedStatement(connection, null)); + } + + @Test + public void testSameTypeStringUdfExpressionInValuesFallsBackToStandardSqlBatch() + throws Exception { + String sql = "INSERT INTO TAB (STR_FIELD) VALUES (MASK_UDF(?));"; + Schema tableSchema = Schema.of(Field.of("STR_FIELD", StandardSQLTypeName.STRING)); + Job dryRunJob = + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + tableSchema, + Collections.singletonList(queryParam("STRING"))); + doReturn(dryRunJob).when(bigQuery).create(any(JobInfo.class)); + + TableResult dmlResult = mock(TableResult.class); + doReturn(StatementType.INSERT).when(dmlResult).getStatementType(); + doReturn(1L).when(dmlResult).getNumDmlAffectedRows(); + doReturn(dmlResult).when(bigQuery).queryWithTimeout(any(), any(), any()); + + BigQueryPreparedStatement ps = new BigQueryPreparedStatement(connection, sql); + assertNull(ps.insertSchema); + + for (int i = 0; i < 3; i++) { + ps.setString(1, "secret-" + i); + ps.addBatch(); + } + + assertFalse(ps.useWriteAPI()); + + int[] counts = ps.executeBatch(); + assertEquals(3, counts.length); + verify(connection, never()).getBigQueryWriteClient(); + verify(bigQuery, times(1)).create(any(JobInfo.class)); + + ArgumentCaptor jobCaptor = + ArgumentCaptor.forClass(QueryJobConfiguration.class); + verify(bigQuery).queryWithTimeout(jobCaptor.capture(), any(), any()); + QueryJobConfiguration executedConfig = jobCaptor.getValue(); + assertTrue(executedConfig.getQuery().contains("MASK_UDF(?)")); + assertEquals(3, executedConfig.getPositionalParameters().size()); + assertEquals("secret-0", executedConfig.getPositionalParameters().get(0).getValue()); + assertEquals("secret-1", executedConfig.getPositionalParameters().get(1).getValue()); + assertEquals("secret-2", executedConfig.getPositionalParameters().get(2).getValue()); + } + + @Test + public void testUdfExpressionInValuesFallsBackToStandardSqlBatch() throws Exception { + String sql = "INSERT INTO TAB (TARG_FIELD) VALUES (UDF(CAST(? AS STRING), 'XXXXXX'));"; + Schema tableSchema = Schema.of(Field.of("TARG_FIELD", StandardSQLTypeName.BYTES)); + Job dryRunJob = + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + tableSchema, + Collections.singletonList(queryParam("STRING"))); + doReturn(dryRunJob).when(bigQuery).create(any(JobInfo.class)); + + TableResult dmlResult = mock(TableResult.class); + doReturn(StatementType.INSERT).when(dmlResult).getStatementType(); + doReturn(1L).when(dmlResult).getNumDmlAffectedRows(); + doReturn(dmlResult).when(bigQuery).queryWithTimeout(any(), any(), any()); + + BigQueryPreparedStatement ps = new BigQueryPreparedStatement(connection, sql); + assertNull(ps.insertSchema); + + ps.setString(1, "plaintext-1"); + ps.addBatch(); + ps.setString(1, "plaintext-2"); + ps.addBatch(); + ps.setString(1, "plaintext-3"); + ps.addBatch(); + + assertFalse(ps.useWriteAPI()); + + int[] counts = ps.executeBatch(); + assertEquals(3, counts.length); + verify(connection, never()).getBigQueryWriteClient(); + // Constructor ran one dry run; executeBatch must NOT fire a second wasted dry run. + verify(bigQuery, times(1)).create(any(JobInfo.class)); + + ArgumentCaptor jobCaptor = + ArgumentCaptor.forClass(QueryJobConfiguration.class); + verify(bigQuery).queryWithTimeout(jobCaptor.capture(), any(), any()); + QueryJobConfiguration executedConfig = jobCaptor.getValue(); + assertTrue(executedConfig.getQuery().contains("UDF(CAST(? AS STRING), 'XXXXXX')")); + assertEquals(3, executedConfig.getPositionalParameters().size()); + assertEquals("plaintext-1", executedConfig.getPositionalParameters().get(0).getValue()); + assertEquals("plaintext-2", executedConfig.getPositionalParameters().get(1).getValue()); + assertEquals("plaintext-3", executedConfig.getPositionalParameters().get(2).getValue()); + } + + @Test + public void testResolveInsertSchemaMapsReorderedColumnsAndRejectsSubsetsOrDuplicates() { + Schema tableSchema = + Schema.of( + Field.of("col_a", StandardSQLTypeName.STRING), + Field.of("col_b", StandardSQLTypeName.INT64)); + + BigQueryPreparedStatement.DirectInsertShape reordered = + BigQueryPreparedStatement.parseDirectInsertValues( + "INSERT INTO TAB (COL_B, col_a) VALUES (?, ?)"); + Schema resolved = BigQueryPreparedStatement.resolveInsertSchema(tableSchema, reordered, 2); + assertNotNull(resolved); + assertEquals("col_b", resolved.getFields().get(0).getName()); + assertEquals("col_a", resolved.getFields().get(1).getName()); + + // Partial column list must return null so DDL DEFAULT expressions run on BigQuery's SQL engine. + BigQueryPreparedStatement.DirectInsertShape subset = + BigQueryPreparedStatement.parseDirectInsertValues("INSERT INTO TAB (col_b) VALUES (?)"); + assertNull(BigQueryPreparedStatement.resolveInsertSchema(tableSchema, subset, 1)); + + // Duplicate column names and unknown/pseudo columns must return null. + BigQueryPreparedStatement.DirectInsertShape duplicate = + BigQueryPreparedStatement.parseDirectInsertValues( + "INSERT INTO TAB (col_a, COL_A) VALUES (?, ?)"); + assertNull(BigQueryPreparedStatement.resolveInsertSchema(tableSchema, duplicate, 2)); + + BigQueryPreparedStatement.DirectInsertShape unknown = + BigQueryPreparedStatement.parseDirectInsertValues( + "INSERT INTO TAB (col_a, _PARTITIONTIME) VALUES (?, ?)"); + assertNull(BigQueryPreparedStatement.resolveInsertSchema(tableSchema, unknown, 2)); + } + + @ParameterizedTest + @ValueSource( + strings = { + "STRING", + "BYTES", + "INT64", + "FLOAT64", + "NUMERIC", + "BIGNUMERIC", + "BOOL", + "DATE", + "TIME", + "DATETIME", + "TIMESTAMP", + "JSON", + "GEOGRAPHY", + "INTERVAL" + }) + public void testResolveInsertSchemaAcceptsSupportedScalarTemporalAndSpecialTypes( + String typeName) { + Schema schema = Schema.of(Field.of("c1", StandardSQLTypeName.valueOf(typeName))); + BigQueryPreparedStatement.DirectInsertShape shape = + BigQueryPreparedStatement.parseDirectInsertValues("INSERT INTO TAB (c1) VALUES (?)"); + assertNotNull(BigQueryPreparedStatement.resolveInsertSchema(schema, shape, 1)); + } + + @Test + public void testResolveInsertSchemaRejectsRangeRepeatedAndStructFields() { + Schema repeatedSchema = + Schema.of( + Field.newBuilder("arr", StandardSQLTypeName.STRING) + .setMode(Field.Mode.REPEATED) + .build()); + BigQueryPreparedStatement.DirectInsertShape shape = + BigQueryPreparedStatement.parseDirectInsertValues("INSERT INTO TAB VALUES (?)"); + assertNull(BigQueryPreparedStatement.resolveInsertSchema(repeatedSchema, shape, 1)); + + Schema structSchema = + Schema.of( + Field.of( + "rec", StandardSQLTypeName.STRUCT, Field.of("sub", StandardSQLTypeName.INT64))); + assertNull(BigQueryPreparedStatement.resolveInsertSchema(structSchema, shape, 1)); + + Schema rangeSchema = + Schema.of( + Field.newBuilder("rng", StandardSQLTypeName.RANGE) + .setRangeElementType(FieldElementType.newBuilder().setType("DATE").build()) + .build()); + assertNull(BigQueryPreparedStatement.resolveInsertSchema(rangeSchema, shape, 1)); + } + + @Test + public void testCreateJsonRowEncodesBytesFloatAndBigDecimalAccurately() throws Exception { + Schema schema = + Schema.of( + Field.of("b_col", StandardSQLTypeName.BYTES), + Field.of("f_col", StandardSQLTypeName.FLOAT64), + Field.of("n_zero", StandardSQLTypeName.NUMERIC), + Field.of("n_val", StandardSQLTypeName.NUMERIC), + Field.of("bool_col", StandardSQLTypeName.BOOL)); + Job dryRunJob = + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + schema, + Arrays.asList( + queryParam("BYTES"), + queryParam("FLOAT64"), + queryParam("NUMERIC"), + queryParam("NUMERIC"), + queryParam("BOOL"))); + doReturn(dryRunJob).when(bigQuery).create(any(JobInfo.class)); + + BigQueryPreparedStatement ps = + new BigQueryPreparedStatement(connection, "INSERT INTO TAB VALUES (?, ?, ?, ?, ?)"); + byte[] rawBytes = new byte[] {-128, 0, 127}; + ps.setBytes(1, rawBytes); + ps.setFloat(2, 0.6f); + // Extreme negative scale zero must format in O(1) without allocating 100M zeros. + ps.setBigDecimal(3, BigDecimal.valueOf(0, -100_000_000)); + ps.setBigDecimal(4, new BigDecimal("123.4500")); + ps.setBoolean(5, true); + + JsonObject row = + BigQueryPreparedStatement.createJsonRow( + schema.getFields(), ps.parameterHandler.parametersList, false); + + JsonArray encodedBytes = row.getAsJsonArray("b_col"); + assertEquals(3, encodedBytes.size()); + assertEquals(-128, encodedBytes.get(0).getAsByte()); + assertEquals(0, encodedBytes.get(1).getAsByte()); + assertEquals(127, encodedBytes.get(2).getAsByte()); + assertEquals(Double.toString(((Float) 0.6f).doubleValue()), row.get("f_col").getAsString()); + assertEquals("0", row.get("n_zero").getAsString()); + assertEquals("123.45", row.get("n_val").getAsString()); + assertTrue(row.get("bool_col").getAsBoolean()); + } + + @Test + public void testTemporalCreateJsonRowAndProtoConversionAcrossLocalesAndZones() throws Exception { + Schema schema = + Schema.of( + Field.of("d_sql", StandardSQLTypeName.DATE), + Field.of("d_local", StandardSQLTypeName.DATE), + Field.of("t_sql", StandardSQLTypeName.TIME), + Field.of("t_local", StandardSQLTypeName.TIME), + Field.of("dt_col", StandardSQLTypeName.DATETIME), + Field.of("ts_sql", StandardSQLTypeName.TIMESTAMP), + Field.of("ts_inst", StandardSQLTypeName.TIMESTAMP), + Field.of("ts_odt", StandardSQLTypeName.TIMESTAMP), + Field.of("ts_zdt", StandardSQLTypeName.TIMESTAMP), + Field.newBuilder("ts_picos", StandardSQLTypeName.TIMESTAMP) + .setTimestampPrecision(12L) + .build()); + Job dryRunJob = + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + schema, + Arrays.asList( + queryParam("DATE"), + queryParam("DATE"), + queryParam("TIME"), + queryParam("TIME"), + queryParam("STRING"), + queryParam("TIMESTAMP"), + queryParam("TIMESTAMP"), + queryParam("TIMESTAMP"), + queryParam("TIMESTAMP"), + queryParam("TIMESTAMP"))); + doReturn(dryRunJob).when(bigQuery).create(any(JobInfo.class)); + + TableSchema protoTableSchema = + TableSchema.newBuilder() + .addFields( + TableFieldSchema.newBuilder() + .setName("d_sql") + .setType(TableFieldSchema.Type.DATE) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build()) + .addFields( + TableFieldSchema.newBuilder() + .setName("d_local") + .setType(TableFieldSchema.Type.DATE) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build()) + .addFields( + TableFieldSchema.newBuilder() + .setName("t_sql") + .setType(TableFieldSchema.Type.TIME) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build()) + .addFields( + TableFieldSchema.newBuilder() + .setName("t_local") + .setType(TableFieldSchema.Type.TIME) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build()) + .addFields( + TableFieldSchema.newBuilder() + .setName("dt_col") + .setType(TableFieldSchema.Type.DATETIME) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build()) + .addFields( + TableFieldSchema.newBuilder() + .setName("ts_sql") + .setType(TableFieldSchema.Type.TIMESTAMP) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build()) + .addFields( + TableFieldSchema.newBuilder() + .setName("ts_inst") + .setType(TableFieldSchema.Type.TIMESTAMP) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build()) + .addFields( + TableFieldSchema.newBuilder() + .setName("ts_odt") + .setType(TableFieldSchema.Type.TIMESTAMP) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build()) + .addFields( + TableFieldSchema.newBuilder() + .setName("ts_zdt") + .setType(TableFieldSchema.Type.TIMESTAMP) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build()) + .addFields( + TableFieldSchema.newBuilder() + .setName("ts_picos") + .setType(TableFieldSchema.Type.TIMESTAMP) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setTimestampPrecision(Int64Value.of(12)) + .build()) + .build(); + Descriptor descriptor = + BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(protoTableSchema); + + Locale originalLocale = Locale.getDefault(); + TimeZone originalZone = TimeZone.getDefault(); + try { + for (Locale locale : Arrays.asList(Locale.US, Locale.FRANCE, Locale.GERMANY, Locale.JAPAN)) { + for (String zoneId : Arrays.asList("UTC", "America/New_York", "Asia/Tokyo")) { + Locale.setDefault(locale); + TimeZone.setDefault(TimeZone.getTimeZone(zoneId)); + + for (boolean picos : new boolean[] {false, true}) { + doReturn(picos).when(connection).isEnableTimestampPicos(); + BigQueryPreparedStatement ps = + new BigQueryPreparedStatement( + connection, "INSERT INTO TAB VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + + Date sqlDate = Date.valueOf("2026-11-18"); + LocalDate localDate = LocalDate.of(1969, 7, 20); + Time sqlTime = Time.valueOf("14:39:36"); + LocalTime localTime = LocalTime.of(12, 34, 56, 789_123_000); + LocalDateTime localDateTime = LocalDateTime.of(2025, 12, 3, 12, 30, 0, 123_456_000); + Timestamp sqlTimestamp = + Timestamp.from(Instant.parse("2026-11-18T19:39:36.123456789Z")); + Instant preEpochInstant = Instant.parse("1969-07-20T20:17:40.123456Z"); + OffsetDateTime odt = OffsetDateTime.parse("2026-11-18T14:39:36.123456-05:00"); + ZonedDateTime zdt = + ZonedDateTime.of( + 2026, 11, 18, 14, 39, 36, 123_456_000, ZoneId.of("America/New_York")); + Timestamp picosTs = Timestamp.from(Instant.parse("2026-11-18T19:39:36.123456789Z")); + + for (int rowIdx = 0; rowIdx < 3; rowIdx++) { + ps.setDate(1, sqlDate); + ps.setObject(2, localDate); + ps.setTime(3, sqlTime); + ps.setObject(4, localTime); + ps.setObject(5, localDateTime); + ps.setTimestamp(6, sqlTimestamp); + ps.setObject(7, preEpochInstant); + ps.setObject(8, odt); + ps.setObject(9, zdt); + ps.setTimestamp(10, picosTs); + ps.addBatch(); + } + + assertTrue(ps.useWriteAPI()); + + JsonObject row = + BigQueryPreparedStatement.createJsonRow( + schema.getFields(), ps.parameterHandler.parametersList, picos); + + assertEquals("2026-11-18", row.get("d_sql").getAsString()); + assertEquals("1969-07-20", row.get("d_local").getAsString()); + assertEquals("14:39:36.000000", row.get("t_sql").getAsString()); + assertEquals("12:34:56.789123", row.get("t_local").getAsString()); + assertEquals("2025-12-03 12:30:00.123456", row.get("dt_col").getAsString()); + assertEquals("2026-11-18 19:39:36.123456Z", row.get("ts_sql").getAsString()); + assertEquals("1969-07-20 20:17:40.123456Z", row.get("ts_inst").getAsString()); + assertEquals("2026-11-18 19:39:36.123456Z", row.get("ts_odt").getAsString()); + assertEquals("2026-11-18 19:39:36.123456Z", row.get("ts_zdt").getAsString()); + assertEquals( + picos ? "2026-11-18 19:39:36.123456789000Z" : "2026-11-18 19:39:36.123456Z", + row.get("ts_picos").getAsString()); + // Door 1 (QueryParameterValue.timestamp) and Door 2 accept the exact same wire format. + assertEquals( + row.get("ts_picos").getAsString(), + QueryParameterValue.timestamp(row.get("ts_picos").getAsString()).getValue()); + for (int colIdx = 0; colIdx < 9; colIdx++) { + String fieldName = schema.getFields().get(colIdx).getName(); + QueryParameterValue door1Value = + BigQueryParameterHandler.toQueryParameterValue( + ps.parameterHandler.parametersList.get(colIdx), false); + assertEquals(door1Value.getValue(), row.get(fieldName).getAsString()); + } + + DynamicMessage protoMsg = + JsonToProtoMessage.INSTANCE.convertToProtoMessage( + descriptor, protoTableSchema, new JSONObject(row.toString())); + assertNotNull(protoMsg); + long expectedTsMicros = + ChronoUnit.MICROS.between( + Instant.EPOCH, Instant.parse("2026-11-18T19:39:36.123456Z")); + long expectedPreEpochMicros = + ChronoUnit.MICROS.between( + Instant.EPOCH, Instant.parse("1969-07-20T20:17:40.123456Z")); + assertEquals( + (int) LocalDate.of(2026, 11, 18).toEpochDay(), + protoMsg.getField(descriptor.findFieldByName("d_sql"))); + assertEquals( + (int) localDate.toEpochDay(), + protoMsg.getField(descriptor.findFieldByName("d_local"))); + assertEquals(expectedTsMicros, protoMsg.getField(descriptor.findFieldByName("ts_sql"))); + assertEquals( + expectedPreEpochMicros, protoMsg.getField(descriptor.findFieldByName("ts_inst"))); + assertEquals(expectedTsMicros, protoMsg.getField(descriptor.findFieldByName("ts_odt"))); + assertEquals(expectedTsMicros, protoMsg.getField(descriptor.findFieldByName("ts_zdt"))); + assertEquals( + picos ? "2026-11-18 19:39:36.123456789000Z" : "2026-11-18T19:39:36.123456+00:00", + protoMsg.getField(descriptor.findFieldByName("ts_picos"))); + } + } + } + } finally { + doReturn(false).when(connection).isEnableTimestampPicos(); + Locale.setDefault(originalLocale); + TimeZone.setDefault(originalZone); + } + } + + @Test + public void testUseWriteApiGatesTransactionsSessionsAndIncompatibleParameterValues() + throws Exception { + Schema schema = + Schema.of( + Field.newBuilder("req_str", StandardSQLTypeName.STRING) + .setMode(Field.Mode.REQUIRED) + .build(), + Field.of("num_col", StandardSQLTypeName.NUMERIC), + Field.of("flt_col", StandardSQLTypeName.FLOAT64)); + Job dryRunJob = + mockInsertDryRunJob( + TableId.of("proj", "_internal_session_ds", "TAB"), + schema, + Arrays.asList(queryParam("STRING"), queryParam("NUMERIC"), queryParam("FLOAT64"))); + doReturn(dryRunJob).when(bigQuery).create(any(JobInfo.class)); + + BigQueryPreparedStatement ps = + new BigQueryPreparedStatement(connection, "INSERT INTO TAB VALUES (?, ?, ?)"); + for (int i = 0; i < 3; i++) { + ps.setString(1, "v" + i); + ps.setBigDecimal(2, new BigDecimal("12.34")); + ps.setDouble(3, 1.5d); + ps.addBatch(); + } + assertTrue(ps.useWriteAPI()); + + // Active SQL transaction must block Storage Write API so rollback() stays atomic. + doReturn(true).when(connection).isTransactionStarted(); + assertFalse(ps.useWriteAPI()); + doReturn(false).when(connection).isTransactionStarted(); + + // Active session with anonymous "_" dataset must block SWA (session TEMP table), + // unless that "_" dataset is explicitly configured as DefaultDataset. + BigQueryConnection.SessionState activeSession = + new BigQueryConnection.SessionState( + ConnectionProperty.of("session_id", "s1"), Collections.emptyList(), true); + doReturn(activeSession).when(connection).getSessionStateSnapshot(); + assertFalse(ps.useWriteAPI()); + + doReturn(DatasetId.of("proj", "_internal_session_ds")).when(connection).getDefaultDataset(); + BigQueryPreparedStatement psWithExplicitHiddenDs = + new BigQueryPreparedStatement(connection, "INSERT INTO TAB VALUES (?, ?, ?)"); + populateBatchAndAssertWriteApi( + psWithExplicitHiddenDs, + 3, + i -> { + psWithExplicitHiddenDs.setString(1, "v" + i); + psWithExplicitHiddenDs.setInt(2, i); // INT64 widening to NUMERIC is valid + psWithExplicitHiddenDs.setDouble(3, 1.5d); + }, + true); + + // Non-finite Double (NaN), out-of-range NUMERIC BigDecimal, BigInteger, NULL on REQUIRED, or + // cross-family INT64->STRING must return false so BigQuery SQL handles them. + populateBatchAndAssertWriteApi( + psWithExplicitHiddenDs, + 3, + i -> { + psWithExplicitHiddenDs.setString(1, "v" + i); + psWithExplicitHiddenDs.setBigDecimal(2, new BigDecimal("1E+40")); + psWithExplicitHiddenDs.setDouble(3, 1.5d); + }, + false); + + populateBatchAndAssertWriteApi( + psWithExplicitHiddenDs, + 3, + i -> { + psWithExplicitHiddenDs.setString(1, "v" + i); + psWithExplicitHiddenDs.setInt(2, 1); + psWithExplicitHiddenDs.setDouble(3, Double.NaN); + }, + false); + + populateBatchAndAssertWriteApi( + psWithExplicitHiddenDs, + 3, + i -> { + psWithExplicitHiddenDs.setNull(1, Types.VARCHAR); // REQUIRED column + psWithExplicitHiddenDs.setInt(2, 1); + psWithExplicitHiddenDs.setDouble(3, 1.0d); + }, + false); + + populateBatchAndAssertWriteApi( + psWithExplicitHiddenDs, + 3, + i -> { + psWithExplicitHiddenDs.setInt(1, 99); // INT64 into STRING column + psWithExplicitHiddenDs.setInt(2, 1); + psWithExplicitHiddenDs.setDouble(3, 1.0d); + }, + false); + + populateBatchAndAssertWriteApi( + psWithExplicitHiddenDs, + 3, + i -> { + psWithExplicitHiddenDs.setString(1, "ok"); + psWithExplicitHiddenDs.setObject(2, BigInteger.TEN, Types.BIGINT); + psWithExplicitHiddenDs.setDouble(3, 1.0d); + }, + false); + } + + @Test + public void testExecuteBatchWriteApiDoesNotCloseConnectionClientAndWrapsRuntimeException() + throws Exception { + Schema schema = Schema.of(Field.of("col1", StandardSQLTypeName.STRING)); + Job dryRunJob = + mockInsertDryRunJob( + TableId.of("ds", "TAB"), schema, Collections.singletonList(queryParam("STRING"))); + doReturn("fallback-proj").when(connection).getCatalog(); + doReturn(dryRunJob).when(bigQuery).create(any(JobInfo.class)); + + BigQueryWriteClient mockWriteClient = mock(BigQueryWriteClient.class); + StatusRuntimeException streamError = + Status.INVALID_ARGUMENT.withDescription("simulated SWA failure").asRuntimeException(); + when(mockWriteClient.createWriteStream(any(CreateWriteStreamRequest.class))) + .thenThrow(streamError); + doReturn(mockWriteClient).when(connection).getBigQueryWriteClient(); + + BigQueryPreparedStatement ps = + new BigQueryPreparedStatement(connection, "INSERT INTO ds.TAB (col1) VALUES (?)"); + for (int i = 0; i < 3; i++) { + ps.setString(1, "row-" + i); + ps.addBatch(); + } + + BigQueryJdbcException ex = assertThrows(BigQueryJdbcException.class, ps::executeBatch); + assertSame(streamError, ex.getCause()); + // Cached BigQueryWriteClient belongs to BigQueryConnection; executeBatch must not close it. + verify(mockWriteClient, never()).close(); + // Failed executeBatch must clear queued batchParameters so a subsequent batch starts clean. + assertArrayEquals(new int[0], ps.executeBatch()); + } + + @Test + public void testRetryDryRunRunsOnceWhenTableCreatedAfterPrepareStatement() throws Exception { + String sql = "INSERT INTO ds.TAB (col1) VALUES (?)"; + Schema schema = Schema.of(Field.of("col1", StandardSQLTypeName.STRING)); + Job dryRunJob = + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + schema, + Collections.singletonList(queryParam("STRING"))); + when(bigQuery.create(any(JobInfo.class))) + .thenThrow(new BigQueryException(404, "Not found: Table proj:ds.TAB")) + .thenReturn(dryRunJob); + + BigQueryWriteClient mockWriteClient = mock(BigQueryWriteClient.class); + when(mockWriteClient.createWriteStream(any(CreateWriteStreamRequest.class))) + .thenThrow(new RuntimeException("simulated stream init stop")); + doReturn(mockWriteClient).when(connection).getBigQueryWriteClient(); + + BigQueryPreparedStatement ps = new BigQueryPreparedStatement(connection, sql); + assertNull(ps.insertSchema); + + for (int i = 0; i < 3; i++) { + ps.setString(1, "v" + i); + ps.addBatch(); + } + // Retry dry run executes during executeBatch, populates insertSchema, and enters SWA path. + assertThrows(BigQueryJdbcException.class, ps::executeBatch); + assertNotNull(ps.insertSchema); + verify(bigQuery, times(2)).create(any(JobInfo.class)); + verify(mockWriteClient, times(1)).createWriteStream(any(CreateWriteStreamRequest.class)); + verify(mockWriteClient, never()).close(); + } + + @Test + public void testUnboundBatchParameterSparseAddBatchClearBatchAndClose() throws Exception { + // Constructor dry run fails -> parameterCount falls back to 2 '?' placeholders. + when(bigQuery.create(any(JobInfo.class))).thenThrow(new BigQueryException(400, "dry run off")); + + BigQueryPreparedStatement ps = + new BigQueryPreparedStatement(connection, "INSERT INTO TAB (a, b) VALUES (?, ?)"); + // Setting parameter 2 before parameter 1 leaves index 0 null in parametersList; addBatch must + // not NPE, and executeBatch must report missing parameter 1. + ps.setInt(2, 42); + ps.addBatch(); + + BigQueryJdbcException missingEx = assertThrows(BigQueryJdbcException.class, ps::executeBatch); + assertTrue(missingEx.getMessage().contains("parameter 1 of 2")); + + // clearBatch empties queued rows. + ps.setString(1, "x"); + ps.setInt(2, 42); + ps.addBatch(); + ps.clearBatch(); + assertArrayEquals(new int[0], ps.executeBatch()); + + // close() also clears queued batch rows without throwing. + ps.addBatch(); + ps.close(); + assertTrue(ps.isClosed()); + } + + @Test + public void testLegacySqlSkipsPositionalDryRun() throws Exception { + doReturn("BIG_QUERY").when(connection).getQueryDialect(); + BigQueryPreparedStatement ps = + new BigQueryPreparedStatement(connection, "SELECT ? FROM [proj:ds.tab]"); + assertEquals(1, ps.parameterCount); + verify(bigQuery, never()).create(any(JobInfo.class)); + } + + @Test + public void testTemporalAndSpecialColumnCompatibilityInUseWriteApi() throws Exception { + Schema schema = + Schema.of( + Field.of("d_col", StandardSQLTypeName.DATE), + Field.of("dt_col", StandardSQLTypeName.DATETIME), + Field.of("ts_col", StandardSQLTypeName.TIMESTAMP), + Field.of("json_col", StandardSQLTypeName.JSON)); + Job dryRunJob = + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + schema, + Arrays.asList( + queryParam("DATE"), + queryParam("STRING"), + queryParam("TIMESTAMP"), + queryParam("JSON"))); + doReturn(dryRunJob).when(bigQuery).create(any(JobInfo.class)); + + BigQueryPreparedStatement ps = + new BigQueryPreparedStatement(connection, "INSERT INTO TAB VALUES (?, ?, ?, ?)"); + + // Nullable NULL values on temporal and JSON columns stay SWA-eligible when null type matches. + populateBatchAndAssertWriteApi( + ps, + 3, + i -> { + ps.setObject(1, LocalDate.of(2026, 11, 18)); + ps.setObject(2, LocalDateTime.of(2026, 11, 18, 10, 20, 30)); + ps.setTimestamp(3, Timestamp.from(Instant.parse("2026-11-18T10:20:30Z"))); + ps.setNull(4, Types.OTHER); + }, + true); + + // Non-null JSON/GEOGRAPHY/INTERVAL bound as String falls back to SQL so BigQuery casts/parses. + populateBatchAndAssertWriteApi( + ps, + 3, + i -> { + ps.setObject(1, LocalDate.of(2026, 11, 18)); + ps.setObject(2, LocalDateTime.of(2026, 11, 18, 10, 20, 30)); + ps.setTimestamp(3, Timestamp.from(Instant.parse("2026-11-18T10:20:30Z"))); + ps.setString(4, "{\"k\":1}"); + }, + false); + + // Year 0 / 10000 LocalDate or out-of-range Date falls back to SQL. + populateBatchAndAssertWriteApi( + ps, + 3, + i -> { + ps.setObject(1, LocalDate.of(10000, 1, 1)); + ps.setObject(2, LocalDateTime.of(2026, 11, 18, 10, 20, 30)); + ps.setTimestamp(3, Timestamp.from(Instant.parse("2026-11-18T10:20:30Z"))); + ps.setNull(4, Types.OTHER); + }, + false); + + // String bound to DATE or invalid DATETIME digits falls back to SQL. + populateBatchAndAssertWriteApi( + ps, + 3, + i -> { + ps.setObject(1, LocalDate.of(2026, 11, 18)); + ps.setString(2, "2026-11-18 10:20:30"); // missing fractional microseconds + ps.setTimestamp(3, Timestamp.from(Instant.parse("2026-11-18T10:20:30Z"))); + ps.setNull(4, Types.OTHER); + }, + false); + + // LocalDateTime bound as STRING into TIMESTAMP falls back to SQL. + populateBatchAndAssertWriteApi( + ps, + 3, + i -> { + ps.setObject(1, LocalDate.of(2026, 11, 18)); + ps.setObject(2, LocalDateTime.of(2026, 11, 18, 10, 20, 30)); + ps.setObject(3, LocalDateTime.of(2026, 11, 18, 10, 20, 30)); + ps.setNull(4, Types.OTHER); + }, + false); + + // Out-of-range Instant (> 9999-12-31) into TIMESTAMP falls back to SQL. + populateBatchAndAssertWriteApi( + ps, + 3, + i -> { + ps.setObject(1, LocalDate.of(2026, 11, 18)); + ps.setObject(2, LocalDateTime.of(2026, 11, 18, 10, 20, 30)); + ps.setTimestamp(3, Timestamp.from(Instant.parse("+10000-01-01T00:00:00Z"))); + ps.setNull(4, Types.OTHER); + }, + false); + } + + @Test + public void testExecuteBatchWriteApiMultiChunkSuccessOffsetsCommitAndTelemetry() + throws Exception { + String streamName = "projects/proj/datasets/ds/tables/TAB/streams/_pending_ok"; + Schema schema = + Schema.of( + Field.of("id", StandardSQLTypeName.INT64), + Field.of("name", StandardSQLTypeName.STRING)); + Job dryRunJob = + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + schema, + Arrays.asList(queryParam("INT64"), queryParam("STRING"))); + doReturn(dryRunJob).when(bigQuery).create(any(JobInfo.class)); + doReturn(2).when(connection).getWriteAPIAppendRowCount(); + + BigQueryWriteClient mockWriteClient = mock(BigQueryWriteClient.class); + when(mockWriteClient.createWriteStream(any(CreateWriteStreamRequest.class))) + .thenReturn( + WriteStream.newBuilder() + .setName(streamName) + .setTableSchema(TableSchema.getDefaultInstance()) + .build()); + when(mockWriteClient.finalizeWriteStream(streamName)) + .thenReturn(FinalizeWriteStreamResponse.newBuilder().setRowCount(5L).build()); + BatchCommitWriteStreamsResponse.Builder commitRespBuilder = + BatchCommitWriteStreamsResponse.newBuilder(); + commitRespBuilder.getCommitTimeBuilder().setSeconds(1700000000L); + when(mockWriteClient.batchCommitWriteStreams(any(BatchCommitWriteStreamsRequest.class))) + .thenReturn(commitRespBuilder.build()); + doReturn(mockWriteClient).when(connection).getBigQueryWriteClient(); + + JsonStreamWriter mockStreamWriter = mock(JsonStreamWriter.class); + when(mockStreamWriter.append(any(JsonArray.class), anyLong())) + .thenReturn(ApiFutures.immediateFuture(AppendRowsResponse.getDefaultInstance())); + + BigQueryPreparedStatement ps = + newWriteApiPreparedStatement("INSERT INTO TAB (name, id) VALUES (?, ?)", mockStreamWriter); + for (int i = 0; i < 5; i++) { + ps.setString(1, "user-" + i); + ps.setInt(2, i + 10); + ps.addBatch(); + } + + int[] counts; + try (MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { + counts = ps.executeBatch(); + + ArgumentCaptor executionCaptor = + ArgumentCaptor.forClass(StatementExecution.Builder.class); + telemetry.verify( + () -> TelemetryManager.recordStatementExecution(executionCaptor.capture(), anyLong()), + times(1)); + assertEquals( + QueryApiType.QUERY_API_TYPE_WRITE_API, executionCaptor.getValue().getQueryApiType()); + assertEquals("STATUS_SUCCESS", executionCaptor.getValue().getStatus().name()); + } + + assertArrayEquals(new int[] {1, 1, 1, 1, 1}, counts); + + ArgumentCaptor batchCaptor = ArgumentCaptor.forClass(JsonArray.class); + ArgumentCaptor offsetCaptor = ArgumentCaptor.forClass(Long.class); + verify(mockStreamWriter, times(3)).append(batchCaptor.capture(), offsetCaptor.capture()); + assertEquals(Arrays.asList(0L, 2L, 4L), offsetCaptor.getAllValues()); + List appendedChunks = batchCaptor.getAllValues(); + assertEquals(2, appendedChunks.get(0).size()); + assertEquals(2, appendedChunks.get(1).size()); + assertEquals(1, appendedChunks.get(2).size()); + assertEquals( + "user-0", appendedChunks.get(0).get(0).getAsJsonObject().get("name").getAsString()); + assertEquals("14", appendedChunks.get(2).get(0).getAsJsonObject().get("id").getAsString()); + + ArgumentCaptor commitCaptor = + ArgumentCaptor.forClass(BatchCommitWriteStreamsRequest.class); + verify(mockWriteClient, times(1)).finalizeWriteStream(streamName); + verify(mockWriteClient, times(1)).batchCommitWriteStreams(commitCaptor.capture()); + assertEquals("projects/proj/datasets/ds/tables/TAB", commitCaptor.getValue().getParent()); + assertEquals( + Collections.singletonList(streamName), commitCaptor.getValue().getWriteStreamsList()); + + verify(mockStreamWriter, times(1)).close(); + verify(mockWriteClient, never()).close(); + assertArrayEquals(new int[0], ps.executeBatch()); + } + + @Test + public void testExecuteBatchWriteApiCommitMissingCommitTimeThrowsAndCleansUp() throws Exception { + String streamName = "projects/proj/datasets/ds/tables/TAB/streams/_pending_commit_fail"; + Schema schema = Schema.of(Field.of("col1", StandardSQLTypeName.STRING)); + Job dryRunJob = + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + schema, + Collections.singletonList(queryParam("STRING"))); + doReturn(dryRunJob).when(bigQuery).create(any(JobInfo.class)); + + BigQueryWriteClient mockWriteClient = mock(BigQueryWriteClient.class); + when(mockWriteClient.createWriteStream(any(CreateWriteStreamRequest.class))) + .thenReturn( + WriteStream.newBuilder() + .setName(streamName) + .setTableSchema(TableSchema.getDefaultInstance()) + .build()); + when(mockWriteClient.finalizeWriteStream(streamName)) + .thenReturn(FinalizeWriteStreamResponse.newBuilder().setRowCount(3L).build()); + when(mockWriteClient.batchCommitWriteStreams(any(BatchCommitWriteStreamsRequest.class))) + .thenReturn(BatchCommitWriteStreamsResponse.getDefaultInstance()); + doReturn(mockWriteClient).when(connection).getBigQueryWriteClient(); + + JsonStreamWriter mockStreamWriter = mock(JsonStreamWriter.class); + when(mockStreamWriter.append(any(JsonArray.class), anyLong())) + .thenReturn(ApiFutures.immediateFuture(AppendRowsResponse.getDefaultInstance())); + + BigQueryPreparedStatement ps = + newWriteApiPreparedStatement("INSERT INTO TAB VALUES (?)", mockStreamWriter); + for (int i = 0; i < 3; i++) { + ps.setString(1, "row-" + i); + ps.addBatch(); + } + + try (MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { + BigQueryJdbcException ex = assertThrows(BigQueryJdbcException.class, ps::executeBatch); + assertTrue(ex.getMessage().contains("Error committing the streams")); + + ArgumentCaptor executionCaptor = + ArgumentCaptor.forClass(StatementExecution.Builder.class); + telemetry.verify( + () -> TelemetryManager.recordStatementExecution(executionCaptor.capture(), anyLong()), + times(1)); + assertEquals("STATUS_ERROR", executionCaptor.getValue().getStatus().name()); + } + + verify(mockStreamWriter, times(1)).close(); + verify(mockWriteClient, never()).close(); + assertArrayEquals(new int[0], ps.executeBatch()); + } + + @Test + public void testExecuteBatchWriteApiAppendFailureClosesWriterOnceWithoutClosingClient() + throws Exception { + String streamName = "projects/proj/datasets/ds/tables/TAB/streams/_pending_append_fail"; + Schema schema = Schema.of(Field.of("col1", StandardSQLTypeName.STRING)); + Job dryRunJob = + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + schema, + Collections.singletonList(queryParam("STRING"))); + doReturn(dryRunJob).when(bigQuery).create(any(JobInfo.class)); + + BigQueryWriteClient mockWriteClient = mock(BigQueryWriteClient.class); + when(mockWriteClient.createWriteStream(any(CreateWriteStreamRequest.class))) + .thenReturn( + WriteStream.newBuilder() + .setName(streamName) + .setTableSchema(TableSchema.getDefaultInstance()) + .build()); + doReturn(mockWriteClient).when(connection).getBigQueryWriteClient(); + + JsonStreamWriter mockStreamWriter = mock(JsonStreamWriter.class); + IOException appendIoEx = new IOException("broken pipe"); + when(mockStreamWriter.append(any(JsonArray.class), anyLong())).thenThrow(appendIoEx); + + BigQueryPreparedStatement ps = + newWriteApiPreparedStatement("INSERT INTO TAB VALUES (?)", mockStreamWriter); + for (int i = 0; i < 3; i++) { + ps.setString(1, "row-" + i); + ps.addBatch(); + } + + try (MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { + BigQueryJdbcException ex = assertThrows(BigQueryJdbcException.class, ps::executeBatch); + assertSame(appendIoEx, ex.getCause()); + + ArgumentCaptor executionCaptor = + ArgumentCaptor.forClass(StatementExecution.Builder.class); + telemetry.verify( + () -> TelemetryManager.recordStatementExecution(executionCaptor.capture(), anyLong()), + times(1)); + assertEquals("STATUS_ERROR", executionCaptor.getValue().getStatus().name()); + } + + verify(mockStreamWriter, times(1)).close(); + verify(mockWriteClient, never()).finalizeWriteStream(anyString()); + verify(mockWriteClient, never()).close(); + assertArrayEquals(new int[0], ps.executeBatch()); + } + + @Test + public void testExecuteBatchWriteApiInterruptedCleanupRestoresInterruptFlagAndRecordsTelemetry() + throws Exception { + String streamName = "projects/proj/datasets/ds/tables/TAB/streams/_pending_interrupt"; + Schema schema = Schema.of(Field.of("col1", StandardSQLTypeName.STRING)); + Job dryRunJob = + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + schema, + Collections.singletonList(queryParam("STRING"))); + doReturn(dryRunJob).when(bigQuery).create(any(JobInfo.class)); + + BigQueryWriteClient mockWriteClient = mock(BigQueryWriteClient.class); + when(mockWriteClient.createWriteStream(any(CreateWriteStreamRequest.class))) + .thenReturn( + WriteStream.newBuilder() + .setName(streamName) + .setTableSchema(TableSchema.getDefaultInstance()) + .build()); + doReturn(mockWriteClient).when(connection).getBigQueryWriteClient(); + + JsonStreamWriter mockStreamWriter = mock(JsonStreamWriter.class); + SettableApiFuture stalledFuture = SettableApiFuture.create(); + when(mockStreamWriter.append(any(JsonArray.class), anyLong())).thenReturn(stalledFuture); + + BigQueryPreparedStatement ps = + newWriteApiPreparedStatement("INSERT INTO TAB VALUES (?)", mockStreamWriter); + for (int i = 0; i < 3; i++) { + ps.setString(1, "row-" + i); + ps.addBatch(); + } + + assertTimeoutPreemptively( + Duration.ofSeconds(2), + () -> { + try (MockedStatic telemetry = + mockStatic(TelemetryManager.class, CALLS_REAL_METHODS)) { + Thread.currentThread().interrupt(); + BigQueryJdbcException ex = assertThrows(BigQueryJdbcException.class, ps::executeBatch); + assertTrue(ex.getCause() instanceof InterruptedException); + assertTrue(Thread.currentThread().isInterrupted()); + + ArgumentCaptor executionCaptor = + ArgumentCaptor.forClass(StatementExecution.Builder.class); + telemetry.verify( + () -> + TelemetryManager.recordStatementExecution(executionCaptor.capture(), anyLong()), + times(1)); + assertEquals("STATUS_ERROR", executionCaptor.getValue().getStatus().name()); + } finally { + Thread.interrupted(); + } + }); + + verify(mockStreamWriter, times(1)).close(); + verify(mockWriteClient, never()).close(); + } + + @Test + public void testRetryDryRunUsesUnboundDescribeForNullsAndRunsOnceWhenDryRunReturnsNull() + throws Exception { + String sql = "INSERT INTO ds.TAB (num_col) VALUES (?)"; + Schema schema = Schema.of(Field.of("num_col", StandardSQLTypeName.INT64)); + Job dryRunJob = + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + schema, + Collections.singletonList(queryParam("INT64"))); + TableResult dmlResult = mock(TableResult.class); + doReturn(StatementType.INSERT).when(dmlResult).getStatementType(); + doReturn(1L).when(dmlResult).getNumDmlAffectedRows(); + doReturn(dmlResult).when(bigQuery).queryWithTimeout(any(), any(), any()); + + when(bigQuery.create(any(JobInfo.class))) + .thenThrow(new BigQueryException(404, "Not found")) + .thenReturn(dryRunJob); + + BigQueryPreparedStatement ps = new BigQueryPreparedStatement(connection, sql); + // First batch binds Types.OTHER (defaults to STRING before dry run); retry dry run must still + // send an unbound POSITIONAL dry-run query without failing on STRING vs INT64 mismatch, and + // this mismatched batch must fall back to SQL DML cleanly. + for (int i = 0; i < 3; i++) { + ps.setNull(1, Types.OTHER); + ps.addBatch(); + } + int[] counts = ps.executeBatch(); + assertEquals(3, counts.length); + assertNotNull(ps.insertSchema); + + ArgumentCaptor jobInfoCaptor = ArgumentCaptor.forClass(JobInfo.class); + verify(bigQuery, times(2)).create(jobInfoCaptor.capture()); + QueryJobConfiguration retryConfig = jobInfoCaptor.getAllValues().get(1).getConfiguration(); + assertTrue( + retryConfig.getPositionalParameters() == null + || retryConfig.getPositionalParameters().isEmpty()); + + // When retry dry run completes with null statistics, insertMetadataEvaluated becomes true so + // subsequent executeBatch calls do not repeat the dry run. + clearInvocations(bigQuery); + Job nullStatsJob = mock(Job.class); + when(bigQuery.create(any(JobInfo.class))) + .thenThrow(new BigQueryException(404, "Not found")) + .thenReturn(nullStatsJob); + BigQueryPreparedStatement psNullStats = new BigQueryPreparedStatement(connection, sql); + for (int b = 0; b < 2; b++) { + for (int i = 0; i < 3; i++) { + psNullStats.setLong(1, i); + psNullStats.addBatch(); + } + psNullStats.executeBatch(); + } + verify(bigQuery, times(2)).create(any(JobInfo.class)); + } + + @Test + public void testResolveTargetProjectDatasetMatchingAndParameterizedNumericBounds() + throws Exception { + doReturn("catalog-proj").when(connection).getCatalog(); + doReturn(DatasetId.of("default-proj", "default_ds")).when(connection).getDefaultDataset(); + + // Unqualified table project on a different dataset ("other_ds" != "default_ds") uses catalog. + Schema numericSchema = + Schema.of( + Field.newBuilder("n_prec_only", StandardSQLTypeName.NUMERIC).setPrecision(5L).build(), + Field.newBuilder("n_prec_scale", StandardSQLTypeName.NUMERIC) + .setPrecision(5L) + .setScale(2L) + .build(), + Field.newBuilder("bn_prec_scale", StandardSQLTypeName.BIGNUMERIC) + .setPrecision(8L) + .setScale(3L) + .build()); + Job otherDsJob = + mockInsertDryRunJob( + TableId.of("other_ds", "TAB"), + numericSchema, + Arrays.asList(queryParam("NUMERIC"), queryParam("NUMERIC"), queryParam("BIGNUMERIC"))); + doReturn(otherDsJob).when(bigQuery).create(any(JobInfo.class)); + + String streamName = "projects/catalog-proj/datasets/other_ds/tables/TAB/streams/_pending1"; + BigQueryWriteClient mockWriteClient = mock(BigQueryWriteClient.class); + when(mockWriteClient.createWriteStream(any(CreateWriteStreamRequest.class))) + .thenReturn( + WriteStream.newBuilder() + .setName(streamName) + .setTableSchema(TableSchema.getDefaultInstance()) + .build()); + when(mockWriteClient.finalizeWriteStream(streamName)) + .thenReturn(FinalizeWriteStreamResponse.newBuilder().setRowCount(3L).build()); + BatchCommitWriteStreamsResponse.Builder okCommit = BatchCommitWriteStreamsResponse.newBuilder(); + okCommit.getCommitTimeBuilder().setSeconds(1700000000L); + when(mockWriteClient.batchCommitWriteStreams(any(BatchCommitWriteStreamsRequest.class))) + .thenReturn(okCommit.build()); + doReturn(mockWriteClient).when(connection).getBigQueryWriteClient(); + + JsonStreamWriter mockStreamWriter = mock(JsonStreamWriter.class); + when(mockStreamWriter.append(any(JsonArray.class), anyLong())) + .thenReturn(ApiFutures.immediateFuture(AppendRowsResponse.getDefaultInstance())); + + BigQueryPreparedStatement ps = + newWriteApiPreparedStatement("INSERT INTO other_ds.TAB VALUES (?, ?, ?)", mockStreamWriter); + + // Trailing zeros ("12.3400") on NUMERIC(5, 2) and trailing-zero scale on NUMERIC(5) succeed. + populateBatchAndAssertWriteApi( + ps, + 3, + i -> { + ps.setBigDecimal(1, new BigDecimal("12300")); + ps.setBigDecimal(2, new BigDecimal("12.3400")); + ps.setBigDecimal(3, new BigDecimal("12345.6780")); + }, + true); + + // Parameterized precision/scale violations fall back to SQL instead of failing on SWA. + populateBatchAndAssertWriteApi( + ps, + 3, + i -> { + ps.setBigDecimal(1, new BigDecimal("12.5")); // NUMERIC(5) has effective scale 0 + ps.setBigDecimal(2, new BigDecimal("12.34")); + ps.setBigDecimal(3, new BigDecimal("1.23")); + }, + false); + populateBatchAndAssertWriteApi( + ps, + 3, + i -> { + ps.setBigDecimal(1, new BigDecimal("100")); + ps.setBigDecimal(2, new BigDecimal("1234.50")); // 4 integer digits > (5 - 2 = 3) + ps.setBigDecimal(3, new BigDecimal("1.23")); + }, + false); + populateBatchAndAssertWriteApi( + ps, + 3, + i -> { + ps.setBigDecimal(1, new BigDecimal("100")); + ps.setBigDecimal(2, new BigDecimal("12.345")); // scale 3 > 2 + ps.setBigDecimal(3, new BigDecimal("1.23")); + }, + false); + populateBatchAndAssertWriteApi( + ps, + 3, + i -> { + ps.setInt(1, 100); + ps.setInt(2, 1000); // 4 integer digits > (5 - 2 = 3) for INT64->NUMERIC(5, 2) + ps.setInt(3, 100); + }, + false); + populateBatchAndAssertWriteApi( + ps, + 3, + i -> { + ps.setInt(1, 100); + ps.setInt(2, 999); // fits 3 integer digits for INT64->NUMERIC(5, 2) + ps.setInt(3, 100); + }, + true); + + // Executing eligible batch creates stream under catalog-proj (not default-proj). + populateBatchAndAssertWriteApi( + ps, + 3, + i -> { + ps.setBigDecimal(1, new BigDecimal("12300")); + ps.setBigDecimal(2, new BigDecimal("12.3400")); + ps.setBigDecimal(3, new BigDecimal("12345.6780")); + }, + true); + ps.executeBatch(); + ArgumentCaptor createCaptor = + ArgumentCaptor.forClass(CreateWriteStreamRequest.class); + verify(mockWriteClient, times(1)).createWriteStream(createCaptor.capture()); + assertEquals( + "projects/catalog-proj/datasets/other_ds/tables/TAB", createCaptor.getValue().getParent()); + + // When tableId.getDataset() matches defaultDataset.getDataset(), defaultDataset project wins. + Job defaultDsJob = + mockInsertDryRunJob( + TableId.of("default_ds", "TAB"), + Schema.of(Field.of("id", StandardSQLTypeName.INT64)), + Collections.singletonList(queryParam("INT64"))); + doReturn(defaultDsJob).when(bigQuery).create(any(JobInfo.class)); + String defaultStreamName = "projects/default-proj/datasets/default_ds/tables/TAB/streams/_p2"; + when(mockWriteClient.createWriteStream(any(CreateWriteStreamRequest.class))) + .thenReturn( + WriteStream.newBuilder() + .setName(defaultStreamName) + .setTableSchema(TableSchema.getDefaultInstance()) + .build()); + when(mockWriteClient.finalizeWriteStream(defaultStreamName)) + .thenReturn(FinalizeWriteStreamResponse.newBuilder().setRowCount(3L).build()); + + BigQueryPreparedStatement psDefaultDs = + newWriteApiPreparedStatement("INSERT INTO TAB VALUES (?)", mockStreamWriter); + populateBatchAndAssertWriteApi(psDefaultDs, 3, i -> psDefaultDs.setInt(1, i), true); + psDefaultDs.executeBatch(); + verify(mockWriteClient, times(2)).createWriteStream(createCaptor.capture()); + assertEquals( + "projects/default-proj/datasets/default_ds/tables/TAB", + createCaptor.getValue().getParent()); + } +} diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistryTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistryTest.java index b3d4a384481d..0b29ba34700b 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistryTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistryTest.java @@ -19,6 +19,7 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; +import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.exception.BigQueryJdbcException; import java.sql.Date; @@ -325,4 +326,52 @@ public void testDatetimeReadCoercionsPreserveCivilDigits(String zone) throws Exc .isEqualTo(Duration.ofHours(2).plusMinutes(30).toMillis()); } } + + @Test + public void testWriteApiHelpersReuseRegistryDescriptorsAndExtractInstant() { + assertThat(BigQueryTypeRegistry.isSupportedWriteApiField(null)).isFalse(); + assertThat( + BigQueryTypeRegistry.isSupportedWriteApiField( + Field.of("s", StandardSQLTypeName.STRING))) + .isTrue(); + assertThat( + BigQueryTypeRegistry.isSupportedWriteApiField( + Field.newBuilder("rep", StandardSQLTypeName.STRING) + .setMode(Field.Mode.REPEATED) + .build())) + .isFalse(); + assertThat( + BigQueryTypeRegistry.isSupportedWriteApiField( + Field.of( + "st", StandardSQLTypeName.STRUCT, Field.of("a", StandardSQLTypeName.STRING)))) + .isFalse(); + + assertThat(BigQueryTypeRegistry.isSupportedJavaValue(null, "x")).isFalse(); + assertThat(BigQueryTypeRegistry.isSupportedJavaValue(StandardSQLTypeName.STRING, null)) + .isFalse(); + assertThat(BigQueryTypeRegistry.isSupportedJavaValue(StandardSQLTypeName.INT64, (byte) 1)) + .isTrue(); + assertThat(BigQueryTypeRegistry.isSupportedJavaValue(StandardSQLTypeName.INT64, (short) 2)) + .isTrue(); + assertThat(BigQueryTypeRegistry.isSupportedJavaValue(StandardSQLTypeName.INT64, 3)).isTrue(); + assertThat(BigQueryTypeRegistry.isSupportedJavaValue(StandardSQLTypeName.INT64, 4L)).isTrue(); + assertThat(BigQueryTypeRegistry.isSupportedJavaValue(StandardSQLTypeName.INT64, "4")).isFalse(); + assertThat( + BigQueryTypeRegistry.isSupportedJavaValue( + StandardSQLTypeName.DATE, LocalDate.of(2026, 11, 18))) + .isTrue(); + assertThat( + BigQueryTypeRegistry.isSupportedJavaValue( + StandardSQLTypeName.TIME, Time.valueOf("10:00:00"))) + .isTrue(); + + Instant expected = Instant.parse("2026-11-18T19:39:36.123456789Z"); + assertThat(BigQueryTypeRegistry.extractInstant(expected)).isEqualTo(expected); + assertThat(BigQueryTypeRegistry.extractInstant(expected.atOffset(ZoneOffset.ofHours(-5)))) + .isEqualTo(expected); + assertThat(BigQueryTypeRegistry.extractInstant(expected.atZone(ZoneId.of("Asia/Tokyo")))) + .isEqualTo(expected); + assertThat(BigQueryTypeRegistry.extractInstant(Timestamp.from(expected))).isEqualTo(expected); + assertThat(BigQueryTypeRegistry.extractInstant(Date.valueOf("2026-11-18"))).isNull(); + } }