From 22875e8be94b02e5176e016dbe631c94703d9094 Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Thu, 19 Nov 2026 19:40:19 -0500 Subject: [PATCH] fix(bigquery-jdbc): gate Storage Write API on direct VALUES inserts and align columns Stack: PR 3 of 3 (base: `fix/swa-2-unified-write-conversion`, stacked on #15084 -> #15085) ### Summary - Fix silent expression/UDF bypass (`b/587228731`) and `AppendSerializationError` when `EnableWriteAPI=true` is used with SQL expressions, `CAST`s, or UDFs inside `INSERT ... VALUES (...)`. - Add a zero-regex whitelist parser (`parseDirectInsertValues`) so only single-row `INSERT [INTO] [()] VALUES (?, ...) [;]` statements with bare `?` placeholders use the Storage Write API; all other `INSERT` forms fall back automatically to BigQuery SQL DML. - Align reordered column lists (`resolveInsertSchema`), reject partial/duplicate/pseudo-column lists, honor `DefaultDataset` propagation and project matching (`BigQueryStatement.generateBigQuerySettings`, `resolveTargetProject`), skip positional dry runs in Legacy SQL mode (`getUseLegacySql`), retry unbound dry runs at most once when tables are created after `prepareStatement()`, and fall back to SQL inside active transactions or `_SESSION` temp tables. ### Testing - `make lint`, `mvn -B -ntp verify -DskipTests -DskipITs`, `mvn -q -Denforcer.skip=true -Dclirr.skip=true test-compile`, and `make unittest` (284 targeted tests passing; full suite 1305/1306 non-skipped passing, 1 pre-existing `BigQueryJdbcOAuthUtilityTest` P12 env failure; ITs require GCP credentials). --- .../jdbc/BigQueryPreparedStatement.java | 373 +++++++++++++++++- .../bigquery/jdbc/BigQueryStatement.java | 8 +- ...BigQueryPreparedStatementWriteApiTest.java | 328 +++++++++++++++ 3 files changed, 687 insertions(+), 22 deletions(-) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java index 92bf77c3e333..ac87eaf5b45b 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java @@ -19,6 +19,7 @@ import com.google.api.gax.retrying.RetrySettings; import com.google.api.services.bigquery.model.QueryParameter; import com.google.cloud.Tuple; +import com.google.cloud.bigquery.DatasetId; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.FieldList; import com.google.cloud.bigquery.JobStatistics.QueryStatistics; @@ -67,9 +68,15 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Queue; +import java.util.Set; class BigQueryPreparedStatement extends BigQueryStatement implements PreparedStatement { private static final BigQueryJdbcCustomLogger LOG = @@ -78,9 +85,11 @@ class BigQueryPreparedStatement extends BigQueryStatement implements PreparedSta // parameterHandler is inherited from BigQueryStatement protected final int parameterCount; protected final String currentQuery; + private final DirectInsertShape directInsertShape; + private boolean insertMetadataEvaluated = false; private Queue> batchParameters = new LinkedList<>(); - // Both hold statistics.getSchema(), which reports different things per statement type: the rows - // a SELECT returns, versus the columns an INSERT writes. + // resultSchema feeds getMetaData() for SELECT statements; insertSchema holds the target columns + // in placeholder order for single-table direct-VALUES INSERT statements on the Storage Write API. Schema resultSchema = null; // Feeds getMetaData(). Schema insertSchema = null; // Feeds the Storage Write API. private TableName insertTableName = null; @@ -91,6 +100,7 @@ class BigQueryPreparedStatement extends BigQueryStatement implements PreparedSta throw new BigQueryJdbcException("SQL query cannot be null."); } this.currentQuery = query; + this.directInsertShape = parseDirectInsertValues(query); QueryStatistics queryStatistics = describeQueryQuietly(); this.parameterCount = resolveParameterCount(query, queryStatistics); this.parameterHandler = @@ -117,7 +127,7 @@ private int resolveParameterCount(String query, QueryStatistics queryStatistics) * of the query text, so a statement stays usable against a service that refuses the dry run. */ private QueryStatistics describeQueryQuietly() { - if (this.currentQuery.indexOf(POSITIONAL_PARAMETER_CHAR) == -1) { + if (this.currentQuery.indexOf(POSITIONAL_PARAMETER_CHAR) == -1 || getUseLegacySql()) { return null; } try { @@ -161,27 +171,320 @@ private static StandardSQLTypeName toStandardSqlType(String typeName) { } } - // Remembers the table and schema of single-table INSERT targets, for the Storage Write API. - // Failure is delegated to executeBatch instead of the constructor. + // Remembers the table and target column schema of direct single-table INSERT ... VALUES (?, ...) + // statements for the Storage Write API. Statements with SQL expressions, partial column lists, or + // unsupported column types leave insertSchema null so executeBatch uses standard SQL execution. private void captureInsertMetadata(QueryStatistics statistics) { + if (statistics == null) { + return; + } + this.insertMetadataEvaluated = true; Schema schema = statistics.getSchema(); - if (!QueryStatistics.StatementType.INSERT.equals(statistics.getStatementType()) + if (this.directInsertShape == null + || !QueryStatistics.StatementType.INSERT.equals(statistics.getStatementType()) || schema == null - || schema.getFields() == null || statistics.getReferencedTables() == null || statistics.getReferencedTables().stream().distinct().count() != 1) { return; } - for (Field field : schema.getFields()) { + TableId tableId = statistics.getReferencedTables().get(0); + String project = resolveTargetProject(tableId); + if (project == null || tableId.getDataset() == null || tableId.getTable() == null) { + return; + } + Schema resolvedSchema = + resolveInsertSchema(schema, this.directInsertShape, this.parameterCount); + if (resolvedSchema == null) { + LOG.fine( + "INSERT target schema is incompatible with direct Storage Write API streaming;" + + " falling back to standard SQL batch execution."); + return; + } + this.insertSchema = resolvedSchema; + this.insertTableName = TableName.of(project, tableId.getDataset(), tableId.getTable()); + LOG.finer("insertTableName: %s, insertSchema: %s", this.insertTableName, this.insertSchema); + } + + private String resolveTargetProject(TableId tableId) { + if (tableId.getProject() != null) { + return tableId.getProject(); + } + DatasetId defaultDataset = this.querySettings.getDefaultDataset(); + String catalog = this.connection != null ? this.connection.getCatalog() : null; + if (defaultDataset != null + && defaultDataset.getProject() != null + && (catalog == null + || (tableId.getDataset() != null + && tableId.getDataset().equals(defaultDataset.getDataset())))) { + return defaultDataset.getProject(); + } + return catalog; + } + + static Schema resolveInsertSchema( + Schema tableSchema, DirectInsertShape shape, int parameterCount) { + if (tableSchema == null || shape == null) { + return null; + } + FieldList tableFields = tableSchema.getFields(); + if (tableFields == null + || shape.placeholderCount == 0 + || shape.placeholderCount != parameterCount + || tableFields.size() != shape.placeholderCount) { + return null; + } + for (Field field : tableFields) { if (!BigQueryTypeRegistry.isSupportedWriteApiField(field)) { - return; + return null; } } - this.insertSchema = schema; - TableId tableId = statistics.getReferencedTables().get(0); - this.insertTableName = - TableName.of(tableId.getProject(), tableId.getDataset(), tableId.getTable()); - LOG.finer("insertTableName: %s, insertSchema: %s", this.insertTableName, this.insertSchema); + if (shape.columnNames == null) { + return tableSchema; + } + if (shape.columnNames.size() != tableFields.size()) { + return null; + } + Map fieldsByLowerName = new HashMap<>(tableFields.size() * 2); + for (Field field : tableFields) { + fieldsByLowerName.put(field.getName().toLowerCase(Locale.ROOT), field); + } + List orderedFields = new ArrayList<>(shape.columnNames.size()); + Set seenColumns = new HashSet<>(shape.columnNames.size() * 2); + for (String columnName : shape.columnNames) { + String normalized = columnName.toLowerCase(Locale.ROOT); + Field matched = fieldsByLowerName.get(normalized); + if (matched == null || !seenColumns.add(normalized)) { + return null; + } + orderedFields.add(matched); + } + return Schema.of(orderedFields); + } + + static final class DirectInsertShape { + final int placeholderCount; + final List columnNames; + + DirectInsertShape(int placeholderCount, List columnNames) { + this.placeholderCount = placeholderCount; + this.columnNames = + columnNames == null ? null : Collections.unmodifiableList(new ArrayList<>(columnNames)); + } + } + + /** + * Recognizes single-row {@code INSERT [INTO]
[(, ...)] VALUES (?, ...) [;]} + * statements whose {@code VALUES} tuple consists strictly of bare positional {@code ?} + * placeholders. Returns {@code null} for any SQL expression, function/UDF call, cast, literal, + * {@code DEFAULT}, {@code SELECT}, multi-row tuple list, or {@code _SESSION}-qualified temporary + * table so the statement executes on BigQuery's SQL engine instead of the Storage Write API. + */ + static DirectInsertShape parseDirectInsertValues(String sql) { + if (sql == null) { + return null; + } + int pos = skipTrivia(sql, 0); + pos = pos < 0 ? -1 : matchKeyword(sql, pos, "INSERT"); + pos = pos < 0 ? -1 : skipTrivia(sql, pos); + if (pos < 0) { + return null; + } + int afterInto = matchKeyword(sql, pos, "INTO"); + if (afterInto >= 0) { + pos = skipTrivia(sql, afterInto); + if (pos < 0) { + return null; + } + } + pos = consumeTablePath(sql, pos); + if (pos < 0 || pos >= sql.length()) { + return null; + } + + List columnNames = null; + if (sql.charAt(pos) == '(') { + columnNames = new ArrayList<>(); + pos++; + while (true) { + pos = skipTrivia(sql, pos); + Tuple column = pos < 0 ? null : readIdentifier(sql, pos, false); + if (column == null) { + return null; + } + columnNames.add(column.x()); + pos = skipTrivia(sql, column.y()); + if (pos < 0 || pos >= sql.length()) { + return null; + } + char sep = sql.charAt(pos++); + if (sep == ')') { + break; + } + if (sep != ',') { + return null; + } + } + } + + pos = skipTrivia(sql, pos); + pos = pos < 0 ? -1 : matchKeyword(sql, pos, "VALUES"); + pos = pos < 0 ? -1 : skipTrivia(sql, pos); + if (pos < 0 || pos >= sql.length() || sql.charAt(pos++) != '(') { + return null; + } + + int placeholderCount = 0; + while (true) { + pos = skipTrivia(sql, pos); + if (pos < 0 || pos >= sql.length() || sql.charAt(pos) != POSITIONAL_PARAMETER_CHAR) { + return null; + } + placeholderCount++; + pos = skipTrivia(sql, pos + 1); + if (pos < 0 || pos >= sql.length()) { + return null; + } + char sep = sql.charAt(pos++); + if (sep == ')') { + break; + } + if (sep != ',') { + return null; + } + } + + pos = skipTrivia(sql, pos); + if (pos >= 0 && pos < sql.length() && sql.charAt(pos) == ';') { + pos = skipTrivia(sql, pos + 1); + } + return pos == sql.length() ? new DirectInsertShape(placeholderCount, columnNames) : null; + } + + private static int consumeTablePath(String sql, int pos) { + Tuple firstSegment = readIdentifier(sql, pos, true); + if (firstSegment == null) { + return -1; + } + String firstName = firstSegment.x(); + pos = firstSegment.y(); + int segmentCount = 1; + while (true) { + int next = skipTrivia(sql, pos); + if (next < 0) { + return -1; + } + if (next < sql.length() && (sql.charAt(next) == '.' || sql.charAt(next) == ':')) { + pos = skipTrivia(sql, next + 1); + Tuple segment = pos < 0 ? null : readIdentifier(sql, pos, true); + if (segment == null) { + return -1; + } + pos = segment.y(); + segmentCount++; + } else { + pos = next; + break; + } + } + if ((segmentCount == 2 && "_session".equalsIgnoreCase(firstName)) + || (segmentCount == 1 + && firstName.length() > 9 + && firstName.regionMatches(true, 0, "_session.", 0, 9))) { + return -1; + } + return pos; + } + + private static Tuple readIdentifier(String sql, int pos, boolean tableSegment) { + if (pos >= sql.length()) { + return null; + } + char first = sql.charAt(pos); + if (first == '`') { + return readBacktickIdentifier(sql, pos); + } + if (!(tableSegment ? isIdentPart(first) : isIdentStart(first))) { + return null; + } + int i = pos + 1; + while (i < sql.length() + && (isIdentPart(sql.charAt(i)) || (tableSegment && sql.charAt(i) == '-'))) { + i++; + } + return Tuple.of(sql.substring(pos, i), i); + } + + private static Tuple readBacktickIdentifier(String sql, int pos) { + StringBuilder sb = new StringBuilder(); + int i = pos + 1; + while (i < sql.length()) { + char c = sql.charAt(i); + if (c == '\\') { + if (i + 1 >= sql.length()) { + return null; + } + char escaped = sql.charAt(i + 1); + if (escaped != '\\' && escaped != '`') { + return null; + } + sb.append(escaped); + i += 2; + } else if (c == '`') { + return sb.length() == 0 ? null : Tuple.of(sb.toString(), i + 1); + } else { + sb.append(c); + i++; + } + } + return null; + } + + private static int matchKeyword(String sql, int pos, String keyword) { + int end = pos + keyword.length(); + if (end > sql.length() + || !sql.regionMatches(true, pos, keyword, 0, keyword.length()) + || (end < sql.length() && isIdentPart(sql.charAt(end)))) { + return -1; + } + return end; + } + + private static int skipTrivia(String sql, int pos) { + int len = sql.length(); + while (pos < len) { + char ch = sql.charAt(pos); + if (Character.isWhitespace(ch)) { + pos++; + } else if (ch == '#') { + pos = skipToLineEnd(sql, pos + 1); + } else if (ch == '-' && pos + 1 < len && sql.charAt(pos + 1) == '-') { + pos = skipToLineEnd(sql, pos + 2); + } else if (ch == '/' && pos + 1 < len && sql.charAt(pos + 1) == '*') { + int closeIdx = sql.indexOf("*/", pos + 2); + if (closeIdx < 0) { + return -1; + } + pos = closeIdx + 2; + } else { + break; + } + } + return pos; + } + + private static int skipToLineEnd(String sql, int pos) { + while (pos < sql.length() && sql.charAt(pos) != '\n' && sql.charAt(pos) != '\r') { + pos++; + } + return pos < sql.length() ? pos + 1 : pos; + } + + private static boolean isIdentStart(char ch) { + return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || ch == '_'; + } + + private static boolean isIdentPart(char ch) { + return isIdentStart(ch) || (ch >= '0' && ch <= '9'); } // This populates the schema of the ResultSet that is being returned by the query. @@ -449,12 +752,19 @@ public int[] executeBatch() throws SQLException { private int[] executeBatchInternal(int[] result) throws SQLException { // The constructor's dry run may have happened before the target table existed. - if (this.insertSchema == null + if (!this.insertMetadataEvaluated + && this.directInsertShape != null && this.querySettings.isUseWriteAPI() + && !getUseLegacySql() + && (this.connection == null || !this.connection.isTransactionStarted()) && this.batchParameters.size() >= this.querySettings.getWriteAPIActivationRowCount()) { + this.insertMetadataEvaluated = true; try { - captureInsertMetadata( - getQueryStatistics(getWriteBatchJobConfiguration(this.batchParameters.peek()))); + QueryStatistics statistics = describePositionalParameterQuery(this.currentQuery); + if (statistics != null) { + applyInferredParameterTypes(statistics); + captureInsertMetadata(statistics); + } } catch (SQLException | RuntimeException ex) { LOG.warning(ex, "Could not describe INSERT target; using the standard batch path."); } @@ -503,6 +813,12 @@ private int[] executeBatchInternal(int[] result) throws SQLException { } } else { + if (this.querySettings.isUseWriteAPI() + && this.batchParameters.size() >= this.querySettings.getWriteAPIActivationRowCount()) { + LOG.fine( + "Batch query is not eligible for Storage Write API; falling back to standard SQL" + + " batch execution."); + } try { LOG.info("Using individual INSERT query runs."); int count = this.batchParameters.size(); @@ -647,13 +963,34 @@ QueryJobConfiguration getStandardBatchJobConfiguration(String query) throws SQLE Boolean useWriteAPI() { LOG.finer("++enter++"); return this.querySettings.isUseWriteAPI() - // the constructor dry run already confirmed a single-table INSERT && this.insertSchema != null && this.batchParameters.size() >= this.querySettings.getWriteAPIActivationRowCount() + && isWriteApiAllowedBySessionAndTransaction() && BigQueryParameterHandler.areBatchParametersCompatibleWithWriteApi( this.insertSchema.getFields(), this.batchParameters); } + private boolean isWriteApiAllowedBySessionAndTransaction() { + if (this.connection == null) { + return true; + } + if (this.connection.isTransactionStarted()) { + return false; + } + if (this.insertTableName != null && this.insertTableName.getDataset().startsWith("_")) { + BigQueryConnection.SessionState snapshot = this.connection.getSessionStateSnapshot(); + boolean sessionActive = + this.connection.isSessionEnabled() || (snapshot != null && snapshot.sessionInfo != null); + DatasetId defaultDataset = this.querySettings.getDefaultDataset(); + if (sessionActive + && (defaultDataset == null + || !this.insertTableName.getDataset().equals(defaultDataset.getDataset()))) { + return false; + } + } + return true; + } + @Override public void setCharacterStream(int parameterIndex, Reader reader, int length) throws SQLException { diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java index 13181dfc4806..a7f26a58891d 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java @@ -192,7 +192,7 @@ private BigQuerySettings generateBigQuerySettings() throws SQLException { BigQuerySettings.Builder querySettings = BigQuerySettings.newBuilder(); DatasetId defaultDataset = this.connection.getDefaultDataset(); if (defaultDataset != null) { - querySettings.setDefaultDataset(this.connection.defaultDataset); + querySettings.setDefaultDataset(defaultDataset); } Long maxBytesBilled = this.connection.getMaxBytesBilled(); if (maxBytesBilled > 0) { @@ -1634,9 +1634,9 @@ QueryJobConfiguration.Builder getJobConfig(String query) { return queryConfigBuilder; } - private boolean getUseLegacySql() { - return QueryDialectType.BIG_QUERY.equals( - QueryDialectType.valueOf(this.querySettings.getQueryDialect())); + boolean getUseLegacySql() { + String dialect = this.querySettings.getQueryDialect(); + return dialect != null && QueryDialectType.BIG_QUERY.equals(QueryDialectType.valueOf(dialect)); } boolean isEnableTimestampPicos() { diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java index 77bb1ee1726d..82afef658758 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementWriteApiTest.java @@ -18,6 +18,8 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -27,6 +29,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; @@ -41,16 +44,20 @@ import com.google.api.services.bigquery.model.QueryParameterType; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.ConnectionProperty; +import com.google.cloud.bigquery.DatasetId; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.FieldElementType; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.JobInfo; import com.google.cloud.bigquery.JobStatistics.QueryStatistics; import com.google.cloud.bigquery.JobStatistics.QueryStatistics.StatementType; +import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.QueryParameterValue; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableResult; import com.google.cloud.bigquery.exception.BigQueryJdbcException; import com.google.cloud.bigquery.jdbc.telemetry.v1.QueryApiType; import com.google.cloud.bigquery.jdbc.telemetry.v1.StatementExecution; @@ -96,9 +103,14 @@ import java.util.List; import java.util.Locale; import java.util.TimeZone; +import java.util.stream.Stream; import org.json.JSONObject; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; @@ -596,4 +608,320 @@ public void testWriteApiCompatibilityAndParameterizedNumericBounds() throws Exce false); } } + + static Stream validDirectInsertQueries() { + return Stream.of( + Arguments.of("INSERT INTO TAB VALUES (?)", 1, null), + Arguments.of( + "insert tab (TARG_FIELD) values (?);", 1, Collections.singletonList("TARG_FIELD")), + Arguments.of( + "/* header */ INSERT INTO `my-proj.my_ds.TAB` (`col_b`, col_a) -- comment\n" + + "VALUES ( ? , ? ) ; # trailing", + 2, + Arrays.asList("col_b", "col_a")), + Arguments.of( + "INSERT INTO proj:123_ds.tab (c1) -- mac line ending\rVALUES (?)", + 1, + Collections.singletonList("c1")), + Arguments.of( + "INSERT INTO `my\\`tab` (`col\\\\1`, `col\\`2`) VALUES (?, ?)", + 2, + Arrays.asList("col\\1", "col`2"))); + } + + @ParameterizedTest + @MethodSource("validDirectInsertQueries") + public void testParseDirectInsertValuesAcceptsValidForms( + String sql, int expectedPlaceholders, List expectedColumns) { + BigQueryPreparedStatement.DirectInsertShape shape = + BigQueryPreparedStatement.parseDirectInsertValues(sql); + assertNotNull(shape); + assertEquals(expectedPlaceholders, shape.placeholderCount); + assertEquals(expectedColumns, shape.columnNames); + } + + @ParameterizedTest + @ValueSource( + strings = { + "", + " ", + "SELECT ?", + "UPDATE TAB SET a = ? WHERE b = ?", + "MERGE INTO TAB USING (SELECT ? AS v) ON FALSE WHEN NOT MATCHED THEN INSERT (a) VALUES (v)", + "INSERT INTO TAB (TARG_FIELD) VALUES (UDF(CAST(? AS STRING), 'XXXXXX'))", + "INSERT INTO TAB (STR_FIELD) VALUES (MASK_UDF(?))", + "INSERT INTO TAB (a) VALUES ((?))", + "INSERT INTO TAB (a, b) VALUES (?, 'constant')", + "INSERT INTO TAB (a, b) VALUES (?, DEFAULT)", + "INSERT INTO TAB (a, b) VALUES (?, NULL)", + "INSERT INTO TAB (a) VALUES (?), (?)", + "INSERT INTO TAB (a) SELECT a FROM TAB WHERE a = ?", + "INSERT INTO _SESSION.temp_tab (a) VALUES (?)", + "INSERT INTO `_SESSION.temp_tab` (a) VALUES (?)", + "INSERT INTO `\\u005fSESSION.temp_tab` (a) VALUES (?)", + "INSERT INTO TAB () VALUES (?)", + "INSERT INTO TAB (s.a) VALUES (?)", + "INSERT INTO TAB (1col) VALUES (?)", + "INSERT INTO TAB (`bad\\nescape`) VALUES (?)", + "INSERT INTO `` VALUES (?)", + "INSERT INTO TAB (a) VALUES (?); INSERT INTO TAB (a) VALUES (?)", + "/* unclosed comment INSERT INTO TAB VALUES (?)" + }) + public void testParseDirectInsertValuesRejectsIneligibleSql(String sql) { + assertNull(BigQueryPreparedStatement.parseDirectInsertValues(null)); + assertNull(BigQueryPreparedStatement.parseDirectInsertValues(sql)); + } + + static Stream expressionFallbackQueries() { + return Stream.of( + Arguments.of( + "INSERT INTO TAB (STR_FIELD) VALUES (MASK_UDF(?));", StandardSQLTypeName.STRING), + Arguments.of( + "INSERT INTO TAB (TARG_FIELD) VALUES (UDF(CAST(? AS STRING), 'XXXXXX'));", + StandardSQLTypeName.BYTES)); + } + + @ParameterizedTest + @MethodSource("expressionFallbackQueries") + public void testExpressionInValuesFallsBackToStandardSqlBatch( + String sql, StandardSQLTypeName targetColType) throws Exception { + doReturn( + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + Schema.of(Field.of("COL", targetColType)), + Collections.singletonList(queryParam("STRING")))) + .when(bigQuery) + .create(any(JobInfo.class)); + + TableResult dmlResult = mock(TableResult.class); + doReturn(StatementType.INSERT).when(dmlResult).getStatementType(); + doReturn(1L).when(dmlResult).getNumDmlAffectedRows(); + doReturn(dmlResult).when(bigQuery).queryWithTimeout(any(), any(), any()); + + BigQueryPreparedStatement ps = new BigQueryPreparedStatement(connection, sql); + assertNull(ps.insertSchema); + for (int i = 0; i < 3; i++) { + ps.setString(1, "val-" + i); + ps.addBatch(); + } + assertFalse(ps.useWriteAPI()); + assertEquals(3, ps.executeBatch().length); + verify(connection, never()).getBigQueryWriteClient(); + verify(bigQuery, times(1)).create(any(JobInfo.class)); + + ArgumentCaptor jobCaptor = + ArgumentCaptor.forClass(QueryJobConfiguration.class); + verify(bigQuery).queryWithTimeout(jobCaptor.capture(), any(), any()); + assertEquals(3, jobCaptor.getValue().getPositionalParameters().size()); + assertEquals("val-0", jobCaptor.getValue().getPositionalParameters().get(0).getValue()); + } + + @Test + public void testResolveInsertSchemaAndReorderedColumnExecution() throws Exception { + Schema twoCol = + Schema.of( + Field.of("id", StandardSQLTypeName.INT64), + Field.of("name", StandardSQLTypeName.STRING)); + BigQueryPreparedStatement.DirectInsertShape reordered = + BigQueryPreparedStatement.parseDirectInsertValues( + "INSERT INTO TAB (NAME, id) VALUES (?, ?)"); + Schema resolved = BigQueryPreparedStatement.resolveInsertSchema(twoCol, reordered, 2); + assertEquals("name", resolved.getFields().get(0).getName()); + assertEquals("id", resolved.getFields().get(1).getName()); + + // Subset, duplicate, and unknown/pseudo columns return null. + for (String badSql : + new String[] { + "INSERT INTO TAB (id) VALUES (?)", + "INSERT INTO TAB (id, ID) VALUES (?, ?)", + "INSERT INTO TAB (id, _PARTITIONTIME) VALUES (?, ?)" + }) { + BigQueryPreparedStatement.DirectInsertShape s = + BigQueryPreparedStatement.parseDirectInsertValues(badSql); + assertNull(BigQueryPreparedStatement.resolveInsertSchema(twoCol, s, s.placeholderCount)); + } + + // All 14 scalar/temporal/special types accepted; REPEATED, STRUCT, and RANGE rejected. + BigQueryPreparedStatement.DirectInsertShape singleShape = + BigQueryPreparedStatement.parseDirectInsertValues("INSERT INTO TAB VALUES (?)"); + for (StandardSQLTypeName t : + new StandardSQLTypeName[] { + StandardSQLTypeName.STRING, StandardSQLTypeName.BYTES, StandardSQLTypeName.INT64, + StandardSQLTypeName.FLOAT64, StandardSQLTypeName.NUMERIC, StandardSQLTypeName.BIGNUMERIC, + StandardSQLTypeName.BOOL, StandardSQLTypeName.DATE, StandardSQLTypeName.TIME, + StandardSQLTypeName.DATETIME, StandardSQLTypeName.TIMESTAMP, StandardSQLTypeName.JSON, + StandardSQLTypeName.GEOGRAPHY, StandardSQLTypeName.INTERVAL + }) { + assertNotNull( + BigQueryPreparedStatement.resolveInsertSchema( + Schema.of(Field.of("c", t)), singleShape, 1)); + } + assertNull( + BigQueryPreparedStatement.resolveInsertSchema( + Schema.of( + Field.newBuilder("arr", StandardSQLTypeName.STRING) + .setMode(Field.Mode.REPEATED) + .build()), + singleShape, + 1)); + assertNull( + BigQueryPreparedStatement.resolveInsertSchema( + Schema.of( + Field.of( + "rec", StandardSQLTypeName.STRUCT, Field.of("s", StandardSQLTypeName.INT64))), + singleShape, + 1)); + + // End-to-end reordered executeBatch maps placeholder 1 -> "name" and placeholder 2 -> "id". + doReturn( + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + twoCol, + Arrays.asList(queryParam("STRING"), queryParam("INT64")))) + .when(bigQuery) + .create(any(JobInfo.class)); + stubWriteClient(3L, true); + JsonStreamWriter mockWriter = mock(JsonStreamWriter.class); + when(mockWriter.append(any(JsonArray.class), anyLong())) + .thenReturn(ApiFutures.immediateFuture(AppendRowsResponse.getDefaultInstance())); + + BigQueryPreparedStatement ps = + newWriteApiPreparedStatement("INSERT INTO TAB (name, id) VALUES (?, ?)", mockWriter); + for (int i = 0; i < 3; i++) { + ps.setString(1, "u-" + i); + ps.setInt(2, 10 + i); + ps.addBatch(); + } + assertArrayEquals(new int[] {1, 1, 1}, ps.executeBatch()); + ArgumentCaptor rowsCaptor = ArgumentCaptor.forClass(JsonArray.class); + verify(mockWriter).append(rowsCaptor.capture(), anyLong()); + JsonObject firstRow = rowsCaptor.getValue().get(0).getAsJsonObject(); + assertEquals("u-0", firstRow.get("name").getAsString()); + assertEquals("10", firstRow.get("id").getAsString()); + } + + @Test + public void testTargetProjectTransactionsSessionsLegacySqlAndRetryDryRun() throws Exception { + // 1. Legacy SQL skips positional dry run in constructor. + doReturn("BIG_QUERY").when(connection).getQueryDialect(); + assertEquals( + 1, new BigQueryPreparedStatement(connection, "SELECT ? FROM [proj:ds.tab]").parameterCount); + verify(bigQuery, never()).create(any(JobInfo.class)); + doReturn("SQL").when(connection).getQueryDialect(); + + // 2. Active transaction & active session anonymous "_" dataset gating. + Schema oneStr = Schema.of(Field.of("c1", StandardSQLTypeName.STRING)); + doReturn( + mockInsertDryRunJob( + TableId.of("proj", "_anon_session_ds", "TAB"), + oneStr, + Collections.singletonList(queryParam("STRING")))) + .when(bigQuery) + .create(any(JobInfo.class)); + BigQueryPreparedStatement psGate = + new BigQueryPreparedStatement(connection, "INSERT INTO TAB VALUES (?)"); + populateBatchAndAssertWriteApi(psGate, 3, i -> psGate.setString(1, "v" + i), true); + doReturn(true).when(connection).isTransactionStarted(); + assertFalse(psGate.useWriteAPI()); + doReturn(false).when(connection).isTransactionStarted(); + + doReturn( + new BigQueryConnection.SessionState( + ConnectionProperty.of("session_id", "s1"), Collections.emptyList(), true)) + .when(connection) + .getSessionStateSnapshot(); + assertFalse(psGate.useWriteAPI()); + doReturn(DatasetId.of("proj", "_anon_session_ds")).when(connection).getDefaultDataset(); + BigQueryPreparedStatement psExplicitHidden = + new BigQueryPreparedStatement(connection, "INSERT INTO TAB VALUES (?)"); + populateBatchAndAssertWriteApi( + psExplicitHidden, 3, i -> psExplicitHidden.setString(1, "v" + i), true); + doReturn(BigQueryConnection.SessionState.empty()).when(connection).getSessionStateSnapshot(); + + // 3. Target project resolution: other_ds uses catalog-proj; default_ds uses default-proj. + doReturn("catalog-proj").when(connection).getCatalog(); + doReturn(DatasetId.of("default-proj", "default_ds")).when(connection).getDefaultDataset(); + BigQueryWriteClient writeClient = stubWriteClient(3L, true); + JsonStreamWriter okWriter = mock(JsonStreamWriter.class); + when(okWriter.append(any(JsonArray.class), anyLong())) + .thenReturn(ApiFutures.immediateFuture(AppendRowsResponse.getDefaultInstance())); + for (String[] dsAndExpectedProj : + new String[][] {{"other_ds", "catalog-proj"}, {"default_ds", "default-proj"}}) { + doReturn( + mockInsertDryRunJob( + TableId.of(dsAndExpectedProj[0], "TAB"), + oneStr, + Collections.singletonList(queryParam("STRING")))) + .when(bigQuery) + .create(any(JobInfo.class)); + BigQueryPreparedStatement psProj = + newWriteApiPreparedStatement( + "INSERT INTO " + dsAndExpectedProj[0] + ".TAB VALUES (?)", okWriter); + for (int i = 0; i < 3; i++) { + psProj.setString(1, "v" + i); + psProj.addBatch(); + } + psProj.executeBatch(); + } + ArgumentCaptor parentCaptor = + ArgumentCaptor.forClass(CreateWriteStreamRequest.class); + verify(writeClient, times(2)).createWriteStream(parentCaptor.capture()); + assertEquals( + "projects/catalog-proj/datasets/other_ds/tables/TAB", + parentCaptor.getAllValues().get(0).getParent()); + assertEquals( + "projects/default-proj/datasets/default_ds/tables/TAB", + parentCaptor.getAllValues().get(1).getParent()); + + // 4. Retry dry run uses unbound query, and runs at most once even if it returns null or throws. + TableResult dmlResult = mock(TableResult.class); + doReturn(StatementType.INSERT).when(dmlResult).getStatementType(); + doReturn(1L).when(dmlResult).getNumDmlAffectedRows(); + doReturn(dmlResult).when(bigQuery).queryWithTimeout(any(), any(), any()); + + clearInvocations(bigQuery); + Job intJob = + mockInsertDryRunJob( + TableId.of("proj", "ds", "TAB"), + Schema.of(Field.of("n", StandardSQLTypeName.INT64)), + Collections.singletonList(queryParam("INT64"))); + when(bigQuery.create(any(JobInfo.class))) + .thenThrow(new BigQueryException(404, "Not found")) + .thenReturn(intJob); + BigQueryPreparedStatement psRetry = + new BigQueryPreparedStatement(connection, "INSERT INTO ds.TAB (n) VALUES (?)"); + for (int i = 0; i < 3; i++) { + psRetry.setNull(1, Types.OTHER); + psRetry.addBatch(); + } + psRetry.executeBatch(); + assertNotNull(psRetry.insertSchema); + ArgumentCaptor jobCaptor = ArgumentCaptor.forClass(JobInfo.class); + verify(bigQuery, times(2)).create(jobCaptor.capture()); + QueryJobConfiguration retryCfg = jobCaptor.getAllValues().get(1).getConfiguration(); + assertTrue( + retryCfg.getPositionalParameters() == null || retryCfg.getPositionalParameters().isEmpty()); + + // Retry dry run returning null statistics OR throwing does NOT retry on 2nd executeBatch. + for (boolean throwOnRetry : new boolean[] {false, true}) { + clearInvocations(bigQuery); + if (throwOnRetry) { + when(bigQuery.create(any(JobInfo.class))).thenThrow(new BigQueryException(403, "Denied")); + } else { + when(bigQuery.create(any(JobInfo.class))) + .thenThrow(new BigQueryException(404, "Not found")) + .thenReturn(mock(Job.class)); + } + BigQueryPreparedStatement psOnce = + new BigQueryPreparedStatement(connection, "INSERT INTO ds.TAB (n) VALUES (?)"); + for (int b = 0; b < 2; b++) { + for (int i = 0; i < 3; i++) { + psOnce.setLong(1, i); + psOnce.addBatch(); + } + psOnce.executeBatch(); + } + verify(bigQuery, times(2)).create(any(JobInfo.class)); + } + } }