diff --git a/google-cloud-jar-parent/pom.xml b/google-cloud-jar-parent/pom.xml index 7ab2d6e857a0..acb8610974d9 100644 --- a/google-cloud-jar-parent/pom.xml +++ b/google-cloud-jar-parent/pom.xml @@ -142,7 +142,7 @@ com.google.apis google-api-services-bigquery - v2-rev20260612-2.0.0 + v2-rev20260707-2.0.0 diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java new file mode 100644 index 000000000000..ab586fe9b2b2 --- /dev/null +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java @@ -0,0 +1,205 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.common.collect.ImmutableList; +import com.google.common.io.BaseEncoding; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorLoader; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.ipc.ReadChannel; +import org.apache.arrow.vector.ipc.message.MessageSerializer; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; + +final class ArrowDeserializer { + + private ArrowDeserializer() {} + + static Schema arrowSchemaToBigQuerySchema(org.apache.arrow.vector.types.pojo.Schema arrowSchema) { + List fields = new ArrayList<>(); + for (Field arrowField : arrowSchema.getFields()) { + fields.add(arrowFieldToBigQueryField(arrowField)); + } + return Schema.of(fields); + } + + private static com.google.cloud.bigquery.Field arrowFieldToBigQueryField(Field arrowField) { + String name = arrowField.getName(); + ArrowType type = arrowField.getType(); + com.google.cloud.bigquery.Field.Builder builder; + + if (type instanceof ArrowType.List) { + Field innerField = arrowField.getChildren().get(0); + LegacySQLTypeName innerType = arrowTypeToLegacySQLTypeName(innerField.getType()); + builder = com.google.cloud.bigquery.Field.newBuilder(name, innerType); + builder.setMode(com.google.cloud.bigquery.Field.Mode.REPEATED); + if (!innerField.getChildren().isEmpty()) { + List subFields = new ArrayList<>(); + for (Field childField : innerField.getChildren()) { + subFields.add(arrowFieldToBigQueryField(childField)); + } + builder.setType(LegacySQLTypeName.RECORD, FieldList.of(subFields)); + } + } else { + LegacySQLTypeName bqType = arrowTypeToLegacySQLTypeName(type); + builder = com.google.cloud.bigquery.Field.newBuilder(name, bqType); + if (arrowField.isNullable()) { + builder.setMode(com.google.cloud.bigquery.Field.Mode.NULLABLE); + } else { + builder.setMode(com.google.cloud.bigquery.Field.Mode.REQUIRED); + } + if (!arrowField.getChildren().isEmpty()) { + List subFields = new ArrayList<>(); + for (Field childField : arrowField.getChildren()) { + subFields.add(arrowFieldToBigQueryField(childField)); + } + builder.setType(LegacySQLTypeName.RECORD, FieldList.of(subFields)); + } + } + return builder.build(); + } + + private static LegacySQLTypeName arrowTypeToLegacySQLTypeName(ArrowType type) { + switch (type.getTypeID()) { + case Int: + return LegacySQLTypeName.INTEGER; + case FloatingPoint: + return LegacySQLTypeName.FLOAT; + case Utf8: + return LegacySQLTypeName.STRING; + case Bool: + return LegacySQLTypeName.BOOLEAN; + case Binary: + return LegacySQLTypeName.BYTES; + case Decimal: + return LegacySQLTypeName.NUMERIC; + case Timestamp: + return LegacySQLTypeName.TIMESTAMP; + case Date: + return LegacySQLTypeName.DATE; + case Time: + return LegacySQLTypeName.TIME; + case Struct: + return LegacySQLTypeName.RECORD; + default: + throw new IllegalArgumentException("Unsupported Arrow type: " + type.getTypeID()); + } + } + + static List deserializeRecordBatch( + byte[] recordBatchBytes, Schema schema, org.apache.arrow.vector.types.pojo.Schema arrowSchema) + throws IOException { + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + List vectors = new ArrayList<>(); + for (Field field : arrowSchema.getFields()) { + vectors.add(field.createVector(allocator)); + } + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + VectorLoader loader = new VectorLoader(root); + try (org.apache.arrow.vector.ipc.message.ArrowRecordBatch deserializedBatch = + MessageSerializer.deserializeRecordBatch( + new ReadChannel(new ByteArrayReadableSeekableByteChannel(recordBatchBytes)), + allocator)) { + loader.load(deserializedBatch); + int rowCount = root.getRowCount(); + List rows = new ArrayList<>(rowCount); + for (int i = 0; i < rowCount; i++) { + rows.add(arrowRootToFieldValueList(root, i, schema)); + } + return ImmutableList.copyOf(rows); + } + } + } + } + + static FieldValueList arrowRootToFieldValueList( + VectorSchemaRoot root, int rowIndex, Schema schema) { + List fieldValues = new ArrayList<>(); + for (int colIndex = 0; colIndex < root.getFieldVectors().size(); colIndex++) { + FieldVector vector = root.getVector(colIndex); + com.google.cloud.bigquery.Field bqField = schema.getFields().get(colIndex); + fieldValues.add(arrowVectorToFieldValue(vector, rowIndex, bqField)); + } + return FieldValueList.of(fieldValues, schema.getFields()); + } + + private static FieldValue arrowVectorToFieldValue( + FieldVector vector, int rowIndex, com.google.cloud.bigquery.Field bqField) { + if (vector.isNull(rowIndex)) { + return FieldValue.of(FieldValue.Attribute.PRIMITIVE, null); + } + + // Handle repeated fields + if (bqField.getMode() == com.google.cloud.bigquery.Field.Mode.REPEATED) { + ListVector listVector = (ListVector) vector; + FieldVector dataVector = (FieldVector) listVector.getDataVector(); + int start = listVector.getElementStartIndex(rowIndex); + int end = listVector.getElementEndIndex(rowIndex); + List elements = new ArrayList<>(end - start); + com.google.cloud.bigquery.Field elementBqField = + com.google.cloud.bigquery.Field.newBuilder(bqField.getName(), bqField.getType()) + .setMode(com.google.cloud.bigquery.Field.Mode.NULLABLE) + .build(); + for (int k = start; k < end; k++) { + elements.add(arrowVectorToFieldValue(dataVector, k, elementBqField)); + } + return FieldValue.of( + FieldValue.Attribute.REPEATED, FieldValueList.of(elements, bqField.getSubFields())); + } + + // Handle RECORD/STRUCT fields + if (bqField.getType() == LegacySQLTypeName.RECORD) { + StructVector structVector = (StructVector) vector; + List elements = new ArrayList<>(structVector.size()); + for (int colIndex = 0; colIndex < structVector.size(); colIndex++) { + FieldVector childVector = (FieldVector) structVector.getChildByOrdinal(colIndex); + com.google.cloud.bigquery.Field childBqField = bqField.getSubFields().get(colIndex); + elements.add(arrowVectorToFieldValue(childVector, rowIndex, childBqField)); + } + return FieldValue.of( + FieldValue.Attribute.RECORD, FieldValueList.of(elements, bqField.getSubFields())); + } + + // Handle primitive types - convert everything to String representations to match BQ standard + Object value = vector.getObject(rowIndex); + String stringVal; + if (value instanceof byte[]) { + stringVal = BaseEncoding.base64().encode((byte[]) value); + } else if (bqField.getType() == LegacySQLTypeName.TIMESTAMP) { + // Arrow timestamps are long values representing epoch micro/milli/nano seconds. + // Standard BigQuery JSON returns timestamps as string of epoch seconds with micro precision + // (e.g. "1408452095.220000"). + long micros = (long) value; + // Convert to seconds with 6 decimal places of precision + stringVal = String.format(Locale.US, "%.6f", micros / 1000000.0); + } else { + stringVal = String.valueOf(value); + } + + return FieldValue.of(FieldValue.Attribute.PRIMITIVE, stringVal); + } +} diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowSerializationOptions.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowSerializationOptions.java new file mode 100644 index 000000000000..457ea4917daf --- /dev/null +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowSerializationOptions.java @@ -0,0 +1,118 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.api.core.BetaApi; +import java.io.Serializable; +import java.util.Objects; + +/** Options specific to the Apache Arrow output format. */ +@BetaApi +public final class ArrowSerializationOptions implements Serializable { + + private static final long serialVersionUID = 1L; + + private final String bufferCompression; + private final String picosTimestampPrecision; + + private ArrowSerializationOptions(Builder builder) { + this.bufferCompression = builder.bufferCompression; + this.picosTimestampPrecision = builder.picosTimestampPrecision; + } + + public String getBufferCompression() { + return bufferCompression; + } + + public String getPicosTimestampPrecision() { + return picosTimestampPrecision; + } + + public static Builder newBuilder() { + return new Builder(); + } + + @Override + public String toString() { + return com.google.common.base.MoreObjects.toStringHelper(this) + .add("bufferCompression", bufferCompression) + .add("picosTimestampPrecision", picosTimestampPrecision) + .toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ArrowSerializationOptions that = (ArrowSerializationOptions) o; + return Objects.equals(bufferCompression, that.bufferCompression) + && Objects.equals(picosTimestampPrecision, that.picosTimestampPrecision); + } + + @Override + public int hashCode() { + return Objects.hash(bufferCompression, picosTimestampPrecision); + } + + com.google.api.services.bigquery.model.ArrowSerializationOptions toPb() { + com.google.api.services.bigquery.model.ArrowSerializationOptions optionsPb = + new com.google.api.services.bigquery.model.ArrowSerializationOptions(); + if (bufferCompression != null) { + optionsPb.setBufferCompression(bufferCompression); + } + if (picosTimestampPrecision != null) { + optionsPb.setPicosTimestampPrecision(picosTimestampPrecision); + } + return optionsPb; + } + + static ArrowSerializationOptions fromPb( + com.google.api.services.bigquery.model.ArrowSerializationOptions optionsPb) { + if (optionsPb == null) { + return null; + } + return newBuilder() + .setBufferCompression(optionsPb.getBufferCompression()) + .setPicosTimestampPrecision(optionsPb.getPicosTimestampPrecision()) + .build(); + } + + public static final class Builder { + private String bufferCompression; + private String picosTimestampPrecision; + + private Builder() {} + + public Builder setBufferCompression(String bufferCompression) { + this.bufferCompression = bufferCompression; + return this; + } + + public Builder setPicosTimestampPrecision(String picosTimestampPrecision) { + this.picosTimestampPrecision = picosTimestampPrecision; + return this; + } + + public ArrowSerializationOptions build() { + return new ArrowSerializationOptions(this); + } + } +} diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java index a62fbb5008d4..0d75d509d1a3 100644 --- a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java @@ -75,6 +75,8 @@ public final class QueryJobConfiguration extends JobConfiguration { private final Long maxResults; private final JobCreationMode jobCreationMode; private final String reservation; + private final QueryResultsFormat queryResultsFormat; + private final ArrowSerializationOptions arrowSerializationOptions; /** * Priority levels for a query. If not specified the priority is assumed to be {@link @@ -144,6 +146,8 @@ public static final class Builder private Long maxResults; private JobCreationMode jobCreationMode; private String reservation; + private QueryResultsFormat queryResultsFormat; + private ArrowSerializationOptions arrowSerializationOptions; private Builder() { super(Type.QUERY); @@ -181,6 +185,8 @@ private Builder(QueryJobConfiguration jobConfiguration) { this.maxResults = jobConfiguration.maxResults; this.jobCreationMode = jobConfiguration.jobCreationMode; this.reservation = jobConfiguration.reservation; + this.queryResultsFormat = jobConfiguration.queryResultsFormat; + this.arrowSerializationOptions = jobConfiguration.arrowSerializationOptions; } private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { @@ -701,6 +707,17 @@ public Builder setReservation(String reservation) { return this; } + public Builder setQueryResultsFormat(QueryResultsFormat queryResultsFormat) { + this.queryResultsFormat = queryResultsFormat; + return this; + } + + public Builder setArrowSerializationOptions( + ArrowSerializationOptions arrowSerializationOptions) { + this.arrowSerializationOptions = arrowSerializationOptions; + return this; + } + public QueryJobConfiguration build() { return new QueryJobConfiguration(this); } @@ -747,6 +764,8 @@ private QueryJobConfiguration(Builder builder) { this.maxResults = builder.maxResults; this.jobCreationMode = builder.jobCreationMode; this.reservation = builder.reservation; + this.queryResultsFormat = builder.queryResultsFormat; + this.arrowSerializationOptions = builder.arrowSerializationOptions; } /** @@ -973,6 +992,14 @@ public Builder toBuilder() { return new Builder(this); } + public QueryResultsFormat getQueryResultsFormat() { + return queryResultsFormat; + } + + public ArrowSerializationOptions getArrowSerializationOptions() { + return arrowSerializationOptions; + } + @Override ToStringHelper toStringHelper() { return super.toStringHelper() @@ -1004,13 +1031,23 @@ ToStringHelper toStringHelper() { .add("rangePartitioning", rangePartitioning) .add("connectionProperties", connectionProperties) .add("jobCreationMode", jobCreationMode) - .add("reservation", reservation); + .add("reservation", reservation) + .add("queryResultsFormat", queryResultsFormat) + .add("arrowSerializationOptions", arrowSerializationOptions); } @Override public boolean equals(Object obj) { - return obj == this - || obj instanceof QueryJobConfiguration && baseEquals((QueryJobConfiguration) obj); + if (obj == this) { + return true; + } + if (obj == null || !(obj instanceof QueryJobConfiguration)) { + return false; + } + QueryJobConfiguration other = (QueryJobConfiguration) obj; + return baseEquals(other) + && Objects.equals(queryResultsFormat, other.queryResultsFormat) + && Objects.equals(arrowSerializationOptions, other.arrowSerializationOptions); } @Override @@ -1043,7 +1080,9 @@ public int hashCode() { labels, rangePartitioning, connectionProperties, - reservation); + reservation, + queryResultsFormat, + arrowSerializationOptions); } @Override diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryResultsFormat.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryResultsFormat.java new file mode 100644 index 000000000000..61c12d34e326 --- /dev/null +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryResultsFormat.java @@ -0,0 +1,29 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.api.core.BetaApi; + +/** The format of the query results. */ +@BetaApi +public enum QueryResultsFormat { + /** Serialized row data in Apache Arrow format. */ + ARROW, + + /** Default encoding of results as JSON struct array. */ + STRUCT_ENCODING +} diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java index 7fe41daa0608..1d60d904bfab 100644 --- a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java +++ b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java @@ -241,6 +241,35 @@ public void testJobCreationMode() { QUERY_JOB_CONFIGURATION_SET_JOB_CREATION_MODE.toBuilder().build()); } + @Test + public void testArrowConfigurations() { + QueryResultsFormat format = QueryResultsFormat.ARROW; + ArrowSerializationOptions options = + ArrowSerializationOptions.newBuilder() + .setBufferCompression("LZ4") + .setPicosTimestampPrecision("PRECISION_MILLIS") + .build(); + QueryJobConfiguration job = + QueryJobConfiguration.newBuilder(QUERY) + .setQueryResultsFormat(format) + .setArrowSerializationOptions(options) + .build(); + + assertEquals(format, job.getQueryResultsFormat()); + assertEquals(options, job.getArrowSerializationOptions()); + + // Test toBuilder + QueryJobConfiguration copiedJob = job.toBuilder().build(); + assertEquals(job, copiedJob); + assertEquals(format, copiedJob.getQueryResultsFormat()); + assertEquals(options, copiedJob.getArrowSerializationOptions()); + + // Test toPb/fromPb (not preserved) + QueryJobConfiguration jobFromPb = QueryJobConfiguration.fromPb(job.toPb()); + assertNull(jobFromPb.getQueryResultsFormat()); + assertNull(jobFromPb.getArrowSerializationOptions()); + } + private void compareQueryJobConfiguration( QueryJobConfiguration expected, QueryJobConfiguration value) { assertEquals(expected, value);