Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,19 @@
package com.google.cloud.bigquery.jdbc;

import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch;
import com.google.cloud.bigquery.storage.v1.ArrowSchema;
import org.apache.arrow.vector.util.JsonStringArrayList;

/** This class acts as a facade layer and wraps Arrow's VectorSchemaRoot & JsonStringArrayList */
class BigQueryArrowBatchWrapper {
// Reference to the current arrowBatch
private final ArrowRecordBatch currentArrowBatch;
// Arrow schema carried with this batch, set only when the source sent one (e.g. the first
// ReadRowsResponse of a stream)
private final ArrowSchema arrowSchema;
// Rows the source reported for this message (ReadRowsResponse.row_count), or null when the
// wrapper was not built from a stream message (markers, nested records)
private final Long reportedRowCount;
// Reference to the nested Records, set as null otherwise (Arrays)
private final JsonStringArrayList nestedRecords;

Expand All @@ -33,34 +40,61 @@ class BigQueryArrowBatchWrapper {

private BigQueryArrowBatchWrapper(
ArrowRecordBatch currentArrowBatch,
ArrowSchema arrowSchema,
Long reportedRowCount,
JsonStringArrayList nestedRecords,
boolean isLast,
Throwable exception) {
this.currentArrowBatch = currentArrowBatch;
this.arrowSchema = arrowSchema;
this.reportedRowCount = reportedRowCount;
this.nestedRecords = nestedRecords;
this.isLast = isLast;
this.exception = exception;
}

static BigQueryArrowBatchWrapper of(ArrowRecordBatch currentArrowBatch, boolean... isLast) {
boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0];
return new BigQueryArrowBatchWrapper(currentArrowBatch, null, isLastFlag, null);
return new BigQueryArrowBatchWrapper(currentArrowBatch, null, null, null, isLastFlag, null);
}

/**
* Wraps one stream message: its record batch, the schema sent with it, and the row count it
* reports. The batch and the schema may each be null (a schema-only message carries no batch;
* later messages carry no schema).
*/
static BigQueryArrowBatchWrapper of(
ArrowRecordBatch currentArrowBatch, ArrowSchema arrowSchema, long reportedRowCount) {
return new BigQueryArrowBatchWrapper(
currentArrowBatch, arrowSchema, reportedRowCount, null, false, null);
}

static BigQueryArrowBatchWrapper getNestedFieldValueListWrapper(
JsonStringArrayList nestedRecords, boolean... isLast) {
boolean isLastFlag = isLast != null && isLast.length == 1 && isLast[0];
return new BigQueryArrowBatchWrapper(null, nestedRecords, isLastFlag, null);
return new BigQueryArrowBatchWrapper(null, null, null, nestedRecords, isLastFlag, null);
}

static BigQueryArrowBatchWrapper ofError(Throwable exception) {
return new BigQueryArrowBatchWrapper(null, null, true, exception);
return new BigQueryArrowBatchWrapper(null, null, null, null, true, exception);
}

ArrowRecordBatch getCurrentArrowBatch() {
return this.currentArrowBatch;
}

ArrowSchema getArrowSchema() {
return this.arrowSchema;
}

/**
* Rows the source reported for this message, or null if this wrapper was not built from a stream
* message.
*/
Long getReportedRowCount() {
return this.reportedRowCount;
}

JsonStringArrayList getNestedRecords() {
return this.nestedRecords;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@

/** {@link ResultSet} Implementation for Arrow datasource (Using Storage Read APIs) */
class BigQueryArrowResultSet extends BigQueryBaseResultSet {
// Number of rows to return. Iteration stops once this many rows have been returned (this is
// also how Statement.setMaxRows is applied).
private final long totalRows;
// count of rows read by the current instance of ResultSet
private long rowCount = 0;
Expand Down Expand Up @@ -112,12 +114,8 @@ private BigQueryArrowResultSet(
this.nestedRowIndex = fromIndex - 1;
this.ownedTask = ownedTask;
this.enableTimestampPicos = enableTimestampPicos;
if (!isNested && arrowSchema != null) {
try {
this.arrowDeserializer = new ArrowDeserializer(arrowSchema);
} catch (IOException ex) {
throw new BigQueryJdbcException("IOException during ArrowDeserializer creation", ex);
}
if (!isNested) {
initDeserializerIfNeeded(arrowSchema);
}
}

Expand Down Expand Up @@ -269,49 +267,34 @@ public boolean next() throws SQLException {
return false;
} else {
/* Non nested */
if (this.hasReachedEnd || this.isLast()) {
if (this.afterLast || this.hasReachedEnd || reachedRowLimit()) {
if (!this.afterLast && this.vectorSchemaRoot != null) {
// Free the last batch's Arrow buffers as soon as iteration ends, not at close().
this.vectorSchemaRoot.clear();
}
this.afterLast = true;
return false;
}
try {
if (this.currentBatchRowIndex == -1
|| this.currentBatchRowIndex == (this.vectorSchemaRoot.getRowCount() - 1)) {
/* Start of iteration or we have exhausted the current batch */
// Advance the cursor. Potentially blocking operation.
try (Scope scope = makeOriginalContextCurrent()) {
BigQueryArrowBatchWrapper batchWrapper = this.buffer.take();
checkClosed();
if (batchWrapper.getException() != null) {
throw backgroundFetchError(batchWrapper.getException());
}
if (batchWrapper.isLast()) {
/* Marks the end of the records */
if (this.vectorSchemaRoot != null) {
// IMP: To avoid memory leak: clear vectorSchemaRoot as it still holds
// the last batch
this.vectorSchemaRoot.clear();
}
this.hasReachedEnd = true;
this.rowCount++;
return false;
}
// Valid batch, process it
ArrowRecordBatch arrowBatch = batchWrapper.getCurrentArrowBatch();
// Populates vectorSchemaRoot
this.arrowDeserializer.deserializeArrowBatch(arrowBatch);
// Pointing to the first row in this fresh batch
this.currentBatchRowIndex = 0;
this.rowCount++;
return true;
}
}
// There are rows left in the current batch.
else if (this.currentBatchRowIndex < this.vectorSchemaRoot.getRowCount()) {
this.currentBatchRowIndex++;
this.rowCount++;
return true;
if (hasMoreRowsInCurrentBatch()) {
this.currentBatchRowIndex++;
this.rowCount++;
return true;
}
// Start of iteration or the current batch is exhausted. Potentially blocking.
try (Scope scope = makeOriginalContextCurrent()) {
if (!loadNextBatch()) {
this.afterLast = true;
return false;
}
// Pointing to the first row in this fresh batch
this.currentBatchRowIndex = 0;
this.rowCount++;
return true;
} catch (SQLException ex) {
// A stream error ends iteration: the next call returns false and no row stays readable.
// The failed message is never skipped, even if the caller ignores this exception.
this.hasReachedEnd = true;
this.currentBatchRowIndex = -1;
throw new BigQueryJdbcException(
"Error occurred while advancing the cursor. This could happen when connection is closed while the next method is being called.",
ex);
Expand All @@ -322,7 +305,94 @@ else if (this.currentBatchRowIndex < this.vectorSchemaRoot.getRowCount()) {
ex);
}
}
return false;
}

/** True once totalRows rows (or the row limit) have been returned. */
private boolean reachedRowLimit() {
return this.rowCount >= this.totalRows;
}

private boolean hasMoreRowsInCurrentBatch() {
return this.currentBatchRowIndex >= 0
&& this.vectorSchemaRoot != null
&& this.currentBatchRowIndex < this.vectorSchemaRoot.getRowCount() - 1;
}

private void initDeserializerIfNeeded(ArrowSchema arrowSchema) throws SQLException {
if (arrowSchema == null || this.arrowDeserializer != null) {
return;
}
try {
this.arrowDeserializer = new ArrowDeserializer(arrowSchema);
} catch (RuntimeException | IOException ex) {
// Arrow throws unchecked exceptions on malformed schema bytes.
throw new BigQueryJdbcException("Failed to create the Arrow deserializer", ex);
}
}

/**
* Loads the next batch that has rows into the vectors. The reader queues every message as-is, so
* this is the single place that decides what to use:
*
* <ul>
* <li>A message without batch data (a schema, statistics or throttling update) is skipped. It
* must not report rows.
* <li>A message with batch data is decoded, and must hold exactly the rows it reports.
* <li>The stream must deliver all {@code totalRows} rows.
* </ul>
*
* Returns false at the end of the stream. A failure ends iteration, see {@link #next()}.
*/
private boolean loadNextBatch() throws SQLException, InterruptedException {
while (true) {
BigQueryArrowBatchWrapper batchWrapper = this.buffer.take();
Comment thread
keshavdandeva marked this conversation as resolved.
Resolved
checkClosed();
if (batchWrapper.getException() != null) {
throw backgroundFetchError(batchWrapper.getException());
}
if (batchWrapper.isLast()) {
/* Marks the end of the records */
if (this.vectorSchemaRoot != null) {
// IMP: To avoid memory leak: clear vectorSchemaRoot as it still holds the last batch
this.vectorSchemaRoot.clear();
}
this.hasReachedEnd = true;
if (this.rowCount < this.totalRows) {
throw new BigQueryJdbcException(
String.format(
"The Arrow stream ended after %d of %d rows.", this.rowCount, this.totalRows));
}
return false;
}
initDeserializerIfNeeded(batchWrapper.getArrowSchema());
ArrowRecordBatch arrowBatch = batchWrapper.getCurrentArrowBatch();
Long reportedRows = batchWrapper.getReportedRowCount();
if (arrowBatch == null || arrowBatch.getSerializedRecordBatch().isEmpty()) {
if (reportedRows != null && reportedRows > 0) {
throw rowCountMismatch(reportedRows, 0);
}
continue; // no rows in this message
}
if (this.arrowDeserializer == null) {
throw new BigQueryJdbcException("Received an Arrow record batch before its schema.");
}
// Populates vectorSchemaRoot
this.arrowDeserializer.deserializeArrowBatch(arrowBatch);
int decodedRows = this.vectorSchemaRoot.getRowCount();
if (reportedRows != null && reportedRows != decodedRows) {
throw rowCountMismatch(reportedRows, decodedRows);
}
if (decodedRows > 0) {
return true;
}
}
}

private static SQLException rowCountMismatch(long reportedRows, long decodedRows) {
return new BigQueryJdbcException(
String.format(
"An Arrow stream message reported %d rows but its batch has %d.",
reportedRows, decodedRows));
}

@Override
Expand Down Expand Up @@ -500,7 +570,8 @@ public boolean isBeforeFirst() throws SQLException {
if (this.isNested) {
return this.nestedRowIndex < this.fromIndex;
} else {
return this.rowCount == 0;
// Per JDBC, false when the result set contains no rows.
return !this.afterLast && this.rowCount == 0 && this.totalRows > 0;
}
}

Expand All @@ -518,7 +589,8 @@ public boolean isFirst() throws SQLException {
if (this.isNested) {
return !this.afterLast && this.nestedRowIndex == this.fromIndex;
} else {
return !this.afterLast && this.rowCount == 1;
// The index is negative after a stream error: the cursor is on no row then.
return !this.afterLast && this.rowCount == 1 && this.currentBatchRowIndex >= 0;
}
}

Expand All @@ -529,7 +601,7 @@ public boolean isLast() throws SQLException {
if (this.isNested) {
return !this.afterLast && this.nestedRowIndex == this.toIndexExclusive - 1;
} else {
return !this.afterLast && this.rowCount == this.totalRows;
return !this.afterLast && this.rowCount > 0 && this.rowCount == this.totalRows;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
BigQueryJdbcUrlUtility.DEFAULT_DATASET_PROPERTY_NAME,
BigQueryJdbcUrlUtility.LOCATION_PROPERTY_NAME,
BigQueryJdbcUrlUtility.ENABLE_HTAPI_PROPERTY_NAME,
BigQueryJdbcUrlUtility.ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME,
BigQueryJdbcUrlUtility.UNSUPPORTED_HTAPI_FALLBACK_PROPERTY_NAME,
BigQueryJdbcUrlUtility.HTAPI_MIN_TABLE_SIZE_PROPERTY_NAME,
BigQueryJdbcUrlUtility.HTAPI_ACTIVATION_RATIO_PROPERTY_NAME,
Expand Down Expand Up @@ -196,6 +197,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
DatasetId defaultDataset;
String location;
boolean enableHighThroughputAPI;
boolean enableQueryResultsArrowFormat;
int highThroughputMinTableSize;
int highThroughputActivationRatio;
boolean enableSession;
Expand Down Expand Up @@ -359,6 +361,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
this.defaultDataset = BigQueryJdbcUrlUtility.parseDefaultDataset(ds.getDefaultDataset());
this.location = ds.getLocation();
this.enableHighThroughputAPI = ds.getEnableHighThroughputAPI();
this.enableQueryResultsArrowFormat = ds.getEnableQueryResultsArrowFormat();
this.highThroughputMinTableSize = ds.getHighThroughputMinTableSize();
this.highThroughputActivationRatio = ds.getHighThroughputActivationRatio();
this.useQueryCache = ds.getUseQueryCache();
Expand Down Expand Up @@ -793,6 +796,10 @@ boolean isEnableHighThroughputAPI() {
return this.enableHighThroughputAPI;
}

boolean isEnableQueryResultsArrowFormat() {
return this.enableQueryResultsArrowFormat;
}

boolean isUseQueryCache() {
return useQueryCache;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,13 +110,16 @@ protected boolean removeEldestEntry(Map.Entry<String, Map<String, String>> eldes
static final String DEFAULT_OAUTH_CLIENT_ID = "977385342095.apps.googleusercontent.com";
static final String DEFAULT_OAUTH_CLIENT_SECRET = "wbER7576mc_1YOII0dGk7jEE";
static final String ENABLE_HTAPI_PROPERTY_NAME = "EnableHighThroughputAPI";
static final String ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME =
"EnableQueryResultsArrowFormat";
static final String PROXY_HOST_PROPERTY_NAME = "ProxyHost";
static final String PROXY_PORT_PROPERTY_NAME = "ProxyPort";
static final String PROXY_USER_ID_PROPERTY_NAME = "ProxyUid";
static final String PROXY_PASSWORD_PROPERTY_NAME = "ProxyPwd";
static final String HTTP_CONNECT_TIMEOUT_PROPERTY_NAME = "HttpConnectTimeout";
static final String HTTP_READ_TIMEOUT_PROPERTY_NAME = "HttpReadTimeout";
static final boolean DEFAULT_ENABLE_HTAPI_VALUE = false;
static final boolean DEFAULT_ENABLE_QUERY_RESULTS_ARROW_FORMAT_VALUE = false;
static final boolean DEFAULT_ENABLE_SESSION_VALUE = false;
static final int DEFAULT_LOG_LEVEL = 0;
static final String LOG_LEVEL_PROPERTY_NAME = "LogLevel";
Expand Down Expand Up @@ -445,6 +448,15 @@ protected boolean removeEldestEntry(Map.Entry<String, Map<String, String>> eldes
+ " to the BigQuery Storage API for faster processing.")
.setDefaultValue(String.valueOf(DEFAULT_HTAPI_MIN_TABLE_SIZE_VALUE))
.build(),
BigQueryConnectionProperty.newBuilder()
.setName(ENABLE_QUERY_RESULTS_ARROW_FORMAT_PROPERTY_NAME)
.setDescription(
"Reads query results in the Apache Arrow format: the first page of"
+ " jobs.query and the query job's Storage Read API stream (Preview)."
+ " Disabled by default.")
.setDefaultValue(
String.valueOf(DEFAULT_ENABLE_QUERY_RESULTS_ARROW_FORMAT_VALUE))
.build(),
BigQueryConnectionProperty.newBuilder()
.setName(ENABLE_SESSION_PROPERTY_NAME)
.setDescription(
Expand Down
Loading
Loading