Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import com.google.cloud.RetryOption;
import com.google.cloud.ServiceOptions;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryException;
import com.google.cloud.bigquery.DatasetId;
import com.google.cloud.bigquery.DatasetInfo;
import com.google.cloud.bigquery.Field;
Expand Down Expand Up @@ -107,6 +108,7 @@
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import org.apache.avro.Conversions;
Expand All @@ -118,7 +120,6 @@
import org.json.JSONObject;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;

/** Integration tests for BigQuery Storage API. */
Expand Down Expand Up @@ -1629,18 +1630,23 @@ void testStructAndArraySqlTypes() throws InterruptedException, IOException {
assertEquals(new Utf8("abc"), structRecord.get("str_field"), rowAssertMessage);
}

@Disabled("Flaky test: b/575539309")
@Test
void testSimpleReadWithBackgroundExecutorProvider() throws IOException {
BigQueryReadSettings.Builder settingsBuilder =
Helper.createBigQueryReadSettingsBuilder()
.setBackgroundExecutorProvider(
InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(14).build());
if (Helper.isRegionalEndpoint()) {
settingsBuilder.setEndpoint(BigQueryReadSettings.getDefaultEndpoint());
}
BigQueryReadClient localClient = BigQueryReadClient.create(settingsBuilder.build());
void testSimpleReadWithBackgroundExecutorProvider() throws InterruptedException, IOException {
String tableName =
"test_simple_read_background_executor_" + UUID.randomUUID().toString().substring(0, 8);
TableId tableId = TableId.of(DATASET, tableName);
String createTableStatement =
String.format(
"CREATE TABLE %s.%s AS SELECT 1 AS int_field, 'abc' AS str_field", DATASET, tableName);
BigQueryReadClient localClient = null;
try {
runQueryJobAndExpectSuccess(QueryJobConfiguration.newBuilder(createTableStatement).build());

BigQueryReadSettings.Builder settingsBuilder =
Helper.createBigQueryReadSettingsBuilder()
.setBackgroundExecutorProvider(
InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(14).build());
localClient = BigQueryReadClient.create(settingsBuilder.build());
assertTrue(
localClient.getStub().getStubSettings().getBackgroundExecutorProvider()
instanceof InstantiatingExecutorProvider);
Expand All @@ -1651,38 +1657,20 @@ void testSimpleReadWithBackgroundExecutorProvider() throws IOException {
.getExecutorThreadCount());
String table =
BigQueryResource.formatTableResource(
/* projectId= */ "bigquery-public-data",
/* datasetId= */ "samples",
/* tableId= */ "shakespeare");

ReadSession session =
localClient.createReadSession(
/* parent= */ parentProjectId,
/* readSession= */ ReadSession.newBuilder()
.setTable(table)
.setDataFormat(DataFormat.AVRO)
.build(),
/* maxStreamCount= */ 1);
assertEquals(
1,
session.getStreamsCount(),
String.format(
"Did not receive expected number of streams for table '%s' CreateReadSession"
+ " response:%n%s",
table, session.toString()));
/* projectId= */ projectName, /* datasetId= */ DATASET, /* tableId= */ tableName);

ReadRowsRequest readRowsRequest =
ReadRowsRequest.newBuilder().setReadStream(session.getStreams(0).getName()).build();

long rowCount = 0;
ServerStream<ReadRowsResponse> stream = localClient.readRowsCallable().call(readRowsRequest);
for (ReadRowsResponse response : stream) {
rowCount += response.getRowCount();
}

assertEquals(SHAKESPEARE_SAMPLE_ROW_COUNT, rowCount);
List<GenericData.Record> rows =
Helper.readAllRows(localClient, parentProjectId, /* table= */ table, /* filter= */ null);
assertEquals(1, rows.size(), "Actual rows read: " + rows);
} finally {
localClient.close();
if (localClient != null) {
localClient.close();
}
try {
bigquery.delete(tableId);
} catch (BigQueryException e) {
LOG.log(Level.WARNING, "Failed to delete temporary table: " + tableId, e);
}
}
Comment thread
lqiu96 marked this conversation as resolved.
Resolved
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -458,7 +458,9 @@ public void accept(GenericData.Record record) {
assertEquals(1_333, rowCount);
}

@Disabled("Flaky test: b/576746567")
@Disabled(
"Disabled on non-GA v1beta1 to avoid overloading BigQuery Storage CreateReadSession quota"
+ " across versions; covered on v1 (b/576746567)")
@Test
void testColumnSelection() throws IOException {
TableReference tableReference =
Expand Down Expand Up @@ -1088,7 +1090,9 @@ void testInvalidUniverseDomainWithMismatchCredentials() throws IOException {
localClient.close();
}

@Disabled("Flaky test: b/575538723")
@Disabled(
"Disabled on non-GA v1beta1 to avoid overloading BigQuery Storage CreateReadSession quota"
+ " across versions; covered on v1 (b/575538723)")
@Test
void testUniverseDomainWithMatchingDomain() throws IOException {
// Test a valid domain using the default credentials and Google default universe domain.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,9 @@ public void accept(GenericData.Record record) {
assertEquals(1_333, rowCount);
}

@Disabled("Flaky test: b/576747708")
@Disabled(
"Disabled on non-GA v1beta2 to avoid overloading BigQuery Storage CreateReadSession quota"
+ " across versions; covered on v1 (b/576747708)")
@Test
void testColumnSelection() throws IOException {
String table =
Expand Down Expand Up @@ -1077,7 +1079,9 @@ void testInvalidUniverseDomainWithMismatchCredentials() throws IOException {
localClient.close();
}

@Disabled("Flaky test: b/575538593")
@Disabled(
"Disabled on non-GA v1beta2 to avoid overloading BigQuery Storage CreateReadSession quota"
+ " across versions; covered on v1 (b/575538593)")
@Test
void testUniverseDomainWithMatchingDomain() throws IOException {
// Test a valid domain using the default credentials and Google default universe domain.
Expand Down
Loading