Skip to content

chore(bigquery): allow WRITE_APPEND in createQueryArrowStream #9648

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ohmayr wants to merge 1 commit into
base: main
Choose a base branch
from
+0 −48
Open

chore(bigquery): allow WRITE_APPEND in createQueryArrowStream #9648

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 0 additions & 10 deletions handwritten/bigquery/src/arrow.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,16 +199,6 @@ export function createQueryArrowStream_(

const queryObj: Query = typeof query === 'string' ? {query} : {...query};

if (queryObj.writeDisposition === 'WRITE_APPEND') {
stream.destroy(
new Error(
'WRITE_APPEND is not supported with Arrow query results because reading ' +
'the destination table via the Storage Read API would include pre-existing rows.',
),
);
return;
}

if (queryObj.destination) {
// TODO: Support explicit destination tables via CreateReadSession.
stream.destroy(
Expand Down
22 changes: 0 additions & 22 deletions handwritten/bigquery/system-test/arrow.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,28 +137,6 @@ describe('Arrow Query Results', () => {
assert.deepStrictEqual(rows, [{x: 2}]);
});

it('should reject WRITE_APPEND before running the query and leave the destination table unchanged', async () => {
// Reading a WRITE_APPEND destination would also return the rows already
// in the table, so the stream rejects it before sending the query.
const tableRef = `\`${dataset.id}.append_target\``;
await bigquery.query(`CREATE TABLE ${tableRef} AS SELECT 1 AS x`);

await assert.rejects(
bigquery
.createQueryArrowStream({
query: 'SELECT 2 AS x',
destination: dataset.table('append_target'),
writeDisposition: 'WRITE_APPEND',
})
.toArray(),
/WRITE_APPEND is not supported with Arrow query results/,
);

// Verify the query never ran: the table still holds only its original row.
const [rows] = await bigquery.query(`SELECT x FROM ${tableRef}`);
assert.deepStrictEqual(rows, [{x: 1}]);
});

it('should raise the same error as query() when a query fails during execution', async () => {
const sql = 'SELECT 1 / 0';
let structErr: (Error & {code?: number}) | undefined;
Expand Down
16 changes: 0 additions & 16 deletions handwritten/bigquery/test/arrow.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -333,22 +333,6 @@ describe('Arrow Support', () => {
assert.strictEqual(batches[0].numCols, 0);
});

it('should fail before sending the query when writeDisposition is WRITE_APPEND', async () => {
const getCapturedReq = stubJobsQuery(bq, {});

// Verify WRITE_APPEND is rejected before executing the query on BigQuery.
await assert.rejects(
bq
.createQueryArrowStream({
query: 'SELECT 1',
writeDisposition: 'WRITE_APPEND',
})
.toArray(),
/WRITE_APPEND is not supported with Arrow query results/,
);
assert.strictEqual(getCapturedReq(), undefined);
});

it('should fail when a SQL query string is not provided', async () => {
// Verify missing query string emits an error on the stream.
await assert.rejects(
Expand Down
Loading