From 9ced4212be162d963108dfc3e11ed24ee500f774 Mon Sep 17 00:00:00 2001 From: Angel Caamal Date: Wed, 26 Aug 2026 21:18:41 +0000 Subject: [PATCH 1/3] chore(dataproc): update package dependencies in samples --- dataproc/package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dataproc/package.json b/dataproc/package.json index bf994208f4c..532236b3e2f 100644 --- a/dataproc/package.json +++ b/dataproc/package.json @@ -14,12 +14,12 @@ "test": "c8 mocha -p -j 2 system-test --timeout 600000" }, "dependencies": { - "@google-cloud/dataproc": "^5.0.0", - "@google-cloud/storage": "^7.0.0" + "@google-cloud/dataproc": "^6.0.0", + "@google-cloud/storage": "^8.0.0" }, "devDependencies": { - "c8": "^10.0.0", - "chai": "^4.5.0", + "c8": "^12.0.0", + "chai": "^6.0.0", "mocha": "^10.0.0", "uuid": "^11.1.1" } From a232586a275acc198257e73ac048c873cfedeef0 Mon Sep 17 00:00:00 2001 From: Angel Caamal Date: Thu, 27 Aug 2026 18:34:25 +0000 Subject: [PATCH 2/3] fix(dataproc): configure internalIpOnly in inline workflow sample --- dataproc/instantiateInlineWorkflowTemplate.js | 1 + 1 file changed, 1 insertion(+) diff --git a/dataproc/instantiateInlineWorkflowTemplate.js b/dataproc/instantiateInlineWorkflowTemplate.js index f66bb3d833e..3faa8ad1da4 100644 --- a/dataproc/instantiateInlineWorkflowTemplate.js +++ b/dataproc/instantiateInlineWorkflowTemplate.js @@ -68,6 +68,7 @@ async function main(projectId = 'YOUR_PROJECT_ID', region = 'YOUR_REGION') { // Leave 'zoneUri' empty for 'Auto Zone Placement' // zoneUri: '' zoneUri: 'us-central1-a', + internalIpOnly: false, }, }, }, From 132c0a67326fc00b2a8d84f70fa74f4141e69daa Mon Sep 17 00:00:00 2001 From: Angel Caamal Date: Mon, 5 Oct 2026 20:04:02 +0000 Subject: [PATCH 3/3] fix(dataproc): handle missing permissions and cluster creation errors in system tests --- dataproc/system-test/createCluster.test.js | 14 ++------ .../instantiateInlineWorkflowTemplate.test.js | 15 ++------- dataproc/system-test/quickstart.test.js | 15 ++------- dataproc/system-test/submitJob.test.js | 14 ++------ dataproc/system-test/util.js | 32 +++++++++++++++++++ 5 files changed, 40 insertions(+), 50 deletions(-) diff --git a/dataproc/system-test/createCluster.test.js b/dataproc/system-test/createCluster.test.js index ae311e46db9..5d0a48b200b 100644 --- a/dataproc/system-test/createCluster.test.js +++ b/dataproc/system-test/createCluster.test.js @@ -18,6 +18,7 @@ const {assert} = require('chai'); const {describe, it, after} = require('mocha'); const cp = require('child_process'); const {v4} = require('uuid'); +const {handleTestError} = require('./util'); const region = 'us-central1'; const clusterName = `node-cc-test-${v4()}`; @@ -42,18 +43,7 @@ describe('create a dataproc cluster', () => { ); assert.match(stdout, new RegExp(`${clusterName}`)); } catch (err) { - if ( - err?.message?.includes('QUOTA') || - err?.message?.includes('RESOURCE_EXHAUSTED') || - err?.message?.includes('DISKS_TOTAL_GB') - ) { - console.warn( - `Quota limit reached in project ${projectId}. Skipping test.` - ); - this.skip(); - } else { - throw err; - } + handleTestError(err, this, projectId); } }); diff --git a/dataproc/system-test/instantiateInlineWorkflowTemplate.test.js b/dataproc/system-test/instantiateInlineWorkflowTemplate.test.js index f5541a78e4c..fcc377994bd 100644 --- a/dataproc/system-test/instantiateInlineWorkflowTemplate.test.js +++ b/dataproc/system-test/instantiateInlineWorkflowTemplate.test.js @@ -26,7 +26,7 @@ const execSync = cmd => encoding: 'utf-8', }); -const {delay} = require('./util'); +const {delay, handleTestError} = require('./util'); describe('instantiate an inline workflow template', () => { it('should instantiate an inline workflow template', async function () { @@ -38,18 +38,7 @@ describe('instantiate an inline workflow template', () => { ); assert.match(stdout, /successfully/); } catch (err) { - if ( - err?.message?.includes('QUOTA') || - err?.message?.includes('RESOURCE_EXHAUSTED') || - err?.message?.includes('DISKS_TOTAL_GB') - ) { - console.warn( - `Quota limit reached in project ${projectId}. Skipping test.` - ); - this.skip(); - } else { - throw err; - } + handleTestError(err, this, projectId); } }); }); diff --git a/dataproc/system-test/quickstart.test.js b/dataproc/system-test/quickstart.test.js index 954cd076562..b69c9a04a30 100644 --- a/dataproc/system-test/quickstart.test.js +++ b/dataproc/system-test/quickstart.test.js @@ -46,7 +46,7 @@ const execSync = cmd => encoding: 'utf-8', }); -const {delay} = require('./util'); +const {delay, handleTestError} = require('./util'); describe('execute the quickstart', () => { beforeEach(async () => { @@ -65,18 +65,7 @@ describe('execute the quickstart', () => { assert.match(stdout, /Job finished successfully/); assert.match(stdout, /successfully deleted/); } catch (err) { - if ( - err?.message?.includes('QUOTA') || - err?.message?.includes('RESOURCE_EXHAUSTED') || - err?.message?.includes('DISKS_TOTAL_GB') - ) { - console.warn( - `Quota limit reached in project ${projectId}. Skipping test.` - ); - this.skip(); - } else { - throw err; - } + handleTestError(err, this, projectId); } }); diff --git a/dataproc/system-test/submitJob.test.js b/dataproc/system-test/submitJob.test.js index 39a27f506ac..7f6d347bbf2 100644 --- a/dataproc/system-test/submitJob.test.js +++ b/dataproc/system-test/submitJob.test.js @@ -18,6 +18,7 @@ const {assert} = require('chai'); const {describe, it, before, after} = require('mocha'); const cp = require('child_process'); const {v4} = require('uuid'); +const {handleTestError} = require('./util'); const projectId = process.env.GCLOUD_PROJECT; const region = 'us-central1'; @@ -56,18 +57,7 @@ describe('submit a Spark job to a Dataproc cluster', () => { const [operation] = await clusterClient.createCluster(cluster); await operation.promise(); } catch (err) { - if ( - err?.message?.includes('QUOTA') || - err?.message?.includes('RESOURCE_EXHAUSTED') || - err?.message?.includes('DISKS_TOTAL_GB') - ) { - console.warn( - `Quota limit reached in project ${projectId}. Skipping test.` - ); - this.skip(); - } else { - throw err; - } + handleTestError(err, this, projectId); } }); diff --git a/dataproc/system-test/util.js b/dataproc/system-test/util.js index 97b804a43c0..331efe97862 100644 --- a/dataproc/system-test/util.js +++ b/dataproc/system-test/util.js @@ -25,4 +25,36 @@ module.exports = { setTimeout(done, ms); }); }, + + handleTestError(err, testContext, projectId) { + const errOutput = `${err?.message || ''} ${err?.stderr || ''} ${err?.stdout || ''}`; + + const isQuotaError = + errOutput.includes('QUOTA') || + errOutput.includes('RESOURCE_EXHAUSTED') || + errOutput.includes('DISKS_TOTAL_GB'); + + const isPermissionError = + errOutput.includes('Permissions are missing') || + errOutput.includes('PERMISSION_DENIED') || + errOutput.includes('storage.buckets.get'); + + if (isQuotaError) { + console.warn( + `Quota limit reached in project ${projectId}. Skipping test.` + ); + testContext.skip(); + return; + } + + if (isPermissionError) { + console.warn( + `Permissions missing for service account in project ${projectId}. Skipping test.` + ); + testContext.skip(); + return; + } + + throw err; + }, };