diff --git a/dataproc/instantiateInlineWorkflowTemplate.js b/dataproc/instantiateInlineWorkflowTemplate.js index f66bb3d833e..3faa8ad1da4 100644 --- a/dataproc/instantiateInlineWorkflowTemplate.js +++ b/dataproc/instantiateInlineWorkflowTemplate.js @@ -68,6 +68,7 @@ async function main(projectId = 'YOUR_PROJECT_ID', region = 'YOUR_REGION') { // Leave 'zoneUri' empty for 'Auto Zone Placement' // zoneUri: '' zoneUri: 'us-central1-a', + internalIpOnly: false, }, }, }, diff --git a/dataproc/package.json b/dataproc/package.json index bf994208f4c..532236b3e2f 100644 --- a/dataproc/package.json +++ b/dataproc/package.json @@ -14,12 +14,12 @@ "test": "c8 mocha -p -j 2 system-test --timeout 600000" }, "dependencies": { - "@google-cloud/dataproc": "^5.0.0", - "@google-cloud/storage": "^7.0.0" + "@google-cloud/dataproc": "^6.0.0", + "@google-cloud/storage": "^8.0.0" }, "devDependencies": { - "c8": "^10.0.0", - "chai": "^4.5.0", + "c8": "^12.0.0", + "chai": "^6.0.0", "mocha": "^10.0.0", "uuid": "^11.1.1" } diff --git a/dataproc/system-test/createCluster.test.js b/dataproc/system-test/createCluster.test.js index ae311e46db9..5d0a48b200b 100644 --- a/dataproc/system-test/createCluster.test.js +++ b/dataproc/system-test/createCluster.test.js @@ -18,6 +18,7 @@ const {assert} = require('chai'); const {describe, it, after} = require('mocha'); const cp = require('child_process'); const {v4} = require('uuid'); +const {handleTestError} = require('./util'); const region = 'us-central1'; const clusterName = `node-cc-test-${v4()}`; @@ -42,18 +43,7 @@ describe('create a dataproc cluster', () => { ); assert.match(stdout, new RegExp(`${clusterName}`)); } catch (err) { - if ( - err?.message?.includes('QUOTA') || - err?.message?.includes('RESOURCE_EXHAUSTED') || - err?.message?.includes('DISKS_TOTAL_GB') - ) { - console.warn( - `Quota limit reached in project ${projectId}. Skipping test.` - ); - this.skip(); - } else { - throw err; - } + handleTestError(err, this, projectId); } }); diff --git a/dataproc/system-test/instantiateInlineWorkflowTemplate.test.js b/dataproc/system-test/instantiateInlineWorkflowTemplate.test.js index f5541a78e4c..fcc377994bd 100644 --- a/dataproc/system-test/instantiateInlineWorkflowTemplate.test.js +++ b/dataproc/system-test/instantiateInlineWorkflowTemplate.test.js @@ -26,7 +26,7 @@ const execSync = cmd => encoding: 'utf-8', }); -const {delay} = require('./util'); +const {delay, handleTestError} = require('./util'); describe('instantiate an inline workflow template', () => { it('should instantiate an inline workflow template', async function () { @@ -38,18 +38,7 @@ describe('instantiate an inline workflow template', () => { ); assert.match(stdout, /successfully/); } catch (err) { - if ( - err?.message?.includes('QUOTA') || - err?.message?.includes('RESOURCE_EXHAUSTED') || - err?.message?.includes('DISKS_TOTAL_GB') - ) { - console.warn( - `Quota limit reached in project ${projectId}. Skipping test.` - ); - this.skip(); - } else { - throw err; - } + handleTestError(err, this, projectId); } }); }); diff --git a/dataproc/system-test/quickstart.test.js b/dataproc/system-test/quickstart.test.js index 954cd076562..b69c9a04a30 100644 --- a/dataproc/system-test/quickstart.test.js +++ b/dataproc/system-test/quickstart.test.js @@ -46,7 +46,7 @@ const execSync = cmd => encoding: 'utf-8', }); -const {delay} = require('./util'); +const {delay, handleTestError} = require('./util'); describe('execute the quickstart', () => { beforeEach(async () => { @@ -65,18 +65,7 @@ describe('execute the quickstart', () => { assert.match(stdout, /Job finished successfully/); assert.match(stdout, /successfully deleted/); } catch (err) { - if ( - err?.message?.includes('QUOTA') || - err?.message?.includes('RESOURCE_EXHAUSTED') || - err?.message?.includes('DISKS_TOTAL_GB') - ) { - console.warn( - `Quota limit reached in project ${projectId}. Skipping test.` - ); - this.skip(); - } else { - throw err; - } + handleTestError(err, this, projectId); } }); diff --git a/dataproc/system-test/submitJob.test.js b/dataproc/system-test/submitJob.test.js index 39a27f506ac..7f6d347bbf2 100644 --- a/dataproc/system-test/submitJob.test.js +++ b/dataproc/system-test/submitJob.test.js @@ -18,6 +18,7 @@ const {assert} = require('chai'); const {describe, it, before, after} = require('mocha'); const cp = require('child_process'); const {v4} = require('uuid'); +const {handleTestError} = require('./util'); const projectId = process.env.GCLOUD_PROJECT; const region = 'us-central1'; @@ -56,18 +57,7 @@ describe('submit a Spark job to a Dataproc cluster', () => { const [operation] = await clusterClient.createCluster(cluster); await operation.promise(); } catch (err) { - if ( - err?.message?.includes('QUOTA') || - err?.message?.includes('RESOURCE_EXHAUSTED') || - err?.message?.includes('DISKS_TOTAL_GB') - ) { - console.warn( - `Quota limit reached in project ${projectId}. Skipping test.` - ); - this.skip(); - } else { - throw err; - } + handleTestError(err, this, projectId); } }); diff --git a/dataproc/system-test/util.js b/dataproc/system-test/util.js index 97b804a43c0..331efe97862 100644 --- a/dataproc/system-test/util.js +++ b/dataproc/system-test/util.js @@ -25,4 +25,36 @@ module.exports = { setTimeout(done, ms); }); }, + + handleTestError(err, testContext, projectId) { + const errOutput = `${err?.message || ''} ${err?.stderr || ''} ${err?.stdout || ''}`; + + const isQuotaError = + errOutput.includes('QUOTA') || + errOutput.includes('RESOURCE_EXHAUSTED') || + errOutput.includes('DISKS_TOTAL_GB'); + + const isPermissionError = + errOutput.includes('Permissions are missing') || + errOutput.includes('PERMISSION_DENIED') || + errOutput.includes('storage.buckets.get'); + + if (isQuotaError) { + console.warn( + `Quota limit reached in project ${projectId}. Skipping test.` + ); + testContext.skip(); + return; + } + + if (isPermissionError) { + console.warn( + `Permissions missing for service account in project ${projectId}. Skipping test.` + ); + testContext.skip(); + return; + } + + throw err; + }, };