chore(dataproc): update package dependencies in dataproc

dataproc/instantiateInlineWorkflowTemplate.js

@@ -68,6 +68,7 @@ async function main(projectId = 'YOUR_PROJECT_ID', region = 'YOUR_REGION') {
               // Leave 'zoneUri' empty for 'Auto Zone Placement'
               // zoneUri: ''
               zoneUri: 'us-central1-a',
+              internalIpOnly: false,
             },
           },
         },

dataproc/package.json

@@ -14,12 +14,12 @@
     "test": "c8 mocha -p -j 2 system-test --timeout 600000"
   },
   "dependencies": {
-    "@google-cloud/dataproc": "^5.0.0",
-    "@google-cloud/storage": "^7.0.0"
+    "@google-cloud/dataproc": "^6.0.0",
+    "@google-cloud/storage": "^8.0.0"
   },
   "devDependencies": {
-    "c8": "^10.0.0",
-    "chai": "^4.5.0",
+    "c8": "^12.0.0",
+    "chai": "^6.0.0",
     "mocha": "^10.0.0",
     "uuid": "^11.1.1"
   }

dataproc/system-test/createCluster.test.js

@@ -18,6 +18,7 @@ const {assert} = require('chai');
 const {describe, it, after} = require('mocha');
 const cp = require('child_process');
 const {v4} = require('uuid');
+const {handleTestError} = require('./util');
 
 const region = 'us-central1';
 const clusterName = `node-cc-test-${v4()}`;
@@ -42,18 +43,7 @@ describe('create a dataproc cluster', () => {
       );
       assert.match(stdout, new RegExp(`${clusterName}`));
     } catch (err) {
-      if (
-        err?.message?.includes('QUOTA') ||
-        err?.message?.includes('RESOURCE_EXHAUSTED') ||
-        err?.message?.includes('DISKS_TOTAL_GB')
-      ) {
-        console.warn(
-          `Quota limit reached in project ${projectId}. Skipping test.`
-        );
-        this.skip();
-      } else {
-        throw err;
-      }
+      handleTestError(err, this, projectId);
     }
   });
 

dataproc/system-test/instantiateInlineWorkflowTemplate.test.js

@@ -26,7 +26,7 @@ const execSync = cmd =>
     encoding: 'utf-8',
   });
 
-const {delay} = require('./util');
+const {delay, handleTestError} = require('./util');
 
 describe('instantiate an inline workflow template', () => {
   it('should instantiate an inline workflow template', async function () {
@@ -38,18 +38,7 @@ describe('instantiate an inline workflow template', () => {
       );
       assert.match(stdout, /successfully/);
     } catch (err) {
-      if (
-        err?.message?.includes('QUOTA') ||
-        err?.message?.includes('RESOURCE_EXHAUSTED') ||
-        err?.message?.includes('DISKS_TOTAL_GB')
-      ) {
-        console.warn(
-          `Quota limit reached in project ${projectId}. Skipping test.`
-        );
-        this.skip();
-      } else {
-        throw err;
-      }
+      handleTestError(err, this, projectId);
     }
   });
 });

dataproc/system-test/quickstart.test.js

@@ -46,7 +46,7 @@ const execSync = cmd =>
     encoding: 'utf-8',
   });
 
-const {delay} = require('./util');
+const {delay, handleTestError} = require('./util');
 
 describe('execute the quickstart', () => {
   beforeEach(async () => {
@@ -65,18 +65,7 @@ describe('execute the quickstart', () => {
       assert.match(stdout, /Job finished successfully/);
       assert.match(stdout, /successfully deleted/);
     } catch (err) {
-      if (
-        err?.message?.includes('QUOTA') ||
-        err?.message?.includes('RESOURCE_EXHAUSTED') ||
-        err?.message?.includes('DISKS_TOTAL_GB')
-      ) {
-        console.warn(
-          `Quota limit reached in project ${projectId}. Skipping test.`
-        );
-        this.skip();
-      } else {
-        throw err;
-      }
+      handleTestError(err, this, projectId);
     }
   });
 

dataproc/system-test/submitJob.test.js

@@ -18,6 +18,7 @@ const {assert} = require('chai');
 const {describe, it, before, after} = require('mocha');
 const cp = require('child_process');
 const {v4} = require('uuid');
+const {handleTestError} = require('./util');
 
 const projectId = process.env.GCLOUD_PROJECT;
 const region = 'us-central1';
@@ -56,18 +57,7 @@ describe('submit a Spark job to a Dataproc cluster', () => {
       const [operation] = await clusterClient.createCluster(cluster);
       await operation.promise();
     } catch (err) {
-      if (
-        err?.message?.includes('QUOTA') ||
-        err?.message?.includes('RESOURCE_EXHAUSTED') ||
-        err?.message?.includes('DISKS_TOTAL_GB')
-      ) {
-        console.warn(
-          `Quota limit reached in project ${projectId}. Skipping test.`
-        );
-        this.skip();
-      } else {
-        throw err;
-      }
+      handleTestError(err, this, projectId);
     }
   });
 

dataproc/system-test/util.js

@@ -25,4 +25,36 @@ module.exports = {
       setTimeout(done, ms);
     });
   },
+
+  handleTestError(err, testContext, projectId) {
+    const errOutput = `${err?.message || ''} ${err?.stderr || ''} ${err?.stdout || ''}`;
+
+    const isQuotaError =
+      errOutput.includes('QUOTA') ||
+      errOutput.includes('RESOURCE_EXHAUSTED') ||
+      errOutput.includes('DISKS_TOTAL_GB');
+
+    const isPermissionError =
+      errOutput.includes('Permissions are missing') ||
+      errOutput.includes('PERMISSION_DENIED') ||
+      errOutput.includes('storage.buckets.get');
+
+    if (isQuotaError) {
+      console.warn(
+        `Quota limit reached in project ${projectId}. Skipping test.`
+      );
+      testContext.skip();
+      return;
+    }
+
+    if (isPermissionError) {
+      console.warn(
+        `Permissions missing for service account in project ${projectId}. Skipping test.`
+      );
+      testContext.skip();
+      return;
+    }
+
+    throw err;
+  },
 };