From d60d8e8fe8c342835f44cbfaaf3968913c81e325 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 17 Nov 2026 17:42:03 +0000 Subject: [PATCH 1/2] ci(bigtable): move emulator ITs to GitHub Actions and clean up CI configs --- .github/workflows/java-bigtable-ci.yaml | 182 ------------------ .github/workflows/java-bigtable-emulator.yaml | 60 ++++++ .kokoro/presubmit/bigtable-integration.cfg | 2 +- java-bigtable/.cloudbuild/samples_build.yaml | 34 ---- java-bigtable/scripts/setup-test-table.sh | 31 ++- 5 files changed, 75 insertions(+), 234 deletions(-) delete mode 100644 .github/workflows/java-bigtable-ci.yaml create mode 100644 .github/workflows/java-bigtable-emulator.yaml delete mode 100644 java-bigtable/.cloudbuild/samples_build.yaml diff --git a/.github/workflows/java-bigtable-ci.yaml b/.github/workflows/java-bigtable-ci.yaml deleted file mode 100644 index 0976188f0263..000000000000 --- a/.github/workflows/java-bigtable-ci.yaml +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Github action job to test core java library features on -# downstream client libraries before they are released. -permissions: - contents: read - -on: - push: - branches: - - main - pull_request: -name: java-bigtable ci -env: - BUILD_SUBDIR: java-bigtable -jobs: - filter: - runs-on: ubuntu-latest - outputs: - library: ${{ steps.filter.outputs.library }} - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - persist-credentials: false - - uses: dorny/paths-filter@d1c1ffe0248fe513906c8e24db8ea791d46f8590 # v3.0.3 - id: filter - with: - filters: | - library: - - 'java-bigtable/**' - units: - needs: filter - if: ${{ needs.filter.outputs.library == 'true' }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - java: [11, 17, 21, 25, 26] - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - persist-credentials: false - - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 - with: - distribution: temurin - java-version: ${{matrix.java}} - cache: maven - - run: java -version - - run: .kokoro/build.sh - env: - JOB_TYPE: test - units-java8: - needs: filter - if: ${{ needs.filter.outputs.library == 'true' }} - # Building using Java 17 and run the tests with Java 8 runtime - name: "units (8)" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - persist-credentials: false - - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 - with: - java-version: 8 - distribution: temurin - cache: maven - - name: "Set jvm system property environment variable for surefire plugin (unit tests)" - # Maven surefire plugin (unit tests) allows us to specify JVM to run the tests. - # https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#jvm - run: echo "SUREFIRE_JVM_OPT=-Djvm=${JAVA_HOME}/bin/java -P !java17" >> $GITHUB_ENV - shell: bash - - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 - with: - java-version: 17 - distribution: temurin - cache: maven - - run: .kokoro/build.sh - env: - JOB_TYPE: test - windows: - needs: filter - if: ${{ needs.filter.outputs.library == 'true' }} - runs-on: windows-latest - steps: - - name: Support longpaths - run: git config --system core.longpaths true - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - persist-credentials: false - - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 - with: - distribution: temurin - java-version: 8 - cache: maven - - run: java -version - - run: .kokoro/build.sh - env: - JOB_TYPE: test - dependencies: - needs: filter - if: ${{ needs.filter.outputs.library == 'true' }} - runs-on: ubuntu-latest - strategy: - matrix: - java: [17] - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - persist-credentials: false - - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 - with: - distribution: temurin - java-version: ${{matrix.java}} - cache: maven - - run: java -version - - run: .kokoro/dependencies.sh - javadoc: - needs: filter - if: ${{ needs.filter.outputs.library == 'true' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - persist-credentials: false - - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 - with: - distribution: temurin - java-version: 17 - cache: maven - - run: java -version - - run: .kokoro/build.sh - env: - JOB_TYPE: javadoc - lint: - needs: filter - if: ${{ needs.filter.outputs.library == 'true' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - fetch-depth: 0 - persist-credentials: false - - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 - with: - distribution: temurin - java-version: 17 - cache: maven - - run: java -version - - run: .kokoro/build.sh - env: - JOB_TYPE: lint - HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} - clirr: - needs: filter - if: ${{ needs.filter.outputs.library == 'true' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - persist-credentials: false - - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 - with: - distribution: temurin - java-version: 11 - cache: maven - - run: java -version - - run: .kokoro/build.sh - env: - JOB_TYPE: clirr - BUILD_SUBDIR: java-bigtable diff --git a/.github/workflows/java-bigtable-emulator.yaml b/.github/workflows/java-bigtable-emulator.yaml new file mode 100644 index 000000000000..1ac9bccb8dd0 --- /dev/null +++ b/.github/workflows/java-bigtable-emulator.yaml @@ -0,0 +1,60 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Github action job to test core java library features on +# downstream client libraries before they are released. +permissions: + contents: read + +on: + push: + branches: + - main + pull_request: +name: java-bigtable emulator +env: + BUILD_SUBDIR: java-bigtable +jobs: + filter: + runs-on: ubuntu-latest + outputs: + library: ${{ steps.filter.outputs.library }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - uses: dorny/paths-filter@d1c1ffe0248fe513906c8e24db8ea791d46f8590 # v3.0.3 + id: filter + with: + filters: | + library: + - 'java-bigtable/**' + emulator: + needs: filter + if: ${{ needs.filter.outputs.library == 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 + with: + distribution: temurin + java-version: 17 + cache: maven + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: integration-single + TEST_ALL_MODULES: "true" + INTEGRATION_TEST_ARGS: "-P bigtable-emulator-it" diff --git a/.kokoro/presubmit/bigtable-integration.cfg b/.kokoro/presubmit/bigtable-integration.cfg index e056914485b9..ca98c4c20eed 100644 --- a/.kokoro/presubmit/bigtable-integration.cfg +++ b/.kokoro/presubmit/bigtable-integration.cfg @@ -8,7 +8,7 @@ env_vars: { env_vars: { key: "INTEGRATION_TEST_ARGS" - value: "-P bigtable-emulator-it,bigtable-prod-it,bigtable-prod-batch-it,enable-verbose-grpc-logs -Dbigtable.project=gcloud-devel -Dbigtable.instance=google-cloud-bigtable -Dbigtable.table=integration-tests -Dbigtable.kms_key_name=projects/gcloud-devel/locations/us-east1/keyRings/cmek-test-key-ring/cryptoKeys/cmek-test-key -Dbigtable.wait-for-cmek-key-status=true" + value: "-P bigtable-prod-it,enable-verbose-grpc-logs -Dbigtable.project=gcloud-devel -Dbigtable.instance=google-cloud-bigtable -Dbigtable.table=integration-tests -Dbigtable.kms_key_name=projects/gcloud-devel/locations/us-east1/keyRings/cmek-test-key-ring/cryptoKeys/cmek-test-key -Dbigtable.wait-for-cmek-key-status=true" } env_vars: { diff --git a/java-bigtable/.cloudbuild/samples_build.yaml b/java-bigtable/.cloudbuild/samples_build.yaml deleted file mode 100644 index a9c79af5b0f6..000000000000 --- a/java-bigtable/.cloudbuild/samples_build.yaml +++ /dev/null @@ -1,34 +0,0 @@ -steps: -- name: gcr.io/cloud-devrel-public-resources/java8 - entrypoint: ls - args: [ - '-alt', - ] -- name: gcr.io/cloud-devrel-public-resources/java8 - entrypoint: curl - args: [ - '--header', - 'Metadata-Flavor: Google', - 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email' - ] -- name: gcr.io/cloud-devrel-public-resources/java8 - entrypoint: pwd -- name: gcr.io/cloud-devrel-public-resources/java8 - entrypoint: bash - args: [ - '.kokoro/build.sh' - ] - env: - - 'BUILD_SUBDIR=java-bigtable' - - 'JOB_TYPE=samples' - - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-sample' - - 'BIGTABLE_TESTING_INSTANCE=instance' - - 'KOKORO_GITHUB_PULL_REQUEST_NUMBER=$_PR_NUMBER' -- name: gcr.io/cloud-devrel-public-resources/java8 - entrypoint: echo - args: [ - 'Sample job succeeded', - ] -timeout: 3600s -options: - defaultLogsBucketBehavior: REGIONAL_USER_OWNED_BUCKET diff --git a/java-bigtable/scripts/setup-test-table.sh b/java-bigtable/scripts/setup-test-table.sh index c8bb1e2a62a6..fb9c74967122 100755 --- a/java-bigtable/scripts/setup-test-table.sh +++ b/java-bigtable/scripts/setup-test-table.sh @@ -2,19 +2,18 @@ # Set up a table to use for integration tests. +# Usage: ./scripts/setup-test-table.sh projects//instances//tables/ [admin-endpoint] +# Environment overrides: ADMIN_HOST, ZONE + set -e set -u -ADMIN_HOSTS=( - "bigtableadmin.googleapis.com:443" - "test-bigtableadmin.sandbox.googleapis.com:443" -) - -ZONE=us-central1-b +ZONE=${ZONE:-us-central1-b} FAMILY=cf # Format: projects//instances//tables/ TABLE_NAME=$1 +ADMIN_HOST=${2:-${ADMIN_HOST:-bigtableadmin.googleapis.com:443}} if [[ ${TABLE_NAME} =~ projects\/([^/]+)\/instances\/([^/]+)\/tables\/([^/]+) ]]; then PROJECT_ID=${BASH_REMATCH[1]} @@ -29,14 +28,12 @@ call_cbt() { cbt -admin-endpoint="${ADMIN_HOST}" -project "${PROJECT_ID}" -instance "${INSTANCE_ID}" "$@" } -for ADMIN_HOST in "${ADMIN_HOSTS[@]}"; do - # Ensure that the instance exists. - if ! call_cbt listinstances | grep -q "${INSTANCE_ID}"; then - call_cbt createinstance "${INSTANCE_ID}" "${INSTANCE_ID}" "${INSTANCE_ID}-c0" "${ZONE}" 1 SSD - fi - - # Ensure that the table exists - if ! call_cbt -instance "$INSTANCE_ID" ls | grep -q "^${TABLE_ID}\$"; then - call_cbt createtable "$TABLE_ID" "families=$FAMILY:maxversions=1||maxage=1h" - fi -done +# Ensure that the instance exists. +if ! call_cbt listinstances | grep -q "^${INSTANCE_ID}\b"; then + call_cbt createinstance "${INSTANCE_ID}" "${INSTANCE_ID}" "${INSTANCE_ID}-c0" "${ZONE}" 1 SSD +fi + +# Ensure that the table exists. +if ! call_cbt -instance "$INSTANCE_ID" ls | grep -q "^${TABLE_ID}\$"; then + call_cbt createtable "$TABLE_ID" "families=$FAMILY:maxversions=1||maxage=1h" +fi From 6bf8c83c256e81e8469bd0ce2f3d964cc63bfc17 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 17 Nov 2026 18:24:00 +0000 Subject: [PATCH 2/2] fix(bigtable): restore samples_build.yaml and use portable grep regex --- java-bigtable/.cloudbuild/samples_build.yaml | 34 ++++++++++++++++++++ java-bigtable/scripts/setup-test-table.sh | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 java-bigtable/.cloudbuild/samples_build.yaml diff --git a/java-bigtable/.cloudbuild/samples_build.yaml b/java-bigtable/.cloudbuild/samples_build.yaml new file mode 100644 index 000000000000..a9c79af5b0f6 --- /dev/null +++ b/java-bigtable/.cloudbuild/samples_build.yaml @@ -0,0 +1,34 @@ +steps: +- name: gcr.io/cloud-devrel-public-resources/java8 + entrypoint: ls + args: [ + '-alt', + ] +- name: gcr.io/cloud-devrel-public-resources/java8 + entrypoint: curl + args: [ + '--header', + 'Metadata-Flavor: Google', + 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email' + ] +- name: gcr.io/cloud-devrel-public-resources/java8 + entrypoint: pwd +- name: gcr.io/cloud-devrel-public-resources/java8 + entrypoint: bash + args: [ + '.kokoro/build.sh' + ] + env: + - 'BUILD_SUBDIR=java-bigtable' + - 'JOB_TYPE=samples' + - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-sample' + - 'BIGTABLE_TESTING_INSTANCE=instance' + - 'KOKORO_GITHUB_PULL_REQUEST_NUMBER=$_PR_NUMBER' +- name: gcr.io/cloud-devrel-public-resources/java8 + entrypoint: echo + args: [ + 'Sample job succeeded', + ] +timeout: 3600s +options: + defaultLogsBucketBehavior: REGIONAL_USER_OWNED_BUCKET diff --git a/java-bigtable/scripts/setup-test-table.sh b/java-bigtable/scripts/setup-test-table.sh index fb9c74967122..ac0050af9d9d 100755 --- a/java-bigtable/scripts/setup-test-table.sh +++ b/java-bigtable/scripts/setup-test-table.sh @@ -29,7 +29,7 @@ call_cbt() { } # Ensure that the instance exists. -if ! call_cbt listinstances | grep -q "^${INSTANCE_ID}\b"; then +if ! call_cbt listinstances | grep -Eq "^${INSTANCE_ID}([[:space:]]|$)"; then call_cbt createinstance "${INSTANCE_ID}" "${INSTANCE_ID}" "${INSTANCE_ID}-c0" "${ZONE}" 1 SSD fi