From 47ea1df546129f3a4cf192ff2f236181be262b2f Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 21:05:21 -0700 Subject: [PATCH 1/8] added back core-deps tests --- .github/workflows/unittest.yml | 51 ++++++++++++++++++++++++++++++++-- ci/get_package_shards.py | 43 +++++++++++++++++++++++----- ci/run_single_test.sh | 16 ++++++++--- 3 files changed, 96 insertions(+), 14 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 6c2206518bc3..2fd658a7e1f5 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -156,12 +156,52 @@ jobs: path: .coverage.${{ matrix.python }}.* include-hidden-files: true + core-deps: + needs: initialize + if: needs.initialize.outputs.matrix != '[]' && needs.initialize.outputs.matrix != '' + runs-on: ubuntu-22.04 + strategy: + fail-fast: true + matrix: + python: ["3.14"] + package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }} + name: ${{ matrix.package_shard.is_sharded && format('core-deps handwritten ({0}, {1})', matrix.python, matrix.package_shard.name) || format('core-deps handwritten ({0})', matrix.python) }} + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` + # See https://github.com/googleapis/google-cloud-python/issues/12013 + # and https://github.com/actions/checkout#checkout-head. + with: + fetch-depth: 2 + persist-credentials: false + - name: Setup Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: ${{ matrix.python }} + cache: 'pip' + allow-prereleases: true + - name: Install nox + run: | + pip install nox + - name: Run core_deps_from_source for ${{ matrix.package_shard.description }} + env: + BUILD_TYPE: presubmit + TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }} + TEST_TYPE: core_deps_from_source + PY_VERSION: ${{ matrix.python }} + PACKAGE_LIST: ${{ matrix.package_shard.packages }} + NOX_DEFAULT_VENV_BACKEND: "virtualenv" + NOXFORCEPYTHON: ${{ matrix.python }} + run: | + ci/run_conditional_tests.sh + all-tests: - needs: [initialize, unit] + needs: [initialize, unit, core-deps] if: always() runs-on: ubuntu-latest steps: - - name: Check unit test results + - name: Check test results run: | # 1. Check initialize job if [[ "${{ needs.initialize.result }}" != "success" ]]; then @@ -173,7 +213,12 @@ jobs: echo "Unit tests failed" exit 1 fi - echo "All unit tests passed or were skipped" + # 3. Check core dependencies test shards + if [[ "${{ needs.core-deps.result }}" != "success" && "${{ needs.core-deps.result }}" != "skipped" ]]; then + echo "Core dependencies tests failed" + exit 1 + fi + echo "All unit and core dependencies tests passed or were skipped" cover: if: always() && !cancelled() && needs.all-tests.result == 'success' diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py index 015adfc2bacd..fd26332648a3 100644 --- a/ci/get_package_shards.py +++ b/ci/get_package_shards.py @@ -22,12 +22,12 @@ preview-packages/foo) are kept aligned in the exact same shard. """ -import os -import subprocess +import collections import json import math +import os +import subprocess import sys -import collections def get_package_directories(): @@ -70,9 +70,11 @@ def get_package_weights(): return weights -def get_packages(): +def get_packages(handwritten_only=False): """Lists all package directory paths in the repository grouped by package name. + If handwritten_only is True, includes only non-GAPIC_AUTO libraries. + Returns: dict: A dictionary mapping package_name -> list of relative directory paths. """ @@ -83,8 +85,19 @@ def get_packages(): continue for d in os.listdir(subdir): full_path = os.path.join(subdir, d) + '/' - if os.path.isdir(full_path): - packages_map[d].append(full_path) + if not os.path.isdir(full_path): + continue + if handwritten_only: + meta_file = os.path.join(full_path, ".repo-metadata.json") + if os.path.exists(meta_file): + try: + with open(meta_file) as f: + data = json.load(f) + if data.get("library_type") == "GAPIC_AUTO": + continue + except Exception: + pass + packages_map[d].append(full_path) return packages_map @@ -132,6 +145,22 @@ def get_packages_to_test(): if full_path not in to_test_paths[pkg_name]: to_test_paths[pkg_name].append(full_path) + # Core dependency packages whose changes require testing across downstream handwritten packages + core_packages = { + "google-api-core", + "google-auth", + "google-auth-httplib2", + "google-auth-oauthlib", + "google-cloud-core", + "googleapis-common-protos", + "grpc-google-iam-v1", + "proto-plus", + "google-crc32c", + } + if any(pkg in core_packages for pkg in to_test_paths): + # When a core package changes, test all handwritten packages (non-GAPIC_AUTO) + return get_packages(handwritten_only=True) + return dict(to_test_paths) @@ -173,7 +202,7 @@ def group_packages(packages_map): # Pack packages alphabetically by package name. for name, paths, weight in pkg_items: - # If adding this package would exceed target weight AND we haven't reached the + # If adding this package would exceed target weight AND we haven't reached the # shard limit, start a new shard. Otherwise, keep "stuffing" the current one. if current_shard_items and (current_shard_weight + weight > target_weight) and len(shards_list) < max_shards - 1: shards_list.append(current_shard_items) diff --git a/ci/run_single_test.sh b/ci/run_single_test.sh index 0ea0b969770d..8f888923e3b2 100755 --- a/ci/run_single_test.sh +++ b/ci/run_single_test.sh @@ -19,7 +19,7 @@ # `PY_VERSION` should be one of ["3.10", "3.11", "3.12", "3.13"] # This script is called by the `ci/run_conditional_tests.sh` script. -# A specific `nox` session will be run, depending on the value of +# A specific `nox` session will be run, depending on the value of # `TEST_TYPE` and `PY_VERSION`. For example, if `TEST_TYPE` is # `lint`, the `nox -s lint` session will be run. @@ -68,6 +68,14 @@ case ${TEST_TYPE} in nox -s prerelease_deps-3.14 retval=$? ;; + core_deps_from_source) + if [[ "$(pwd)" == */preview-packages/* ]]; then + echo "Skipping core_deps_from_source for preview package $(pwd)" + exit 0 + fi + nox --stop-on-first-error -s core_deps_from_source + retval=$? + ;; unit) case ${PY_VERSION} in "3.10") @@ -131,12 +139,12 @@ case ${TEST_TYPE} in source .venv-profiler/bin/activate export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 python -m pip install --upgrade pip setuptools - + PROFILER_TEMP_DIR=$(mktemp -d) cp ../../scripts/import_profiler/profiler.py "${PROFILER_TEMP_DIR}/profiler.py" PROFILER_SCRIPT="${PROFILER_TEMP_DIR}/profiler.py" BASELINE_CSV="${PROFILER_TEMP_DIR}/baseline_${PACKAGE_NAME}.csv" - + if [ -n "${TARGET_BRANCH}" ]; then # Fetch history for the target branch without --depth=1 in case it was shallowly fetched if [ -f "$(git rev-parse --git-dir)/shallow" ]; then @@ -180,7 +188,7 @@ case ${TEST_TYPE} in echo "Could not find baseline commit for ${TARGET_BRANCH:-main}. Skipping baseline generation." fi fi - + # TODO(https://github.com/googleapis/google-cloud-python/issues/18035): # Clean up this fallback once Python 3.15 is officially released and upstream binary wheels are available on PyPI. # On pre-release Python versions, packages with complex C/Rust dependencies (e.g. bigframes) fail during pip install due to missing pre-built wheels. From f81172aaa7756b5bdfd38b829edaef20950bd266 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 21:05:33 -0700 Subject: [PATCH 2/8] rebalanced package weights --- .github/workflows/unittest.yml | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 2fd658a7e1f5..68ccc4969d7f 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -40,15 +40,33 @@ jobs: # Each weight is roughly 1 minute of expected execution time # Default for unset packages is 1 PACKAGE_WEIGHTS: | - google-ai-generativelanguage: 4 - google-auth: 5 + django-google-spanner: 2 + gapic-generator: 3 + google-ai-generativelanguage: 3 + google-api-core: 2 + google-auth: 2 + google-cloud-bigquery: 4 + google-cloud-bigquery-storage: 2 + google-cloud-bigtable: 4 google-cloud-compute: 12 google-cloud-compute-v1beta: 12 - google-cloud-dialogflow: 6 - google-cloud-dialogflow-cx: 6 - google-cloud-discoveryengine: 8 - google-cloud-retail: 5 - google-shopping-merchant-accounts: 4 + google-cloud-datastore: 3 + google-cloud-dialogflow: 4 + google-cloud-dialogflow-cx: 4 + google-cloud-discoveryengine: 4 + google-cloud-firestore: 3 + google-cloud-logging: 3 + google-cloud-monitoring: 3 + google-cloud-ndb: 2 + google-cloud-pubsub: 2 + google-cloud-retail: 3 + google-cloud-spanner: 4 + google-cloud-storage: 4 + google-shopping-merchant-accounts: 3 + pandas-gbq: 2 + proto-plus: 1 + sqlalchemy-bigquery: 2 + sqlalchemy-spanner: 2 steps: - name: Checkout uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 From 267bad68ab52442b14928b11dbdd845421211585 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 21:06:11 -0700 Subject: [PATCH 3/8] updated noxfiles to use local installations --- packages/db-dtypes/noxfile.py | 27 +++++++++++++------ .../noxfile.py | 21 +++++++++------ packages/google-cloud-ndb/noxfile.py | 21 +++++++++------ packages/googleapis-common-protos/noxfile.py | 21 +++++++++------ packages/pandas-gbq/noxfile.py | 21 +++++++++------ packages/proto-plus/noxfile.py | 16 ++++++----- packages/sqlalchemy-bigquery/noxfile.py | 21 +++++++++------ packages/sqlalchemy-spanner/noxfile.py | 21 +++++++++------ 8 files changed, 107 insertions(+), 62 deletions(-) diff --git a/packages/db-dtypes/noxfile.py b/packages/db-dtypes/noxfile.py index b26f72f7a285..d7fc9b977ee3 100644 --- a/packages/db-dtypes/noxfile.py +++ b/packages/db-dtypes/noxfile.py @@ -512,16 +512,27 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [ + str(deps_dir / dep) + for dep in core_dependencies_from_source + if (deps_dir / dep).exists() + ] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print( + f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}" + ) tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/google-cloud-documentai-toolbox/noxfile.py b/packages/google-cloud-documentai-toolbox/noxfile.py index 18a537cb9bda..1502ae07a03b 100644 --- a/packages/google-cloud-documentai-toolbox/noxfile.py +++ b/packages/google-cloud-documentai-toolbox/noxfile.py @@ -536,16 +536,21 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/google-cloud-ndb/noxfile.py b/packages/google-cloud-ndb/noxfile.py index cb957a674ed1..7a90848f63c2 100644 --- a/packages/google-cloud-ndb/noxfile.py +++ b/packages/google-cloud-ndb/noxfile.py @@ -504,16 +504,21 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/googleapis-common-protos/noxfile.py b/packages/googleapis-common-protos/noxfile.py index 78fd0cbf4ff9..79587afeca21 100644 --- a/packages/googleapis-common-protos/noxfile.py +++ b/packages/googleapis-common-protos/noxfile.py @@ -558,16 +558,21 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - f"{CURRENT_DIRECTORY}/../googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - f"{CURRENT_DIRECTORY}/../grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") session.run( "py.test", diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 0021226416cd..ed751fe7e104 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -584,16 +584,21 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/proto-plus/noxfile.py b/packages/proto-plus/noxfile.py index 5f56955863c8..63cfe15796a6 100644 --- a/packages/proto-plus/noxfile.py +++ b/packages/proto-plus/noxfile.py @@ -167,14 +167,18 @@ def core_deps_from_source(session, implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - # dependency of google-api-core - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", + "google-api-core", + "googleapis-common-protos", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") # TODO(https://github.com/googleapis/google-cloud-python/issues/15115): Install protobuf from source at HEAD session.install("--pre", "--upgrade", "protobuf") diff --git a/packages/sqlalchemy-bigquery/noxfile.py b/packages/sqlalchemy-bigquery/noxfile.py index 506a5bfcfb8f..5e7d20291107 100644 --- a/packages/sqlalchemy-bigquery/noxfile.py +++ b/packages/sqlalchemy-bigquery/noxfile.py @@ -706,16 +706,21 @@ def core_deps_from_source(session, protobuf_implementation): session.install("-e", install_target, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/sqlalchemy-spanner/noxfile.py b/packages/sqlalchemy-spanner/noxfile.py index df709c96e50a..84a348c76e61 100644 --- a/packages/sqlalchemy-spanner/noxfile.py +++ b/packages/sqlalchemy-spanner/noxfile.py @@ -547,16 +547,21 @@ def core_deps_from_source(session, protobuf_implementation): session.install(".") core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( From 21afd0d80898f0b4c1208eeaf0f0f6a9a50a72f3 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 21:06:46 -0700 Subject: [PATCH 4/8] updated core_deps noxfile step to support preview package directories --- .../gapic/templates/noxfile.py.j2 | 22 ++++++++++--------- .../integration/goldens/asset/noxfile.py | 12 +++++----- .../goldens/credentials/noxfile.py | 12 +++++----- .../integration/goldens/eventarc/noxfile.py | 12 +++++----- .../integration/goldens/logging/noxfile.py | 12 +++++----- .../goldens/logging_internal/noxfile.py | 12 +++++----- .../integration/goldens/redis/noxfile.py | 12 +++++----- .../goldens/redis_selective/noxfile.py | 12 +++++----- .../goldens/storagebatchoperations/noxfile.py | 12 +++++----- 9 files changed, 68 insertions(+), 50 deletions(-) diff --git a/packages/gapic-generator/gapic/templates/noxfile.py.j2 b/packages/gapic-generator/gapic/templates/noxfile.py.j2 index d4176bdf4222..47205dd39819 100644 --- a/packages/gapic-generator/gapic/templates/noxfile.py.j2 +++ b/packages/gapic-generator/gapic/templates/noxfile.py.j2 @@ -6,9 +6,8 @@ import os import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -163,7 +162,7 @@ def lint(session): "ruff", "format", "--check", f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", - "--line-length=88", + "--line-length=88", *LINT_PATHS, ) @@ -179,7 +178,7 @@ def lint(session): def blacken(session): """(Deprecated) Legacy session. Please use 'nox -s format'.""" session.log("WARNING: The 'blacken' session is deprecated and will be removed in a future release. Please use 'nox -s format' in the future.") - + # Just run the ruff formatter (keeping legacy behavior of only formatting, not sorting imports) session.install(RUFF_VERSION) session.run( @@ -518,14 +517,14 @@ def prerelease_deps(session, protobuf_implementation): # Extract the base package name, safely ignoring version bounds and spaces # (e.g., "grpcio>=1.75.1" becomes "grpcio") parsed_deps = { - dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) for dep in prerel_deps } # Dynamically sort local packages vs PyPI dependencies local_paths = [] pypi_deps = [] - + for dep, pkg_name in parsed_deps.items(): if (deps_dir / pkg_name).exists(): local_paths.append(str(deps_dir / pkg_name)) @@ -624,13 +623,16 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] - + session.install(*dep_paths, "--no-deps", "--ignore-installed") print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") diff --git a/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py b/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py index 0098c6989836..e876bb4f2cfd 100755 --- a/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py b/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py index 0744db4ab69b..0e277e5dfd2b 100755 --- a/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py b/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py index f163aac8e963..738850b16b99 100755 --- a/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py b/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py index 1510142f6d7d..df1eb716d12c 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py b/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py index 1510142f6d7d..df1eb716d12c 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py b/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py index 8b1386af4435..555a4745454b 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py index 8b1386af4435..555a4745454b 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py index 75fded704a26..1ac80d2800c0 100755 --- a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] From 704550ff714b741a75ba4bf6392afbe28c844036 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 21:24:45 -0700 Subject: [PATCH 5/8] if test modified, run all handwritten tests --- ci/get_package_shards.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py index fd26332648a3..71c43009ddac 100644 --- a/ci/get_package_shards.py +++ b/ci/get_package_shards.py @@ -136,7 +136,16 @@ def get_packages_to_test(): package_dirs = set(get_package_directories()) to_test_paths = collections.defaultdict(list) + has_ci_change = False + + CI_INFRASTRUCTURE_PREFIXES = ( + ".github/", + "ci/", + ) + for f in changed_files: + if f.startswith(CI_INFRASTRUCTURE_PREFIXES): + has_ci_change = True parts = f.split('/') if len(parts) >= 2 and parts[0] in package_dirs: pkg_name = parts[1] @@ -157,9 +166,14 @@ def get_packages_to_test(): "proto-plus", "google-crc32c", } - if any(pkg in core_packages for pkg in to_test_paths): - # When a core package changes, test all handwritten packages (non-GAPIC_AUTO) - return get_packages(handwritten_only=True) + has_core_change = any(pkg in core_packages for pkg in to_test_paths) + + # If CI infrastructure or a core dependency was touched, merge all handwritten packages + if has_ci_change or has_core_change: + for pkg, paths in get_packages(handwritten_only=True).items(): + for path in paths: + if path not in to_test_paths[pkg]: + to_test_paths[pkg].append(path) return dict(to_test_paths) From 71e17aa8db70e439f98930b35df610a397098365 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 21:29:41 -0700 Subject: [PATCH 6/8] moved package lists --- ci/get_package_shards.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py index 71c43009ddac..f7c1305c5fbf 100644 --- a/ci/get_package_shards.py +++ b/ci/get_package_shards.py @@ -29,6 +29,25 @@ import subprocess import sys +# CI infrastructure and workflow paths that affect test execution +CI_INFRASTRUCTURE_PREFIXES = ( + ".github/", + "ci/", +) + +# Core dependency packages whose changes affect all downstream handwritten packages +CORE_PACKAGES = { + "google-api-core", + "google-auth", + "google-auth-httplib2", + "google-auth-oauthlib", + "google-cloud-core", + "googleapis-common-protos", + "grpc-google-iam-v1", + "proto-plus", + "google-crc32c", +} + def get_package_directories(): """Parses package directory roots from the PACKAGE_DIRS environment variable. @@ -138,11 +157,6 @@ def get_packages_to_test(): to_test_paths = collections.defaultdict(list) has_ci_change = False - CI_INFRASTRUCTURE_PREFIXES = ( - ".github/", - "ci/", - ) - for f in changed_files: if f.startswith(CI_INFRASTRUCTURE_PREFIXES): has_ci_change = True @@ -154,19 +168,7 @@ def get_packages_to_test(): if full_path not in to_test_paths[pkg_name]: to_test_paths[pkg_name].append(full_path) - # Core dependency packages whose changes require testing across downstream handwritten packages - core_packages = { - "google-api-core", - "google-auth", - "google-auth-httplib2", - "google-auth-oauthlib", - "google-cloud-core", - "googleapis-common-protos", - "grpc-google-iam-v1", - "proto-plus", - "google-crc32c", - } - has_core_change = any(pkg in core_packages for pkg in to_test_paths) + has_core_change = any(pkg in CORE_PACKAGES for pkg in to_test_paths) # If CI infrastructure or a core dependency was touched, merge all handwritten packages if has_ci_change or has_core_change: From f97e1b35b08aa882d0d166274cdc3b1836ac4d3d Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 21:39:17 -0700 Subject: [PATCH 7/8] address gemini comments --- ci/get_package_shards.py | 14 +++++++------- packages/db-dtypes/noxfile.py | 8 +++++--- packages/django-google-spanner/noxfile.py | 16 ++++++++++------ .../gapic/templates/noxfile.py.j2 | 9 ++++++--- .../tests/integration/goldens/asset/noxfile.py | 9 ++++++--- .../integration/goldens/credentials/noxfile.py | 9 ++++++--- .../integration/goldens/eventarc/noxfile.py | 9 ++++++--- .../tests/integration/goldens/logging/noxfile.py | 9 ++++++--- .../goldens/logging_internal/noxfile.py | 9 ++++++--- .../tests/integration/goldens/redis/noxfile.py | 9 ++++++--- .../goldens/redis_selective/noxfile.py | 9 ++++++--- .../goldens/storagebatchoperations/noxfile.py | 9 ++++++--- .../google-cloud-documentai-toolbox/noxfile.py | 8 +++++--- packages/google-cloud-ndb/noxfile.py | 8 +++++--- packages/googleapis-common-protos/noxfile.py | 8 +++++--- packages/pandas-gbq/noxfile.py | 8 +++++--- packages/proto-plus/noxfile.py | 8 +++++--- packages/sqlalchemy-bigquery/noxfile.py | 8 +++++--- packages/sqlalchemy-spanner/noxfile.py | 16 ++++++++++------ 19 files changed, 116 insertions(+), 67 deletions(-) diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py index f7c1305c5fbf..4f3804895754 100644 --- a/ci/get_package_shards.py +++ b/ci/get_package_shards.py @@ -30,10 +30,10 @@ import sys # CI infrastructure and workflow paths that affect test execution -CI_INFRASTRUCTURE_PREFIXES = ( - ".github/", - "ci/", -) +CI_INFRASTRUCTURE_DIRS = { + ".github", + "ci", +} # Core dependency packages whose changes affect all downstream handwritten packages CORE_PACKAGES = { @@ -112,7 +112,7 @@ def get_packages(handwritten_only=False): try: with open(meta_file) as f: data = json.load(f) - if data.get("library_type") == "GAPIC_AUTO": + if isinstance(data, dict) and data.get("library_type") == "GAPIC_AUTO": continue except Exception: pass @@ -158,9 +158,9 @@ def get_packages_to_test(): has_ci_change = False for f in changed_files: - if f.startswith(CI_INFRASTRUCTURE_PREFIXES): + parts = os.path.normpath(f).split(os.sep) + if parts and parts[0] in CI_INFRASTRUCTURE_DIRS: has_ci_change = True - parts = f.split('/') if len(parts) >= 2 and parts[0] in package_dirs: pkg_name = parts[1] full_path = f"{parts[0]}/{parts[1]}/" diff --git a/packages/db-dtypes/noxfile.py b/packages/db-dtypes/noxfile.py index d7fc9b977ee3..fb12833198ae 100644 --- a/packages/db-dtypes/noxfile.py +++ b/packages/db-dtypes/noxfile.py @@ -519,9 +519,11 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) local_paths = [ str(deps_dir / dep) diff --git a/packages/django-google-spanner/noxfile.py b/packages/django-google-spanner/noxfile.py index 612fafe793f9..bb99a5b2d459 100644 --- a/packages/django-google-spanner/noxfile.py +++ b/packages/django-google-spanner/noxfile.py @@ -379,9 +379,11 @@ def core_deps_from_source(session, protobuf_implementation): "google-cloud-spanner", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] @@ -434,9 +436,11 @@ def prerelease_deps(session, protobuf_implementation): "google-cloud-spanner", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) parsed_deps = { dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) for dep in prerel_deps diff --git a/packages/gapic-generator/gapic/templates/noxfile.py.j2 b/packages/gapic-generator/gapic/templates/noxfile.py.j2 index 47205dd39819..d233a4fe603f 100644 --- a/packages/gapic-generator/gapic/templates/noxfile.py.j2 +++ b/packages/gapic-generator/gapic/templates/noxfile.py.j2 @@ -510,9 +510,12 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Extract the base package name, safely ignoring version bounds and spaces # (e.g., "grpcio>=1.75.1" becomes "grpcio") diff --git a/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py b/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py index e876bb4f2cfd..368799a04f58 100755 --- a/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py @@ -502,9 +502,12 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Extract the base package name, safely ignoring version bounds and spaces # (e.g., "grpcio>=1.75.1" becomes "grpcio") diff --git a/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py b/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py index 0e277e5dfd2b..f1fd34b5e0d5 100755 --- a/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py @@ -502,9 +502,12 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Extract the base package name, safely ignoring version bounds and spaces # (e.g., "grpcio>=1.75.1" becomes "grpcio") diff --git a/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py b/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py index 738850b16b99..1b4c2b120539 100755 --- a/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py @@ -502,9 +502,12 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Extract the base package name, safely ignoring version bounds and spaces # (e.g., "grpcio>=1.75.1" becomes "grpcio") diff --git a/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py b/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py index df1eb716d12c..aec023750e54 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py @@ -502,9 +502,12 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Extract the base package name, safely ignoring version bounds and spaces # (e.g., "grpcio>=1.75.1" becomes "grpcio") diff --git a/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py b/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py index df1eb716d12c..aec023750e54 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py @@ -502,9 +502,12 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Extract the base package name, safely ignoring version bounds and spaces # (e.g., "grpcio>=1.75.1" becomes "grpcio") diff --git a/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py b/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py index 555a4745454b..58859884d368 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py @@ -502,9 +502,12 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Extract the base package name, safely ignoring version bounds and spaces # (e.g., "grpcio>=1.75.1" becomes "grpcio") diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py index 555a4745454b..58859884d368 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py @@ -502,9 +502,12 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Extract the base package name, safely ignoring version bounds and spaces # (e.g., "grpcio>=1.75.1" becomes "grpcio") diff --git a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py index 1ac80d2800c0..0bd3bf67a42c 100755 --- a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py @@ -502,9 +502,12 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Extract the base package name, safely ignoring version bounds and spaces # (e.g., "grpcio>=1.75.1" becomes "grpcio") diff --git a/packages/google-cloud-documentai-toolbox/noxfile.py b/packages/google-cloud-documentai-toolbox/noxfile.py index 1502ae07a03b..0a79c7097ff2 100644 --- a/packages/google-cloud-documentai-toolbox/noxfile.py +++ b/packages/google-cloud-documentai-toolbox/noxfile.py @@ -543,9 +543,11 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] if local_paths: diff --git a/packages/google-cloud-ndb/noxfile.py b/packages/google-cloud-ndb/noxfile.py index 7a90848f63c2..e498b5b854b8 100644 --- a/packages/google-cloud-ndb/noxfile.py +++ b/packages/google-cloud-ndb/noxfile.py @@ -511,9 +511,11 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] if local_paths: diff --git a/packages/googleapis-common-protos/noxfile.py b/packages/googleapis-common-protos/noxfile.py index 79587afeca21..778ac1d74104 100644 --- a/packages/googleapis-common-protos/noxfile.py +++ b/packages/googleapis-common-protos/noxfile.py @@ -565,9 +565,11 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] if local_paths: diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index ed751fe7e104..bda1d06e6de0 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -591,9 +591,11 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] if local_paths: diff --git a/packages/proto-plus/noxfile.py b/packages/proto-plus/noxfile.py index 63cfe15796a6..a6eac6f95ab7 100644 --- a/packages/proto-plus/noxfile.py +++ b/packages/proto-plus/noxfile.py @@ -171,9 +171,11 @@ def core_deps_from_source(session, implementation): "googleapis-common-protos", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] if local_paths: diff --git a/packages/sqlalchemy-bigquery/noxfile.py b/packages/sqlalchemy-bigquery/noxfile.py index 5e7d20291107..d44c5faf4a08 100644 --- a/packages/sqlalchemy-bigquery/noxfile.py +++ b/packages/sqlalchemy-bigquery/noxfile.py @@ -713,9 +713,11 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] if local_paths: diff --git a/packages/sqlalchemy-spanner/noxfile.py b/packages/sqlalchemy-spanner/noxfile.py index 84a348c76e61..029597af08e8 100644 --- a/packages/sqlalchemy-spanner/noxfile.py +++ b/packages/sqlalchemy-spanner/noxfile.py @@ -554,9 +554,11 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] if local_paths: @@ -601,9 +603,11 @@ def prerelease_deps(session, protobuf_implementation): "google-cloud-spanner", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) parsed_deps = { dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) for dep in prerel_deps From 40cbb8a8c9a32f83efd97576453f514fd0bb491c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 27 Aug 2026 09:33:43 -0700 Subject: [PATCH 8/8] fix lint --- packages/db-dtypes/noxfile.py | 4 +--- packages/django-google-spanner/noxfile.py | 8 ++------ .../google-cloud-documentai-toolbox/noxfile.py | 14 +++++++++----- packages/google-cloud-ndb/noxfile.py | 14 +++++++++----- packages/googleapis-common-protos/noxfile.py | 14 +++++++++----- packages/pandas-gbq/noxfile.py | 14 +++++++++----- packages/proto-plus/noxfile.py | 14 +++++++++----- packages/sqlalchemy-bigquery/noxfile.py | 14 +++++++++----- packages/sqlalchemy-spanner/noxfile.py | 18 ++++++++++-------- 9 files changed, 67 insertions(+), 47 deletions(-) diff --git a/packages/db-dtypes/noxfile.py b/packages/db-dtypes/noxfile.py index fb12833198ae..20466fbb0755 100644 --- a/packages/db-dtypes/noxfile.py +++ b/packages/db-dtypes/noxfile.py @@ -520,9 +520,7 @@ def core_deps_from_source(session, protobuf_implementation): ] deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() + p / "packages" for p in CURRENT_DIRECTORY.parents if (p / "packages").is_dir() ) local_paths = [ diff --git a/packages/django-google-spanner/noxfile.py b/packages/django-google-spanner/noxfile.py index bb99a5b2d459..fbb6a88f6dd0 100644 --- a/packages/django-google-spanner/noxfile.py +++ b/packages/django-google-spanner/noxfile.py @@ -380,9 +380,7 @@ def core_deps_from_source(session, protobuf_implementation): ] deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() + p / "packages" for p in CURRENT_DIRECTORY.parents if (p / "packages").is_dir() ) # Batch the pip installation to avoid sequential overhead @@ -437,9 +435,7 @@ def prerelease_deps(session, protobuf_implementation): ] deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() + p / "packages" for p in CURRENT_DIRECTORY.parents if (p / "packages").is_dir() ) parsed_deps = { diff --git a/packages/google-cloud-documentai-toolbox/noxfile.py b/packages/google-cloud-documentai-toolbox/noxfile.py index 0a79c7097ff2..4e15fcc1ca36 100644 --- a/packages/google-cloud-documentai-toolbox/noxfile.py +++ b/packages/google-cloud-documentai-toolbox/noxfile.py @@ -544,15 +544,19 @@ def core_deps_from_source(session, protobuf_implementation): ] deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() + p / "packages" for p in CURRENT_DIRECTORY.parents if (p / "packages").is_dir() ) - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + local_paths = [ + str(deps_dir / dep) + for dep in core_dependencies_from_source + if (deps_dir / dep).exists() + ] if local_paths: session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + print( + f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}" + ) tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/google-cloud-ndb/noxfile.py b/packages/google-cloud-ndb/noxfile.py index e498b5b854b8..6ebaa1548f86 100644 --- a/packages/google-cloud-ndb/noxfile.py +++ b/packages/google-cloud-ndb/noxfile.py @@ -512,15 +512,19 @@ def core_deps_from_source(session, protobuf_implementation): ] deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() + p / "packages" for p in CURRENT_DIRECTORY.parents if (p / "packages").is_dir() ) - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + local_paths = [ + str(deps_dir / dep) + for dep in core_dependencies_from_source + if (deps_dir / dep).exists() + ] if local_paths: session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + print( + f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}" + ) tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/googleapis-common-protos/noxfile.py b/packages/googleapis-common-protos/noxfile.py index 778ac1d74104..29a733be0d48 100644 --- a/packages/googleapis-common-protos/noxfile.py +++ b/packages/googleapis-common-protos/noxfile.py @@ -566,15 +566,19 @@ def core_deps_from_source(session, protobuf_implementation): ] deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() + p / "packages" for p in CURRENT_DIRECTORY.parents if (p / "packages").is_dir() ) - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + local_paths = [ + str(deps_dir / dep) + for dep in core_dependencies_from_source + if (deps_dir / dep).exists() + ] if local_paths: session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + print( + f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}" + ) session.run( "py.test", diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index bda1d06e6de0..a46e996f170a 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -592,15 +592,19 @@ def core_deps_from_source(session, protobuf_implementation): ] deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() + p / "packages" for p in CURRENT_DIRECTORY.parents if (p / "packages").is_dir() ) - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + local_paths = [ + str(deps_dir / dep) + for dep in core_dependencies_from_source + if (deps_dir / dep).exists() + ] if local_paths: session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + print( + f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}" + ) tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/proto-plus/noxfile.py b/packages/proto-plus/noxfile.py index a6eac6f95ab7..5de46a57f295 100644 --- a/packages/proto-plus/noxfile.py +++ b/packages/proto-plus/noxfile.py @@ -172,15 +172,19 @@ def core_deps_from_source(session, implementation): ] deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() + p / "packages" for p in CURRENT_DIRECTORY.parents if (p / "packages").is_dir() ) - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + local_paths = [ + str(deps_dir / dep) + for dep in core_dependencies_from_source + if (deps_dir / dep).exists() + ] if local_paths: session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + print( + f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}" + ) # TODO(https://github.com/googleapis/google-cloud-python/issues/15115): Install protobuf from source at HEAD session.install("--pre", "--upgrade", "protobuf") diff --git a/packages/sqlalchemy-bigquery/noxfile.py b/packages/sqlalchemy-bigquery/noxfile.py index d44c5faf4a08..088d9bc5f952 100644 --- a/packages/sqlalchemy-bigquery/noxfile.py +++ b/packages/sqlalchemy-bigquery/noxfile.py @@ -714,15 +714,19 @@ def core_deps_from_source(session, protobuf_implementation): ] deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() + p / "packages" for p in CURRENT_DIRECTORY.parents if (p / "packages").is_dir() ) - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + local_paths = [ + str(deps_dir / dep) + for dep in core_dependencies_from_source + if (deps_dir / dep).exists() + ] if local_paths: session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + print( + f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}" + ) tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/sqlalchemy-spanner/noxfile.py b/packages/sqlalchemy-spanner/noxfile.py index 029597af08e8..8e6214c2e526 100644 --- a/packages/sqlalchemy-spanner/noxfile.py +++ b/packages/sqlalchemy-spanner/noxfile.py @@ -555,15 +555,19 @@ def core_deps_from_source(session, protobuf_implementation): ] deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() + p / "packages" for p in CURRENT_DIRECTORY.parents if (p / "packages").is_dir() ) - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + local_paths = [ + str(deps_dir / dep) + for dep in core_dependencies_from_source + if (deps_dir / dep).exists() + ] if local_paths: session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + print( + f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}" + ) tests_path = os.path.join("tests", "unit") session.run( @@ -604,9 +608,7 @@ def prerelease_deps(session, protobuf_implementation): ] deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() + p / "packages" for p in CURRENT_DIRECTORY.parents if (p / "packages").is_dir() ) parsed_deps = {