diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 6c2206518bc3..68ccc4969d7f 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -40,15 +40,33 @@ jobs: # Each weight is roughly 1 minute of expected execution time # Default for unset packages is 1 PACKAGE_WEIGHTS: | - google-ai-generativelanguage: 4 - google-auth: 5 + django-google-spanner: 2 + gapic-generator: 3 + google-ai-generativelanguage: 3 + google-api-core: 2 + google-auth: 2 + google-cloud-bigquery: 4 + google-cloud-bigquery-storage: 2 + google-cloud-bigtable: 4 google-cloud-compute: 12 google-cloud-compute-v1beta: 12 - google-cloud-dialogflow: 6 - google-cloud-dialogflow-cx: 6 - google-cloud-discoveryengine: 8 - google-cloud-retail: 5 - google-shopping-merchant-accounts: 4 + google-cloud-datastore: 3 + google-cloud-dialogflow: 4 + google-cloud-dialogflow-cx: 4 + google-cloud-discoveryengine: 4 + google-cloud-firestore: 3 + google-cloud-logging: 3 + google-cloud-monitoring: 3 + google-cloud-ndb: 2 + google-cloud-pubsub: 2 + google-cloud-retail: 3 + google-cloud-spanner: 4 + google-cloud-storage: 4 + google-shopping-merchant-accounts: 3 + pandas-gbq: 2 + proto-plus: 1 + sqlalchemy-bigquery: 2 + sqlalchemy-spanner: 2 steps: - name: Checkout uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 @@ -156,12 +174,52 @@ jobs: path: .coverage.${{ matrix.python }}.* include-hidden-files: true + core-deps: + needs: initialize + if: needs.initialize.outputs.matrix != '[]' && needs.initialize.outputs.matrix != '' + runs-on: ubuntu-22.04 + strategy: + fail-fast: true + matrix: + python: ["3.14"] + package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }} + name: ${{ matrix.package_shard.is_sharded && format('core-deps handwritten ({0}, {1})', matrix.python, matrix.package_shard.name) || format('core-deps handwritten ({0})', matrix.python) }} + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` + # See https://github.com/googleapis/google-cloud-python/issues/12013 + # and https://github.com/actions/checkout#checkout-head. + with: + fetch-depth: 2 + persist-credentials: false + - name: Setup Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: ${{ matrix.python }} + cache: 'pip' + allow-prereleases: true + - name: Install nox + run: | + pip install nox + - name: Run core_deps_from_source for ${{ matrix.package_shard.description }} + env: + BUILD_TYPE: presubmit + TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }} + TEST_TYPE: core_deps_from_source + PY_VERSION: ${{ matrix.python }} + PACKAGE_LIST: ${{ matrix.package_shard.packages }} + NOX_DEFAULT_VENV_BACKEND: "virtualenv" + NOXFORCEPYTHON: ${{ matrix.python }} + run: | + ci/run_conditional_tests.sh + all-tests: - needs: [initialize, unit] + needs: [initialize, unit, core-deps] if: always() runs-on: ubuntu-latest steps: - - name: Check unit test results + - name: Check test results run: | # 1. Check initialize job if [[ "${{ needs.initialize.result }}" != "success" ]]; then @@ -173,7 +231,12 @@ jobs: echo "Unit tests failed" exit 1 fi - echo "All unit tests passed or were skipped" + # 3. Check core dependencies test shards + if [[ "${{ needs.core-deps.result }}" != "success" && "${{ needs.core-deps.result }}" != "skipped" ]]; then + echo "Core dependencies tests failed" + exit 1 + fi + echo "All unit and core dependencies tests passed or were skipped" cover: if: always() && !cancelled() && needs.all-tests.result == 'success' diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py index 015adfc2bacd..f7c1305c5fbf 100644 --- a/ci/get_package_shards.py +++ b/ci/get_package_shards.py @@ -22,12 +22,31 @@ preview-packages/foo) are kept aligned in the exact same shard. """ -import os -import subprocess +import collections import json import math +import os +import subprocess import sys -import collections + +# CI infrastructure and workflow paths that affect test execution +CI_INFRASTRUCTURE_PREFIXES = ( + ".github/", + "ci/", +) + +# Core dependency packages whose changes affect all downstream handwritten packages +CORE_PACKAGES = { + "google-api-core", + "google-auth", + "google-auth-httplib2", + "google-auth-oauthlib", + "google-cloud-core", + "googleapis-common-protos", + "grpc-google-iam-v1", + "proto-plus", + "google-crc32c", +} def get_package_directories(): @@ -70,9 +89,11 @@ def get_package_weights(): return weights -def get_packages(): +def get_packages(handwritten_only=False): """Lists all package directory paths in the repository grouped by package name. + If handwritten_only is True, includes only non-GAPIC_AUTO libraries. + Returns: dict: A dictionary mapping package_name -> list of relative directory paths. """ @@ -83,8 +104,19 @@ def get_packages(): continue for d in os.listdir(subdir): full_path = os.path.join(subdir, d) + '/' - if os.path.isdir(full_path): - packages_map[d].append(full_path) + if not os.path.isdir(full_path): + continue + if handwritten_only: + meta_file = os.path.join(full_path, ".repo-metadata.json") + if os.path.exists(meta_file): + try: + with open(meta_file) as f: + data = json.load(f) + if data.get("library_type") == "GAPIC_AUTO": + continue + except Exception: + pass + packages_map[d].append(full_path) return packages_map @@ -123,7 +155,11 @@ def get_packages_to_test(): package_dirs = set(get_package_directories()) to_test_paths = collections.defaultdict(list) + has_ci_change = False + for f in changed_files: + if f.startswith(CI_INFRASTRUCTURE_PREFIXES): + has_ci_change = True parts = f.split('/') if len(parts) >= 2 and parts[0] in package_dirs: pkg_name = parts[1] @@ -132,6 +168,15 @@ def get_packages_to_test(): if full_path not in to_test_paths[pkg_name]: to_test_paths[pkg_name].append(full_path) + has_core_change = any(pkg in CORE_PACKAGES for pkg in to_test_paths) + + # If CI infrastructure or a core dependency was touched, merge all handwritten packages + if has_ci_change or has_core_change: + for pkg, paths in get_packages(handwritten_only=True).items(): + for path in paths: + if path not in to_test_paths[pkg]: + to_test_paths[pkg].append(path) + return dict(to_test_paths) @@ -173,7 +218,7 @@ def group_packages(packages_map): # Pack packages alphabetically by package name. for name, paths, weight in pkg_items: - # If adding this package would exceed target weight AND we haven't reached the + # If adding this package would exceed target weight AND we haven't reached the # shard limit, start a new shard. Otherwise, keep "stuffing" the current one. if current_shard_items and (current_shard_weight + weight > target_weight) and len(shards_list) < max_shards - 1: shards_list.append(current_shard_items) diff --git a/ci/run_single_test.sh b/ci/run_single_test.sh index 0ea0b969770d..8f888923e3b2 100755 --- a/ci/run_single_test.sh +++ b/ci/run_single_test.sh @@ -19,7 +19,7 @@ # `PY_VERSION` should be one of ["3.10", "3.11", "3.12", "3.13"] # This script is called by the `ci/run_conditional_tests.sh` script. -# A specific `nox` session will be run, depending on the value of +# A specific `nox` session will be run, depending on the value of # `TEST_TYPE` and `PY_VERSION`. For example, if `TEST_TYPE` is # `lint`, the `nox -s lint` session will be run. @@ -68,6 +68,14 @@ case ${TEST_TYPE} in nox -s prerelease_deps-3.14 retval=$? ;; + core_deps_from_source) + if [[ "$(pwd)" == */preview-packages/* ]]; then + echo "Skipping core_deps_from_source for preview package $(pwd)" + exit 0 + fi + nox --stop-on-first-error -s core_deps_from_source + retval=$? + ;; unit) case ${PY_VERSION} in "3.10") @@ -131,12 +139,12 @@ case ${TEST_TYPE} in source .venv-profiler/bin/activate export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 python -m pip install --upgrade pip setuptools - + PROFILER_TEMP_DIR=$(mktemp -d) cp ../../scripts/import_profiler/profiler.py "${PROFILER_TEMP_DIR}/profiler.py" PROFILER_SCRIPT="${PROFILER_TEMP_DIR}/profiler.py" BASELINE_CSV="${PROFILER_TEMP_DIR}/baseline_${PACKAGE_NAME}.csv" - + if [ -n "${TARGET_BRANCH}" ]; then # Fetch history for the target branch without --depth=1 in case it was shallowly fetched if [ -f "$(git rev-parse --git-dir)/shallow" ]; then @@ -180,7 +188,7 @@ case ${TEST_TYPE} in echo "Could not find baseline commit for ${TARGET_BRANCH:-main}. Skipping baseline generation." fi fi - + # TODO(https://github.com/googleapis/google-cloud-python/issues/18035): # Clean up this fallback once Python 3.15 is officially released and upstream binary wheels are available on PyPI. # On pre-release Python versions, packages with complex C/Rust dependencies (e.g. bigframes) fail during pip install due to missing pre-built wheels. diff --git a/packages/db-dtypes/noxfile.py b/packages/db-dtypes/noxfile.py index b26f72f7a285..d7fc9b977ee3 100644 --- a/packages/db-dtypes/noxfile.py +++ b/packages/db-dtypes/noxfile.py @@ -512,16 +512,27 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [ + str(deps_dir / dep) + for dep in core_dependencies_from_source + if (deps_dir / dep).exists() + ] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print( + f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}" + ) tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/gapic-generator/gapic/templates/noxfile.py.j2 b/packages/gapic-generator/gapic/templates/noxfile.py.j2 index d4176bdf4222..47205dd39819 100644 --- a/packages/gapic-generator/gapic/templates/noxfile.py.j2 +++ b/packages/gapic-generator/gapic/templates/noxfile.py.j2 @@ -6,9 +6,8 @@ import os import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -163,7 +162,7 @@ def lint(session): "ruff", "format", "--check", f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", - "--line-length=88", + "--line-length=88", *LINT_PATHS, ) @@ -179,7 +178,7 @@ def lint(session): def blacken(session): """(Deprecated) Legacy session. Please use 'nox -s format'.""" session.log("WARNING: The 'blacken' session is deprecated and will be removed in a future release. Please use 'nox -s format' in the future.") - + # Just run the ruff formatter (keeping legacy behavior of only formatting, not sorting imports) session.install(RUFF_VERSION) session.run( @@ -518,14 +517,14 @@ def prerelease_deps(session, protobuf_implementation): # Extract the base package name, safely ignoring version bounds and spaces # (e.g., "grpcio>=1.75.1" becomes "grpcio") parsed_deps = { - dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) for dep in prerel_deps } # Dynamically sort local packages vs PyPI dependencies local_paths = [] pypi_deps = [] - + for dep, pkg_name in parsed_deps.items(): if (deps_dir / pkg_name).exists(): local_paths.append(str(deps_dir / pkg_name)) @@ -624,13 +623,16 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] - + session.install(*dep_paths, "--no-deps", "--ignore-installed") print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") diff --git a/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py b/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py index 0098c6989836..e876bb4f2cfd 100755 --- a/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py b/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py index 0744db4ab69b..0e277e5dfd2b 100755 --- a/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py b/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py index f163aac8e963..738850b16b99 100755 --- a/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py b/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py index 1510142f6d7d..df1eb716d12c 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py b/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py index 1510142f6d7d..df1eb716d12c 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py b/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py index 8b1386af4435..555a4745454b 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py index 8b1386af4435..555a4745454b 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py index 75fded704a26..1ac80d2800c0 100755 --- a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/google-cloud-documentai-toolbox/noxfile.py b/packages/google-cloud-documentai-toolbox/noxfile.py index 18a537cb9bda..1502ae07a03b 100644 --- a/packages/google-cloud-documentai-toolbox/noxfile.py +++ b/packages/google-cloud-documentai-toolbox/noxfile.py @@ -536,16 +536,21 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/google-cloud-ndb/noxfile.py b/packages/google-cloud-ndb/noxfile.py index cb957a674ed1..7a90848f63c2 100644 --- a/packages/google-cloud-ndb/noxfile.py +++ b/packages/google-cloud-ndb/noxfile.py @@ -504,16 +504,21 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/googleapis-common-protos/noxfile.py b/packages/googleapis-common-protos/noxfile.py index 78fd0cbf4ff9..79587afeca21 100644 --- a/packages/googleapis-common-protos/noxfile.py +++ b/packages/googleapis-common-protos/noxfile.py @@ -558,16 +558,21 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - f"{CURRENT_DIRECTORY}/../googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - f"{CURRENT_DIRECTORY}/../grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") session.run( "py.test", diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 0021226416cd..ed751fe7e104 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -584,16 +584,21 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/proto-plus/noxfile.py b/packages/proto-plus/noxfile.py index 5f56955863c8..63cfe15796a6 100644 --- a/packages/proto-plus/noxfile.py +++ b/packages/proto-plus/noxfile.py @@ -167,14 +167,18 @@ def core_deps_from_source(session, implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - # dependency of google-api-core - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", + "google-api-core", + "googleapis-common-protos", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") # TODO(https://github.com/googleapis/google-cloud-python/issues/15115): Install protobuf from source at HEAD session.install("--pre", "--upgrade", "protobuf") diff --git a/packages/sqlalchemy-bigquery/noxfile.py b/packages/sqlalchemy-bigquery/noxfile.py index 506a5bfcfb8f..5e7d20291107 100644 --- a/packages/sqlalchemy-bigquery/noxfile.py +++ b/packages/sqlalchemy-bigquery/noxfile.py @@ -706,16 +706,21 @@ def core_deps_from_source(session, protobuf_implementation): session.install("-e", install_target, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/sqlalchemy-spanner/noxfile.py b/packages/sqlalchemy-spanner/noxfile.py index df709c96e50a..84a348c76e61 100644 --- a/packages/sqlalchemy-spanner/noxfile.py +++ b/packages/sqlalchemy-spanner/noxfile.py @@ -547,16 +547,21 @@ def core_deps_from_source(session, protobuf_implementation): session.install(".") core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run(