[DRAFT] chore(tests): enable core_deps presubmit check

.github/workflows/unittest.yml

@@ -40,15 +40,33 @@ jobs:
       # Each weight is roughly 1 minute of expected execution time
       # Default for unset packages is 1
       PACKAGE_WEIGHTS: |
-        google-ai-generativelanguage: 4
-        google-auth: 5
+        django-google-spanner: 2
+        gapic-generator: 3
+        google-ai-generativelanguage: 3
+        google-api-core: 2
+        google-auth: 2
+        google-cloud-bigquery: 4
+        google-cloud-bigquery-storage: 2
+        google-cloud-bigtable: 4
         google-cloud-compute: 12
         google-cloud-compute-v1beta: 12
-        google-cloud-dialogflow: 6
-        google-cloud-dialogflow-cx: 6
-        google-cloud-discoveryengine: 8
-        google-cloud-retail: 5
-        google-shopping-merchant-accounts: 4
+        google-cloud-datastore: 3
+        google-cloud-dialogflow: 4
+        google-cloud-dialogflow-cx: 4
+        google-cloud-discoveryengine: 4
+        google-cloud-firestore: 3
+        google-cloud-logging: 3
+        google-cloud-monitoring: 3
+        google-cloud-ndb: 2
+        google-cloud-pubsub: 2
+        google-cloud-retail: 3
+        google-cloud-spanner: 4
+        google-cloud-storage: 4
+        google-shopping-merchant-accounts: 3
+        pandas-gbq: 2
+        proto-plus: 1
+        sqlalchemy-bigquery: 2
+        sqlalchemy-spanner: 2
     steps:
     - name: Checkout
       uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -156,12 +174,52 @@ jobs:
         path: .coverage.${{ matrix.python }}.*
         include-hidden-files: true
 
+  core-deps:
+    needs: initialize
+    if: needs.initialize.outputs.matrix != '[]' && needs.initialize.outputs.matrix != ''
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: true
+      matrix:
+        python: ["3.14"]
+        package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }}
+    name: ${{ matrix.package_shard.is_sharded && format('core-deps handwritten ({0}, {1})', matrix.python, matrix.package_shard.name) || format('core-deps handwritten ({0})', matrix.python) }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
+      # See https://github.com/googleapis/google-cloud-python/issues/12013
+      # and https://github.com/actions/checkout#checkout-head.
+      with:
+        fetch-depth: 2
+        persist-credentials: false
+    - name: Setup Python
+      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+      with:
+        python-version: ${{ matrix.python }}
+        cache: 'pip'
+        allow-prereleases: true
+    - name: Install nox
+      run: |
+        pip install nox
+    - name: Run core_deps_from_source for ${{ matrix.package_shard.description }}
+      env:
+        BUILD_TYPE: presubmit
+        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
+        TEST_TYPE: core_deps_from_source
+        PY_VERSION: ${{ matrix.python }}
+        PACKAGE_LIST: ${{ matrix.package_shard.packages }}
+        NOX_DEFAULT_VENV_BACKEND: "virtualenv"
+        NOXFORCEPYTHON: ${{ matrix.python }}
+      run: |
+        ci/run_conditional_tests.sh
+
   all-tests:
-    needs: [initialize, unit]
+    needs: [initialize, unit, core-deps]
     if: always()
     runs-on: ubuntu-latest
     steps:
-      - name: Check unit test results
+      - name: Check test results
         run: |
           # 1. Check initialize job
           if [[ "${{ needs.initialize.result }}" != "success" ]]; then
@@ -173,7 +231,12 @@ jobs:
             echo "Unit tests failed"
             exit 1
           fi
-          echo "All unit tests passed or were skipped"
+          # 3. Check core dependencies test shards
+          if [[ "${{ needs.core-deps.result }}" != "success" && "${{ needs.core-deps.result }}" != "skipped" ]]; then
+            echo "Core dependencies tests failed"
+            exit 1
+          fi
+          echo "All unit and core dependencies tests passed or were skipped"
 
   cover:
     if: always() && !cancelled() && needs.all-tests.result == 'success'

ci/get_package_shards.py

@@ -22,12 +22,31 @@
 preview-packages/foo) are kept aligned in the exact same shard.
 """
 
-import os
-import subprocess
+import collections
 import json
 import math
+import os
+import subprocess
 import sys
-import collections
+
+# CI infrastructure and workflow paths that affect test execution
+CI_INFRASTRUCTURE_PREFIXES = (
+    ".github/",
+    "ci/",
+)
+
+# Core dependency packages whose changes affect all downstream handwritten packages
+CORE_PACKAGES = {
+    "google-api-core",
+    "google-auth",
+    "google-auth-httplib2",
+    "google-auth-oauthlib",
+    "google-cloud-core",
+    "googleapis-common-protos",
+    "grpc-google-iam-v1",
+    "proto-plus",
+    "google-crc32c",
+}
 
 
 def get_package_directories():
@@ -70,9 +89,11 @@ def get_package_weights():
     return weights
 
 
-def get_packages():
+def get_packages(handwritten_only=False):
     """Lists all package directory paths in the repository grouped by package name.
 
+    If handwritten_only is True, includes only non-GAPIC_AUTO libraries.
+
     Returns:
         dict: A dictionary mapping package_name -> list of relative directory paths.
     """
@@ -83,8 +104,19 @@ def get_packages():
             continue
         for d in os.listdir(subdir):
             full_path = os.path.join(subdir, d) + '/'
-            if os.path.isdir(full_path):
-                packages_map[d].append(full_path)
+            if not os.path.isdir(full_path):
+                continue
+            if handwritten_only:
+                meta_file = os.path.join(full_path, ".repo-metadata.json")
+                if os.path.exists(meta_file):
+                    try:
+                        with open(meta_file) as f:
+                            data = json.load(f)
+                            if data.get("library_type") == "GAPIC_AUTO":
+                                continue
+                    except Exception:
+                        pass
+            packages_map[d].append(full_path)
     return packages_map
 
 
@@ -123,7 +155,11 @@ def get_packages_to_test():
 
     package_dirs = set(get_package_directories())
     to_test_paths = collections.defaultdict(list)
+    has_ci_change = False
+
     for f in changed_files:
+        if f.startswith(CI_INFRASTRUCTURE_PREFIXES):
+            has_ci_change = True
         parts = f.split('/')
         if len(parts) >= 2 and parts[0] in package_dirs:
             pkg_name = parts[1]
@@ -132,6 +168,15 @@ def get_packages_to_test():
                 if full_path not in to_test_paths[pkg_name]:
                     to_test_paths[pkg_name].append(full_path)
 
+    has_core_change = any(pkg in CORE_PACKAGES for pkg in to_test_paths)
+
+    # If CI infrastructure or a core dependency was touched, merge all handwritten packages
+    if has_ci_change or has_core_change:
+        for pkg, paths in get_packages(handwritten_only=True).items():
+            for path in paths:
+                if path not in to_test_paths[pkg]:
+                    to_test_paths[pkg].append(path)
+
     return dict(to_test_paths)
 
 
@@ -173,7 +218,7 @@ def group_packages(packages_map):
 
     # Pack packages alphabetically by package name.
     for name, paths, weight in pkg_items:
-        # If adding this package would exceed target weight AND we haven't reached the 
+        # If adding this package would exceed target weight AND we haven't reached the
         # shard limit, start a new shard. Otherwise, keep "stuffing" the current one.
         if current_shard_items and (current_shard_weight + weight > target_weight) and len(shards_list) < max_shards - 1:
             shards_list.append(current_shard_items)

ci/run_single_test.sh

@@ -19,7 +19,7 @@
 # `PY_VERSION` should be one of ["3.10", "3.11", "3.12", "3.13"]
 
 # This script is called by the `ci/run_conditional_tests.sh` script.
-# A specific `nox` session will be run, depending on the value of 
+# A specific `nox` session will be run, depending on the value of
 # `TEST_TYPE` and `PY_VERSION`. For example, if `TEST_TYPE` is
 # `lint`, the `nox -s lint` session will be run.
 
@@ -68,6 +68,14 @@ case ${TEST_TYPE} in
         nox -s prerelease_deps-3.14
         retval=$?
         ;;
+    core_deps_from_source)
+        if [[ "$(pwd)" == */preview-packages/* ]]; then
+            echo "Skipping core_deps_from_source for preview package $(pwd)"
+            exit 0
+        fi
+        nox --stop-on-first-error -s core_deps_from_source
+        retval=$?
+        ;;
     unit)
         case ${PY_VERSION} in
         "3.10")
@@ -131,12 +139,12 @@ case ${TEST_TYPE} in
             source .venv-profiler/bin/activate
             export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1
             python -m pip install --upgrade pip setuptools
-            
+
             PROFILER_TEMP_DIR=$(mktemp -d)
             cp ../../scripts/import_profiler/profiler.py "${PROFILER_TEMP_DIR}/profiler.py"
             PROFILER_SCRIPT="${PROFILER_TEMP_DIR}/profiler.py"
             BASELINE_CSV="${PROFILER_TEMP_DIR}/baseline_${PACKAGE_NAME}.csv"
-            
+
             if [ -n "${TARGET_BRANCH}" ]; then
                 # Fetch history for the target branch without --depth=1 in case it was shallowly fetched
                 if [ -f "$(git rev-parse --git-dir)/shallow" ]; then
@@ -180,7 +188,7 @@ case ${TEST_TYPE} in
                     echo "Could not find baseline commit for ${TARGET_BRANCH:-main}. Skipping baseline generation."
                 fi
             fi
-            
+
             # TODO(https://github.com/googleapis/google-cloud-python/issues/18035):
             # Clean up this fallback once Python 3.15 is officially released and upstream binary wheels are available on PyPI.
             # On pre-release Python versions, packages with complex C/Rust dependencies (e.g. bigframes) fail during pip install due to missing pre-built wheels.

packages/db-dtypes/noxfile.py

@@ -512,16 +512,27 @@ def core_deps_from_source(session, protobuf_implementation):
     install_unittest_dependencies(session, "-c", constraints_path)
 
     core_dependencies_from_source = [
-        "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos",
-        "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core",
-        "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth",
-        "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1",
-        "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus",
+        "googleapis-common-protos",
+        "google-api-core",
+        "google-auth",
+        "grpc-google-iam-v1",
+        "proto-plus",
     ]
 
-    for dep in core_dependencies_from_source:
-        session.install(dep, "--no-deps", "--ignore-installed")
-        print(f"Installed {dep}")
+    deps_dir = CURRENT_DIRECTORY.parent
+    while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
+        deps_dir = deps_dir.parent
+
+    local_paths = [
+        str(deps_dir / dep)
+        for dep in core_dependencies_from_source
+        if (deps_dir / dep).exists()
+    ]
+    if local_paths:
+        session.install(*local_paths, "--no-deps", "--ignore-installed")
+        print(
+            f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}"
+        )
 
     tests_path = os.path.join("tests", "unit")
     session.run(

packages/gapic-generator/gapic/templates/noxfile.py.j2

@@ -6,9 +6,8 @@ import os
 import pathlib
 import re
 import shutil
-
-from typing import Dict, List
 import warnings
+from typing import Dict, List
 
 import nox
 
@@ -163,7 +162,7 @@ def lint(session):
         "ruff", "format",
         "--check",
         f"--target-version=py{ALL_PYTHON[0].replace('.', '')}",
-        "--line-length=88", 
+        "--line-length=88",
         *LINT_PATHS,
     )
 
@@ -179,7 +178,7 @@ def lint(session):
 def blacken(session):
     """(Deprecated) Legacy session. Please use 'nox -s format'."""
     session.log("WARNING: The 'blacken' session is deprecated and will be removed in a future release. Please use 'nox -s format' in the future.")
-    
+
     # Just run the ruff formatter (keeping legacy behavior of only formatting, not sorting imports)
     session.install(RUFF_VERSION)
     session.run(
@@ -518,14 +517,14 @@ def prerelease_deps(session, protobuf_implementation):
     # Extract the base package name, safely ignoring version bounds and spaces
     # (e.g., "grpcio>=1.75.1" becomes "grpcio")
     parsed_deps = {
-        dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) 
+        dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1)
         for dep in prerel_deps
     }
 
     # Dynamically sort local packages vs PyPI dependencies
     local_paths = []
     pypi_deps = []
-    
+
     for dep, pkg_name in parsed_deps.items():
         if (deps_dir / pkg_name).exists():
             local_paths.append(str(deps_dir / pkg_name))
@@ -624,13 +623,16 @@ def core_deps_from_source(session, protobuf_implementation):
         "proto-plus",
     ]
 
-    deps_dir = CURRENT_DIRECTORY.parent
-    while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
-        deps_dir = deps_dir.parent
+    # Locate the monorepo 'packages' directory containing core dependencies
+    deps_dir = next(
+        p / "packages"
+        for p in CURRENT_DIRECTORY.parents
+        if (p / "packages").is_dir()
+    )
 
     # Batch the pip installation to avoid sequential overhead
     dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source]
-    
+
     session.install(*dep_paths, "--no-deps", "--ignore-installed")
     print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}")
 

packages/gapic-generator/tests/integration/goldens/asset/noxfile.py

@@ -17,9 +17,8 @@
 import pathlib
 import re
 import shutil
-
-from typing import Dict, List
 import warnings
+from typing import Dict, List
 
 import nox
 
@@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation):
         "proto-plus",
     ]
 
-    deps_dir = CURRENT_DIRECTORY.parent
-    while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
-        deps_dir = deps_dir.parent
+    # Locate the monorepo 'packages' directory containing core dependencies
+    deps_dir = next(
+        p / "packages"
+        for p in CURRENT_DIRECTORY.parents
+        if (p / "packages").is_dir()
+    )
 
     # Batch the pip installation to avoid sequential overhead
     dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source]