Merge the updated #24299 cherry-pick into the #24314 cherry-pick

Author: shoumikhin

.ci/scripts/tests/test_filter_cuda_matrix.py

@@ -327,6 +327,8 @@ def test_supported_toolkits_select_the_matching_torch_index(self):
                     INSTALL_UTILS.platform, "system", return_value="Linux"
                 ), mock.patch.object(
                     INSTALL_UTILS.subprocess, "run", return_value=detected
+                ), mock.patch.dict(
+                    os.environ, {"CMAKE_ARGS": "-DEXECUTORCH_BUILD_CUDA=ON"}
                 ):
                     self.assertEqual(
                         INSTALL_UTILS.determine_torch_url(base_url),

.ci/scripts/wheel/test_shared_libraries.py

@@ -92,6 +92,7 @@
 _AOTI_SHIM_SYMBOLS = (
     "aoti_torch_empty_strided",
     "aoti_torch_delete_tensor_object",
+    "aoti_torch_dtype_element_size",
     "executorch::backends::cuda::CUDAStreamGuard::create",
     # From the CUDA sources rather than the C++ ones. The build drops every .cu file when no working
     # compiler is found, and the library is still produced from its .cpp sources, so a check that

.github/workflows/pull.yml

@@ -213,7 +213,13 @@ jobs:
           EXECUTORCH_PYBIND_TEST_BFLOAT16_CHANNELS_LAST_PTE=cmake-out-no-aten/add_bfloat16_channels_last.pte \
           PYTHONPATH=cmake-out-no-aten:.. \
           python extension/pybindings/test/test_pybindings_no_aten.py
-        if ldd cmake-out-no-aten/_C*.so | grep -Eiq 'torch|aten|c10'; then
+        NO_ATEN_LINKS="$(ldd cmake-out-no-aten/_C*.so)"
+        printf '%s\n' "$NO_ATEN_LINKS"
+        if grep -q 'not found' <<<"$NO_ATEN_LINKS"; then
+          echo "torch-free pybindings have a missing runtime dependency"
+          exit 1
+        fi
+        if grep -Eiq '^[[:space:]]*(lib(torch|aten|c10)[^[:space:]/]*\.so|[^[:space:]]*/lib(torch|aten|c10)[^[:space:]/]*\.so)' <<<"$NO_ATEN_LINKS"; then
           echo "torch-free pybindings unexpectedly link against ATen"
           exit 1
         fi

.github/workflows/release-runner-groups-refresh.yml

@@ -23,8 +23,20 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  dispatch:
+    if: github.repository_owner == 'pytorch' && github.event_name == 'push'
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+    steps:
+      - name: Refresh runner groups from protected main
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+        run: gh workflow run release-runner-groups-refresh.yml --ref main -f apply=true
+
   reconcile:
-    if: github.repository_owner == 'pytorch'
+    if: github.repository_owner == 'pytorch' && github.event_name != 'push'
     runs-on: ubuntu-latest
     environment: ${{ github.event_name != 'pull_request' && 'runner-group' || '' }}
     steps:

backends/aoti/common_shims_slim.cpp

@@ -177,6 +177,11 @@ int32_t aoti_torch_dtype_bool() {
   return 11; // ScalarType::Bool
 }
 
+size_t aoti_torch_dtype_element_size(int32_t dtype) {
+  using namespace executorch::backends::aoti::slim::c10;
+  return elementSize(static_cast<ScalarType>(dtype));
+}
+
 // ============================================================
 // Device Type Constants - Implementations
 // ============================================================

backends/aoti/common_shims_slim.h

@@ -92,6 +92,8 @@ AOTI_SHIM_EXPORT int32_t aoti_torch_dtype_int8();
 AOTI_SHIM_EXPORT int32_t aoti_torch_dtype_uint8();
 AOTI_SHIM_EXPORT int32_t aoti_torch_dtype_bool();
 
+AOTI_SHIM_EXPORT size_t aoti_torch_dtype_element_size(int32_t dtype);
+
 // ============================================================
 // Device Type Constants - Declarations
 // ============================================================

…blic_api_manifests/api_manifest_1_4.toml …blic_api_manifests/api_manifest_1_6.tomlbackends/arm/public_api_manifests/api_manifest_1_4.toml renamed to backends/arm/public_api_manifests/api_manifest_1_6.toml backends/arm/public_api_manifests/api_manifest_1_4.toml renamed to backends/arm/public_api_manifests/api_manifest_1_6.toml

@@ -18,7 +18,7 @@ signature = "EthosUBackend.preprocess(edge_program: torch.export.exported_progra
 
 [python.EthosUCompileSpec]
 kind = "class"
-signature = "EthosUCompileSpec(target: str, system_config: str | None = None, memory_mode: str | None = None, extra_flags: list[str] | None = None, config_ini: str | None = 'Arm/vela.ini')"
+signature = "EthosUCompileSpec(target: str, system_config: str | None = None, memory_mode: str | None = None, extra_flags: list[str] | None = None, config_ini: str | None = 'Arm/vela.ini', external_block_placements: executorch.backends.arm.ethosu.compile_spec.VelaExternalBlockPlacements | None = None, max_scratch_size: int | None = None)"
 
 [python.EthosUCompileSpec.DebugMode]
 kind = "enum"
@@ -60,6 +60,10 @@ signature = "EthosUPartitioner.partition(self, exported_program: torch.export.ex
 kind = "function"
 signature = "EthosUPartitioner.register_custom_partition_op(self, op: torch._ops.OpOverload) -> None"
 
+[python.EthosUPartitioner.transform_for_pre_decomposition]
+kind = "function"
+signature = "EthosUPartitioner.transform_for_pre_decomposition(self, exported_program: torch.export.exported_program.ExportedProgram) -> torch.export.exported_program.ExportedProgram"
+
 [python.EthosUQuantizer]
 kind = "class"
 signature = "EthosUQuantizer(compile_spec: 'EthosUCompileSpec', use_composable_quantizer: 'bool' = True) -> 'None'"
@@ -92,6 +96,38 @@ signature = "EthosUQuantizer.transform_for_annotation(self, model: 'GraphModule'
 kind = "function"
 signature = "EthosUQuantizer.validate(self, model: 'GraphModule') -> 'None'"
 
+[python.VelaExternalBlockPlacements]
+kind = "class"
+signature = "VelaExternalBlockPlacements(cmd_data: str | None = None, weight_data: str | None = None) -> None"
+
+[python.VelaExternalBlockPlacements.__delattr__]
+kind = "function"
+signature = "VelaExternalBlockPlacements.__delattr__(self, name)"
+
+[python.VelaExternalBlockPlacements.__eq__]
+kind = "function"
+signature = "VelaExternalBlockPlacements.__eq__(self, other)"
+
+[python.VelaExternalBlockPlacements.__hash__]
+kind = "function"
+signature = "VelaExternalBlockPlacements.__hash__(self)"
+
+[python.VelaExternalBlockPlacements.__post_init__]
+kind = "function"
+signature = "VelaExternalBlockPlacements.__post_init__(self) -> None"
+
+[python.VelaExternalBlockPlacements.__repr__]
+kind = "function"
+signature = "VelaExternalBlockPlacements.__repr__(self)"
+
+[python.VelaExternalBlockPlacements.__setattr__]
+kind = "function"
+signature = "VelaExternalBlockPlacements.__setattr__(self, name, value)"
+
+[python.VelaExternalBlockPlacements.to_block_placements]
+kind = "function"
+signature = "VelaExternalBlockPlacements.to_block_placements(self) -> dict[str, str]"
+
 [python.VgfBackend]
 kind = "class"
 signature = "VgfBackend()"
@@ -102,7 +138,7 @@ signature = "VgfBackend.preprocess(edge_program: torch.export.exported_program.E
 
 [python.VgfCompileSpec]
 kind = "class"
-signature = "VgfCompileSpec(tosa_spec: executorch.backends.arm.tosa.specification.TosaSpecification | str | None = None, compiler_flags: list[str] | None = None)"
+signature = "VgfCompileSpec(tosa_spec: executorch.backends.arm.tosa.specification.TosaSpecification | str | None = None, compiler_flags: list[str] | None = None, emit_debug_info: bool = False)"
 
 [python.VgfCompileSpec.DebugMode]
 kind = "enum"
@@ -148,6 +184,10 @@ signature = "VgfPartitioner.partition(self, exported_program: torch.export.expor
 kind = "function"
 signature = "VgfPartitioner.register_custom_partition_op(self, op: torch._ops.OpOverload) -> None"
 
+[python.VgfPartitioner.transform_for_pre_decomposition]
+kind = "function"
+signature = "VgfPartitioner.transform_for_pre_decomposition(self, exported_program: torch.export.exported_program.ExportedProgram) -> torch.export.exported_program.ExportedProgram"
+
 [python.VgfQuantizer]
 kind = "class"
 signature = "VgfQuantizer(compile_spec: 'VgfCompileSpec', use_composable_quantizer: 'bool' = True) -> 'None'"
@@ -187,3 +227,61 @@ signature = "get_symmetric_a16w8_quantization_config(is_per_channel: 'bool' = Tr
 [python.get_symmetric_quantization_config]
 kind = "function"
 signature = "get_symmetric_quantization_config(is_per_channel: 'bool' = True, is_qat: 'bool' = False, is_dynamic: 'bool' = False, act_qmin: 'int' = -128, act_qmax: 'int' = 127, weight_qmin: 'int' = -127, weight_qmax: 'int' = 127, eps: 'float' = 1.52587890625e-05) -> 'QuantizationConfig'"
+
+[cmake]
+
+[cmake.arm_runner_add_minimal_executable]
+kind = "function"
+signature = "arm_runner_add_minimal_executable(*, TARGET, SOURCE, OPS_PREFIX, COMPILE_DEFINITIONS=())"
+
+[cmake.arm_runner_add_standalone_executorch]
+kind = "macro"
+signature = "arm_runner_add_standalone_executorch()"
+
+[cmake.arm_runner_configure_ethos_u_platform]
+kind = "function"
+signature = "arm_runner_configure_ethos_u_platform(*, SDK_PATH, SYSTEM_CONFIG, MEMORY_MODE)"
+
+[cmake.arm_runner_configure_linker_script]
+kind = "function"
+signature = "arm_runner_configure_linker_script(*, TARGET, SYSTEM_CONFIG, OUTPUT_NAME=None)"
+
+[cmake.arm_runner_configure_model]
+kind = "function"
+signature = "arm_runner_configure_model(*, TARGET, PTE_FILE=None, MODEL_PTE_ADDR=None, MODEL_PTE_SIZE=None, PUBLIC=False)"
+
+[cmake.arm_runner_configure_runtime_output]
+kind = "function"
+signature = "arm_runner_configure_runtime_output(TARGET_NAME, FALLBACK_DIR)"
+
+[cmake.arm_runner_create_default_selected_ops_libs]
+kind = "function"
+signature = "arm_runner_create_default_selected_ops_libs(*, PREFIX, SUFFIX=None, OP_LIST=None, OPS_FROM_MODEL=None, DTYPE_SELECTIVE_BUILD=None, OUT_LIBS=None, DEPS=())"
+
+[cmake.arm_runner_create_selected_ops_lib]
+kind = "function"
+signature = "arm_runner_create_selected_ops_lib(*, LIB_NAME, FUNCTIONS_YAML=None, CUSTOM_OPS_YAML=None, OP_LIST=None, OPS_FROM_MODEL=None, DTYPE_SELECTIVE_BUILD=None, KERNEL_LIBS=(), DEPS=(), INCLUDE_ALL_OPS=False, PRIM_OPS=False)"
+
+[cmake.arm_runner_define_cache_options]
+kind = "function"
+signature = "arm_runner_define_cache_options(*, METHOD_ALLOCATOR_SIZE=None)"
+
+[cmake.arm_runner_link_minimal_specs]
+kind = "function"
+signature = "arm_runner_link_minimal_specs(TARGET_NAME)"
+
+[cmake.arm_runner_link_registration_libraries]
+kind = "function"
+signature = "arm_runner_link_registration_libraries(*, TARGET, SCOPE=None, BASE_LIBS=(), REGISTRATION_LIBS=(), NORMAL_LIBS=(), SUPPRESS_LIBS=())"
+
+[cmake.arm_runner_require_baremetal_targets]
+kind = "function"
+signature = "arm_runner_require_baremetal_targets()"
+
+[cmake.arm_runner_require_python]
+kind = "macro"
+signature = "arm_runner_require_python()"
+
+[cmake.arm_runner_validate_model_source]
+kind = "function"
+signature = "arm_runner_validate_model_source(*, ALLOW_SEMIHOSTING=False)"

backends/cuda/runtime/aoti_cuda_shims.lib.md

@@ -15,7 +15,7 @@ nm --defined-only aoti_cuda_shims.lib \
 # add the new names to exports.txt, then
 { echo 'LIBRARY aoti_cuda_shims.dll'; echo 'EXPORTS'; sed 's/^/    /' exports.txt; } \
   > aoti_cuda_shims.def
-x86_64-w64-mingw32-dlltool -d aoti_cuda_shims.def -l aoti_cuda_shims.lib \
+x86_64-w64-mingw32-dlltool -m i386:x86-64 -d aoti_cuda_shims.def -l aoti_cuda_shims.lib \
   --dllname aoti_cuda_shims.dll
 # zero the archive metadata so the file is reproducible
 mkdir extract && cd extract && x86_64-w64-mingw32-ar x ../aoti_cuda_shims.lib \
@@ -27,6 +27,9 @@ The archive has to be built at a fresh path. Updating it in place keeps the reve
 member order the generator produced, so the same export list would not give the same
 bytes twice.
 
+Use the relative output name shown above in each build directory. The generator embeds
+the output name in its object names, so absolute temporary paths change the bytes.
+
 Then check the result is a superset of what it replaced and that no member carries a
 timestamp, since this file ships in the wheel and a stamped one makes two builds of
 the same source differ.

backends/cuda/tests/test_int6_large_quantized_gemm.py

@@ -76,8 +76,9 @@ def test_buckets_and_generic_candidate_counts(self):
         for bucket, expected in expected_counts.items():
             configs = tile_autotune_configs(bucket)
             self.assertEqual(len(configs), expected)
-            self.assertEqual(len(legal_configs(bucket, 256)), expected // 3)
-            self.assertEqual(len(legal_configs(bucket, 5376)), expected)
+            legal_expected = 18 if torch.version.hip is not None else expected
+            self.assertEqual(len(legal_configs(bucket, 256)), legal_expected // 3)
+            self.assertEqual(len(legal_configs(bucket, 5376)), legal_expected)
             self.assertTrue(hasattr(torch.ops.triton, f"int6_quantized_gemm_m{bucket}"))
 
     @unittest.skipUnless(torch.cuda.is_available(), "CUDA required")

backends/cuda/triton/kernels/int6_quantized_gemm.py

@@ -1239,6 +1239,10 @@ def _tile_main_loop_trips(args) -> int:
 
 
 def _prune_tiles(configs, named_args, **kwargs):
+    if torch.version.hip is not None:
+        # Larger M tiles can access invalid memory on masked tails with ROCm
+        # 7.14. Keep the eight-row path for the release.
+        configs = [config for config in configs if config.kwargs["BLOCK_M"] == 8]
     return prune_by_main_loop_trips(
         _tile_main_loop_trips, configs, named_args, **kwargs
     )