From 78b8e0b1ea86269eb208464d1e8ecb98841014a4 Mon Sep 17 00:00:00 2001 From: PyTorch Bot Date: Sun, 6 Sep 2026 14:32:33 -0700 Subject: [PATCH 1/2] Move the NXP op alias table out of the test package The NXP backend keeps a table of short names for edge operators, for example `AddTensor = exir_ops.edge.aten.add.Tensor`. It lives in the backend's test package, but three modules that ship in the wheel read it at import time: backend/edge_helper.py backend/node_format_inference.py edge_passes/move_auxiliary_operator_into_separate_qdq_cluster_pass.py Two dozen production files import those modules, so the backend cannot load at all without a test package. The dependency points the wrong way: the tests should depend on the backend, not the backend on the tests. The table has no test logic in it, so this moves it to backend/ops_aliases.py, next to the code that reads it. That follows the shape the Arm backend already uses for shared operator constants in backends/arm/constants.py. No behavior changes. The names, values, and every importer stay the same, including the roughly fifty tests that use the table. The Buck target that published the table from the test package is removed, since the backend library already globs the directory it moves into. Test plan: imported the moved module and every updated importer from an installed wheel, and confirmed the table exposes the same names from the same values. Checked that every NXP test target can still reach the table through the backend package. Ran the formatter and linter over the changed files. --- backends/nxp/BUCK | 1 - backends/nxp/backend/edge_helper.py | 3 +-- backends/nxp/backend/node_format_inference.py | 3 +-- .../nxp/{tests => backend}/ops_aliases.py | 7 ++++-- ...operator_into_separate_qdq_cluster_pass.py | 3 +-- backends/nxp/tests/BUCK | 11 ---------- ...add_batch_size_for_3d_input_pool_2d_ops.py | 14 ++++++------ .../generic_tests/test_convert_div_to_mul.py | 2 +- .../test_convert_scalar_to_attr.py | 2 +- .../test_decompose_split_to_slices.py | 2 +- .../tests/generic_tests/test_integration.py | 2 +- .../test_node_format_inference.py | 8 +++---- .../test_quantized_input_data.py | 2 +- backends/nxp/tests/graph_verifier.py | 11 +++++----- .../node_converter/test_abs_converter.py | 2 +- .../test_adaptive_avg_pool2d_converter.py | 8 +++---- .../test_add_tensor_converter.py | 12 +++++----- .../node_converter/test_addmm_converter.py | 14 ++++++------ .../node_converter/test_amax_converter.py | 14 ++++++------ .../node_converter/test_amin_converter.py | 14 ++++++------ .../test_avg_pool2d_converter.py | 10 ++++----- .../node_converter/test_bmm_converter.py | 6 ++++- .../node_converter/test_cat_converter.py | 12 +++++----- .../node_converter/test_clamp_converter.py | 10 ++++----- .../test_constant_pad_nd_converter.py | 2 +- .../node_converter/test_conv_converter.py | 10 ++++----- .../node_converter/test_exp_converter.py | 2 +- .../test_hardswish_converter.py | 14 ++++++------ .../node_converter/test_hardtanh_converter.py | 10 ++++----- .../test_leaky_relu_converter.py | 2 +- .../node_converter/test_log_converter.py | 2 +- .../test_max_pool_2d_converter.py | 12 +++++----- .../node_converter/test_maximum_converter.py | 12 +++++----- .../node_converter/test_mean_dim_converter.py | 14 ++++++------ .../node_converter/test_minimum_converter.py | 12 +++++----- .../node_converter/test_mm_converter.py | 2 +- .../test_mul_tensor_converter.py | 12 +++++----- .../node_converter/test_neg_converter.py | 2 +- .../node_converter/test_pad_converter.py | 2 +- .../test_permute_copy_converter.py | 12 +++++----- .../node_converter/test_prelu_converter.py | 22 +++++++++---------- .../node_converter/test_relu_converter.py | 14 ++++++------ .../node_converter/test_rsqrt_converter.py | 2 +- .../node_converter/test_sigmoid_converter.py | 2 +- .../test_slice_copy_tensor_converter.py | 10 ++++----- .../node_converter/test_softmax_converter.py | 12 +++++----- .../test_sub_tensor_converter.py | 12 +++++----- .../test_sum_dim_int_list_converter.py | 14 ++++++------ .../node_converter/test_tanh_converter.py | 2 +- .../test_upsample_bilinear2d.py | 10 ++++----- .../node_converter/test_upsample_nearest2d.py | 10 ++++----- .../test_view_copy_converter.py | 18 +++++++-------- .../test_convert_reshaping_nodes_to_view.py | 2 +- backends/nxp/tests/nsys_testing.py | 2 +- 54 files changed, 210 insertions(+), 217 deletions(-) rename backends/nxp/{tests => backend}/ops_aliases.py (88%) diff --git a/backends/nxp/BUCK b/backends/nxp/BUCK index 6dec42f04d7..81a1e04cfee 100644 --- a/backends/nxp/BUCK +++ b/backends/nxp/BUCK @@ -68,7 +68,6 @@ fbcode_target(_kind = runtime.python_library, "fbsource//third-party/pypi/neutron_converter:neutron_converter", "//caffe2:torch", "//executorch/exir:lib", - "//executorch/backends/nxp/tests:ops_aliases", ], ) diff --git a/backends/nxp/backend/edge_helper.py b/backends/nxp/backend/edge_helper.py index 408b90e264d..4e94981e776 100644 --- a/backends/nxp/backend/edge_helper.py +++ b/backends/nxp/backend/edge_helper.py @@ -7,8 +7,7 @@ import operator import torch - -from executorch.backends.nxp.tests.ops_aliases import ( +from executorch.backends.nxp.backend.ops_aliases import ( AddTensor, Amax, Amin, diff --git a/backends/nxp/backend/node_format_inference.py b/backends/nxp/backend/node_format_inference.py index 689de41f3e4..f49dbdae158 100644 --- a/backends/nxp/backend/node_format_inference.py +++ b/backends/nxp/backend/node_format_inference.py @@ -7,7 +7,6 @@ import operator import torch - from executorch.backends.nxp.backend.data_format import DataFormat, NXP_NODE_FORMAT from executorch.backends.nxp.backend.edge_helper import ( input_rank, @@ -15,7 +14,7 @@ try_get_arg, ) from executorch.backends.nxp.backend.edge_program_converter import functions_converters -from executorch.backends.nxp.tests.ops_aliases import ( +from executorch.backends.nxp.backend.ops_aliases import ( AdaptiveAvgPool2D, Amax, Amin, diff --git a/backends/nxp/tests/ops_aliases.py b/backends/nxp/backend/ops_aliases.py similarity index 88% rename from backends/nxp/tests/ops_aliases.py rename to backends/nxp/backend/ops_aliases.py index 5ccc1d67de0..dc06fd7e4d7 100644 --- a/backends/nxp/tests/ops_aliases.py +++ b/backends/nxp/backend/ops_aliases.py @@ -3,8 +3,11 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -# This file defines ops aliases for shorter and more readable test description. List is sorted alphabetically. -# When finding a missing alias, add it at the correct place. +# This file defines ops aliases for shorter and more readable descriptions of edge operators. +# List is sorted alphabetically. When finding a missing alias, add it at the correct place. +# +# It lives in the backend package, not in tests, because the converters and passes below it +# read these aliases at import time, and production code should not depend on test modules. import operator diff --git a/backends/nxp/edge_passes/move_auxiliary_operator_into_separate_qdq_cluster_pass.py b/backends/nxp/edge_passes/move_auxiliary_operator_into_separate_qdq_cluster_pass.py index dafea5d259b..ffcc07dc56d 100644 --- a/backends/nxp/edge_passes/move_auxiliary_operator_into_separate_qdq_cluster_pass.py +++ b/backends/nxp/edge_passes/move_auxiliary_operator_into_separate_qdq_cluster_pass.py @@ -6,10 +6,9 @@ import operator import torch - +from executorch.backends.nxp.backend.ops_aliases import PermuteCopy from executorch.backends.nxp.edge_passes.neutron_edge_pass import NeutronEdgePass from executorch.backends.nxp.neutron_partitioner import QDQClusterRecognizer -from executorch.backends.nxp.tests.ops_aliases import PermuteCopy # noinspection PyProtectedMember from executorch.exir.dialects._ops import ops as exir_ops diff --git a/backends/nxp/tests/BUCK b/backends/nxp/tests/BUCK index 7879e1e3db5..a3a5adb215c 100644 --- a/backends/nxp/tests/BUCK +++ b/backends/nxp/tests/BUCK @@ -4,17 +4,6 @@ load("@fbcode_macros//build_defs:python_pytest.bzl", "python_pytest") oncall("executorch") -fbcode_target(_kind = runtime.python_library, - name = "ops_aliases", - srcs = [ - "ops_aliases.py", - ], - deps = [ - "//caffe2:torch", - "//executorch/exir:lib", - ], -) - fbcode_target(_kind = runtime.python_library, name = "models", srcs = [ diff --git a/backends/nxp/tests/generic_tests/test_add_batch_size_for_3d_input_pool_2d_ops.py b/backends/nxp/tests/generic_tests/test_add_batch_size_for_3d_input_pool_2d_ops.py index b426c34c260..679e7365577 100644 --- a/backends/nxp/tests/generic_tests/test_add_batch_size_for_3d_input_pool_2d_ops.py +++ b/backends/nxp/tests/generic_tests/test_add_batch_size_for_3d_input_pool_2d_ops.py @@ -15,6 +15,13 @@ from executorch.backends.nxp.aten_passes.neutron_aten_pass_manager import ( NeutronAtenPassManager, ) +from executorch.backends.nxp.backend.ops_aliases import ( + AdaptiveAvgPool2D, + AvgPool2D, + GetItem, + MaxPool2DWithIndices, + ViewCopy, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import neutron_target_spec from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -28,13 +35,6 @@ MaxPool2dModule, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AdaptiveAvgPool2D, - AvgPool2D, - GetItem, - MaxPool2DWithIndices, - ViewCopy, -) @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/generic_tests/test_convert_div_to_mul.py b/backends/nxp/tests/generic_tests/test_convert_div_to_mul.py index 3415b79a39d..044467ca223 100644 --- a/backends/nxp/tests/generic_tests/test_convert_div_to_mul.py +++ b/backends/nxp/tests/generic_tests/test_convert_div_to_mul.py @@ -13,6 +13,7 @@ ConvertDivToMulPass, NeutronAtenPassManager, ) +from executorch.backends.nxp.backend.ops_aliases import MulTensor from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import neutron_target_spec from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -22,7 +23,6 @@ StaticDivLinearModel, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import MulTensor @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/generic_tests/test_convert_scalar_to_attr.py b/backends/nxp/tests/generic_tests/test_convert_scalar_to_attr.py index b3d68eeeb56..c68cde0c23b 100644 --- a/backends/nxp/tests/generic_tests/test_convert_scalar_to_attr.py +++ b/backends/nxp/tests/generic_tests/test_convert_scalar_to_attr.py @@ -18,6 +18,7 @@ from executorch.backends.nxp.backend.edge_helper import ( try_get_tensor_constant_from_node, ) +from executorch.backends.nxp.backend.ops_aliases import AddTensor, MulTensor, SubTensor from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import neutron_target_spec from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -31,7 +32,6 @@ AllCloseOutputComparator, lower_run_compare, ) -from executorch.backends.nxp.tests.ops_aliases import AddTensor, MulTensor, SubTensor @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/generic_tests/test_decompose_split_to_slices.py b/backends/nxp/tests/generic_tests/test_decompose_split_to_slices.py index de4e684405c..3c3c29f95b5 100644 --- a/backends/nxp/tests/generic_tests/test_decompose_split_to_slices.py +++ b/backends/nxp/tests/generic_tests/test_decompose_split_to_slices.py @@ -12,6 +12,7 @@ NeutronAtenPassManager, SplitGRUBasedOnNumLayers, ) +from executorch.backends.nxp.backend.ops_aliases import SliceCopy from executorch.backends.nxp.tests.executorch_pipeline import neutron_target_spec from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier @@ -21,7 +22,6 @@ SplitWithSize, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import SliceCopy @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/generic_tests/test_integration.py b/backends/nxp/tests/generic_tests/test_integration.py index 9916cba5bdd..f0cb25548ec 100644 --- a/backends/nxp/tests/generic_tests/test_integration.py +++ b/backends/nxp/tests/generic_tests/test_integration.py @@ -5,8 +5,8 @@ import executorch.extension.pybindings.portable_lib import executorch.kernels.quantized # noqa F401 +from executorch.backends.nxp.backend.ops_aliases import AddMM, Convolution from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops -from executorch.backends.nxp.tests.ops_aliases import AddMM, Convolution from executorch.backends.nxp.tests.use_qat import * # noqa F401 from executorch.backends.nxp.tests.executorch_pipeline import ( diff --git a/backends/nxp/tests/generic_tests/test_node_format_inference.py b/backends/nxp/tests/generic_tests/test_node_format_inference.py index 18d5f874aab..5206a029e01 100644 --- a/backends/nxp/tests/generic_tests/test_node_format_inference.py +++ b/backends/nxp/tests/generic_tests/test_node_format_inference.py @@ -13,6 +13,10 @@ NodeFormatInference, NXP_NODE_FORMAT, ) +from executorch.backends.nxp.backend.ops_aliases import ( + ExecutorchDelegateCall, + MaxPool2DWithIndices, +) from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -21,10 +25,6 @@ MaxPool2dModule, SoftmaxModule, ) -from executorch.backends.nxp.tests.ops_aliases import ( - ExecutorchDelegateCall, - MaxPool2DWithIndices, -) def test_convolution(): diff --git a/backends/nxp/tests/generic_tests/test_quantized_input_data.py b/backends/nxp/tests/generic_tests/test_quantized_input_data.py index a9f9f3e47e6..bd2a6056a0b 100644 --- a/backends/nxp/tests/generic_tests/test_quantized_input_data.py +++ b/backends/nxp/tests/generic_tests/test_quantized_input_data.py @@ -5,6 +5,7 @@ import executorch.backends.nxp.tests.nsys_testing as nsys_testing import torch +from executorch.backends.nxp.backend.ops_aliases import AvgPool2D, MulTensor from executorch.backends.nxp.tests.executorch_pipeline import ModelInputSpec from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier @@ -14,7 +15,6 @@ OUTPUTS_DIR, ReferenceModel, ) -from executorch.backends.nxp.tests.ops_aliases import AvgPool2D, MulTensor def test__single_quantized_inputs(mocker, request): diff --git a/backends/nxp/tests/graph_verifier.py b/backends/nxp/tests/graph_verifier.py index 44900b6a11b..8dad390d6ff 100644 --- a/backends/nxp/tests/graph_verifier.py +++ b/backends/nxp/tests/graph_verifier.py @@ -10,17 +10,18 @@ from dataclasses import dataclass from typing import Callable, Union -from executorch.backends.nxp.neutron_partitioner import ( - NeutronPartitioner, - NXP_DELEGATION_TAG, -) -from executorch.backends.nxp.tests.ops_aliases import ( +from executorch.backends.nxp.backend.ops_aliases import ( DequantizePerChannel, DequantizePerTensor, QuantizePerChannel, QuantizePerTensor, ) +from executorch.backends.nxp.neutron_partitioner import ( + NeutronPartitioner, + NXP_DELEGATION_TAG, +) + from executorch.exir.dialects.edge._ops import EdgeOpOverload from pytest_mock import MockerFixture diff --git a/backends/nxp/tests/ir/converter/node_converter/test_abs_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_abs_converter.py index d42ef4c6e7d..94a0aafc2a7 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_abs_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_abs_converter.py @@ -8,12 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import Abs from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import ( lower_run_compare, RandomDatasetCreator, ) -from executorch.backends.nxp.tests.ops_aliases import Abs from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_adaptive_avg_pool2d_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_adaptive_avg_pool2d_converter.py index 9646c04a3f2..65e2b82555c 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_adaptive_avg_pool2d_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_adaptive_avg_pool2d_converter.py @@ -8,6 +8,10 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + AdaptiveAvgPool2D, + ExecutorchDelegateCall, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program @@ -18,10 +22,6 @@ ) from executorch.backends.nxp.tests.models import AdaptiveAvgPool2dModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AdaptiveAvgPool2D, - ExecutorchDelegateCall, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_add_tensor_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_add_tensor_converter.py index c01d0ca818d..237d15aefbd 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_add_tensor_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_add_tensor_converter.py @@ -8,6 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import ( @@ -21,12 +27,6 @@ ) from executorch.backends.nxp.tests.models import AddTensorModule, MaxPoolAddTensorModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_addmm_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_addmm_converter.py index 1db604d5b1e..6ebb8ac128c 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_addmm_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_addmm_converter.py @@ -8,6 +8,13 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + AddMM, + ExecutorchDelegateCall, + MM, + PermuteCopy, + ViewCopy, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program @@ -15,13 +22,6 @@ from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier, Operator from executorch.backends.nxp.tests.models import AddmmModule, LinearModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddMM, - ExecutorchDelegateCall, - MM, - PermuteCopy, - ViewCopy, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_amax_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_amax_converter.py index d348c12102e..892ea70018e 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_amax_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_amax_converter.py @@ -21,6 +21,13 @@ from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options.transpose_options import ( Transpose, ) +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + Amax, + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -29,13 +36,6 @@ AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - Amax, - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_amin_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_amin_converter.py index e2490d9c2c4..3ae13bafe40 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_amin_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_amin_converter.py @@ -21,6 +21,13 @@ from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options.transpose_options import ( Transpose, ) +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + Amin, + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -29,13 +36,6 @@ AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - Amin, - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_avg_pool2d_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_avg_pool2d_converter.py index 3db1158d637..c132c9e509d 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_avg_pool2d_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_avg_pool2d_converter.py @@ -8,17 +8,17 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + AvgPool2D, + ExecutorchDelegateCall, + ViewCopy, +) from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.models import AvgPool2dModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AvgPool2D, - ExecutorchDelegateCall, - ViewCopy, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_bmm_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_bmm_converter.py index c564c024623..ee46ad94d44 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_bmm_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_bmm_converter.py @@ -6,6 +6,11 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + BMM, + GetItem, + MaxPool2DWithIndices, +) from executorch.backends.nxp.edge_passes.move_auxiliary_operator_into_separate_qdq_cluster_pass import ( ViewCopy, @@ -21,7 +26,6 @@ BatchMatMulModel, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import BMM, GetItem, MaxPool2DWithIndices from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_cat_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_cat_converter.py index b28a431e3ca..30983f4a666 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_cat_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_cat_converter.py @@ -8,6 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + Cat, + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, +) from executorch.backends.nxp.tests.executorch_pipeline import ( ModelInputSpec, @@ -16,12 +22,6 @@ from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - Cat, - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_clamp_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_clamp_converter.py index b2147a0d984..04ae8757110 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_clamp_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_clamp_converter.py @@ -18,6 +18,11 @@ from executorch.backends.nxp.backend.ir.lib.tflite.BuiltinOperator import ( BuiltinOperator as Ops, ) +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + Clamp, + ExecutorchDelegateCall, +) from executorch.backends.nxp.tests.executorch_pipeline import ( ModelInputSpec, to_quantized_edge_program, @@ -25,11 +30,6 @@ from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - Clamp, - ExecutorchDelegateCall, -) from executorch.backends.nxp.tests.use_qat import * # noqa: F403 F401 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_constant_pad_nd_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_constant_pad_nd_converter.py index b4a64447aa6..730334932d7 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_constant_pad_nd_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_constant_pad_nd_converter.py @@ -12,10 +12,10 @@ from executorch.backends.nxp.backend.ir.converter.builder.model_builder import ( ModelBuilder, ) +from executorch.backends.nxp.backend.ops_aliases import ConstantPadND, Convolution from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.models import PadConvModule, PadModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ConstantPadND, Convolution from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_conv_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_conv_converter.py index 3d20d38bb54..f0fb0fdb7b4 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_conv_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_conv_converter.py @@ -6,6 +6,11 @@ import numpy as np import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + Convolution, + ExecutorchDelegateCall, + ViewCopy, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -16,11 +21,6 @@ lower_run_compare, ReferenceModel, ) -from executorch.backends.nxp.tests.ops_aliases import ( - Convolution, - ExecutorchDelegateCall, - ViewCopy, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_exp_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_exp_converter.py index b304dce2c94..9f8068f67d4 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_exp_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_exp_converter.py @@ -8,10 +8,10 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import Exp from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import Exp from executorch.backends.nxp.tests.use_qat import * # noqa F403 from executorch.backends.nxp.tests.dataset_creator import ( LinearRampDatasetCreator, diff --git a/backends/nxp/tests/ir/converter/node_converter/test_hardswish_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_hardswish_converter.py index deff4e12f0c..76aaab7520e 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_hardswish_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_hardswish_converter.py @@ -8,6 +8,13 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + AddMM, + Convolution, + Hardswish, + PermuteCopy, + ViewCopy, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier @@ -20,13 +27,6 @@ LinearHardswishModule, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddMM, - Convolution, - Hardswish, - PermuteCopy, - ViewCopy, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_hardtanh_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_hardtanh_converter.py index 66a052dba4f..1199609bb17 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_hardtanh_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_hardtanh_converter.py @@ -18,17 +18,17 @@ from executorch.backends.nxp.backend.ir.lib.tflite.BuiltinOperator import ( BuiltinOperator as Ops, ) +from executorch.backends.nxp.backend.ops_aliases import ( + Convolution, + ExecutorchDelegateCall, + HardTanh, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.models import Conv2dWithActivation, HardTanhModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - Convolution, - ExecutorchDelegateCall, - HardTanh, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_leaky_relu_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_leaky_relu_converter.py index 567cf85ebe5..5176a8c61db 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_leaky_relu_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_leaky_relu_converter.py @@ -8,11 +8,11 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import LeakyRelu from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import LeakyRelu from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_log_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_log_converter.py index 0b7fe88cffc..ebf6a8bfb9b 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_log_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_log_converter.py @@ -8,10 +8,10 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import Log from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import Log from executorch.backends.nxp.tests.use_qat import * # noqa F403 from executorch.backends.nxp.tests.dataset_creator import ( LinearRampDatasetCreator, diff --git a/backends/nxp/tests/ir/converter/node_converter/test_max_pool_2d_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_max_pool_2d_converter.py index 55a47146bfc..7d3769c896d 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_max_pool_2d_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_max_pool_2d_converter.py @@ -8,17 +8,17 @@ # noinspection PyUnusedImports import pytest import torch - -from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program -from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops -from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier -from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( +from executorch.backends.nxp.backend.ops_aliases import ( ExecutorchDelegateCall, GetItem, MaxPool2DWithIndices, ViewCopy, ) + +from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program +from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops +from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier +from executorch.backends.nxp.tests.nsys_testing import lower_run_compare from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_maximum_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_maximum_converter.py index e25ed98fb3f..349a624242a 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_maximum_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_maximum_converter.py @@ -8,6 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + ExecutorchDelegateCall, + GetItem, + Maximum, + MaxPool2DWithIndices, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import ( @@ -21,12 +27,6 @@ ) from executorch.backends.nxp.tests.models import MaximumModule, MaxPoolMaximumModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - ExecutorchDelegateCall, - GetItem, - Maximum, - MaxPool2DWithIndices, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_mean_dim_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_mean_dim_converter.py index 1674153540f..14f55de94d4 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_mean_dim_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_mean_dim_converter.py @@ -21,6 +21,13 @@ from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options.transpose_options import ( Transpose, ) +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, + MeanDim, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -29,13 +36,6 @@ AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, - MeanDim, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_minimum_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_minimum_converter.py index 9dc2b8d77d7..7c74fd4a1c9 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_minimum_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_minimum_converter.py @@ -8,6 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, + Minimum, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import ( @@ -21,12 +27,6 @@ ) from executorch.backends.nxp.tests.models import MaxPoolMinimumModule, MinimumModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, - Minimum, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_mm_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_mm_converter.py index 423999dc7ec..405e955a5ca 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_mm_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_mm_converter.py @@ -8,12 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import MM, PermuteCopy, ViewCopy from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier, Operator from executorch.backends.nxp.tests.models import LinearModule, MmModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import MM, PermuteCopy, ViewCopy from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_mul_tensor_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_mul_tensor_converter.py index 718383284be..b031ab7e47c 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_mul_tensor_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_mul_tensor_converter.py @@ -8,6 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, + MulTensor, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import ( @@ -21,12 +27,6 @@ ) from executorch.backends.nxp.tests.models import MaxPoolMulTensorModule, MulTensorModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, - MulTensor, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_neg_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_neg_converter.py index 691cb3bd2ca..182e7754013 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_neg_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_neg_converter.py @@ -8,6 +8,7 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import Convolution, Neg from executorch.backends.nxp.tests.dataset_creator import ( LinearRampDatasetCreator, @@ -15,7 +16,6 @@ ) from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import Convolution, Neg from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_pad_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_pad_converter.py index 266260f9e1f..f5c0114c81e 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_pad_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_pad_converter.py @@ -10,10 +10,10 @@ from executorch.backends.nxp.backend.ir.converter.builder.model_builder import ( ModelBuilder, ) +from executorch.backends.nxp.backend.ops_aliases import Convolution, Pad from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.models import PadConvModule, PadModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import Convolution, Pad @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/ir/converter/node_converter/test_permute_copy_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_permute_copy_converter.py index bdfd1e9da25..2317efd45d9 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_permute_copy_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_permute_copy_converter.py @@ -9,17 +9,17 @@ import pytest import torch from _pytest.mark import ParameterSet - -from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program -from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops -from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier -from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( +from executorch.backends.nxp.backend.ops_aliases import ( ExecutorchDelegateCall, GetItem, MaxPool2DWithIndices, PermuteCopy, ) + +from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program +from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops +from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier +from executorch.backends.nxp.tests.nsys_testing import lower_run_compare from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_prelu_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_prelu_converter.py index 884e95ec20c..358320c30c7 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_prelu_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_prelu_converter.py @@ -11,6 +11,17 @@ from executorch.backends.nxp.backend.edge_program_converter import ( EdgeProgramToIRConverter, ) +from executorch.backends.nxp.backend.ops_aliases import ( + AddMM, + Convolution, + ExecutorchDelegateCall, + GtScalar, + MulTensor, + PermuteCopy, + Prelu, + ViewCopy, + WhereSelf, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier @@ -25,17 +36,6 @@ ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddMM, - Convolution, - ExecutorchDelegateCall, - GtScalar, - MulTensor, - PermuteCopy, - Prelu, - ViewCopy, - WhereSelf, -) from torch.export import ExportedProgram from executorch.backends.nxp.tests.use_qat import * # noqa F403 from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program diff --git a/backends/nxp/tests/ir/converter/node_converter/test_relu_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_relu_converter.py index 1f274576767..4bf9263e93f 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_relu_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_relu_converter.py @@ -7,13 +7,7 @@ import pytest import torch from executorch.backends.nxp.backend.edge_program_converter import exir_ops -from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator -from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program -from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops -from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier -from executorch.backends.nxp.tests.models import Conv2dModule, LinearModule, ReLUModule -from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( +from executorch.backends.nxp.backend.ops_aliases import ( AddMM, Convolution, DequantizePerChannel, @@ -23,6 +17,12 @@ Relu, ViewCopy, ) +from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator +from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program +from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops +from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier +from executorch.backends.nxp.tests.models import Conv2dModule, LinearModule, ReLUModule +from executorch.backends.nxp.tests.nsys_testing import lower_run_compare from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_rsqrt_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_rsqrt_converter.py index 67101410d9d..916ab5ba518 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_rsqrt_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_rsqrt_converter.py @@ -8,6 +8,7 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import Rsqrt from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier @@ -15,7 +16,6 @@ AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import Rsqrt from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_sigmoid_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_sigmoid_converter.py index 5c4e4f4f007..c6ea90cb5e2 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_sigmoid_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_sigmoid_converter.py @@ -9,6 +9,7 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import Sigmoid from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier @@ -16,7 +17,6 @@ AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import Sigmoid from torch import nn from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_slice_copy_tensor_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_slice_copy_tensor_converter.py index 56d0b4bbd64..4a66c000505 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_slice_copy_tensor_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_slice_copy_tensor_converter.py @@ -8,6 +8,11 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + Convolution, + ExecutorchDelegateCall, + SliceCopy, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program @@ -21,11 +26,6 @@ SliceTensorModule, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - Convolution, - ExecutorchDelegateCall, - SliceCopy, -) @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/ir/converter/node_converter/test_softmax_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_softmax_converter.py index 2ce0790fc98..8cec44b5274 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_softmax_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_softmax_converter.py @@ -6,6 +6,12 @@ import numpy as np import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + Convolution, + ExecutorchDelegateCall, + Softmax, + ViewCopy, +) from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -17,12 +23,6 @@ ) from executorch.backends.nxp.tests.models import SoftmaxModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - Convolution, - ExecutorchDelegateCall, - Softmax, - ViewCopy, -) @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/ir/converter/node_converter/test_sub_tensor_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_sub_tensor_converter.py index 1601c1e19c2..6e88bc270e4 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_sub_tensor_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_sub_tensor_converter.py @@ -8,6 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, + SubTensor, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import ( @@ -21,12 +27,6 @@ ) from executorch.backends.nxp.tests.models import MaxPoolSubTensorModule, SubTensorModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, - SubTensor, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_sum_dim_int_list_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_sum_dim_int_list_converter.py index 8b28142b63a..5726776c78a 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_sum_dim_int_list_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_sum_dim_int_list_converter.py @@ -21,6 +21,13 @@ from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options.transpose_options import ( Transpose, ) +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, + SumDimIntList, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -29,13 +36,6 @@ AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, - SumDimIntList, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_tanh_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_tanh_converter.py index 51b7ee484a7..2795f607494 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_tanh_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_tanh_converter.py @@ -7,12 +7,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import Convolution, Tanh from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.models import Conv2dWithActivation from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import Convolution, Tanh from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_upsample_bilinear2d.py b/backends/nxp/tests/ir/converter/node_converter/test_upsample_bilinear2d.py index 949f193b267..8513b87a1a3 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_upsample_bilinear2d.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_upsample_bilinear2d.py @@ -8,6 +8,11 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + ExecutorchDelegateCall, + UpsampleBilinear2D, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program @@ -17,11 +22,6 @@ AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - ExecutorchDelegateCall, - UpsampleBilinear2D, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_upsample_nearest2d.py b/backends/nxp/tests/ir/converter/node_converter/test_upsample_nearest2d.py index b3e28a7b2f8..dd3d7c9c147 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_upsample_nearest2d.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_upsample_nearest2d.py @@ -8,17 +8,17 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + ExecutorchDelegateCall, + UpsampleNearest2D, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - ExecutorchDelegateCall, - UpsampleNearest2D, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py index 2a2d270e30a..200ffb6fc99 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py @@ -8,16 +8,8 @@ import numpy as np import pytest import torch -from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator -from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program -from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops -from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier -from executorch.backends.nxp.tests.model_output_comparator import ( - AllCloseOutputComparator, -) -from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( +from executorch.backends.nxp.backend.ops_aliases import ( AddMM, AddTensor, AvgPool2D, @@ -28,6 +20,14 @@ Relu, ViewCopy, ) +from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator +from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program +from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops +from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier +from executorch.backends.nxp.tests.model_output_comparator import ( + AllCloseOutputComparator, +) +from executorch.backends.nxp.tests.nsys_testing import lower_run_compare from torch import nn from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/edge_passes/test_convert_reshaping_nodes_to_view.py b/backends/nxp/tests/ir/edge_passes/test_convert_reshaping_nodes_to_view.py index f0489b151f7..c1850f625dd 100644 --- a/backends/nxp/tests/ir/edge_passes/test_convert_reshaping_nodes_to_view.py +++ b/backends/nxp/tests/ir/edge_passes/test_convert_reshaping_nodes_to_view.py @@ -6,6 +6,7 @@ import numpy as np import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import AddTensor, ViewCopy from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.models import SqueezeAddModel, UnsqueezeAddModel @@ -13,7 +14,6 @@ AllCloseOutputComparator, lower_run_compare, ) -from executorch.backends.nxp.tests.ops_aliases import AddTensor, ViewCopy @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/nsys_testing.py b/backends/nxp/tests/nsys_testing.py index a8038083c37..6ec0d821bae 100644 --- a/backends/nxp/tests/nsys_testing.py +++ b/backends/nxp/tests/nsys_testing.py @@ -24,6 +24,7 @@ from executorch.backends.nxp.backend.ir.converter.conversion.translator import ( torch_type_to_numpy_type, ) +from executorch.backends.nxp.backend.ops_aliases import ExecutorchDelegateCall from executorch.backends.nxp.neutron_partitioner import NeutronPartitioner from executorch.backends.nxp.tests.config_importer import test_config from executorch.backends.nxp.tests.dataset_creator import ( @@ -44,7 +45,6 @@ from executorch.backends.nxp.tests.model_output_comparator import ( AllCloseOutputComparator, ) -from executorch.backends.nxp.tests.ops_aliases import ExecutorchDelegateCall from executorch.backends.nxp.tests.outputs_dir_importer import outputs_dir from executorch.backends.nxp.tests.utils import save_pte_program, store_txt_input_tensor From a3588651b644efd324566ee7d99137036d096ab0 Mon Sep 17 00:00:00 2001 From: PyTorch Bot Date: Sun, 6 Sep 2026 14:32:33 -0700 Subject: [PATCH 2/2] Stop shipping vendored third-party sources in the wheel A pip install of ExecuTorch carries files copied out of the vendored third-party checkouts: most of the MLX Python source tree, the XNNPACK codegen scripts and their operator yaml, the Vulkan header registry, and two git submodules checked out under ordinary names. Those exist to build the C++ targets. Once the libraries are built nothing in an installed wheel imports them, and the nested submodule copies cannot satisfy the imports the code does use: the Cadence helper imports facto.specdb from the top level, and the tokenizers are already declared as pytorch-tokenizers. The cause is one missing setting. setup.py passes a `packages` list only for the minimal build, so the full build falls back to setuptools auto discovery and picks up everything under src/executorch. This adds the list for the full build and closes the two ways those files arrive: - `packages` excludes directories named third-party or third_party, plus the submodule paths read from .gitmodules. A submodule under an ordinary name cannot be recognized by its name, so it is read from the file that defines it rather than hardcoded here. - The data-file manifest is filtered with the same rule. A directory left out of `packages` is not simply skipped: setuptools walks up to the nearest listed package and records the file as that package's data, so a vendored yaml still arrives under its parent without this. One rule feeds both, so they cannot drift apart. Test packages deliberately stay. The suites here import each other through the installed name, for example `from executorch.backends.arm.test import common`, so dropping them stops a non-editable install from collecting most tests. Pruning them needs that cross-import removed first, and the note in pyproject.toml now says so. Test plan: built wheels for macOS arm64, Linux x86_64, and Linux x86_64 with CUDA. Confirmed no vendored directory or submodule remains, the test packages are still present, and the shipped headers, cmake files and flatbuffer schemas are unchanged in number. Installed into a clean environment and, from outside the checkout, exported, loaded and ran a model through the portable kernels and the XNNPACK delegate, checking outputs against eager. Added a unit test beside the existing wheel checks. It fails when the package list stops excluding the vendored directories, when the submodule rule is broken, and when the line that hands the list to setuptools is removed, each checked by deleting that piece and watching the suite go red. --- .../tests/test_wheel_vendored_packages.py | 220 ++++++++++++++++++ pyproject.toml | 7 +- setup.py | 83 ++++++- 3 files changed, 307 insertions(+), 3 deletions(-) create mode 100644 .ci/scripts/tests/test_wheel_vendored_packages.py diff --git a/.ci/scripts/tests/test_wheel_vendored_packages.py b/.ci/scripts/tests/test_wheel_vendored_packages.py new file mode 100644 index 00000000000..a8a10d3e5b2 --- /dev/null +++ b/.ci/scripts/tests/test_wheel_vendored_packages.py @@ -0,0 +1,220 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Unit tests for the packages the full wheel publishes. + +The wheel used to carry the Python files and codegen scripts of every vendored third-party +checkout, because the full build passed no `packages` list and setuptools then discovered +everything under src/executorch. Those files exist to build the C++ targets, so nothing in +an installed wheel imports them. + +Asserting on the discovery result rather than on a built wheel, because the behaviour under +test is a pure function of the source tree plus the exclude patterns, and a full build takes +minutes to exercise one filter. `.ci/scripts/test_minimal_wheel.sh` already covers the +built-artifact side for the minimal wheel. + +setup.py is read rather than imported. It calls setup() at module scope, so importing it under +a test runner hands setup() the runner's own arguments and the session dies on an invalid +command name. +""" + +import ast +import functools +import unittest +from pathlib import Path +from typing import Dict, List, Tuple + +from setuptools import find_namespace_packages + +SETUP_PY = Path(__file__).resolve().parents[3] / "setup.py" +# Discovery is anchored on this file's location, not on the working directory, so the result +# does not depend on where the runner was started. +PACKAGE_ROOT = str(SETUP_PY.parent / "src") + + +def _setup_py_module() -> ast.Module: + return ast.parse(SETUP_PY.read_text()) + + +def _load_from_setup_py() -> Dict[str, object]: + """The vendored-path helpers and the package list builder, from setup.py's source. + + Only those definitions are executed, so none of setup.py's module level build logic runs. + """ + wanted = ( + "_VENDORED_DIR_NAMES", + "_vendored_prefixes", + "_is_vendored_path", + "_full_packages", + ) + + selected: List[ast.stmt] = [] + found = set() + for node in _setup_py_module().body: + if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and node.name in wanted: + selected.append(node) + found.add(node.name) + elif isinstance(node, ast.Assign): + names = { + target.id + for target in node.targets + if isinstance(target, ast.Name) and target.id in wanted + } + if names: + selected.append(node) + found |= names + + assert found == set( + wanted + ), f"setup.py no longer defines {sorted(set(wanted) - found)}, so this test checks nothing" + + namespace: Dict[str, object] = { + "__file__": str(SETUP_PY), + "Path": Path, + "List": List, + "Tuple": Tuple, + "functools": functools, + "find_namespace_packages": find_namespace_packages, + } + exec( + compile(ast.Module(body=selected, type_ignores=[]), str(SETUP_PY), "exec"), + namespace, + ) + return namespace + + +_NAMESPACE = _load_from_setup_py() +_VENDORED_DIR_NAMES = _NAMESPACE["_VENDORED_DIR_NAMES"] +_vendored_prefixes = _NAMESPACE["_vendored_prefixes"] +_is_vendored_path = _NAMESPACE["_is_vendored_path"] +_full_packages = _NAMESPACE["_full_packages"] + + +def _discover(exclude_vendored: bool) -> List[str]: + """Package discovery over the real tree, with and without the exclusion.""" + patterns = ( + [ + pattern + for name in sorted(_VENDORED_DIR_NAMES) + for pattern in (f"*.{name}", f"*.{name}.*") + ] + if exclude_vendored + else [] + ) + return sorted( + find_namespace_packages( + where=PACKAGE_ROOT, + include=["executorch", "executorch.*"], + exclude=patterns, + ) + ) + + +def _vendored(packages: List[str]) -> List[str]: + return [ + package for package in packages if _is_vendored_path(package.replace(".", "/")) + ] + + +class TestFullWheelPackages(unittest.TestCase): + def test_the_tree_has_vendored_packages_to_exclude(self) -> None: + """Fail rather than skip when there is nothing to exclude. + + Every other test here is vacuous on a tree with no vendored checkouts: an empty + package list contains no vendored package, so the exclusion would look correct even + if it had been deleted. Assert the premise instead of quietly passing on it. + """ + discovered = _discover(exclude_vendored=False) + self.assertNotEqual( + discovered, [], f"no packages discovered under {PACKAGE_ROOT}" + ) + self.assertNotEqual( + _vendored(discovered), + [], + "no vendored third-party packages in this tree, so the exclusion below cannot " + "be shown to do anything. Initialize the submodules before running this.", + ) + + def test_no_vendored_package_ships(self) -> None: + """No package under a vendored third-party checkout is published.""" + leaked = _vendored(_full_packages()) + # Only the count and a few names, because a regression here leaks hundreds of + # packages and the default diff would bury the message. + self.assertEqual( + len(leaked), + 0, + f"the wheel would publish {len(leaked)} vendored packages, " + f"e.g. {leaked[:3]}", + ) + + def test_the_exclusion_is_load_bearing(self) -> None: + """Discovery without the exclusion finds the packages the exclusion removes.""" + self.assertLess( + len(_discover(exclude_vendored=True)), + len(_discover(exclude_vendored=False)), + "the exclusion dropped nothing, so it is no longer doing any work", + ) + + def test_setup_passes_the_package_list(self) -> None: + """The helper is actually wired into the full build. + + Without this, every test above still passes when the assignment that hands the list + to setuptools is deleted, which is the whole of the change. The sibling wheel test + asserts its own wiring the same way and for the same reason. + """ + assigned = [ + node + for node in ast.walk(_setup_py_module()) + if isinstance(node, ast.Assign) + for target in node.targets + if isinstance(target, ast.Subscript) + and isinstance(target.value, ast.Name) + and target.value.id == "setup_kwargs" + and isinstance(target.slice, ast.Constant) + and target.slice.value == "packages" + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Name) + and node.value.func.id == "_full_packages" + ] + self.assertEqual( + len(assigned), + 1, + "setup.py does not assign _full_packages() to setup_kwargs['packages'], " + "so the full build falls back to discovering every package", + ) + + def test_is_vendored_path_matches_whole_components(self) -> None: + """The filter matches a path component, not a substring.""" + self.assertTrue( + _is_vendored_path( + "src/executorch/backends/xnnpack/third-party/XNNPACK/a.py" + ) + ) + self.assertTrue(_is_vendored_path("src/executorch/x/third_party/y.yaml")) + self.assertFalse(_is_vendored_path("src/executorch/exir/program/_program.py")) + # "third-party" as part of a longer name is a different directory. + self.assertFalse(_is_vendored_path("src/executorch/x/third-party-tools/y.py")) + + def test_submodules_outside_a_vendored_dir_are_recognized(self) -> None: + """A submodule checked out under an ordinary name is still another repository. + + These are not matched by the directory name, so they are read from .gitmodules. Their + nested copies also cannot satisfy the imports the code uses: the FACTO helper imports + facto.specdb from the top level, and the tokenizers ship as a declared dependency. + """ + prefixes = _vendored_prefixes() + self.assertIn("backends/cadence/utils/FACTO", prefixes) + self.assertIn("extension/llm/tokenizers", prefixes) + for prefix in ("backends/cadence/utils/FACTO", "extension/llm/tokenizers"): + self.assertTrue(_is_vendored_path(f"executorch/{prefix}")) + self.assertTrue(_is_vendored_path(f"src/executorch/{prefix}/setup.py")) + self.assertFalse( + _is_vendored_path("executorch/extension/llm/custom_ops/op_sdpa.py") + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyproject.toml b/pyproject.toml index 105565862df..66176b35608 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,8 +118,11 @@ license-files = ["LICENSE"] "executorch" = "src/executorch" [tool.setuptools.package-data] -# TODO(dbort): Prune /test[s]/ dirs, /third-party/ dirs, yaml files that we -# don't need. +# Vendored third-party dirs are excluded in setup.py, so the globs below only reach the dirs +# the wheel ships. +# TODO(dbort): Prune /test[s]/ dirs and yaml files that we don't need. Test dirs cannot simply +# be dropped yet: the suites here import each other through the installed name, for example +# `from executorch.backends.arm.test import common`, so a non-editable install stops collecting. # TODO(RobertKalmar): When test[s] dirs pruned the PROJECT_DIR resolution in backends.nxp.tests_models.config.py can # avoid exporting and reading env variable. "*" = [ diff --git a/setup.py b/setup.py index 8980d1bd94a..ede5b0a3658 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ # derivative works thereof, in binary and source code form. import contextlib +import functools # Import this before distutils so that setuptools can intercept the distuils # imports. @@ -64,7 +65,7 @@ from distutils import log # type: ignore[import-not-found] from distutils.sysconfig import get_python_lib # type: ignore[import-not-found] from pathlib import Path, PurePosixPath -from typing import List, Optional +from typing import List, Optional, Tuple # Clean dynamic import using importlib _install_utils_path = Path(__file__).parent / "install_utils.py" @@ -177,6 +178,50 @@ def _minimal_cmake_flags() -> List[str]: ] +_VENDORED_DIR_NAMES = frozenset({"third-party", "third_party"}) + + +@functools.lru_cache(maxsize=None) +def _vendored_prefixes() -> Tuple[str, ...]: + """Source-tree prefixes holding code from another repository. + + Two shapes reach the wheel. Most vendored code sits in a directory named third-party, + which the name above covers wherever it appears. The rest are git submodules checked out + under an ordinary name, so they can only be recognized by asking git what they are. Both + exist to build the C++ targets, and their nested copies cannot satisfy the top level + imports the code actually uses: backends/cadence/utils/facto_util.py imports facto.specdb, + and the tokenizers are declared as pytorch-tokenizers in the dependency list. + """ + root = Path(__file__).parent + modules = root / ".gitmodules" + prefixes = [] + if modules.is_file(): + for line in modules.read_text().splitlines(): + name, separator, value = line.partition("=") + if separator and name.strip() == "path": + path = value.strip() + if not any(part in _VENDORED_DIR_NAMES for part in Path(path).parts): + prefixes.append(path) + return tuple(sorted(prefixes)) + + +def _is_vendored_path(path: str) -> bool: + """Whether a source-tree path holds code from another repository.""" + parts = Path(path).parts + if any(part in _VENDORED_DIR_NAMES for part in parts): + return True + # A submodule path is relative to the repository root, while a path here may be relative + # to src/executorch or carry a src/executorch prefix, so match on any suffix boundary. + posix = "/".join(parts) + return any( + posix == prefix + or posix.startswith(f"{prefix}/") + or posix.endswith(f"/{prefix}") + or f"/{prefix}/" in posix + for prefix in _vendored_prefixes() + ) + + def _minimal_packages() -> List[str]: return sorted( find_namespace_packages( @@ -204,6 +249,36 @@ def _minimal_packages() -> List[str]: ) +def _full_packages() -> List[str]: + """Every package the full wheel ships. + + Without an explicit list setuptools discovers all of src/executorch, which pulls in the + Python files and codegen scripts of the vendored third-party checkouts. Those exist to + build the C++ targets, so once the libraries are built no shipped module imports them. + + Test packages deliberately stay. The suites in this repository import each other through + the installed name, for example `from executorch.backends.arm.test import common`, so + dropping them from the wheel stops the suites collecting under a non-editable install. + """ + return sorted( + package + for package in find_namespace_packages( + where="src", + include=["executorch", "executorch.*"], + # A vendored dir like "third-party" is a literal path component, so it needs no + # escaping here even though it is not a valid identifier. + exclude=[ + pattern + for name in sorted(_VENDORED_DIR_NAMES) + for pattern in (f"*.{name}", f"*.{name}.*") + ], + ) + # The submodules checked out under an ordinary name are not matched by the patterns + # above, so they are filtered on the path they map to. + if not _is_vendored_path(package.replace(".", "/")) + ) + + # The published project names for the CUDA runtime components a CUDA wheel links but # does not bundle, keyed by CUDA major version. Not derivable from a suffix rule: the # CUDA 12 wheels carry a "-cu12" suffix while the CUDA 13 ones are published under @@ -1642,6 +1717,11 @@ def analyze_manifest(self): _f for _f in self.manifest_files[_pkg] if os.path.isfile(os.path.join(_root, _f)) + # A directory left out of `packages` is not simply skipped. setuptools + # walks up to the nearest listed package and records the file as that + # package's data, so a vendored *.yaml still arrives under its parent. + # Filter with the same list so the two agree. + and not _is_vendored_path(_f) ] def _copy_extra_files(self, src_to_dst, dst_root: str) -> None: @@ -2325,6 +2405,7 @@ def iter_distribution_names(self): setup_kwargs["packages"] = _minimal_packages() setup_kwargs["install_requires"] = _minimal_dependencies() else: + setup_kwargs["packages"] = _full_packages() # A CUDA wheel links the CUDA runtime but does not bundle it, so the wheels that # carry it are declared here. A CPU wheel adds nothing. setup_kwargs["install_requires"] = _base_dependencies() + _cuda_dependencies()