diff --git a/backends/openvino/README.md b/backends/openvino/README.md index 331aebf87ff..40a245f12a0 100644 --- a/backends/openvino/README.md +++ b/backends/openvino/README.md @@ -48,6 +48,51 @@ print(_get_registered_backend_names()) # Should include 'OpenvinoBackend' ``` +## Precision + +When the device is `CPU`, a program computes in float32 by default. Without this default, +OpenVINO would compute in bfloat16 on x86 CPUs with bfloat16 units, which keeps only two to +three significant digits, and in float16 on ARM CPUs, which keeps about three. +Both are faster. To trade accuracy for speed, ask for the faster precision when +lowering: + +```python +from executorch.backends.openvino.partitioner import OpenvinoPartitioner +from executorch.exir.backend.compile_spec_schema import CompileSpec + +OpenvinoPartitioner( + [CompileSpec("device", b"CPU"), CompileSpec("inference_precision", b"bf16")] +) +``` + +On an ARM CPU, ask for `f16` instead: there float32 is about 1.8x slower than +float16, as measured on an Orin Nano. + +The value is OpenVINO's `INFERENCE_PRECISION_HINT`: `f32`, `f16` or `bf16`. An +empty value means the default, and any other value fails at lowering. A CPU +without float16 support computes an `f16` program in float32. OpenVINO has no +bfloat16 on ARM CPUs, so there `bf16` keeps the ARM default, float16 where the +CPU supports it. Which x86 CPUs compute in bfloat16 is up to OpenVINO; see its +[Precision Control](https://docs.openvino.ai/2026/openvino-workflow/running-inference/optimize-inference/precision-control.html) +page. Other devices, such as GPUs, keep OpenVINO's default unless the option is +set. + +The default also applies to a program exported in float16: on a CPU it computes +in float32 unless you lower it with `inference_precision`. + +On x86 CPUs, OpenVINO applies the precision when the runtime loads a program. +So a program lowered without `inference_precision` computes in float32, even +one exported with an ExecuTorch release that did not have this default. That +is slower and can use more memory. Export it again with `inference_precision` +set to `bf16` to get the earlier speed. On ARM CPUs, OpenVINO applies the +precision when the program is lowered, so a program exported with such an +earlier release keeps computing in float16 until it is exported again. + +OpenVINO's CPU plugin computes 64-bit integers in 32 bits. An integer result +outside the int32 range wraps around without an error. This applies to every +int64 tensor, including the int64 result of `cumsum` and `prod` over smaller +integer types. + ## Directory Structure ``` diff --git a/backends/openvino/_passes/__init__.py b/backends/openvino/_passes/__init__.py index 0db94ca1ec8..bc2853a73e6 100644 --- a/backends/openvino/_passes/__init__.py +++ b/backends/openvino/_passes/__init__.py @@ -4,6 +4,12 @@ # except in compliance with the License. See the license file found in the # LICENSE file in the root directory of this source tree. +from .cast_reduction_input_pass import CastReductionInputPass from .decompose_floor_divide_pass import DecomposeFloorDividePass +from .fold_constant_cumsum_pass import FoldConstantCumsumPass -__all__ = ["DecomposeFloorDividePass"] +__all__ = [ + "CastReductionInputPass", + "DecomposeFloorDividePass", + "FoldConstantCumsumPass", +] diff --git a/backends/openvino/_passes/cast_reduction_input_pass.py b/backends/openvino/_passes/cast_reduction_input_pass.py new file mode 100644 index 00000000000..36e3f3795c0 --- /dev/null +++ b/backends/openvino/_passes/cast_reduction_input_pass.py @@ -0,0 +1,45 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import torch +from executorch.exir.dialects._ops import ops as exir_ops +from torch.fx.passes.infra.pass_base import PassBase, PassResult + +REDUCTION_OPS = { + exir_ops.edge.aten.cumsum.default, + exir_ops.edge.aten.prod.default, + exir_ops.edge.aten.prod.dim_int, +} + + +class CastReductionInputPass(PassBase): + """Cast the input of cumsum and prod to the dtype of their result. + + PyTorch reduces bool and integer tensors in int64, but OpenVINO keeps a + narrower type for these two, though not for sum. A cumsum over a bool mask + is computed in 8 bits and wraps after 255, and a cumsum or prod over int32 + returns int32. + """ + + def call(self, graph_module: torch.fx.GraphModule) -> PassResult: + modified = False + for node in graph_module.graph.nodes: + if node.op != "call_function" or node.target not in REDUCTION_OPS: + continue + source = node.args[0] + dtype = node.meta["val"].dtype + if source.meta["val"].dtype == dtype: + continue + with graph_module.graph.inserting_before(node): + cast = graph_module.graph.call_function( + exir_ops.edge.aten._to_copy.default, (source,), {"dtype": dtype} + ) + cast.meta["val"] = source.meta["val"].to(dtype) + node.replace_input_with(source, cast) + modified = True + if modified: + graph_module.recompile() + return PassResult(graph_module, modified) diff --git a/backends/openvino/_passes/fold_constant_cumsum_pass.py b/backends/openvino/_passes/fold_constant_cumsum_pass.py new file mode 100644 index 00000000000..fabc030b06a --- /dev/null +++ b/backends/openvino/_passes/fold_constant_cumsum_pass.py @@ -0,0 +1,156 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import logging +import operator + +import torch +from executorch.exir.dialects._ops import ops as exir_ops +from executorch.exir.passes.constant_prop_pass import _nodes_written_in_place +from torch.fx.passes.infra.pass_base import PassBase, PassResult +from torch.utils import _pytree as pytree + + +def _reads_only_shape(node: torch.fx.Node) -> bool: + """full_like with a constant fill value reads only the shape and dtype of + its input. With a static shape its result is a constant, and OpenVINO + folds it into one too. + """ + if node.target != exir_ops.edge.aten.full_like.default: + return False + if any( + isinstance(a, torch.fx.Node) for a in (*node.args[1:], *node.kwargs.values()) + ): + return False + val = node.meta.get("val") + return isinstance(val, torch.Tensor) and all(isinstance(d, int) for d in val.shape) + + +def _inputs(node: torch.fx.Node) -> list: + return [] if _reads_only_shape(node) else node.all_input_nodes + + +def _on_cpu(value): + if isinstance(value, torch.Tensor): + return value.cpu() + if isinstance(value, torch.device): + return torch.device("cpu") + return value + + +def _constant_nodes(graph_module: torch.fx.GraphModule) -> set: + """Nodes that depend only on constants. Graph order puts inputs first.""" + constants = set() + written = _nodes_written_in_place(graph_module) + for node in graph_module.graph.nodes: + # Only tensors: a higher-order op such as cond reads its subgraphs, + # which may hold a random op, through get_attr nodes too. + if node.op == "get_attr": + value = operator.attrgetter(node.target)(graph_module) + if isinstance(value, torch.Tensor) and node not in written: + constants.add(node) + elif node.op == "call_function" and ( + _reads_only_shape(node) + or ( + torch.Tag.nondeterministic_seeded + not in getattr(node.target, "tags", ()) + and all(n in constants for n in node.all_input_nodes) + ) + ): + constants.add(node) + return constants + + +def _evaluate(graph_module: torch.fx.GraphModule, targets: list) -> dict: + """Values of targets, computing only what they read, on the CPU.""" + needed = set() + stack = list(targets) + while stack: + node = stack.pop() + if node not in needed: + needed.add(node) + stack.extend(_inputs(node)) + order = [n for n in graph_module.graph.nodes if n in needed] + last_use = {i: n for n in order for i in _inputs(n)} + values = {} + for node in order: + if node.op == "get_attr": + values[node] = operator.attrgetter(node.target)(graph_module) + continue + if _reads_only_shape(node): + like = node.args[0].meta["val"] + args = (torch.empty(like.shape, dtype=like.dtype), *node.args[1:]) + kwargs = dict(node.kwargs) + else: + args, kwargs = torch.fx.node.map_arg( + (node.args, node.kwargs), lambda n: values[n] + ) + # The values are the same on any device, and lowering for OpenVINO + # must not need the GPU a constant was made for. + args, kwargs = pytree.tree_map(_on_cpu, (args, kwargs)) + if kwargs.get("pin_memory"): + kwargs = {**kwargs, "pin_memory": False} + values[node] = node.target(*args, **kwargs) + for i in _inputs(node): + if last_use[i] is node and i not in targets: + del values[i] + return values + + +class FoldConstantCumsumPass(PassBase): + """Replace an integer cumsum that reads only constants with its result. + + With an integer cumsum over a constant, OpenVINO's CPU plugin (2025.4 to + 2026.4) gives wrong results from the second inference on: masks built from + it keep only what earlier calls allowed. pi0 builds its attention masks + this way. The OpenVINO nightly no longer does this, so the pass can go once + the oldest supported OpenVINO release has the fix. Only these cumsums are + folded; constants that feed no such cumsum, such as compressed weights, are + left as they are. Runs on an unlifted module, where constants are get_attr + nodes; buffers the graph writes to are not constants, and full_like of an + input with a static shape is one. + """ + + def call(self, graph_module: torch.fx.GraphModule) -> PassResult: + graph = graph_module.graph + constants = _constant_nodes(graph_module) + cumsums = [ + n + for n in graph.nodes + if n in constants + and n.target == exir_ops.edge.aten.cumsum.default + and not n.meta["val"].dtype.is_floating_point + and not n.meta["val"].dtype.is_complex + ] + modified = False + # One at a time, so a cumsum that cannot be evaluated does not keep + # the others from being folded. + for node in cumsums: + try: + value = _evaluate(graph_module, [node])[node] + except RuntimeError as e: + # Some ops have modes the CPU lacks. + logging.warning( + "OpenVINO: could not fold %s, so on a CPU some OpenVINO " + "releases give wrong results for it from the second " + "inference on: %s", + node.name, + e, + ) + continue + name = f"_folded_cumsum_{node.name}" + while hasattr(graph_module, name): + name += "_" + graph_module.register_buffer(name, value) + with graph.inserting_before(node): + folded = graph.get_attr(name) + folded.meta = node.meta.copy() + node.replace_all_uses_with(folded) + modified = True + if modified: + graph.eliminate_dead_code() + graph_module.recompile() + return PassResult(graph_module, modified) diff --git a/backends/openvino/preprocess.py b/backends/openvino/preprocess.py index 7a3f490b258..83537f26f38 100644 --- a/backends/openvino/preprocess.py +++ b/backends/openvino/preprocess.py @@ -9,7 +9,11 @@ from typing import final, List import torch -from executorch.backends.openvino._passes import DecomposeFloorDividePass +from executorch.backends.openvino._passes import ( + CastReductionInputPass, + DecomposeFloorDividePass, + FoldConstantCumsumPass, +) from executorch.exir.backend.backend_details import ( BackendDetails, @@ -25,6 +29,10 @@ ) from openvino.preprocess import PrePostProcessor # type: ignore[import-untyped] +# The values of the inference_precision compile spec. The runtime +# (OpenvinoBackend.cpp) accepts the same ones. +_PRECISIONS = ("f32", "f16", "bf16") + @final class OpenvinoBackend(BackendDetails): @@ -43,7 +51,25 @@ def preprocess( Returns: PreprocessResult: The result of preprocessing, including the compiled model bytes. """ - for pass_cls in [DimOrderOpsRevertPass, DecomposeFloorDividePass]: + compile_options = {} + for spec in module_compile_spec: + compile_options[spec.key] = spec.value.decode() + device = compile_options.get("device", "CPU") + requested_precision = compile_options.get("inference_precision") + # Some OpenVINO releases accept an unknown value. Checking before the + # conversion fails fast and keeps it out of the program, where the + # runtime would only fail at load. + if requested_precision and requested_precision not in _PRECISIONS: + raise ValueError( + f"OpenVINO backend: unsupported inference_precision " + f"'{requested_precision}', expected one of {', '.join(_PRECISIONS)}" + ) + + for pass_cls in [ + DimOrderOpsRevertPass, + DecomposeFloorDividePass, + CastReductionInputPass, + ]: result = pass_cls()(edge_program.graph_module) if result and result.graph_module: edge_program._graph_module = result.graph_module @@ -70,14 +96,12 @@ def preprocess( f"{type(val).__name__}; only tensors and ints are supported" ) - compile_options = {} - for spec in module_compile_spec: - compile_options[spec.key] = spec.value.decode() - # The converter chooses some lowerings (e.g. boolean-mask indexing) from # the input types, so they must be known before conversion starts. + module = edge_program.module() + FoldConstantCumsumPass()(module) decoder = TorchFXPythonDecoder( - edge_program.module(), input_shapes=input_shapes, input_types=input_types + module, input_shapes=input_shapes, input_types=input_types ) frontend = FrontEndManager().load_by_framework("pytorch") ppp = PrePostProcessor(frontend.convert(frontend.load(decoder))) @@ -86,7 +110,11 @@ def preprocess( for i in int_inputs: ppp.input(i).tensor().set_shape([1]) ppp.input(i).preprocess().custom(lambda x: opset13.squeeze(x, 0).output(0)) - compiled = compile_model(ppp.build(), compile_options.get("device", "CPU")) + # On ARM CPUs OpenVINO fixes the precision here, not when the runtime + # imports the model, so this must follow the same rule as the runtime. + precision = requested_precision or ("f32" if device.startswith("CPU") else None) + config = {"INFERENCE_PRECISION_HINT": precision} if precision else {} + compiled = compile_model(ppp.build(), device, config) model_bytes = compiled.export_model() return PreprocessResult(processed_bytes=model_bytes.getvalue()) diff --git a/backends/openvino/runtime/OpenvinoApi.h b/backends/openvino/runtime/OpenvinoApi.h index e4c3e7f737a..b4aa25f9d32 100644 --- a/backends/openvino/runtime/OpenvinoApi.h +++ b/backends/openvino/runtime/OpenvinoApi.h @@ -76,6 +76,9 @@ using ov_core_free_fn = void (*)(ov_core_t*); using ov_core_get_available_devices_fn = ov_status_e (*)(const ov_core_t*, ov_available_devices_t*); using ov_available_devices_free_fn = void (*)(ov_available_devices_t*); +// Takes one property key and value, both strings, after the device name. +using ov_core_set_property_fn = + ov_status_e (*)(const ov_core_t*, const char*, ...); using ov_core_import_model_fn = ov_status_e (*)( const ov_core_t*, const char*, @@ -94,10 +97,17 @@ using ov_infer_request_set_input_tensor_by_index_fn = using ov_infer_request_set_output_tensor_by_index_fn = ov_status_e (*)(ov_infer_request_t*, size_t, const ov_tensor_t*); using ov_infer_request_infer_fn = ov_status_e (*)(ov_infer_request_t*); +using ov_infer_request_get_output_tensor_by_index_fn = + ov_status_e (*)(const ov_infer_request_t*, size_t, ov_tensor_t**); using ov_infer_request_free_fn = void (*)(ov_infer_request_t*); using ov_tensor_create_from_host_ptr_fn = ov_status_e (*)(ov_element_type_e, ov_shape_t, void*, ov_tensor_t**); +using ov_tensor_create_fn = + ov_status_e (*)(ov_element_type_e, ov_shape_t, ov_tensor_t**); using ov_tensor_free_fn = void (*)(ov_tensor_t*); +using ov_tensor_get_shape_fn = ov_status_e (*)(const ov_tensor_t*, ov_shape_t*); +using ov_tensor_get_byte_size_fn = ov_status_e (*)(const ov_tensor_t*, size_t*); +using ov_tensor_data_fn = ov_status_e (*)(const ov_tensor_t*, void**); using ov_shape_create_fn = ov_status_e (*)(int64_t, const int64_t*, ov_shape_t*); using ov_shape_free_fn = ov_status_e (*)(ov_shape_t*); @@ -115,6 +125,7 @@ struct OpenvinoFunctions { ov_core_free_fn core_free = nullptr; ov_core_get_available_devices_fn core_get_available_devices = nullptr; ov_available_devices_free_fn available_devices_free = nullptr; + ov_core_set_property_fn core_set_property = nullptr; ov_core_import_model_fn core_import_model = nullptr; ov_compiled_model_create_infer_request_fn compiled_model_create_infer_request = nullptr; @@ -126,9 +137,15 @@ struct OpenvinoFunctions { ov_infer_request_set_output_tensor_by_index_fn infer_request_set_output_tensor_by_index = nullptr; ov_infer_request_infer_fn infer_request_infer = nullptr; + ov_infer_request_get_output_tensor_by_index_fn + infer_request_get_output_tensor_by_index = nullptr; ov_infer_request_free_fn infer_request_free = nullptr; ov_tensor_create_from_host_ptr_fn tensor_create_from_host_ptr = nullptr; + ov_tensor_create_fn tensor_create = nullptr; ov_tensor_free_fn tensor_free = nullptr; + ov_tensor_get_shape_fn tensor_get_shape = nullptr; + ov_tensor_get_byte_size_fn tensor_get_byte_size = nullptr; + ov_tensor_data_fn tensor_data = nullptr; ov_shape_create_fn shape_create = nullptr; ov_shape_free_fn shape_free = nullptr; }; diff --git a/backends/openvino/runtime/OpenvinoBackend.cpp b/backends/openvino/runtime/OpenvinoBackend.cpp index fad6ef91cb1..da9fa43c325 100644 --- a/backends/openvino/runtime/OpenvinoBackend.cpp +++ b/backends/openvino/runtime/OpenvinoBackend.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include "OpenvinoBackend.h" @@ -236,6 +237,7 @@ bool OpenvinoBackend::ensure_loaded() const { LOAD_SYM(core_free, ov_core_free) LOAD_SYM(core_get_available_devices, ov_core_get_available_devices) LOAD_SYM(available_devices_free, ov_available_devices_free) + LOAD_SYM(core_set_property, ov_core_set_property) LOAD_SYM(core_import_model, ov_core_import_model) LOAD_SYM( compiled_model_create_infer_request, @@ -250,9 +252,16 @@ bool OpenvinoBackend::ensure_loaded() const { infer_request_set_output_tensor_by_index, ov_infer_request_set_output_tensor_by_index) LOAD_SYM(infer_request_infer, ov_infer_request_infer) + LOAD_SYM( + infer_request_get_output_tensor_by_index, + ov_infer_request_get_output_tensor_by_index) LOAD_SYM(infer_request_free, ov_infer_request_free) LOAD_SYM(tensor_create_from_host_ptr, ov_tensor_create_from_host_ptr) + LOAD_SYM(tensor_create, ov_tensor_create) LOAD_SYM(tensor_free, ov_tensor_free) + LOAD_SYM(tensor_get_shape, ov_tensor_get_shape) + LOAD_SYM(tensor_get_byte_size, ov_tensor_get_byte_size) + LOAD_SYM(tensor_data, ov_tensor_data) LOAD_SYM(shape_create, ov_shape_create) LOAD_SYM(shape_free, ov_shape_free) @@ -321,17 +330,55 @@ exr::Result OpenvinoBackend::init( size_t data_size = processed->size(); std::string device = "CPU"; + std::string precision; for (auto& compile_spec : compile_specs) { - if (std::strcmp(compile_spec.key, "device") == 0) { - const char* buf = static_cast(compile_spec.value.buffer); - size_t len = compile_spec.value.nbytes; - // Strip trailing null bytes that may be included in nbytes. - while (len > 0 && buf[len - 1] == '\0') { - --len; - } - if (len > 0) { - device.assign(buf, len); - } + const bool is_device = std::strcmp(compile_spec.key, "device") == 0; + if (!is_device && + std::strcmp(compile_spec.key, "inference_precision") != 0) { + continue; + } + const char* buf = static_cast(compile_spec.value.buffer); + size_t len = compile_spec.value.nbytes; + // Strip trailing null bytes that may be included in nbytes. + while (len > 0 && buf[len - 1] == '\0') { + --len; + } + if (len > 0) { + (is_device ? device : precision).assign(buf, len); + } + } + + // The same rule as preprocess.py, because on x86 CPUs OpenVINO applies the + // precision here, on import. + if (precision.empty() && device.rfind("CPU", 0) == 0) { + precision = "f32"; + } + // The same values as in preprocess.py. Some OpenVINO releases accept an + // unknown value, and others only reject it on import, with a generic + // error. + if (!precision.empty() && precision != "f32" && precision != "f16" && + precision != "bf16") { + ET_LOG( + Error, + "OpenVINO: unsupported inference_precision '%s', expected one of " + "f32, f16, bf16", + precision.c_str()); + ov_.core_free(core); + return exr::Error::InvalidArgument; + } + if (!precision.empty()) { + status = ov_.core_set_property( + core, device.c_str(), "INFERENCE_PRECISION_HINT", precision.c_str()); + if (status != OV_STATUS_OK) { + ET_LOG( + Error, + "OpenVINO: failed to set inference precision '%s' for device '%s' " + "(status=%d)", + precision.c_str(), + device.c_str(), + status); + ov_.core_free(core); + return exr::Error::InvalidArgument; } } @@ -379,9 +426,26 @@ exr::Result OpenvinoBackend::init( ov_.compiled_model_free(compiled_model); return exr::Error::MemoryAllocationFailed; } + size_t num_outputs = 0; + status = ov_.compiled_model_outputs_size(compiled_model, &num_outputs); + ov_tensor_t** owned_outputs = nullptr; + if (status == OV_STATUS_OK && num_outputs > 0) { + owned_outputs = allocator->allocateList(num_outputs); + } + if (status != OV_STATUS_OK || (num_outputs > 0 && !owned_outputs)) { + ET_LOG(Error, "OpenVINO: failed to set up outputs (status=%d)", status); + ov_.infer_request_free(infer_request); + ov_.compiled_model_free(compiled_model); + return exr::Error::Internal; + } + for (size_t i = 0; i < num_outputs; i++) { + owned_outputs[i] = nullptr; + } new (handle) ExecutionHandle; handle->compiled_model = compiled_model; handle->infer_request = infer_request; + handle->owned_outputs = owned_outputs; + handle->num_outputs = num_outputs; return handle; } @@ -469,16 +533,35 @@ exr::Error OpenvinoBackend::execute( "OpenVINO: expected tensor for output %zu", i); auto output_tensor = args[num_inputs + i]->toTensor(); - auto result = create_ov_tensor(output_tensor); - if (!result.ok()) { - return result.error(); + ov_tensor_t*& owned = execution_handle->owned_outputs[i]; + ov_tensor_t* tensor = nullptr; + if (output_tensor.shape_dynamism() == exa::TensorShapeDynamism::STATIC) { + auto result = create_ov_tensor(output_tensor); + if (!result.ok()) { + return result.error(); + } + tensor = result.get(); + } else { + // OpenVINO cannot write more than a borrowed buffer's shape holds, and + // an earlier call may have shrunk that shape. So an output whose size + // can change goes to a buffer OpenVINO owns and grows, and is copied + // out after inference. + if (!owned) { + auto result = create_ov_tensor(output_tensor, /*own_memory=*/true); + if (!result.ok()) { + return result.error(); + } + owned = result.get(); + } + tensor = owned; } - ov_tensor_t* tensor = result.get(); status = ov_.infer_request_set_output_tensor_by_index( execution_handle->infer_request, i, tensor); // Safe to free: see shared_ptr ownership comment on input tensor above. - ov_.tensor_free(tensor); + if (tensor != owned) { + ov_.tensor_free(tensor); + } if (status != OV_STATUS_OK) { return exr::Error::Internal; } @@ -490,6 +573,27 @@ exr::Error OpenvinoBackend::execute( return exr::Error::Internal; } + for (size_t i = 0; i < num_outputs; i++) { + auto output_tensor = args[num_inputs + i]->toTensor(); + if (output_tensor.shape_dynamism() == exa::TensorShapeDynamism::STATIC) { + continue; + } + ov_tensor_t* ov_output = nullptr; + status = ov_.infer_request_get_output_tensor_by_index( + execution_handle->infer_request, i, &ov_output); + if (status != OV_STATUS_OK) { + ET_LOG( + Error, "OpenVINO: failed to get output %zu (status=%d)", i, status); + return exr::Error::Internal; + } + exr::Error error = copy_output(ov_output, output_tensor); + ov_.tensor_free(ov_output); + if (error != exr::Error::Ok) { + ET_LOG(Error, "OpenVINO: failed to copy output %zu", i); + return error; + } + } + return exr::Error::Ok; } @@ -504,6 +608,13 @@ void OpenvinoBackend::destroy(exr::DelegateHandle* handle) const { ExecutionHandle* execution_handle = static_cast(handle); + for (size_t i = 0; i < execution_handle->num_outputs; i++) { + if (execution_handle->owned_outputs[i] && ov_.tensor_free) { + ov_.tensor_free(execution_handle->owned_outputs[i]); + execution_handle->owned_outputs[i] = nullptr; + } + } + if (execution_handle->infer_request && ov_.infer_request_free) { ov_.infer_request_free(execution_handle->infer_request); execution_handle->infer_request = nullptr; @@ -515,8 +626,45 @@ void OpenvinoBackend::destroy(exr::DelegateHandle* handle) const { } } +exr::Error OpenvinoBackend::copy_output( + const ov_tensor_t* ov_output, + const exa::Tensor& output) const { + ov_shape_t shape = {}; + ov_status_e status = ov_.tensor_get_shape(ov_output, &shape); + if (status != OV_STATUS_OK) { + ET_LOG( + Error, "OpenVINO: failed to get an output shape (status=%d)", status); + return exr::Error::Internal; + } + exr::Error error = exr::resize_tensor( + output, + exa::ArrayRef(shape.dims, static_cast(shape.rank))); + ov_.shape_free(&shape); + if (error != exr::Error::Ok) { + return error; + } + size_t nbytes = 0; + void* data = nullptr; + if (ov_.tensor_get_byte_size(ov_output, &nbytes) != OV_STATUS_OK || + ov_.tensor_data(ov_output, &data) != OV_STATUS_OK) { + ET_LOG(Error, "OpenVINO: failed to read an output"); + return exr::Error::Internal; + } + ET_CHECK_OR_RETURN_ERROR( + nbytes == output.nbytes(), + Internal, + "OpenVINO: output has %zu bytes, expected %zu", + nbytes, + output.nbytes()); + if (nbytes > 0) { + std::memcpy(output.mutable_data_ptr(), data, nbytes); + } + return exr::Error::Ok; +} + exr::Result OpenvinoBackend::create_ov_tensor( - const exa::Tensor& tensor) const { + const exa::Tensor& tensor, + bool own_memory) const { ov_element_type_e ov_type = convert_to_openvino_type(tensor.scalar_type()); if (ov_type == OV_ELEMENT_UNDEFINED) { return exr::Error::NotSupported; @@ -537,22 +685,33 @@ exr::Result OpenvinoBackend::create_ov_tensor( dims = dims_heap.get(); } for (int64_t d = 0; d < rank; d++) { - dims[d] = sizes[d]; + dims[d] = own_memory ? 0 : sizes[d]; } - // shape is zero-initialized; shape_free is only needed after a successful - // shape_create (the zero state is safe to skip). + // shape_create refuses rank 0, so a 0-d tensor keeps the zero-initialized + // shape, which describes a scalar. ov_shape_t shape = {}; - ov_status_e status = ov_.shape_create(rank, dims, &shape); + ov_status_e status = OV_STATUS_OK; + if (rank > 0) { + status = ov_.shape_create(rank, dims, &shape); + } dims_heap.reset(); // Release heap dims (no-op if stack was used). if (status != OV_STATUS_OK) { + ET_LOG( + Error, + "OpenVINO: failed to create a shape of rank %" PRId64 " (status=%d)", + rank, + status); return exr::Error::Internal; } ov_tensor_t* ov_tensor = nullptr; - status = ov_.tensor_create_from_host_ptr( - ov_type, shape, tensor.mutable_data_ptr(), &ov_tensor); + status = own_memory + ? ov_.tensor_create(ov_type, shape, &ov_tensor) + : ov_.tensor_create_from_host_ptr( + ov_type, shape, tensor.mutable_data_ptr(), &ov_tensor); ov_.shape_free(&shape); if (status != OV_STATUS_OK || !ov_tensor) { + ET_LOG(Error, "OpenVINO: failed to create a tensor (status=%d)", status); return exr::Error::Internal; } return ov_tensor; diff --git a/backends/openvino/runtime/OpenvinoBackend.h b/backends/openvino/runtime/OpenvinoBackend.h index 510e49ee690..d75fa725ab0 100644 --- a/backends/openvino/runtime/OpenvinoBackend.h +++ b/backends/openvino/runtime/OpenvinoBackend.h @@ -28,6 +28,10 @@ namespace openvino { struct ExecutionHandle { ov_compiled_model_t* compiled_model = nullptr; ov_infer_request_t* infer_request = nullptr; + // A buffer OpenVINO owns for each output whose size can change, made on + // first use and reused by every call; null for the others. + ov_tensor_t** owned_outputs = nullptr; + size_t num_outputs = 0; }; class OpenvinoBackend final : public ::exr::BackendInterface { @@ -49,7 +53,15 @@ class OpenvinoBackend final : public ::exr::BackendInterface { private: bool ensure_loaded() const; ov_element_type_e convert_to_openvino_type(exa::ScalarType scalar_type) const; - exr::Result create_ov_tensor(const exa::Tensor& tensor) const; + // With own_memory, the OpenVINO tensor gets its own buffer instead of + // borrowing the tensor's. That buffer starts empty, with the tensor's rank, + // and OpenVINO grows it to the size it writes. + exr::Result create_ov_tensor( + const exa::Tensor& tensor, + bool own_memory = false) const; + exr::Error copy_output( + const ov_tensor_t* ov_output, + const exa::Tensor& output) const; mutable DlHandle lib_handle_; mutable OpenvinoFunctions ov_; diff --git a/backends/openvino/tests/ops/base_openvino_op_test.py b/backends/openvino/tests/ops/base_openvino_op_test.py index f6bc928e4d4..4e20b41cbf6 100644 --- a/backends/openvino/tests/ops/base_openvino_op_test.py +++ b/backends/openvino/tests/ops/base_openvino_op_test.py @@ -25,6 +25,8 @@ def execute_layer_test( sample_inputs: tuple[torch.Tensor], expected_partitions: int = 1, assert_output_equal: bool = True, + compile_specs: tuple[CompileSpec, ...] = (), + more_inputs: tuple[tuple[torch.Tensor, ...], ...] = (), ): module = module.eval() @@ -32,7 +34,7 @@ def execute_layer_test( aten_dialect: ExportedProgram = export(module, sample_inputs) # Convert to edge dialect and lower the module to the backend with a custom partitioner - compile_spec = [CompileSpec("device", self.device.encode())] + compile_spec = [CompileSpec("device", self.device.encode()), *compile_specs] lowered_module: EdgeProgramManager = to_edge_transform_and_lower( aten_dialect, partitioner=[ @@ -59,12 +61,8 @@ def execute_layer_test( # Execute the model and compare the outputs with the reference outputs if assert_output_equal: - # Execute the module in eager mode to calculate the reference outputs - ref_output = module(*sample_inputs) - if isinstance(ref_output, torch.Tensor): - ref_output = [ - ref_output, - ] + # Later inputs run on the same loaded method, as an app would. + all_inputs = (sample_inputs, *more_inputs) # Load from a file: unlike a buffer, it really frees the delegate # data when the backend frees it. @@ -75,20 +73,28 @@ def execute_layer_test( program = Runtime.get().load_program(pte_path) method = program.load_method("forward") assert method is not None - outputs = method.execute(sample_inputs) + all_outputs = [method.execute(inputs) for inputs in all_inputs] # Windows cannot delete a mapped file, so release it first. del method, program - # Compare the outputs with the reference outputs - self.assertTrue(len(ref_output) == len(outputs)) - for i in range(len(ref_output)): - self.assertTrue( - torch.allclose( - outputs[i], - ref_output[i], - atol=self.atol, - rtol=self.rtol, - equal_nan=True, - ), - msg=f"ref_output:\n{ref_output[i]}\n\ntest_output:\n{outputs[i]}", - ) + for inputs, outputs in zip(all_inputs, all_outputs): + # Execute the module in eager mode to calculate the reference outputs + ref_output = module(*inputs) + if isinstance(ref_output, torch.Tensor): + ref_output = [ + ref_output, + ] + + # Compare the outputs with the reference outputs + self.assertTrue(len(ref_output) == len(outputs)) + for i in range(len(ref_output)): + self.assertTrue( + torch.allclose( + outputs[i], + ref_output[i], + atol=self.atol, + rtol=self.rtol, + equal_nan=True, + ), + msg=f"ref_output:\n{ref_output[i]}\n\ntest_output:\n{outputs[i]}", + ) diff --git a/backends/openvino/tests/ops/test_add.py b/backends/openvino/tests/ops/test_add.py index 5b68d0ff149..a976f7cc7fd 100644 --- a/backends/openvino/tests/ops/test_add.py +++ b/backends/openvino/tests/ops/test_add.py @@ -20,3 +20,12 @@ def test_add(self): module = self.create_model() sample_input = (torch.randn(2, 5, 1, 3), torch.randn(2, 5, 1, 3)) self.execute_layer_test(module, sample_input) + + def test_add_0d_inputs(self): + module = self.create_model() + for sample_input in ( + (torch.randn(2, 3), torch.tensor(2.5)), + (torch.tensor(1.5), torch.tensor(2.5)), + ): + with self.subTest(shapes=[list(x.shape) for x in sample_input]): + self.execute_layer_test(module, sample_input) diff --git a/backends/openvino/tests/ops/test_addmm.py b/backends/openvino/tests/ops/test_addmm.py index 51c1314db0d..4a690e08b98 100644 --- a/backends/openvino/tests/ops/test_addmm.py +++ b/backends/openvino/tests/ops/test_addmm.py @@ -1,7 +1,15 @@ +# mypy: disable-error-code=import-not-found + + +from unittest import mock + import torch +from executorch.backends.openvino import preprocess from executorch.backends.openvino.tests.ops.base_openvino_op_test import ( BaseOpenvinoOpTest, ) +from executorch.exir.backend.backend_details import CompileSpec +from openvino import Core, Type # type: ignore[import-untyped] class TestAddMMOperator(BaseOpenvinoOpTest): @@ -26,3 +34,51 @@ def test_addmm(self): input_z = torch.randn(4, 4, dtype=torch.float32) sample_input = (input_x, input_y, input_z) self.execute_layer_test(module, sample_input) + + def test_addmm_computes_in_float32(self): + if not self.device.startswith("CPU"): + self.skipTest("only CPUs compute in float32 by default") + if Core().get_property("CPU", "INFERENCE_PRECISION_HINT") == Type.f32: + self.skipTest("OpenVINO already computes in float32 on this CPU") + # OpenVINO would otherwise compute this in bfloat16 on x86 CPUs with + # bfloat16 units, or in float16 on ARM CPUs, far outside 1e-4. + self.atol = self.rtol = 1e-4 + sample_input = tuple(torch.randn(256, 256) for _ in range(3)) + self.execute_layer_test(self.create_model(), sample_input) + # On x86 only the runtime passes the precision on, so this fails if it + # drops an explicit one. + with self.assertRaisesRegex(AssertionError, "ref_output"): + self.execute_layer_test( + self.create_model(), + sample_input, + compile_specs=(CompileSpec("inference_precision", b"bf16"),), + ) + + def test_addmm_passes_the_precision_to_openvino(self): + if not self.device.startswith("CPU"): + self.skipTest("only CPUs compute in float32 by default") + # Checks the lowering half too: on x86 the runtime half alone decides. + self.atol = self.rtol = 0.05 + sample_input = tuple(torch.randn(4, 4) for _ in range(3)) + bf16 = (CompileSpec("inference_precision", b"bf16"),) + for compile_specs, hint in (((), "f32"), (bf16, "bf16")): + with self.subTest(hint=hint), mock.patch.object( + preprocess, "compile_model", wraps=preprocess.compile_model + ) as compile_model: + self.execute_layer_test( + self.create_model(), sample_input, compile_specs=compile_specs + ) + self.assertEqual( + compile_model.call_args.args[2], {"INFERENCE_PRECISION_HINT": hint} + ) + + def test_addmm_refuses_an_unknown_inference_precision_before_converting(self): + sample_input = tuple(torch.randn(4, 4) for _ in range(3)) + with mock.patch.object( + preprocess, "FrontEndManager", side_effect=AssertionError("converted") + ), self.assertRaisesRegex(ValueError, "unsupported inference_precision 'f7'"): + self.execute_layer_test( + self.create_model(), + sample_input, + compile_specs=(CompileSpec("inference_precision", b"f7"),), + ) diff --git a/backends/openvino/tests/ops/test_reductions.py b/backends/openvino/tests/ops/test_reductions.py new file mode 100644 index 00000000000..66378bafb02 --- /dev/null +++ b/backends/openvino/tests/ops/test_reductions.py @@ -0,0 +1,220 @@ +# mypy: disable-error-code=import-not-found + +import itertools +import unittest +from unittest import mock + +import torch +from executorch.backends.openvino._passes import ( + fold_constant_cumsum_pass, + FoldConstantCumsumPass, +) +from executorch.backends.openvino.tests.ops.base_openvino_op_test import ( + BaseOpenvinoOpTest, +) +from executorch.exir import to_edge +from executorch.exir.dialects._ops import ops as exir_ops +from openvino.frontend.pytorch.torchdynamo.op_support import ( # type: ignore[import-untyped] + OperatorSupport, +) + + +class TestIntegerReductions(BaseOpenvinoOpTest): + """PyTorch reduces bool and small integer tensors to int64.""" + + def test_cumsum(self): + class CumSum(torch.nn.Module): + def forward(self, x): + return torch.cumsum(x, 1) + + for x in ( + torch.rand(2, 7) > 0.5, + torch.randint(0, 5, (2, 7), dtype=torch.int32), + ): + with self.subTest(dtype=x.dtype): + self.execute_layer_test(CumSum(), (x,)) + + def test_sum(self): + class Sum(torch.nn.Module): + def forward(self, x): + return x.sum(1) + + # More than 255 true values in a row, which 8 bits cannot count. + mask = torch.arange(800).reshape(2, 400) % 4 != 0 + for x in (mask, torch.randint(0, 5, (2, 7), dtype=torch.int32)): + with self.subTest(dtype=x.dtype): + self.execute_layer_test(Sum(), (x,)) + + def test_prod(self): + class Prod(torch.nn.Module): + def __init__(self, dim): + super().__init__() + self.dim = dim + + def forward(self, x): + return x.prod() if self.dim is None else x.prod(self.dim) + + support = OperatorSupport([])._support_dict + x = torch.randint(1, 4, (2, 5), dtype=torch.int32) + for dim, op in ((1, "prod.dim_int"), (None, "prod.default")): + with self.subTest(op=op): + if f"torch.ops.aten.{op}" not in support: + self.skipTest(f"this OpenVINO release does not delegate {op}") + self.execute_layer_test(Prod(dim), (x,)) + + def test_attention_masks_with_more_valid_tokens(self): + class AttentionMasks(torch.nn.Module): + """Masks and positions from padding, as pi0 builds them.""" + + def __init__(self, dtype, from_input): + super().__init__() + self.dtype = dtype + self.from_input = from_input + + def forward(self, pad_masks): + if self.from_input: + # Constant too, but only once OpenVINO folds the shape. + att_masks = torch.ones_like(pad_masks[0]) + else: + att_masks = torch.tensor([0] * 765 + [1] * 51, dtype=torch.bool) + cumsum = torch.cumsum(att_masks[None, :], dim=1, dtype=self.dtype) + att_2d = cumsum[:, None, :] <= cumsum[:, :, None] + pad_2d = pad_masks[:, None, :] * pad_masks[:, :, None] + return att_2d & pad_2d, torch.cumsum(pad_masks, dim=1) - 1 + + def pad_masks(valid): + return (torch.arange(816) < valid)[None, :] + + for dtype, from_input in itertools.product( + (torch.int64, torch.int32), (False, True) + ): + with self.subTest(dtype=dtype, from_input=from_input): + # One loaded program, run again with more valid tokens. + self.execute_layer_test( + AttentionMasks(dtype, from_input), + (pad_masks(300),), + more_inputs=((pad_masks(700),),), + ) + + +class TestFoldConstantCumsumPass(unittest.TestCase): + def fold(self, module, inputs): + """Edge program module of module, with constant cumsums folded.""" + edge = to_edge(torch.export.export(module, inputs)) + folded = edge.exported_program().module() + FoldConstantCumsumPass()(folded) + return folded + + def test_buffer_written_by_the_graph_is_not_folded(self): + class State(torch.nn.Module): + def __init__(self): + super().__init__() + self.register_buffer("s", torch.arange(3)) + + def forward(self, x): + self.s.add_(1) + return torch.cumsum(self.s, 0) + x + + x = torch.zeros(3, dtype=torch.int64) + folded = self.fold(State(), (x,)) + eager = State() + for _ in range(2): + torch.testing.assert_close(folded(x), eager(x)) + + def test_folded_name_does_not_replace_a_buffer(self): + class Model(torch.nn.Module): + def __init__(self): + super().__init__() + self.register_buffer( + "_folded_cumsum_aten_cumsum_default", torch.ones(3) + ) + + def forward(self, x): + counts = torch.cumsum(torch.ones(3, dtype=torch.int64), 0) + return counts + x + self._folded_cumsum_aten_cumsum_default + + x = torch.zeros(3) + folded = self.fold(Model(), (x,)) + # The folded cumsum would have taken this name. + self.assertTrue(hasattr(folded, "_folded_cumsum_aten_cumsum_default_")) + torch.testing.assert_close(folded(x), Model()(x)) + + def test_a_cumsum_that_cannot_be_evaluated_leaves_the_others_folded(self): + class Model(torch.nn.Module): + def forward(self, x): + ones = torch.ones(3, dtype=torch.int64) + return torch.cumsum(ones, 0) + torch.cumsum(ones * 2, 0) + x + + evaluate = fold_constant_cumsum_pass._evaluate + + def fail_on_the_doubled_one(graph_module, targets): + if any("mul" in str(n.args[0].target) for n in targets): + raise RuntimeError("not on the CPU") + return evaluate(graph_module, targets) + + x = torch.zeros(3, dtype=torch.int64) + with mock.patch.object( + fold_constant_cumsum_pass, "_evaluate", side_effect=fail_on_the_doubled_one + ), self.assertLogs(level="WARNING") as logs: + folded = self.fold(Model(), (x,)) + targets = [n.target for n in folded.graph.nodes] + self.assertEqual(targets.count(exir_ops.edge.aten.cumsum.default), 1) + self.assertIn("wrong results", logs.output[0]) + torch.testing.assert_close(folded(x), Model()(x)) + + def test_float_cumsum_is_not_folded(self): + class Model(torch.nn.Module): + def forward(self, x): + return torch.cumsum(torch.ones(3), 0) + x + + folded = self.fold(Model(), (torch.zeros(3),)) + self.assertIn( + exir_ops.edge.aten.cumsum.default, + [n.target for n in folded.graph.nodes], + ) + + def test_random_draws_are_not_folded(self): + class Model(torch.nn.Module): + def forward(self, x): + c = torch.ones(3) + r = torch.cond( + torch.tensor(True), + lambda c: (torch.rand(3) * 100 + c).long(), + lambda c: c.long(), + (c,), + ) + direct = (torch.rand(3) * 100).long() + return torch.cumsum(r, 0) + torch.cumsum(direct, 0) + x + + folded = self.fold(Model(), (torch.zeros(3, dtype=torch.int64),)) + targets = [n.target for n in folded.graph.nodes] + self.assertEqual(targets.count(exir_ops.edge.aten.cumsum.default), 2) + + def test_cumsum_over_a_split_constant_is_folded(self): + class Model(torch.nn.Module): + def forward(self, x): + part = torch.ones(6, dtype=torch.int64).split(3)[1] + return torch.cumsum(part, 0) + x + + x = torch.zeros(3, dtype=torch.int64) + folded = self.fold(Model(), (x,)) + self.assertNotIn( + exir_ops.edge.aten.cumsum.default, + [n.target for n in folded.graph.nodes], + ) + torch.testing.assert_close(folded(x), Model()(x)) + + def test_constants_are_folded_without_a_gpu(self): + class Model(torch.nn.Module): + def forward(self, x): + ones = torch.ones(3, dtype=torch.int64, device="cuda") + pinned = torch.ones(3, dtype=torch.int64, pin_memory=True) + return torch.cumsum(ones, 0).cpu() + torch.cumsum(pinned, 0) + x + + x = torch.zeros(3, dtype=torch.int64) + folded = self.fold(Model(), (x,)) + self.assertNotIn( + exir_ops.edge.aten.cumsum.default, + [n.target for n in folded.graph.nodes], + ) + torch.testing.assert_close(folded(x), torch.tensor([2, 4, 6])) diff --git a/backends/openvino/tests/ops/test_view.py b/backends/openvino/tests/ops/test_view.py index 5ea2f8cb1dc..792f39a9056 100644 --- a/backends/openvino/tests/ops/test_view.py +++ b/backends/openvino/tests/ops/test_view.py @@ -1,7 +1,16 @@ +import os +import tempfile + import torch +from executorch.backends.openvino.preprocess import OpenvinoBackend from executorch.backends.openvino.tests.ops.base_openvino_op_test import ( BaseOpenvinoOpTest, ) +from executorch.exir import to_edge +from executorch.exir.backend.backend_api import to_backend +from executorch.exir.backend.backend_details import CompileSpec +from executorch.runtime import Runtime +from torch.export import Dim, export op_params = [ {"input_shape": [2, 3, 2], "target_shape": [2, 6]}, @@ -36,11 +45,72 @@ def test_view(self): def test_view_data_dependent_size(self): class View(torch.nn.Module): + def __init__(self, reduce): + super().__init__() + self.reduce = reduce + def forward(self, x, n): n = n.item() torch._check(n >= 1) torch._check(n <= x.shape[0]) - return x[:n].view(n, -1).sum(0) + y = x[:n].view(n, -1) + return y.sum(0) if self.reduce else y + + # The size reaches the OpenVINO partition as a SymInt input. The second + # call needs a larger output than the first one returned, unless the + # sum makes the output static. + x = torch.randn(4, 8) + for reduce in (False, True): + with self.subTest(reduce=reduce): + self.execute_layer_test( + View(reduce), + (x, torch.tensor(3)), + more_inputs=((x, torch.tensor(4)),), + ) + + def test_one_delegate_called_with_different_output_sizes(self): + class Double(torch.nn.Module): + def forward(self, x): + return x * 2 + + x, y = torch.randn(2, 8), torch.randn(4, 8) + double = to_backend( + OpenvinoBackend.__name__, + to_edge( + export(Double(), (y,), dynamic_shapes=({0: Dim("n", min=1, max=4)},)) + ).exported_program(), + [CompileSpec("device", self.device.encode())], + ) + # Programs from ExecuTorch 0.7 and earlier run every call of a lowered + # module on one delegate handle, each call with its own outputs. + double._share_handle_across_calls = True + + class TwoCalls(torch.nn.Module): + def __init__(self): + super().__init__() + self.lowered = double + + def forward(self, x, y): + return self.lowered(x), self.lowered(y) + + dynamic_shapes = ( + {0: Dim("rows_x", min=1, max=2)}, + {0: Dim("rows_y", min=1, max=4)}, + ) + program = to_edge( + export(TwoCalls(), (x, y), dynamic_shapes=dynamic_shapes) + ).to_executorch() + self.assertEqual(len(program.executorch_program.execution_plan[0].delegates), 1) - # The size reaches the OpenVINO partition as a SymInt input. - self.execute_layer_test(View(), (torch.randn(4, 8), torch.tensor(3))) + with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp_dir: + pte_path = os.path.join(tmp_dir, "model.pte") + with open(pte_path, "wb") as f: + program.write_to_file(f) + loaded = Runtime.get().load_program(pte_path) + method = loaded.load_method("forward") + assert method is not None + for inputs in ((x, y), (y[:1], y[:3])): + outputs = method.execute(inputs) + for output, value in zip(outputs, inputs): + torch.testing.assert_close(output, value * 2) + del method, loaded diff --git a/docs/source/build-run-openvino.md b/docs/source/build-run-openvino.md index 69a73672ac1..c5f38a2d0d7 100644 --- a/docs/source/build-run-openvino.md +++ b/docs/source/build-run-openvino.md @@ -102,6 +102,51 @@ Follow the steps below to setup your build environment: ./openvino_build.sh ``` +## Precision + +When the device is `CPU`, a program computes in float32 by default. Without this default, +OpenVINO would compute in bfloat16 on x86 CPUs with bfloat16 units, which keeps only two to +three significant digits, and in float16 on ARM CPUs, which keeps about three. +Both are faster. To trade accuracy for speed, ask for the faster precision when +lowering: + +```python +from executorch.backends.openvino.partitioner import OpenvinoPartitioner +from executorch.exir.backend.compile_spec_schema import CompileSpec + +OpenvinoPartitioner( + [CompileSpec("device", b"CPU"), CompileSpec("inference_precision", b"bf16")] +) +``` + +On an ARM CPU, ask for `f16` instead: there float32 is about 1.8x slower than +float16, as measured on an Orin Nano. + +The value is OpenVINO's `INFERENCE_PRECISION_HINT`: `f32`, `f16` or `bf16`. An +empty value means the default, and any other value fails at lowering. A CPU +without float16 support computes an `f16` program in float32. OpenVINO has no +bfloat16 on ARM CPUs, so there `bf16` keeps the ARM default, float16 where the +CPU supports it. Which x86 CPUs compute in bfloat16 is up to OpenVINO; see its +[Precision Control](https://docs.openvino.ai/2026/openvino-workflow/running-inference/optimize-inference/precision-control.html) +page. Other devices, such as GPUs, keep OpenVINO's default unless the option is +set. + +The default also applies to a program exported in float16: on a CPU it computes +in float32 unless you lower it with `inference_precision`. + +On x86 CPUs, OpenVINO applies the precision when the runtime loads a program. +So a program lowered without `inference_precision` computes in float32, even +one exported with an ExecuTorch release that did not have this default. That +is slower and can use more memory. Export it again with `inference_precision` +set to `bf16` to get the earlier speed. On ARM CPUs, OpenVINO applies the +precision when the program is lowered, so a program exported with such an +earlier release keeps computing in float16 until it is exported again. + +OpenVINO's CPU plugin computes 64-bit integers in 32 bits. An integer result +outside the int32 range wraps around without an error. This applies to every +int64 tensor, including the int64 result of `cumsum` and `prod` over smaller +integer types. + ## Build Instructions for Examples ### AOT step: