-
Notifications
You must be signed in to change notification settings - Fork 844
NXP backend: Add support for aten.upsample_nearest2d.vec.
#17147
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
MartinPavella
merged 1 commit into
pytorch:main
from
nxp-upstream:nxg01483/EIEX-703-add-aten.upsample_nearest2d.vec-support-to-nxp-backend
Feb 9, 2026
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
107 changes: 107 additions & 0 deletions
107
...s/nxp/backend/ir/converter/node_converters/ops_converters/upsample_nearest2d_converter.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| # Copyright 2026 NXP | ||
| # | ||
| # This source code is licensed under the BSD-style license found in the | ||
| # LICENSE file in the root directory of this source tree. | ||
|
|
||
| import numpy as np | ||
|
|
||
| from executorch.backends.nxp.backend.data_format import DataFormat, NXP_NODE_FORMAT | ||
| from executorch.backends.nxp.backend.edge_helper import node_has_well_defined_shape | ||
| from executorch.backends.nxp.backend.ir.converter.node_converter import ( | ||
| CustomDelegationOptions, | ||
| NodeConverter, | ||
| ) | ||
| from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options.resize_nearest_neighbor_options import ( | ||
| ResizeNearestNeighbor, | ||
| ) | ||
| from executorch.backends.nxp.backend.neutron_target_spec import NeutronTargetSpec | ||
| from torch.fx import Node | ||
| from torch.nn import Parameter | ||
|
|
||
|
|
||
| # noinspection SpellCheckingInspection | ||
| class UpsampleNearest2DConverter(NodeConverter): | ||
|
|
||
| @staticmethod | ||
| def _is_supported_in_IR( | ||
| node: Node, | ||
| parameters_mapping: dict[str, Parameter], | ||
| custom_delegation_options: CustomDelegationOptions, | ||
| ) -> bool: | ||
|
|
||
| if node.meta.get(NXP_NODE_FORMAT, DataFormat.NONE) != DataFormat.CHANNELS_FIRST: | ||
| # This should never happen. | ||
| raise NotImplementedError( | ||
| "NXP backend: `aten.upsample_nearest2d.vec` didn't have correctly identified data" | ||
| " format. Please report this." | ||
| ) | ||
|
|
||
| return True | ||
|
|
||
| @staticmethod | ||
| def _is_supported_on_target( | ||
| node: Node, | ||
| neutron_target_spec: NeutronTargetSpec, | ||
| parameters_mapping: dict[str, Parameter], | ||
| custom_delegation_options: CustomDelegationOptions, | ||
| ) -> bool: | ||
| # Neutron requires static shapes. | ||
| # neutron-converter/src/OperatorC/UpsamplePlugin.cpp?at=NEUTRON_SOFTWARE_2.2.3#74 | ||
| if not node_has_well_defined_shape(node): | ||
| return False | ||
|
|
||
| if len(node.meta["val"].shape) != 4: | ||
| # Unexpected case. The input should always be 4D. | ||
| return False | ||
|
|
||
| # The tensors here use the channels first format (NCHW). | ||
| _, in_c, in_h, in_w = node.all_input_nodes[0].meta["val"].shape | ||
| _, _, out_h, out_w = node.meta["val"].shape | ||
|
|
||
| # Neutron supports only the doubling and quadrupleing of both height and width at the same time. | ||
| # neutron-library/src/utils/NeutronLibraryInterrogation.cpp?at=refs%2Ftags%2FNEUTRON_SOFTWARE_2.2.3#768 | ||
| # neutron-library/src/utils/NeutronLibraryInterrogation.cpp?at=refs%2Ftags%2FNEUTRON_SOFTWARE_2.2.3#778 | ||
| supported_scales = [2, 4] | ||
| if not any( | ||
| in_h * scale == out_h and in_w * scale == out_w | ||
| for scale in supported_scales | ||
| ): | ||
| return False | ||
|
|
||
| # Neutron requires the input channels to be a multiple of `num_macs`. | ||
| # neutron-library/src/utils/NeutronLibraryInterrogation.cpp?at=refs%2Ftags%2FNEUTRON_SOFTWARE_2.2.3#767 | ||
| if in_c % neutron_target_spec.get_num_macs() != 0: | ||
| return False | ||
|
|
||
| return True | ||
|
|
||
| def convert(self, node: Node): | ||
| """Convert the `aten.upsample_nearest2d.vec` operator to Neutron IR `ResizeNearestNeighbor`. | ||
| The schema is: | ||
| aten::upsample_nearest2d.vec( | ||
| Tensor input, | ||
| SymInt[]? output_size, | ||
| float[]? scale_factors | ||
| ) -> Tensor | ||
| """ | ||
| self.assert_convertible(node) | ||
|
|
||
| t_op = self._create_tflite_op_with_io_tensors(node) | ||
| x = t_op.tmp_inputs[0] | ||
| y = t_op.tmp_outputs[0] | ||
|
|
||
| t_op.builtin_options = ResizeNearestNeighbor(False, False) | ||
|
|
||
| # The `aten.upsample_nearest2d` can use either the `size` attribute or the `scale_factor` to define the output | ||
| # size. The Neutron IR `ResizeNearestNeighbor` only supports the `sizes` (output spatial dimensions). | ||
| # Both `size` and `scale_factor` can be easily supported by extracting the output spatial size from the output | ||
| # tensor's shape and using it as the `sizes`. | ||
| # The `self.assert_convertible(node)` call guarantees that the shape is 4D, channels last (NHWC), and static. | ||
| _, out_h, out_w, _ = y.shape | ||
| sizes = self.builder.create_tensor_for_data( | ||
| np.array([out_h, out_w], np.int32), "sizes" | ||
| ) | ||
|
|
||
| t_op.tmp_inputs = [x, sizes] # Assign the NeutronIR inputs. | ||
|
|
||
| self.builder.append_operators([t_op]) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
MartinPavella marked this conversation as resolved.
Show resolved
Hide resolved
|
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,181 @@ | ||
| # Copyright 2026 NXP | ||
| # | ||
| # This source code is licensed under the BSD-style license found in the | ||
| # LICENSE file in the root directory of this source tree. | ||
|
|
||
| import numpy as np | ||
| import pytest | ||
| import torch | ||
|
|
||
| from executorch.backends.nxp.backend.edge_program_converter import ( | ||
| EdgeProgramToIRConverter, | ||
| ) | ||
| from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program | ||
| from executorch.backends.nxp.tests.executors import ( | ||
| convert_run_compare, | ||
| graph_contains_any_of_ops, | ||
| ToChannelFirstPreprocess, | ||
| ToChannelLastPreprocess, | ||
| ) | ||
| from executorch.exir.dialects._ops import ops as exir_ops | ||
|
|
||
|
|
||
| @pytest.fixture(autouse=True) | ||
| def reseed_model_per_test_run(): | ||
| torch.manual_seed(42) | ||
| np.random.seed(23) | ||
|
|
||
|
|
||
| # noinspection PyProtectedMember | ||
| ExecutorchDelegateCall = torch._higher_order_ops.executorch_call_delegate | ||
| UpsampleNearest2D = exir_ops.edge.aten.upsample_nearest2d.vec | ||
|
|
||
|
|
||
| class UpsampleNearestModule(torch.nn.Module): | ||
|
|
||
| def __init__(self, size=None, scale=None): | ||
| super().__init__() | ||
| self.upsample = torch.nn.Upsample(size=size, scale_factor=scale, mode="nearest") | ||
|
|
||
| def forward(self, x): | ||
| return self.upsample(x) | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "input_shape, size", | ||
| [ | ||
| pytest.param((1, 8, 2, 3), (4, 6), id="2x upscale, 8 channels, tuple size"), | ||
| pytest.param((1, 8, 3, 3), 6, id="2x upscale, 8 channels, scalar size"), | ||
| pytest.param((1, 8, 2, 3), (8, 12), id="4x upscale, 8 channels, tuple size"), | ||
| pytest.param((1, 8, 3, 3), 12, id="4x upscale, 8 channels, scalar size"), | ||
| ], | ||
| ) | ||
| def test_convert_upsample_nearest2d__size(mocker, input_shape, size): | ||
| model = UpsampleNearestModule(size=size) | ||
|
|
||
| converter_spy = mocker.spy(EdgeProgramToIRConverter, "convert_program") | ||
| delegated_ep = to_quantized_edge_program( | ||
| model, input_shape, use_neutron_for_format_conversion=False | ||
| ).exported_program() | ||
|
|
||
| # Make sure the `upsample` was delegated. | ||
| assert graph_contains_any_of_ops(delegated_ep.graph, [ExecutorchDelegateCall]) | ||
| assert not graph_contains_any_of_ops(delegated_ep.graph, [UpsampleNearest2D]) | ||
|
|
||
| # Verify correct behavior of the converted NeutronIR model. | ||
| intermediate_ep = converter_spy.call_args.args[1] | ||
| neutron_ir_model, _ = converter_spy.spy_return | ||
|
|
||
| input_data = ( | ||
| np.random.random(input_shape).astype(np.float32) * 256.0 - 128.0 | ||
| ).astype(np.int8) | ||
|
|
||
| # Make sure the tested program contains the `upsample`. | ||
| assert graph_contains_any_of_ops(intermediate_ep.graph, [UpsampleNearest2D]) | ||
|
|
||
| convert_run_compare( | ||
| intermediate_ep, | ||
| tfl_model=neutron_ir_model, | ||
| input_data=input_data, | ||
| tflite_input_preprocess=ToChannelLastPreprocess(), | ||
| tflite_output_preprocess=ToChannelFirstPreprocess(), | ||
| ) | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "input_shape, scale_factor", | ||
| [ | ||
| pytest.param((1, 8, 2, 3), 2, id="2x upscale, 8 channels, scalar scale"), | ||
| pytest.param((1, 8, 3, 3), 2.0, id="2x upscale, 8 channels, float scale"), | ||
| pytest.param((1, 8, 4, 5), (2, 2), id="2x upscale, 8 channels, tuple scale"), | ||
| pytest.param((1, 8, 2, 3), 4, id="4x upscale, 8 channels, scalar scale"), | ||
| pytest.param((1, 8, 2, 3), (4, 4), id="4x upscale, 8 channels, tuple scale"), | ||
| ], | ||
| ) | ||
| def test_convert_upsample_nearest2d__scale_factor(mocker, input_shape, scale_factor): | ||
| model = UpsampleNearestModule(scale=scale_factor) | ||
|
|
||
| converter_spy = mocker.spy(EdgeProgramToIRConverter, "convert_program") | ||
| delegated_ep = to_quantized_edge_program( | ||
| model, input_shape, use_neutron_for_format_conversion=False | ||
| ).exported_program() | ||
|
|
||
| # Make sure the `upsample` was delegated. | ||
| assert graph_contains_any_of_ops(delegated_ep.graph, [ExecutorchDelegateCall]) | ||
| assert not graph_contains_any_of_ops(delegated_ep.graph, [UpsampleNearest2D]) | ||
|
|
||
| # Verify correct behavior of the converted NeutronIR model. | ||
| intermediate_ep = converter_spy.call_args.args[1] | ||
| neutron_ir_model, _ = converter_spy.spy_return | ||
|
|
||
| input_data = ( | ||
| np.random.random(input_shape).astype(np.float32) * 256.0 - 128.0 | ||
| ).astype(np.int8) | ||
|
|
||
| # Make sure the tested program contains the `upsample`. | ||
| assert graph_contains_any_of_ops(intermediate_ep.graph, [UpsampleNearest2D]) | ||
|
|
||
| convert_run_compare( | ||
| intermediate_ep, | ||
| tfl_model=neutron_ir_model, | ||
| input_data=input_data, | ||
| tflite_input_preprocess=ToChannelLastPreprocess(), | ||
| tflite_output_preprocess=ToChannelFirstPreprocess(), | ||
| ) | ||
|
|
||
|
|
||
| def test_convert_upsample_nearest2d__no_delegation__unsupported_channels(): | ||
| size = 6 | ||
| input_shape = (1, 2, size // 2, size // 2) # 2 channels, not `num_macs`. | ||
| model = UpsampleNearestModule(size=size) | ||
|
|
||
| delegated_ep = to_quantized_edge_program( | ||
| model, input_shape, use_neutron_for_format_conversion=False | ||
| ).exported_program() | ||
|
|
||
| # Make sure the `upsample` was NOT delegated (channels != 8). | ||
| assert not graph_contains_any_of_ops(delegated_ep.graph, [ExecutorchDelegateCall]) | ||
| assert graph_contains_any_of_ops(delegated_ep.graph, [UpsampleNearest2D]) | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "input_shape, scale_factor", | ||
| [ | ||
| pytest.param((1, 8, 4, 4), 3, id="3x upscale"), | ||
| pytest.param((1, 8, 4, 4), 1.5, id="1.5x upscale"), | ||
| pytest.param((1, 8, 4, 4), (2, 4), id="2x and 4x mixed upscale"), | ||
| pytest.param((1, 8, 10, 10), 1.99, id="1.99x upscale"), | ||
| ], | ||
| ) | ||
| def test_convert_upsample_nearest2d__no_delegation__unsupported_scale( | ||
| input_shape, scale_factor | ||
| ): | ||
| model = UpsampleNearestModule(scale=scale_factor) | ||
|
|
||
| delegated_ep = to_quantized_edge_program( | ||
| model, input_shape, use_neutron_for_format_conversion=False | ||
| ).exported_program() | ||
|
|
||
| # Make sure the `upsample` was NOT delegated (scale != 2). | ||
| assert not graph_contains_any_of_ops(delegated_ep.graph, [ExecutorchDelegateCall]) | ||
| assert graph_contains_any_of_ops(delegated_ep.graph, [UpsampleNearest2D]) | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "input_shape, size", | ||
| [ | ||
| pytest.param((1, 8, 2, 3), (6, 9), id="3x upscale"), | ||
| pytest.param((1, 8, 2, 4), (3, 6), id="1.5x upscale"), | ||
| pytest.param((1, 8, 3, 4), 6, id="non-uniform upscale"), | ||
| ], | ||
| ) | ||
| def test_convert_upsample_nearest2d__no_delegation__unsupported_size(input_shape, size): | ||
| model = UpsampleNearestModule(size=size) | ||
|
|
||
| delegated_ep = to_quantized_edge_program( | ||
| model, input_shape, use_neutron_for_format_conversion=False | ||
| ).exported_program() | ||
|
|
||
| # Make sure the `upsample` was NOT delegated (size != double of input). | ||
| assert not graph_contains_any_of_ops(delegated_ep.graph, [ExecutorchDelegateCall]) | ||
| assert graph_contains_any_of_ops(delegated_ep.graph, [UpsampleNearest2D]) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.