From e7c60bb18a6f0e524890cbacc71adc7bb3837b05 Mon Sep 17 00:00:00 2001 From: Surya Siddharth Pemmaraju Date: Wed, 22 May 2024 12:59:36 +0530 Subject: [PATCH] Fixed bug with inlined inputs (#24448) ### Details: - Fixed bug with inlined inputs for fx decoder which were being added as parameters and are not needed. ### Tickets: - (https://jira.devtools.intel.com/browse/CVS-116702) --------- Co-authored-by: Maxim Vafin --- .../openvino/frontend/pytorch/node_context.hpp | 10 +--------- src/frontends/pytorch/src/node_context.cpp | 18 ++++++++++++++++++ .../pytorch/src/translate_session.cpp | 4 ++++ src/frontends/pytorch/src/utils.cpp | 6 +++--- .../py_frontend_tests/test_torch_frontend.py | 17 ++++++++++++++++- .../pytorch_tests/pytorch_layer_test_class.py | 10 ++++++++++ tests/layer_tests/pytorch_tests/test_arange.py | 4 ++-- 7 files changed, 54 insertions(+), 15 deletions(-) diff --git a/src/frontends/pytorch/include/openvino/frontend/pytorch/node_context.hpp b/src/frontends/pytorch/include/openvino/frontend/pytorch/node_context.hpp index 02c4b72d088..b1f9153835e 100644 --- a/src/frontends/pytorch/include/openvino/frontend/pytorch/node_context.hpp +++ b/src/frontends/pytorch/include/openvino/frontend/pytorch/node_context.hpp @@ -83,15 +83,7 @@ public: Any get_values_from_const_input(int index) const override; - // TODO: upstream to base class - OutputVector inputs() const { - OutputVector res; - for (auto input : m_decoder_inputs) { - FRONT_END_GENERAL_CHECK(m_tensor_map->count(input), "No tensor corresponding index: ", input, " exist."); - res.push_back(m_tensor_map->at(input)); - } - return res; - } + OutputVector inputs() const; Any get_input_type(size_t index) const { return m_decoder->get_input_type(index); diff --git a/src/frontends/pytorch/src/node_context.cpp b/src/frontends/pytorch/src/node_context.cpp index ff66b31ec7d..5b10de3e084 100644 --- a/src/frontends/pytorch/src/node_context.cpp +++ b/src/frontends/pytorch/src/node_context.cpp @@ -147,6 +147,24 @@ std::shared_ptr NodeContext::convert_subgraph(size_t index) const { return model; } +OutputVector NodeContext::inputs() const { + OutputVector res; + for (size_t i = 0; i < m_decoder_inputs.size(); i++) { + auto input = m_decoder_inputs.at(i); + if (input == 0) { + // Case when input can be inlined (possible only for fx decoder) + if (m_decoder->is_input_inlined(i)) { + auto inlined_input = m_decoder->inlined_input(i); + FRONT_END_GENERAL_CHECK(inlined_input.size() == 1, "Incorrect inlined input with index:", i); + res.push_back(inlined_input[0]); + } + } + FRONT_END_GENERAL_CHECK(m_tensor_map->count(input), "No tensor corresponding input: ", input, " exist."); + res.push_back(m_tensor_map->at(input)); + } + return res; +} + bool NodeContext::input_is_none(size_t index) const { bool res = index >= m_inputs_is_none.size() || m_inputs_is_none.at(index); if (!res) { diff --git a/src/frontends/pytorch/src/translate_session.cpp b/src/frontends/pytorch/src/translate_session.cpp index 96484d4726f..60410f97c33 100644 --- a/src/frontends/pytorch/src/translate_session.cpp +++ b/src/frontends/pytorch/src/translate_session.cpp @@ -129,6 +129,10 @@ std::shared_ptr TranslateSession::convert_pytorch_model( auto raw_inputs = node->inputs(); for (size_t i = 0; i < raw_inputs.size(); ++i) { auto input = raw_inputs.at(i); + // If inputs are inlined (possible only for fx decoder) we shouldn't add a Parameter for it + if (input == 0 && node->is_input_inlined(i)) { + continue; + } if (tensor_map->find(input) == tensor_map->end()) { // Input refers value in the outer scope, need to create a new Parameter in the current scope // Linkage to external scope will be performed on the level of the parent operation (if or loop) diff --git a/src/frontends/pytorch/src/utils.cpp b/src/frontends/pytorch/src/utils.cpp index d788d47f5dc..0e6d0535763 100644 --- a/src/frontends/pytorch/src/utils.cpp +++ b/src/frontends/pytorch/src/utils.cpp @@ -18,9 +18,9 @@ namespace frontend { namespace pytorch { void num_inputs_check(const NodeContext& context, size_t min_inputs, size_t max_inputs) { - auto inputs = context.inputs(); - FRONT_END_OP_CONVERSION_CHECK(inputs.size() >= min_inputs, "Got less inputs than expected"); - for (auto i = max_inputs; i < inputs.size(); i++) { + auto num_inputs = context.get_input_size(); + FRONT_END_OP_CONVERSION_CHECK(num_inputs >= min_inputs, "Got less inputs than expected"); + for (auto i = max_inputs; i < num_inputs; i++) { FRONT_END_OP_CONVERSION_CHECK(context.input_is_none(i), "Got more inputs than expected."); } } diff --git a/tests/layer_tests/py_frontend_tests/test_torch_frontend.py b/tests/layer_tests/py_frontend_tests/test_torch_frontend.py index 0d0d0d3e6e1..18b3a4776f6 100644 --- a/tests/layer_tests/py_frontend_tests/test_torch_frontend.py +++ b/tests/layer_tests/py_frontend_tests/test_torch_frontend.py @@ -720,7 +720,7 @@ def test_patched_16bit_model_converts(): res_fp16 = cm_fp16([x.numpy() for x in example]) np.testing.assert_allclose(res_fp16[0], res_ref[0].numpy(), atol=1e-2) np.testing.assert_allclose(res_fp16[1], res_ref[1].numpy(), atol=1e-2) - + model_bf16 = copy.deepcopy(model_ref).bfloat16() patch_model.__make_16bit_traceable(model_bf16) # the approach with patching only works for node with no grad @@ -731,3 +731,18 @@ def test_patched_16bit_model_converts(): res_bf16 = cm_bf16([x.numpy() for x in example]) np.testing.assert_allclose(res_bf16[0], res_ref[0].numpy(), atol=1e-2) np.testing.assert_allclose(res_bf16[1], res_ref[1].numpy(), atol=1e-2) + + +class InlinedInputsModel(torch.nn.Module): + def __init__(self): + super().__init__() + + def forward(self): + return torch.arange(2048) + + +def test_inlined_inputs(): + model = InlinedInputsModel() + model.eval() + model = torch.compile(model, backend="openvino", options={"testing": 1}) + model() diff --git a/tests/layer_tests/pytorch_tests/pytorch_layer_test_class.py b/tests/layer_tests/pytorch_tests/pytorch_layer_test_class.py index e364b9a7f05..6662bb78ca0 100644 --- a/tests/layer_tests/pytorch_tests/pytorch_layer_test_class.py +++ b/tests/layer_tests/pytorch_tests/pytorch_layer_test_class.py @@ -17,9 +17,19 @@ import torch from packaging import version import pytest + +def skip_check(param): + return skip_if_export(param) if PytorchLayerTest.use_torch_export() else skip_if_fx(param) + + def skip_if_export(param, reason="Unsupported on torch.export"): return pytest.param(param, marks=pytest.mark.skipif(PytorchLayerTest.use_torch_export(), reason=reason)) + +def skip_if_fx(param, reason="Unsupported on torch.fx"): + return pytest.param(param, marks=pytest.mark.skipif(PytorchLayerTest.use_torch_compile_backend(), reason=reason)) + + class PytorchLayerTest: _type_map = { "float64": Type.f64, diff --git a/tests/layer_tests/pytorch_tests/test_arange.py b/tests/layer_tests/pytorch_tests/test_arange.py index d871b09c0e4..7f35ea974b3 100644 --- a/tests/layer_tests/pytorch_tests/test_arange.py +++ b/tests/layer_tests/pytorch_tests/test_arange.py @@ -3,7 +3,7 @@ import pytest -from pytorch_layer_test_class import PytorchLayerTest, skip_if_export +from pytorch_layer_test_class import PytorchLayerTest, skip_check class TestArange(PytorchLayerTest): @@ -112,7 +112,7 @@ class TestArange(PytorchLayerTest): @pytest.mark.precommit_fx_backend @pytest.mark.parametrize("dtype", [None, "float32", "float64", "int32", "int64", "int8", "uin8"]) @pytest.mark.parametrize("end", [1, 2, 3]) - @pytest.mark.parametrize("use_out", [skip_if_export(True), False]) + @pytest.mark.parametrize("use_out", [skip_check(True), False]) def test_arange_end_only(self, dtype, end, use_out, ie_device, precision, ir_version): self._test(*self.create_model(dtype, 1, use_out), ie_device, precision, ir_version, kwargs_to_prepare_input={"end": end})