From ee2962a50ce8ac0583b5b0d40dd46ff04428c8aa Mon Sep 17 00:00:00 2001 From: Maxim Vafin Date: Wed, 3 Apr 2024 07:56:46 +0200 Subject: [PATCH] [OVC] Allow models obtained from torch.export (#23815) ### Details: - *Allow models obtained from `torch.export`* ### Tickets: - *ticket-id* --- .../openvino/frontend/pytorch/fx_decoder.py | 20 ++++++++-- src/frontends/pytorch/src/op/sum.cpp | 5 +++ src/frontends/pytorch/src/op_table.cpp | 1 + src/frontends/pytorch/src/place.cpp | 16 ++++---- .../pytorch_tests/pytorch_layer_test_class.py | 10 +---- .../test_scaled_dot_product_attention.py | 2 + .../pytorch/hf_transformers_models | 2 +- tests/model_hub_tests/pytorch/torch_utils.py | 37 +++++++------------ .../moc_frontend/pytorch_frontend_utils.py | 6 ++- 9 files changed, 55 insertions(+), 44 deletions(-) diff --git a/src/bindings/python/src/openvino/frontend/pytorch/fx_decoder.py b/src/bindings/python/src/openvino/frontend/pytorch/fx_decoder.py index 826f41984d8..70c8c66d42a 100644 --- a/src/bindings/python/src/openvino/frontend/pytorch/fx_decoder.py +++ b/src/bindings/python/src/openvino/frontend/pytorch/fx_decoder.py @@ -37,15 +37,29 @@ class TorchFXPythonDecoder (Decoder): self._nodes = list(pt_module.graph.nodes) self._inputs = [] self._outputs = [] + found_types = [] + found_shapes = [] for i in range(len(self._nodes)): if self._nodes[i].op == 'placeholder': self._inputs.append(i) - self._input_signature.append(self._nodes[i].name) + value = self._nodes[i] + self._input_signature.append(value.name) + if hasattr(value, "meta") and ('tensor_meta' in value.meta.keys()) and value.meta['tensor_meta']: + found_shapes.append(value.meta['tensor_meta'].shape) + found_types.append(OVAny(pt_to_ov_type_map[str(value.meta['tensor_meta'].dtype)])) + else: + found_shapes.append(None) + found_types.append(None) elif self._nodes[i].op == 'output': # Instead of putting output index, refer to its target uargs = self.unpack_containers(self._nodes[i].args) self._outputs = [(arg[0], self._nodes.index(arg[1])) for arg in uargs if arg[1] is not None] + if not input_shapes or len(input_shapes) == 0: + self.input_shapes = found_shapes + if not input_types or len(input_types) == 0: + self.input_types = found_types + elif issubclass(type(pt_module), torch.fx.Node): self._nodes = nodes # passed from outer context @@ -142,7 +156,7 @@ class TorchFXPythonDecoder (Decoder): return self.get_input_debug_name(index) def get_input_shape(self, index): - if index < len(self.input_shapes): + if index < len(self.input_shapes) and self.input_shapes[index] is not None: return PartialShape(self.input_shapes[index]) input = self._raw_input(index) return self.get_shape_for_value(input) @@ -158,7 +172,7 @@ class TorchFXPythonDecoder (Decoder): return [] def get_input_type(self, index): - if index < len(self.input_types): + if index < len(self.input_types) and self.input_types[index] is not None: return self.input_types[index] input = self._raw_input(index) return self.get_type_for_value(input) diff --git a/src/frontends/pytorch/src/op/sum.cpp b/src/frontends/pytorch/src/op/sum.cpp index 1ad2734c2bf..363ba4cb4ef 100644 --- a/src/frontends/pytorch/src/op/sum.cpp +++ b/src/frontends/pytorch/src/op/sum.cpp @@ -89,6 +89,11 @@ OutputVector translate_sum_fx(const NodeContext& context) { axes = get_axes_range(context, 0); } else { axes = context.get_input(1); + // empty constant means default axes + if (const auto constant = ov::as_type_ptr(axes.get_node_shared_ptr())) { + if (constant->get_byte_size() == 0) + axes = get_axes_range(context, 0); + } } if (!context.input_is_none(2)) { keep_dims = context.const_input(2); diff --git a/src/frontends/pytorch/src/op_table.cpp b/src/frontends/pytorch/src/op_table.cpp index aa10e8d602e..638305ff96e 100644 --- a/src/frontends/pytorch/src/op_table.cpp +++ b/src/frontends/pytorch/src/op_table.cpp @@ -775,6 +775,7 @@ const std::map get_supported_ops_fx() { {"aten.ceil.default", op::translate_1to1_match_1_inputs}, {"aten.celu.default", op::translate_celu}, {"aten.clamp.default", op::translate_clamp}, + {"aten.clamp.Tensor", op::translate_clamp}, {"aten.clamp_max.default", op::translate_1to1_match_2_inputs_align_types}, {"aten.clamp_max.Tensor", op::translate_1to1_match_2_inputs_align_types}, {"aten.clamp_min.default", op::translate_1to1_match_2_inputs_align_types}, diff --git a/src/frontends/pytorch/src/place.cpp b/src/frontends/pytorch/src/place.cpp index ad7e46acf24..968cad19988 100644 --- a/src/frontends/pytorch/src/place.cpp +++ b/src/frontends/pytorch/src/place.cpp @@ -38,15 +38,17 @@ Place::Place(const ov::frontend::InputModel& input_model, size_t tensor_index) auto out_it = std::find(outputs.begin(), outputs.end(), tensor_index); if (out_it != outputs.end()) { m_is_output = true; - auto idx = std::distance(outputs.begin(), out_it); - const auto& debug_name = decoder->get_output_debug_name(idx); - m_names.push_back(debug_name); + if (!m_is_input) { + auto idx = std::distance(outputs.begin(), out_it); + const auto& debug_name = decoder->get_output_debug_name(idx); + m_names.push_back(debug_name); - auto type_any = simplified_type_interpret(decoder->get_output_type(idx)); - if (type_any.is()) { - m_type = type_any.as(); + auto type_any = simplified_type_interpret(decoder->get_output_type(idx)); + if (type_any.is()) { + m_type = type_any.as(); + } + m_pshape = decoder->get_output_shape(idx); } - m_pshape = decoder->get_output_shape(idx); } if (m_is_input && m_is_output) { OPENVINO_DEBUG << "[WARNING] Place " << tensor_index << " is input and output at a same time."; diff --git a/tests/layer_tests/pytorch_tests/pytorch_layer_test_class.py b/tests/layer_tests/pytorch_tests/pytorch_layer_test_class.py index b0c804a659b..ffd49015afc 100644 --- a/tests/layer_tests/pytorch_tests/pytorch_layer_test_class.py +++ b/tests/layer_tests/pytorch_tests/pytorch_layer_test_class.py @@ -102,16 +102,8 @@ class PytorchLayerTest: gm = em.module() print(gm.code) - input_shapes = [] - input_types = [] - for input_data in torch_inputs: - input_types.append(input_data.type()) - input_shapes.append(input_data.size()) - - decoder = TorchFXPythonDecoder( - gm, gm, input_shapes=input_shapes, input_types=input_types) converted_model = convert_model( - decoder, example_input=torch_inputs) + em, example_input=torch_inputs) self._resolve_input_shape_dtype( converted_model, ov_inputs, dynamic_shapes) smodel = model diff --git a/tests/layer_tests/pytorch_tests/test_scaled_dot_product_attention.py b/tests/layer_tests/pytorch_tests/test_scaled_dot_product_attention.py index 3d1374ea2d8..bd6fc584885 100644 --- a/tests/layer_tests/pytorch_tests/test_scaled_dot_product_attention.py +++ b/tests/layer_tests/pytorch_tests/test_scaled_dot_product_attention.py @@ -43,5 +43,7 @@ class TestScaledDotProductAttention(PytorchLayerTest): @pytest.mark.parametrize(['mask', 'is_causal'], [(False, False), (False, True), (True, True), (True, False)]) @pytest.mark.parametrize("dtype", (np.float32, np.float64)) def test_scaled_dot_product_atten(self, ie_device, precision, ir_version, mask, is_causal, dtype): + if PytorchLayerTest.use_torch_export() and not mask and is_causal: + pytest.xfail(reason="Unsupported case for torch.export") self._test(*self.create_model(mask, is_causal, dtype), ie_device, precision, ir_version, kwargs_to_prepare_input={"dtype": dtype}) diff --git a/tests/model_hub_tests/pytorch/hf_transformers_models b/tests/model_hub_tests/pytorch/hf_transformers_models index adbbfc468b3..f2596873cac 100644 --- a/tests/model_hub_tests/pytorch/hf_transformers_models +++ b/tests/model_hub_tests/pytorch/hf_transformers_models @@ -50,7 +50,7 @@ connor-henderson/fastspeech2_conformer_with_hifigan,fastspeech2_conformer_with_h csarron/meter-vqa2-ft,meter,skip,Load problem ctrl,ctrl cwkeam/mctc-large,mctc,skip,Load problem -dandelin/vilt-b32-finetuned-vqa,vilt,xfail,Unsupported op aten::_unique2 aten::multinomial +dandelin/vilt-b32-finetuned-vqa,vilt,xfail,Accuracy due to random dangkhoadl/custom_CNN_1D,cnn,skip,Load problem declare-lab/segue-w2v2-base,segue,skip,Load problem deepesh0x/autotrain-mlsec-1013333726,julien,skip,Load problem diff --git a/tests/model_hub_tests/pytorch/torch_utils.py b/tests/model_hub_tests/pytorch/torch_utils.py index d7f02044eb8..2baa62f5cbd 100644 --- a/tests/model_hub_tests/pytorch/torch_utils.py +++ b/tests/model_hub_tests/pytorch/torch_utils.py @@ -66,37 +66,28 @@ class TestTorchConvertModel(TestConvertModel): if hasattr(self, "mode") and self.mode == "export": from torch.export import export from packaging import version - from openvino.frontend.pytorch.fx_decoder import TorchFXPythonDecoder - input_shapes = [] - input_types = [] model_obj.eval() - # need to infer before export to initialize everything, otherwise it will be initialized with FakeTensors if isinstance(self.example, dict): + from openvino.frontend.pytorch.fx_decoder import TorchFXPythonDecoder + + # need to infer before export to initialize everything, otherwise it will be initialized with FakeTensors pt_res = model_obj(**self.example) + graph = export(model_obj, tuple(), self.example) + if version.parse(torch.__version__) >= version.parse("2.2"): + graph = graph.run_decompositions() + + gm = graph.module() + print(gm.code) + + decoder = TorchFXPythonDecoder(gm, gm) + decoder._input_signature = list(self.example.keys()) + ov_model = convert_model(decoder, verbose=True) else: pt_res = model_obj(*self.example) - if isinstance(self.example, dict): - graph = export(model_obj, tuple(), self.example) - for input_data in self.example.values(): - input_types.append(input_data.type()) - input_shapes.append(input_data.size()) - else: graph = export(model_obj, self.example) - for input_data in self.example: - input_types.append(input_data.type()) - input_shapes.append(input_data.size()) - if version.parse(torch.__version__) >= version.parse("2.2"): - graph = graph.run_decompositions() + ov_model = convert_model(graph, verbose=True) - gm = graph.module() - print(gm.code) - - decoder = TorchFXPythonDecoder(gm, gm, input_shapes=input_shapes, input_types=input_types) - print(list(gm.graph.nodes)[-1].args) - if isinstance(self.example, dict): - decoder._input_signature = list(self.example.keys()) - ov_model = convert_model(decoder, example_input=self.example) if isinstance(pt_res, dict): for i, k in enumerate(pt_res.keys()): ov_model.outputs[i].get_tensor().set_names({k}) diff --git a/tools/ovc/openvino/tools/ovc/moc_frontend/pytorch_frontend_utils.py b/tools/ovc/openvino/tools/ovc/moc_frontend/pytorch_frontend_utils.py index 67eb07c1a25..c1696c9898e 100644 --- a/tools/ovc/openvino/tools/ovc/moc_frontend/pytorch_frontend_utils.py +++ b/tools/ovc/openvino/tools/ovc/moc_frontend/pytorch_frontend_utils.py @@ -44,7 +44,11 @@ def get_pytorch_decoder(model, example_inputs, args): inputs = prepare_torch_inputs(example_inputs) if not isinstance(model, (TorchScriptPythonDecoder, TorchFXPythonDecoder)): if hasattr(torch, "export") and isinstance(model, (torch.export.ExportedProgram)): - raise RuntimeError("Models received from torch.export are not yet supported by convert_model.") + from packaging import version + if version.parse(torch.__version__) >= version.parse("2.2"): + model = model.run_decompositions() + gm = model.module() + decoder = TorchFXPythonDecoder(gm, gm) else: decoder = TorchScriptPythonDecoder( model,