diff --git a/src/frontends/pytorch/src/op/full.cpp b/src/frontends/pytorch/src/op/full.cpp index cf60d096555..e8bfa1c7ce9 100644 --- a/src/frontends/pytorch/src/op/full.cpp +++ b/src/frontends/pytorch/src/op/full.cpp @@ -3,10 +3,19 @@ // #include "openvino/frontend/pytorch/node_context.hpp" +#include "openvino/op/add.hpp" #include "openvino/op/broadcast.hpp" #include "openvino/op/constant.hpp" #include "openvino/op/convert_like.hpp" +#include "openvino/op/divide.hpp" +#include "openvino/op/gather.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/power.hpp" +#include "openvino/op/range.hpp" +#include "openvino/op/reshape.hpp" +#include "openvino/op/scatter_elements_update.hpp" #include "openvino/op/shape_of.hpp" +#include "openvino/op/squeeze.hpp" #include "utils.hpp" namespace ov { @@ -71,12 +80,17 @@ OutputVector translate_full_like(const NodeContext& context) { return {base_translate_full_with_convertlike(context, sizes, value, out)}; }; -OutputVector translate_fill_(const NodeContext& context) { - num_inputs_check(context, 2, 2); +OutputVector translate_fill(const NodeContext& context) { + num_inputs_check(context, 2, 3); auto input = context.get_input(0); auto value = context.get_input(1); auto sizes = context.mark_node(std::make_shared(input, element::i32)); - return {base_translate_full_with_convertlike(context, sizes, value, input)}; + auto out = context.input_is_none(2) ? input : context.get_input(2); + auto result = base_translate_full_with_convertlike(context, sizes, value, out); + if (!context.input_is_none(2)) { + context.mutate_input(2, result); + } + return {result}; }; OutputVector translate_new_full(const NodeContext& context) { @@ -187,6 +201,67 @@ OutputVector translate_empty(const NodeContext& context) { } return {empty}; }; + +OutputVector translate_fill_diagonal(const NodeContext& context) { + // aten::fill_diagonal_(Tensor(a!) self, Scalar fill_value, bool wrap=False) -> Tensor(a!) + // realization inspired by numpy: + // https://github.com/numpy/numpy/blob/c236e694d222ae6b812cb8dab54471bc4c912f0f/numpy/lib/_index_tricks_impl.py#L787-L918 + num_inputs_check(context, 3, 3); + auto input_tensor = context.get_input(0); + auto fill_value = context.get_input(1); + auto input_shape = context.mark_node(std::make_shared(input_tensor, element::i32)); + auto input_rank = input_tensor.get_partial_shape().rank(); + auto const_one = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {1})); + auto const_zero = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {0})); + auto const_one_s = context.mark_node(v0::Constant::create(element::i32, Shape{}, {1})); + auto const_zero_s = context.mark_node(v0::Constant::create(element::i32, Shape{}, {0})); + auto const_neg_one = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {-1})); + if (input_rank.is_dynamic() || input_rank.get_length() < 2) { + FRONT_END_OP_CONVERSION_CHECK(false, "aten::fill_diagonal_ required tensor with static rank >= 2 "); + } + auto flatten_input = context.mark_node(std::make_shared(input_tensor, const_neg_one, false)); + auto wrap = context.const_input(2); + Output step; + // default value for end - number of elements in input tensor + Output end; + auto flatten_shape = context.mark_node(std::make_shared(flatten_input, element::i32)); + end = context.mark_node(std::make_shared(flatten_shape, const_neg_one, const_zero)); + auto last_dim = context.mark_node(std::make_shared(input_shape, const_neg_one, const_zero)); + if (input_rank.get_length() == 2) { + // step = a.shape[1] + 1 + step = context.mark_node(std::make_shared(last_dim, const_one_s)); + if (!wrap) { + // if not wrap. and non squared matrix, do not fill tail by cutting end to square + end = context.mark_node(std::make_shared(last_dim, last_dim)); + } + } else { + // step = 1 + (cumprod(a.shape[:-1])).sum() + // cumprod operation is not supported by ov, but with condition that >2D tensors supported only if all dims + // equals cumprod can be represented as finite geometric serial and its sum can be found by formula + // b0 * (bn * q - 1) / (q - 1), where in this particual case q = b0, bn = b0 ^ n + auto rank_minus_one = + context.mark_node(v0::Constant::create(element::i32, Shape{}, {input_rank.get_length() - 1})); + auto dim_power = context.mark_node(std::make_shared(last_dim, rank_minus_one)); + auto dim_power_minus_one = context.mark_node(std::make_shared(dim_power, const_neg_one)); + auto dim_minus_one = context.mark_node(std::make_shared(last_dim, const_neg_one)); + auto q = context.mark_node(std::make_shared(dim_power_minus_one, dim_minus_one, true)); + auto cumprod_sum = context.mark_node(std::make_shared(last_dim, q)); + step = context.mark_node(std::make_shared(const_one_s, cumprod_sum)); + // wrap parameter is not applicable in this case as supported only equal dims on pytorch side + } + step = context.mark_node(std::make_shared(step, const_zero)); + end = context.mark_node(std::make_shared(end, const_zero)); + auto indices = context.mark_node(std::make_shared(const_zero_s, end, step, element::i32)); + auto indices_shape = context.mark_node(std::make_shared(indices, element::i32)); + fill_value = context.mark_node(std::make_shared(fill_value, input_tensor)); + fill_value = context.mark_node(std::make_shared(fill_value, indices_shape)); + // fill values + auto filled_tensor = + context.mark_node(std::make_shared(flatten_input, indices, fill_value, const_zero)); + // reshape back to original shape + filled_tensor = context.mark_node(std::make_shared(filled_tensor, input_shape, false)); + return {filled_tensor}; +} } // namespace op } // namespace pytorch } // namespace frontend diff --git a/src/frontends/pytorch/src/op_table.cpp b/src/frontends/pytorch/src/op_table.cpp index 47969ddb57d..75665ffe8d4 100644 --- a/src/frontends/pytorch/src/op_table.cpp +++ b/src/frontends/pytorch/src/op_table.cpp @@ -66,7 +66,8 @@ OP_CONVERTER(translate_expand_as); OP_CONVERTER(translate_eye); OP_CONVERTER(translate_fake_quantize_per_channel_affine); OP_CONVERTER(translate_fake_quantize_per_tensor_affine); -OP_CONVERTER(translate_fill_); +OP_CONVERTER(translate_fill); +OP_CONVERTER(translate_fill_diagonal); OP_CONVERTER(translate_flatten); OP_CONVERTER(translate_flip); OP_CONVERTER(translate_floor_divide); @@ -323,7 +324,9 @@ const std::map get_supported_ops_ts() { {"aten::fake_quantize_per_channel_affine", op::translate_fake_quantize_per_channel_affine}, {"aten::fake_quantize_per_tensor_affine", op::translate_fake_quantize_per_tensor_affine}, {"aten::feature_dropout", op::skip_node}, - {"aten::fill_", op::inplace_op}, + {"aten::fill", op::translate_fill}, + {"aten::fill_", op::inplace_op}, + {"aten::fill_diagonal_", op::inplace_op}, {"aten::flatten", op::quantizable_op}, {"aten::flip", op::translate_flip}, {"aten::floor", op::translate_1to1_match_1_inputs}, diff --git a/tests/layer_tests/pytorch_tests/test_full.py b/tests/layer_tests/pytorch_tests/test_full.py index 4ce42db7fa9..c564b1bb373 100644 --- a/tests/layer_tests/pytorch_tests/test_full.py +++ b/tests/layer_tests/pytorch_tests/test_full.py @@ -104,31 +104,94 @@ class TestFull(PytorchLayerTest): ir_version, kwargs_to_prepare_input={'value': value}) class TestFill(PytorchLayerTest): - def _prepare_input(self, value, shape, input_dtype, value_dtype): - return (np.random.randn(*shape).astype(input_dtype), np.array(value, dtype=value_dtype),) + def _prepare_input(self, value, shape, input_dtype, value_dtype, out=False): + if not out: + return (np.random.randn(*shape).astype(input_dtype), np.array(value, dtype=value_dtype),) + return (np.random.randn(*shape).astype(input_dtype), np.array(value, dtype=value_dtype), np.zeros(shape, dtype=input_dtype)) - def create_model(self): + + def create_model(self, mode): import torch class aten_fill(torch.nn.Module): + def __init__(self, mode) -> None: + super().__init__() + if mode == "inplace": + self.forward = self.forward_inplace + if mode == "out": + self.forward = self.forward_out - def forward(self, input_t: torch.Tensor, x: float): + + def forward_inplace(self, input_t: torch.Tensor, x: float): return input_t.fill_(x) + + def forward_out(self, input_t: torch.Tensor, x: float, out: torch.Tensor): + return input_t.fill(x, out=out), out + + def forward(self, input_t: torch.Tensor, x:float): + return input_t.fill(x) + ref_net = None - model = aten_fill() + model = aten_fill(mode) - return model, ref_net, "aten::fill_" + return model, ref_net, "aten::fill_" if mode == "inplace" else "aten::fill" @pytest.mark.parametrize("shape", [[1], [1, 2], [1, 2, 3], [1, 2, 3, 4], [2, 3, 4, 5, 6]]) @pytest.mark.parametrize("value", [0, 1, -1, 0.5]) @pytest.mark.parametrize("input_dtype", ["int8", "int32", "int64", "float32", "float64"]) @pytest.mark.parametrize("value_dtype", ["int8", "int32", "int64", "float32", "float64"]) + @pytest.mark.parametrize("mode", ["", "inplace", "out"]) @pytest.mark.nightly @pytest.mark.precommit - def test_fill(self, shape, value, input_dtype, value_dtype, ie_device, precision, ir_version): - self._test(*self.create_model(), ie_device, precision, ir_version, - kwargs_to_prepare_input={'value': value, 'shape': shape, "input_dtype": input_dtype, "value_dtype": value_dtype}) + def test_fill(self, shape, value, input_dtype, value_dtype, mode, ie_device, precision, ir_version): + self._test(*self.create_model(mode), ie_device, precision, ir_version, + kwargs_to_prepare_input={ + 'value': value, + 'shape': shape, + "input_dtype": input_dtype, + "value_dtype": value_dtype, + "out": mode == "out" + }) + +class TestFillDiagonal(PytorchLayerTest): + def _prepare_input(self, shape, input_dtype, value, value_dtype): + return np.zeros(shape).astype(input_dtype), np.array(value, dtype=value_dtype) + + def create_model(self, shape, wrap): + import torch + + class aten_fill_diagonal(torch.nn.Module): + def __init__(self, input_shape, wrap=False) -> None: + super().__init__() + self.wrap = wrap + self.input_shape = input_shape + + def forward(self, x:torch.Tensor, y:float): + x = x.reshape(self.input_shape) + return x.fill_diagonal_(y, wrap=self.wrap), x + + ref_net = None + + model = aten_fill_diagonal(shape, wrap) + return model, "aten::fill_diagonal_", ref_net + + @pytest.mark.parametrize("shape", ([4, 4], [5, 4], [8, 4], [4, 3], [5, 5, 5], [3, 3, 3, 3], [4, 4, 4, 4, 4])) + @pytest.mark.parametrize("value", [0, 1, -1, 2.5]) + @pytest.mark.parametrize("input_dtype", ["int8", "int32", "int64", "float32", "float64"]) + @pytest.mark.parametrize("value_dtype", ["int8", "int32", "int64", "float32", "float64"]) + @pytest.mark.parametrize("wrap", [True, False]) + @pytest.mark.nightly + @pytest.mark.precommit + def test_fill_diagonal(self, shape, value, input_dtype, value_dtype, wrap, ie_device, precision, ir_version): + self._test(*self.create_model(shape, wrap), ie_device, precision, ir_version, + kwargs_to_prepare_input={ + 'value': value, + 'shape': shape, + "input_dtype": input_dtype, + "value_dtype": value_dtype + }) + class TestZero(PytorchLayerTest): def _prepare_input(self, shape, input_dtype): diff --git a/tests/model_hub_tests/torch_tests/hf_transformers_models b/tests/model_hub_tests/torch_tests/hf_transformers_models index 112aedeb60d..0618d98a4d9 100644 --- a/tests/model_hub_tests/torch_tests/hf_transformers_models +++ b/tests/model_hub_tests/torch_tests/hf_transformers_models @@ -242,7 +242,7 @@ microsoft/deberta-base,deberta microsoft/git-large-coco,git,skip,Load problem microsoft/layoutlm-base-uncased,layoutlm microsoft/layoutlmv2-base-uncased,layoutlmv2,skip,Load problem -microsoft/layoutlmv3-base,layoutlmv3,xfail,Unsupported op aten::amax aten::clip +microsoft/layoutlmv3-base,layoutlmv3 microsoft/markuplm-base,markuplm microsoft/resnet-50,resnet microsoft/speecht5_hifigan,hifigan,skip,Load problem @@ -251,7 +251,7 @@ microsoft/swinv2-tiny-patch4-window8-256,swinv2,xfail,Unsupported op aten::adapt microsoft/table-transformer-detection,table-transformer microsoft/wavlm-large,wavlm,skip,Load problem microsoft/xclip-base-patch32,xclip,skip,Load problem -microsoft/xprophetnet-large-wiki100-cased,xlm-prophetnet,xfail,Unsupported op aten::fill_diagonal_ +microsoft/xprophetnet-large-wiki100-cased,xlm-prophetnet miguelvictor/python-fromzero-lstmlm,lstmlm,skip,Load problem mingzi151/test-hf-wav2vec2bert,wav2vec2bert,skip,Load problem MIT/ast-finetuned-audioset-10-10-0.4593,audio-spectrogram-transformer,skip,Load problem @@ -348,7 +348,7 @@ SteveZhan/my-resnet50d,resnet_steve,skip,Load problem suno/bark,bark,skip,Load problem surajnair/r3m-50,r3m,skip,Load problem susnato/clvp_dev,clvp,skip,Load problem -Tanrei/GPTSAN-japanese,gptsan-japanese,xfail,Unsupported op aten::clip aten::index_put_ prim::TupleConstruct +Tanrei/GPTSAN-japanese,gptsan-japanese,xfail,Unsupported op aten::index_put_ prim::TupleConstruct tau/bart-large-sled-govreport,tau/sled,skip,Load problem taufeeque/best-cb-model,codebook,skip,Load problem Team-PIXEL/pixel-base,pixel,skip,Load problem @@ -357,7 +357,7 @@ thomwolf/vqgan_imagenet_f16_1024,vqgan_model,skip,Load problem thu-ml/zh-clip-vit-roberta-large-patch14,zhclip,skip,Load problem tifa-benchmark/promptcap-coco-vqa,ofa,skip,Load problem tli8hf/robertabase_snli,transformerfornli,skip,Load problem -transfo-xl-wt103,transfo-xl,xfail,Unsupported op aten::clamp_ aten::index_copy_ +transfo-xl-wt103,transfo-xl,xfail,Unsupported op aten::index_copy_ transZ/BART_shared_clean,shared_bart,skip,Load problem transZ/BART_shared_v2,shared_bart_v2,skip,Load problem transZ/misecom,misecom,skip,Load problem diff --git a/tests/model_hub_tests/torch_tests/test_hf_transformers.py b/tests/model_hub_tests/torch_tests/test_hf_transformers.py index 3a677353c86..184e725a04f 100644 --- a/tests/model_hub_tests/torch_tests/test_hf_transformers.py +++ b/tests/model_hub_tests/torch_tests/test_hf_transformers.py @@ -298,7 +298,10 @@ class TestTransformersModel(TestConvertModel): ("google/tapas-large-finetuned-wtq", "tapas"), ("gpt2", "gpt2"), ("openai/clip-vit-large-patch14", "clip"), - ("RWKV/rwkv-4-169m-pile", "rwkv")]) + ("RWKV/rwkv-4-169m-pile", "rwkv"), + ("microsoft/layoutlmv3-base", "layoutlmv3"), + ("microsoft/xprophetnet-large-wiki100-cased", "xlm-prophetnet"), + ]) @pytest.mark.precommit def test_convert_model_precommit(self, name, type, ie_device): self.run(model_name=name, model_link=type, ie_device=ie_device)