diff --git a/src/frontends/tensorflow/src/frontend.cpp b/src/frontends/tensorflow/src/frontend.cpp index b1d827edcc5..5011b229cd8 100644 --- a/src/frontends/tensorflow/src/frontend.cpp +++ b/src/frontends/tensorflow/src/frontend.cpp @@ -9,7 +9,6 @@ #include "graph_iterator_proto_txt.hpp" #include "graph_iterator_saved_model.hpp" #include "helper_ops/internal_operation.hpp" -#include "helper_transforms/block_lstm_replacer.hpp" #include "helper_transforms/const_to_result_remover.hpp" #include "helper_transforms/embedding_segments_feature_fusing.hpp" #include "helper_transforms/saved_model_unused_remover.hpp" @@ -565,7 +564,6 @@ void FrontEnd::normalize(const std::shared_ptr& model) const { manager.register_pass(); manager.register_pass(); manager.register_pass(); - manager.register_pass(); manager.register_pass(); manager.register_pass(); manager.register_pass(); diff --git a/src/frontends/tensorflow/src/op/block_lstm.cpp b/src/frontends/tensorflow/src/op/block_lstm.cpp index 34118f3114b..663f5079447 100644 --- a/src/frontends/tensorflow/src/op/block_lstm.cpp +++ b/src/frontends/tensorflow/src/op/block_lstm.cpp @@ -6,8 +6,147 @@ #include "common_op_table.hpp" #include "openvino/frontend/tensorflow/node_context.hpp" +#include "openvino/op/add.hpp" +#include "openvino/op/broadcast.hpp" +#include "openvino/op/concat.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/gather.hpp" +#include "openvino/op/loop.hpp" +#include "openvino/op/lstm_cell.hpp" +#include "openvino/op/lstm_sequence.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/reshape.hpp" +#include "openvino/op/shape_of.hpp" +#include "openvino/op/squeeze.hpp" +#include "openvino/op/strided_slice.hpp" +#include "openvino/op/subtract.hpp" +#include "openvino/op/unsqueeze.hpp" +#include "openvino/op/variadic_split.hpp" +#include "utils.hpp" using namespace std; +using namespace ov; +using namespace ov::op; +using namespace ov::frontend::tensorflow; + +namespace { +void create_decomposed_block_lstm(const Output& x, + const Output& h_init, + const Output& c_init, + const Output& w, + const Output& r, + const Output& b, + const Output& seq_len_max, + const element::Type& x_type, + const Dimension& hidden_size, + Output& hs, + Output& cs) { + // inputs: + // x - [time_len, batch_size, input_size] shape + // h_init - [batch_size, hidden_size] shape + // c_init - [batch_size, hidden_size] shape + // w - [4 * hidden_size, input_size] shape + // r - [4 * hidden_size, input_size] shape + // b - [4 * hidden_size] shape + // + // outputs: + // hs - [time_len, batch_size, hidden_size] shape + // cs - [time_len, batch_size, hidden_size] shape + auto hidden_size_value = hidden_size.get_length(); + + // create a body graph with LSTMCell + auto xi_param = + std::make_shared(x_type, + PartialShape{Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic()}); + auto h_prev_param = + std::make_shared(x_type, ov::PartialShape{ov::Dimension::dynamic(), hidden_size_value}); + auto c_prev_param = + std::make_shared(x_type, ov::PartialShape{ov::Dimension::dynamic(), hidden_size_value}); + auto w_param = + std::make_shared(x_type, ov::PartialShape{4 * hidden_size_value, ov::Dimension::dynamic()}); + auto r_param = std::make_shared(x_type, ov::PartialShape{4 * hidden_size_value, hidden_size_value}); + auto b_param = std::make_shared(x_type, ov::PartialShape{4 * hidden_size_value}); + + // adjust xi since it comes after slicing and slicing axis needs to be squeezed + auto squeeze_axis = std::make_shared(element::i32, Shape{1}, 0); + auto xi = std::make_shared(xi_param, squeeze_axis); + + auto lstm_cell = std::make_shared(xi, + h_prev_param, + c_prev_param, + w_param, + r_param, + b_param, + static_cast(hidden_size_value)); + + auto h = lstm_cell->output(0); + auto c = lstm_cell->output(1); + + // unsqueeze current cell and hidden states + // for concatenation along time dimension + auto axis = std::make_shared(ov::element::i32, ov::Shape{1}, 0); + auto h_concat = std::make_shared(h, axis)->output(0); + auto c_concat = std::make_shared(c, axis)->output(0); + auto body_condition = std::make_shared(element::boolean, Shape{1}, true); + + ov::ParameterVector body_params({xi_param, h_prev_param, c_prev_param, w_param, r_param, b_param}); + ov::OutputVector body_results({body_condition, h, c, h_concat, c_concat}); + auto lstm_body = std::make_shared(body_results, body_params); + + // create Loop node and put lstm body graph inside + // it will represent BlockLSTM operation + auto execution_cond = std::make_shared(ov::element::boolean, ov::Shape{}, true); + auto seq_len_max_shape = std::make_shared(ov::element::i32, ov::Shape{1}, 1); + auto new_seq_len_max = std::make_shared(seq_len_max, seq_len_max_shape, false); + auto loop_node = std::make_shared(new_seq_len_max, execution_cond); + + loop_node->set_function(lstm_body); + loop_node->set_special_body_ports(ov::op::v5::Loop::SpecialBodyPorts{-1, 0}); + + // set inputs for Loop + // x input will be sliced for each time step + loop_node->set_sliced_input(xi_param, x, 0, 1, 1, -1, 0); + // set back edges for cell and hidden states + // since they are changing through timeline + loop_node->set_merged_input(h_prev_param, h_init, h); + loop_node->set_merged_input(c_prev_param, c_init, c); + + loop_node->set_invariant_input(w_param, w); + loop_node->set_invariant_input(r_param, r); + loop_node->set_invariant_input(b_param, b); + + // set external outputs for Loop node + // concatenated cell and hidden states from all time steps + hs = loop_node->get_concatenated_slices(h_concat, 0, 1, 1, -1, 0); + cs = loop_node->get_concatenated_slices(c_concat, 0, 1, 1, -1, 0); + + // clarify shapes inside body graphs and on loop outputs + loop_node->validate_and_infer_types(); + + // compute time_len it is needed for further padding + // of concatenated cell and hidden states + auto x_shape = std::make_shared(x, element::i64); + auto ss_start = std::make_shared(element::i64, Shape{1}, 0); + auto ss_stop = std::make_shared(element::i64, Shape{1}, 1); + auto ss_step = std::make_shared(element::i64, Shape{1}, 1); + auto time_len = std::make_shared(x_shape, + ss_start, + ss_stop, + ss_step, + std::vector{0}, + std::vector{0}); + + // since seq_len_max can be less that time length + // output tensors needs to be padded + auto h_init_shape = std::make_shared(h_init, element::i64); + auto dummy_size = std::make_shared(time_len, new_seq_len_max); + auto dummy_tensor_shape = make_shared(OutputVector{dummy_size, h_init_shape}, 0); + auto zero_element = create_same_type_const_scalar(x, 0); + auto dummy_tensor = make_shared(zero_element, dummy_tensor_shape); + hs = make_shared(OutputVector{hs, dummy_tensor}, 0); + cs = make_shared(OutputVector{cs, dummy_tensor}, 0); +} +} // namespace namespace ov { namespace frontend { @@ -15,36 +154,141 @@ namespace tensorflow { namespace op { OutputVector translate_block_lstm_op(const ov::frontend::tensorflow::NodeContext& node) { default_op_checks(node, 9, {"BlockLSTM"}); + auto node_name = node.get_name(); + auto seq_len_max = node.get_input(0); auto x = node.get_input(1); auto cs_prev = node.get_input(2); auto h_prev = node.get_input(3); - auto w = node.get_input(4); + auto weights = node.get_input(4); auto wci = node.get_input(5); auto wcf = node.get_input(6); auto wco = node.get_input(7); - auto b = node.get_input(8); + auto bias = node.get_input(8); // retrieve attributes - auto forget_bias = node.get_attribute("forget_bias"); - auto cell_clip = node.get_attribute("cell_clip"); - auto use_peephole = node.get_attribute("use_peephole"); + auto forget_bias = node.get_attribute("forget_bias", 1.0f); + auto cell_clip = node.get_attribute("cell_clip", 3.0f); + auto use_peephole = node.get_attribute("use_peephole", false); + TENSORFLOW_OP_VALIDATION( + node, + !use_peephole, + "[TensorFlow Frontend] internal error: BlockLSTM is supported only for false use_peephole"); + TENSORFLOW_OP_VALIDATION( + node, + cell_clip == -1.0f, + "[TensorFlow Frontend] internal error: BlockLSTM is supported only for cell_clip equal to -1"); + + // extract hidden_size + // we assume that this dimension will not be reshaped + // and this is feasible assumption because it seems ridiculous to reshape in real model + auto hidden_size = ov::Dimension::dynamic(); + auto w_shape = weights.get_partial_shape(); + auto w_rank = w_shape.rank(); + auto b_shape = bias.get_partial_shape(); + auto b_rank = b_shape.rank(); + if (w_rank.is_static()) { + hidden_size = w_shape[1].is_static() ? w_shape[1].get_length() / 4 : ov::Dimension::dynamic(); + } + if (b_rank.is_static()) { + hidden_size = b_shape[0].is_static() ? b_shape[0].get_length() / 4 : hidden_size; + } + TENSORFLOW_OP_VALIDATION( + node, + hidden_size.is_static(), + "[TensorFlow Frontend] internal error: BlockLSTM is supported only for static hidden size"); + + // x has a format [timelen, batch_size, input_size] + // retrieve input_size + auto x_shape = std::make_shared(x, element::i64); + auto ss_start = std::make_shared(element::i64, Shape{1}, 2); + auto ss_stop = std::make_shared(element::i64, Shape{1}, 3); + auto ss_step = std::make_shared(element::i64, Shape{1}, 1); + auto input_size = std::make_shared(x_shape, + ss_start, + ss_stop, + ss_step, + std::vector{0}, + std::vector{0}); + + // retrieve the batch size + // now x is in a format [time_len, batch_size, input_size] + auto ss_start2 = std::make_shared(element::i64, Shape{1}, 1); + auto ss_stop2 = std::make_shared(element::i64, Shape{1}, 2); + auto batch_size = std::make_shared(x_shape, + ss_start2, + ss_stop2, + ss_step, + std::vector{0}, + std::vector{0}); + + auto hidden_size_const = + std::make_shared(element::i64, Shape{1}, std::vector{hidden_size.get_length()}); + + // adjust weights and bias + // 1. reshape weights and bias to highlight channel dimension + auto new_weight_shape = std::make_shared(element::i64, Shape{3}, std::vector{0, 4, -1}); + auto weight_reshape = std::make_shared(weights, new_weight_shape, true); + auto new_bias_shape = std::make_shared(element::i64, Shape{2}, std::vector{4, -1}); + auto bias_reshape = std::make_shared(bias, new_bias_shape, true); + // 2. reorder gates icfo --> fico for both weights and biases + auto reorder_const = std::make_shared(element::i64, Shape{4}, std::vector{2, 0, 1, 3}); + auto weights_axis = std::make_shared(element::i64, Shape{}, 1); + auto weights_reorder = std::make_shared(weight_reshape, reorder_const, weights_axis); + auto bias_axis = std::make_shared(element::i64, Shape{}, 0); + auto bias_reorder = std::make_shared(bias_reshape, reorder_const, bias_axis); + // 3. shift_const value should be added to the first 1 / 4th part of the biases(f - gate : 0) + auto shift_const = std::make_shared(element::f32, Shape{}, forget_bias); + auto bias_split_lens = std::make_shared(element::i64, Shape{2}, std::vector{1, 3}); + auto bias_split = std::make_shared(bias_reorder, bias_axis, bias_split_lens); + auto bias_first_shift = std::make_shared(bias_split->output(0), shift_const); + auto bias_shift = std::make_shared(OutputVector{bias_first_shift, bias_split->output(1)}, 0); + // 4. return to the original shapes + auto new_weight_shape2 = std::make_shared(element::i64, Shape{2}, std::vector{0, -1}); + auto weight_reshape2 = std::make_shared(weights_reorder, new_weight_shape2, true); + // 5. normalize weights and bias + auto transpose_order = std::make_shared(element::i64, Shape{2}, std::vector{1, 0}); + auto new_bias_shape2 = std::make_shared(element::i64, Shape{1}, std::vector{-1}); + auto weights_normalize = std::make_shared(weight_reshape2, transpose_order); + auto bias_normalized = std::make_shared(bias_shift, new_bias_shape2, true); + // 6. split weights into W and R inputs + auto WR_split_axis = std::make_shared(element::i64, Shape{}, 1); + auto WR_split_lens = std::make_shared(OutputVector{input_size, hidden_size_const}, 0); + auto WR_split = std::make_shared(weights_normalize, WR_split_axis, WR_split_lens); + // 7. unsqueeze weights and bias to have a dimension for a number of directions + auto W = WR_split->output(0); + auto R = WR_split->output(1); + auto B = bias_normalized; + + ov::Output hs, cs; + auto x_type = x.get_element_type(); + TENSORFLOW_OP_VALIDATION(node, + x_type.is_static(), + "[TensorFlow Frontend] internal error: BlockLSTM is supported only for x of static type"); + create_decomposed_block_lstm(x, h_prev, cs_prev, W, R, B, seq_len_max, x_type, hidden_size, hs, cs); + cs.set_names({node_name + ":1"}); + hs.set_names({node_name + ":6"}); + + // for other outputs, it uses internal operation BlockLSTM auto block_lstm = make_shared(seq_len_max, x, cs_prev, h_prev, - w, + weights, wci, wcf, wco, - b, + bias, forget_bias, cell_clip, use_peephole, node.get_decoder()); - set_node_name(node.get_name(), block_lstm); - return block_lstm->outputs(); + ov::OutputVector results = block_lstm->outputs(); + results[1] = cs; + results[6] = hs; + + return results; } } // namespace op } // namespace tensorflow diff --git a/src/frontends/tensorflow_common/include/helper_transforms/block_lstm_replacer.hpp b/src/frontends/tensorflow_common/include/helper_transforms/block_lstm_replacer.hpp deleted file mode 100644 index 07ac88eaeb0..00000000000 --- a/src/frontends/tensorflow_common/include/helper_transforms/block_lstm_replacer.hpp +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include - -#include "openvino/pass/graph_rewrite.hpp" -#include "openvino/pass/pass.hpp" - -namespace ov { -namespace frontend { -namespace tensorflow { -namespace pass { - -// This transformation replaces BlockLSTM with such outputs as concatenated hidden states -// and cell state from the last time step. -class BlockLSTMReplacer : public ov::pass::MatcherPass { -public: - OPENVINO_RTTI("ov::frontend::tensorflow::pass::BlockLSTMReplacer"); - BlockLSTMReplacer(); -}; - -} // namespace pass -} // namespace tensorflow -} // namespace frontend -} // namespace ov diff --git a/src/frontends/tensorflow_common/src/helper_transforms/block_lstm_replacer.cpp b/src/frontends/tensorflow_common/src/helper_transforms/block_lstm_replacer.cpp deleted file mode 100644 index 6011720c1f2..00000000000 --- a/src/frontends/tensorflow_common/src/helper_transforms/block_lstm_replacer.cpp +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "helper_transforms/block_lstm_replacer.hpp" - -#include -#include - -#include "helper_ops/block_lstm.hpp" -#include "openvino/op/add.hpp" -#include "openvino/op/broadcast.hpp" -#include "openvino/op/concat.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/gather.hpp" -#include "openvino/op/gather_nd.hpp" -#include "openvino/op/lstm_sequence.hpp" -#include "openvino/op/reshape.hpp" -#include "openvino/op/shape_of.hpp" -#include "openvino/op/squeeze.hpp" -#include "openvino/op/strided_slice.hpp" -#include "openvino/op/unsqueeze.hpp" -#include "openvino/op/variadic_split.hpp" -#include "openvino/pass/pattern/matcher.hpp" -#include "openvino/pass/pattern/op/or.hpp" -#include "openvino/pass/pattern/op/wrap_type.hpp" -#include "transformations/utils/utils.hpp" -#include "utils.hpp" - -using namespace std; -using namespace ov::pass; -using namespace ov::op; -using namespace ov::pass::pattern; -using namespace ov::frontend::tensorflow; - -namespace { -std::function)> can_have_outputs(const std::vector& allowed_output_indices) { - return [=](ov::Output output) -> bool { - auto block_lstm_node = output.get_node_shared_ptr(); - auto output_size = block_lstm_node->get_output_size(); - for (size_t output_ind = 0; output_ind < output_size; ++output_ind) { - if (std::find(allowed_output_indices.begin(), allowed_output_indices.end(), output_ind) != - allowed_output_indices.end()) { - continue; - } - if (block_lstm_node->output(output_ind).get_target_inputs().size() > 0) { - return false; - } - } - return true; - }; -} -} // namespace - -pass::BlockLSTMReplacer::BlockLSTMReplacer() { - // Pattern 1: BlockLSTM with last state cell output (BlockLSTM -> Concat -> GatherND) - // used in DeepSpeech model - auto block_lstm_1 = pattern::wrap_type(can_have_outputs({1, 6})); - auto states_cell_1 = pattern::wrap_type({pattern::any_input(), block_lstm_1}); - auto pattern1 = pattern::wrap_type({states_cell_1, pattern::any_input()}); - - // Pattern 2: BlockLSTM with just one output, concatenated hidden states (BlockLSTM) - auto pattern2 = pattern::wrap_type(can_have_outputs({6})); - - auto root = std::make_shared(OutputVector{pattern1, pattern2}); - - matcher_pass_callback callback = [=](pattern::Matcher& m) { - auto pattern_map = m.get_pattern_map(); - auto is_pattern1 = (pattern_map.find(pattern1) != std::end(pattern_map)); - auto is_pattern2 = (pattern_map.find(pattern2) != std::end(pattern_map)); - - // find for each pattern BlockLSTM node for which we adjust inputs - // and check its attributes before the transformation - std::shared_ptr block_lstm_node; - std::shared_ptr last_state_c_node; - ov::NodeVector rt_info_from; - if (is_pattern1) { - block_lstm_node = std::dynamic_pointer_cast(pattern_map.at(block_lstm_1)); - auto concat_node = std::dynamic_pointer_cast(pattern_map.at(states_cell_1)); - if (!concat_node || concat_node->get_axis() != 0) { - // timestep is the first dimension - return false; - } - last_state_c_node = pattern_map.at(pattern1); - rt_info_from = {block_lstm_node, concat_node, last_state_c_node}; - } else if (is_pattern2) { - block_lstm_node = std::dynamic_pointer_cast(pattern_map.at(pattern2)); - rt_info_from = {block_lstm_node}; - } - if (!block_lstm_node) { - return false; - } - - NodeRegistry rg; - // currently, LSTMSequence does not support peephole and cell clip - if (block_lstm_node->get_use_peephole()) { - return false; - } - if (block_lstm_node->get_cell_clip() != -1.0f) { - return false; - } - - // currently, OpenVINO support only static hidden_size - // since this is an attribute of LSTMSequence operation - auto hidden_size = block_lstm_node->get_hidden_size(); - if (hidden_size.is_dynamic()) { - return false; - } - - auto block_lstm_node_name = block_lstm_node->get_friendly_name(); - auto seq_len_max = block_lstm_node->input_value(0); - auto x = block_lstm_node->input_value(1); - auto cs_prev = block_lstm_node->input_value(2); - auto h_prev = block_lstm_node->input_value(3); - auto weights = block_lstm_node->input_value(4); - auto wci = block_lstm_node->input_value(5); - auto wcf = block_lstm_node->input_value(6); - auto wco = block_lstm_node->input_value(7); - auto bias = block_lstm_node->input_value(8); - - // retrieve input_size - auto x_shape = rg.make(x, element::i64); - auto ss_start = rg.make(element::i64, Shape{1}, 2); - auto ss_stop = rg.make(element::i64, Shape{1}, 3); - auto ss_step = rg.make(element::i64, Shape{1}, 1); - auto input_size = rg.make(x_shape, - ss_start, - ss_stop, - ss_step, - std::vector{0}, - std::vector{0}); - - // retrieve the batch size - // now x is in a format [time_len, batch_size, input_size] - auto ss_start2 = rg.make(element::i64, Shape{1}, 1); - auto ss_stop2 = rg.make(element::i64, Shape{1}, 2); - auto batch_size = rg.make(x_shape, - ss_start2, - ss_stop2, - ss_step, - std::vector{0}, - std::vector{0}); - - auto hidden_size_const = - rg.make(element::i64, Shape{1}, std::vector{hidden_size.get_length()}); - - // adjust weights and bias - // 1. reshape weights and bias to highlight channel dimension - auto new_weight_shape = rg.make(element::i64, Shape{3}, std::vector{0, 4, -1}); - auto weight_reshape = rg.make(weights, new_weight_shape, true); - auto new_bias_shape = rg.make(element::i64, Shape{2}, std::vector{4, -1}); - auto bias_reshape = rg.make(bias, new_bias_shape, true); - // 2. reorder gates icfo --> fico for both weights and biases - auto reorder_const = rg.make(element::i64, Shape{4}, std::vector{2, 0, 1, 3}); - auto weights_axis = rg.make(element::i64, Shape{}, 1); - auto weights_reorder = rg.make(weight_reshape, reorder_const, weights_axis); - auto bias_axis = rg.make(element::i64, Shape{}, 0); - auto bias_reorder = rg.make(bias_reshape, reorder_const, bias_axis); - // 3. shift_const.value should be added to the first 1 / 4th part of the biases(f - gate : 0) - auto shift_const = rg.make(element::f32, Shape{}, block_lstm_node->get_forget_bias()); - auto bias_split_lens = rg.make(element::i64, Shape{2}, std::vector{1, 3}); - auto bias_split = rg.make(bias_reorder, bias_axis, bias_split_lens); - auto bias_first_shift = rg.make(bias_split->output(0), shift_const); - auto bias_shift = rg.make(OutputVector{bias_first_shift, bias_split->output(1)}, 0); - // 4. return to the original shapes - auto new_weight_shape2 = rg.make(element::i64, Shape{2}, std::vector{0, -1}); - auto weight_reshape2 = rg.make(weights_reorder, new_weight_shape2, true); - // 5. normalize weights and bias - auto transpose_order = rg.make(element::i64, Shape{2}, std::vector{1, 0}); - auto new_bias_shape2 = rg.make(element::i64, Shape{1}, std::vector{-1}); - auto weights_normalize = rg.make(weight_reshape2, transpose_order); - auto bias_normalized = rg.make(bias_shift, new_bias_shape2, true); - // 6. split weights into W and R inputs - auto WR_split_axis = rg.make(element::i64, Shape{}, 1); - auto WR_split_lens = rg.make(OutputVector{input_size, hidden_size_const}, 0); - auto WR_split = rg.make(weights_normalize, WR_split_axis, WR_split_lens); - // 7. unsqueeze weights and bias to have a dimension for a number of directions - auto num_direct_axis = rg.make(element::i64, Shape{1}, std::vector{0}); - auto W = rg.make(WR_split->output(0), num_direct_axis); - auto R = rg.make(WR_split->output(1), num_direct_axis); - auto B = rg.make(bias_normalized, num_direct_axis); - - // normalize initial hidden and cell states - auto unsqueeze_axis = rg.make(element::i64, Shape{1}, std::vector{1}); - auto init_hidden_state = rg.make(h_prev, unsqueeze_axis); - auto init_cell_state = rg.make(cs_prev, unsqueeze_axis); - - // prepare sequence length input for LSTMSequence - auto seq_len_max_adjusted = rg.make(seq_len_max, batch_size); - - // prepare input data since LSTMSequence accept it in a format [batch_size, time_len, input_size] - auto x_order = rg.make(element::i64, Shape{3}, std::vector{1, 0, 2}); - auto x_adjusted = rg.make(x, x_order); - - // create LSTMSequence node and reconnect inputs and normalized weights and bias - auto lstm_sequence = rg.make(x_adjusted, - init_hidden_state, - init_cell_state, - seq_len_max_adjusted, - W, - R, - B, - hidden_size.get_length(), - v5::LSTMSequence::direction::FORWARD); - - if (block_lstm_node->output(1).get_target_inputs().size() > 0) { - // adjust output with the last state cell and connect to the main graph - // squeeze extra dimension - num_directions - auto squeeze_axis = rg.make(element::i64, Shape{1}, std::vector{1}); - auto squeeze_last_state_cell = rg.make(lstm_sequence->output(2), squeeze_axis); - - // preserve names of the node and the output tensor - squeeze_last_state_cell->set_friendly_name(last_state_c_node->get_friendly_name()); - - ov::replace_node(last_state_c_node, squeeze_last_state_cell); - } - - if (block_lstm_node->output(6).get_target_inputs().size() > 0) { - // adjust output of concatenated of hidden states from LSTMSequence - // to have it in a format [time_len, batch_size, hidden_size] - // 1. squeeze extra dimension - num_directions - auto squeeze_axis = rg.make(element::i64, Shape{1}, std::vector{1}); - auto squeeze_output_hidden_states = rg.make(lstm_sequence->output(0), squeeze_axis); - // 2. transpose the output to rotate batch and time dimensions - auto output_hidden_states_order = - rg.make(element::i64, Shape{3}, std::vector{1, 0, 2}); - auto output_hidden_states = - rg.make(squeeze_output_hidden_states, output_hidden_states_order); - - // preserve names of the node and the output tensor - output_hidden_states->set_friendly_name(block_lstm_node->get_friendly_name() + ":6"); - - // replace BlockLSTM with LSTMSequence manually instead of calling - // ov::replace_node(m.get_match_root(), lstm_sequence); - // because BlockLSTM has 7 outputs and LSTMSequence has three outputs - block_lstm_node->output(6).replace(output_hidden_states->output(0)); - } - - copy_runtime_info(rt_info_from, rg.get()); - - return true; - }; - - auto m = std::make_shared(root, "ov::frontend::tensorflow::pass::BlockLSTMReplacer"); - register_matcher(m, callback); -} diff --git a/src/frontends/tensorflow_common/tests/block_lstm_replacer.cpp b/src/frontends/tensorflow_common/tests/block_lstm_replacer.cpp deleted file mode 100644 index b4ccb9de02c..00000000000 --- a/src/frontends/tensorflow_common/tests/block_lstm_replacer.cpp +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "helper_transforms/block_lstm_replacer.hpp" - -#include - -#include "conversion_with_reference.hpp" -#include "helper_ops/block_lstm.hpp" -#include "openvino/frontend/manager.hpp" -#include "openvino/op/add.hpp" -#include "openvino/op/broadcast.hpp" -#include "openvino/op/concat.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/gather.hpp" -#include "openvino/op/gather_nd.hpp" -#include "openvino/op/lstm_sequence.hpp" -#include "openvino/op/parameter.hpp" -#include "openvino/op/reshape.hpp" -#include "openvino/op/shape_of.hpp" -#include "openvino/op/squeeze.hpp" -#include "openvino/op/strided_slice.hpp" -#include "openvino/op/transpose.hpp" -#include "openvino/op/unsqueeze.hpp" -#include "openvino/op/variadic_split.hpp" -#include "openvino/pass/manager.hpp" - -using namespace std; -using namespace ov; -using namespace ov::op; -using namespace element; -using namespace frontend::tensorflow; -using namespace frontend::tensorflow::pass; - -namespace { -shared_ptr gen_model(Dimension batch_size, - Dimension time_len, - int64_t hidden_size, - Dimension input_size, - float forget_bias, - float cell_clip, - bool use_peephole, - bool with_two_outputs = false) { - auto seq_len_max = make_shared(i64, Shape{}); - auto x = make_shared(f32, PartialShape{time_len, batch_size, input_size}); - auto cs_prev = make_shared(f32, PartialShape::dynamic()); - auto h_prev = make_shared(f32, PartialShape::dynamic()); - auto w = make_shared(f32, PartialShape{Dimension::dynamic(), 4 * hidden_size}); - auto wci = make_shared(f32, PartialShape::dynamic()); - auto wcf = make_shared(f32, PartialShape::dynamic()); - auto wco = make_shared(f32, PartialShape::dynamic()); - auto b = make_shared(f32, PartialShape::dynamic()); - - auto block_lstm = make_shared< - BlockLSTM>(seq_len_max, x, cs_prev, h_prev, w, wci, wcf, wco, b, forget_bias, cell_clip, use_peephole); - - if (with_two_outputs) { - auto prev_cell_states = make_shared( - ov::element::f32, - ov::Shape{1, static_cast(batch_size.get_length()), static_cast(hidden_size)}, - 0); - auto concat = make_shared(OutputVector{prev_cell_states, block_lstm->output(1)}, 0); - auto indices_const = make_shared(ov::element::i32, - ov::Shape{2}, - vector{static_cast(time_len.get_length()), 0}); - auto gather_nd = make_shared(concat, indices_const); - return make_shared(OutputVector{gather_nd->output(0), block_lstm->output(6)}, - ParameterVector{seq_len_max, x, cs_prev, h_prev, w, wci, wcf, wco, b}); - } - - return make_shared(OutputVector{block_lstm->output(6)}, - ParameterVector{seq_len_max, x, cs_prev, h_prev, w, wci, wcf, wco, b}); -} - -shared_ptr gen_model_ref(Dimension m_batch_size, - Dimension m_time_len, - int64_t m_hidden_size, - Dimension m_input_size, - float forget_bias, - bool with_two_outputs = false) { - auto seq_len_max = make_shared(i64, Shape{}); - auto x = make_shared(f32, PartialShape{m_time_len, m_batch_size, m_input_size}); - auto cs_prev = make_shared(f32, PartialShape::dynamic()); - auto h_prev = make_shared(f32, PartialShape::dynamic()); - auto weights = make_shared(f32, PartialShape{Dimension::dynamic(), 4 * m_hidden_size}); - auto bias = make_shared(f32, PartialShape::dynamic()); - - auto x_shape = make_shared(x, element::i64); - auto ss_start = make_shared(element::i64, Shape{1}, 2); - auto ss_stop = make_shared(element::i64, Shape{1}, 3); - auto ss_step = make_shared(element::i64, Shape{1}, 1); - auto input_size = make_shared(x_shape, - ss_start, - ss_stop, - ss_step, - std::vector{0}, - std::vector{0}); - - // retrieve the batch size - // now x is in a format [time_len, batch_size, input_size] - auto ss_start2 = make_shared(element::i64, Shape{1}, 1); - auto ss_stop2 = make_shared(element::i64, Shape{1}, 2); - auto batch_size = make_shared(x_shape, - ss_start2, - ss_stop2, - ss_step, - std::vector{0}, - std::vector{0}); - auto hidden_size_const = make_shared(element::i64, Shape{1}, std::vector{m_hidden_size}); - - // adjust weights and bias - // 1. reshape weights and bias to highlight channel dimension - auto new_weight_shape = make_shared(element::i64, Shape{3}, std::vector{0, 4, -1}); - auto weight_reshape = make_shared(weights, new_weight_shape, true); - auto new_bias_shape = make_shared(element::i64, Shape{2}, std::vector{4, -1}); - auto bias_reshape = make_shared(bias, new_bias_shape, true); - // 2. reorder gates icfo --> fico for both weights and biases - auto reorder_const = make_shared(element::i64, Shape{4}, std::vector{2, 0, 1, 3}); - auto weights_axis = make_shared(element::i64, Shape{}, 1); - auto weights_reorder = make_shared(weight_reshape, reorder_const, weights_axis); - auto bias_axis = make_shared(element::i64, Shape{}, 0); - auto bias_reorder = make_shared(bias_reshape, reorder_const, bias_axis); - // 3. shift_const.value should be added to the first 1 / 4th part of the biases(f - gate : 0) - auto shift_const = make_shared(element::f32, Shape{}, forget_bias); - auto bias_split_lens = make_shared(element::i64, Shape{2}, std::vector{1, 3}); - auto bias_split = make_shared(bias_reorder, bias_axis, bias_split_lens); - auto bias_first_shift = make_shared(bias_split->output(0), shift_const); - auto bias_shift = make_shared(OutputVector{bias_first_shift, bias_split->output(1)}, 0); - // 4. return to the original shapes - auto new_weight_shape2 = make_shared(element::i64, Shape{2}, std::vector{0, -1}); - auto weight_reshape2 = make_shared(weights_reorder, new_weight_shape2, true); - // 5. normalize weights and bias - auto transpose_order = make_shared(element::i64, Shape{2}, std::vector{1, 0}); - auto new_bias_shape2 = make_shared(element::i64, Shape{1}, std::vector{-1}); - auto weights_normalize = make_shared(weight_reshape2, transpose_order); - auto bias_normalized = make_shared(bias_shift, new_bias_shape2, true); - // 6. split weights into W and R inputs - auto WR_split_axis = make_shared(element::i64, Shape{}, 1); - auto WR_split_lens = make_shared(OutputVector{input_size, hidden_size_const}, 0); - auto WR_split = make_shared(weights_normalize, WR_split_axis, WR_split_lens); - // 7. unsqueeze weights and bias to have a dimension for a number of directions - auto num_direct_axis = make_shared(element::i64, Shape{1}, std::vector{0}); - auto W = make_shared(WR_split->output(0), num_direct_axis); - auto R = make_shared(WR_split->output(1), num_direct_axis); - auto B = make_shared(bias_normalized, num_direct_axis); - - // normalize initial hidden and cell states - auto unsqueeze_axis = make_shared(element::i64, Shape{1}, std::vector{1}); - auto init_hidden_state = make_shared(h_prev, unsqueeze_axis); - auto init_cell_state = make_shared(cs_prev, unsqueeze_axis); - - // prepare sequence length input for LSTMSequence - auto seq_len_max_adjusted = make_shared(seq_len_max, batch_size); - - // prepare input data since LSTMSequence accept it in a format [batch_size, time_len, input_size] - auto x_order = make_shared(element::i64, Shape{3}, std::vector{1, 0, 2}); - auto x_adjusted = make_shared(x, x_order); - - // create LSTMSequence node and reconnect inputs and normalized weights and bias - auto lstm_sequence = make_shared(x_adjusted, - init_hidden_state, - init_cell_state, - seq_len_max_adjusted, - W, - R, - B, - m_hidden_size, - v5::LSTMSequence::direction::FORWARD); - - // adjust output of concatenated of hidden states from LSTMSequence to have it in a format [time_len, - // batch_size, hidden_size] - // 1. squeeze extra dimension - num_directions - auto squeeze_axis = make_shared(element::i64, Shape{1}, std::vector{1}); - auto squeeze_output_hidden_states = make_shared(lstm_sequence->output(0), squeeze_axis); - // 2. transpose the output to rotate batch and time dimensions - auto output_hidden_states_order = make_shared(element::i64, Shape{3}, std::vector{1, 0, 2}); - auto output_hidden_states = make_shared(squeeze_output_hidden_states, output_hidden_states_order); - - if (with_two_outputs) { - // adjust output with the last state cell and connect to the main graph - // squeeze extra dimension - num_directions - auto squeeze_axis = make_shared(element::i64, Shape{1}, std::vector{1}); - auto squeeze_last_state_cell = make_shared(lstm_sequence->output(2), squeeze_axis); - return make_shared(OutputVector{squeeze_last_state_cell->output(0), output_hidden_states->output(0)}, - ParameterVector{seq_len_max, x, cs_prev, h_prev, weights, bias}); - } - - return make_shared(OutputVector{output_hidden_states->output(0)}, - ParameterVector{seq_len_max, x, cs_prev, h_prev, weights, bias}); -} - -} // namespace - -TEST_F(FrontEndConversionWithReferenceTestsF, BlockLSTMReplacerWithHiddenOutput) { - { - model = gen_model(2, 10, 120, 20, 1.0f, -1.0f, false); - manager.register_pass(); - } - { model_ref = gen_model_ref(2, 10, 120, 20, 1.0f); } -} - -TEST_F(FrontEndConversionWithReferenceTestsF, BlockLSTMReplacerWithHiddenOutputAndLastCellState) { - { - model = gen_model(2, 10, 120, 20, 1.0f, -1.0f, false, true); - manager.register_pass(); - } - { model_ref = gen_model_ref(2, 10, 120, 20, 1.0f, true); } -} - -TEST_F(FrontEndConversionWithReferenceTestsF, BlockLSTMReplacerWithPeepHole) { - { - model = gen_model(2, 10, 120, 20, 1.0f, -1.0f, true); - manager.register_pass(); - } - { - // the transformation is not applied for the peep hole case - model_ref = gen_model(2, 10, 120, 20, 1.0f, -1.0f, true); - } -} diff --git a/tests/layer_tests/tensorflow_tests/test_tf_BlockLSTM.py b/tests/layer_tests/tensorflow_tests/test_tf_BlockLSTM.py new file mode 100644 index 00000000000..eb842b265df --- /dev/null +++ b/tests/layer_tests/tensorflow_tests/test_tf_BlockLSTM.py @@ -0,0 +1,65 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import numpy as np +import pytest +import tensorflow as tf +from common.tf_layer_test_class import CommonTFLayerTest + +rng = np.random.default_rng(2024) + + +class TestBlockLSTM(CommonTFLayerTest): + def _prepare_input(self, inputs_info): + assert 'x:0' in inputs_info, "Test error: inputs_info must contain `x`" + x_shape = inputs_info['x:0'] + inputs_data = {} + inputs_data['x:0'] = rng.uniform(0, 1, x_shape).astype(np.float32) + return inputs_data + + def create_block_lstm(self, time_len, seq_len_max, input_size, hidden_size, batch_size, + forget_bias, cell_clip, use_peephole): + tf.compat.v1.reset_default_graph() + # Create the graph and model + with tf.compat.v1.Session() as sess: + x = tf.compat.v1.placeholder(np.float32, [time_len, batch_size, input_size], 'x') + cs_prev = rng.uniform(0, 1, [batch_size, hidden_size]).astype(np.float32) + h_prev = rng.uniform(0, 1, [batch_size, hidden_size]).astype(np.float32) + w = rng.uniform(0, 1, [input_size + hidden_size, 4 * hidden_size]).astype(np.float32) + wci = rng.uniform(0, 1, [hidden_size]).astype(np.float32) + wcf = rng.uniform(0, 1, [hidden_size]).astype(np.float32) + wco = rng.uniform(0, 1, [hidden_size]).astype(np.float32) + b = rng.uniform(0, 1, [4 * hidden_size]).astype(np.float32) + _, cs_output, _, _, _, _, h_output = tf.raw_ops.BlockLSTM(x=x, seq_len_max=seq_len_max, cs_prev=cs_prev, + h_prev=h_prev, w=w, wci=wci, + wcf=wcf, wco=wco, b=b, forget_bias=forget_bias, + cell_clip=cell_clip, + use_peephole=use_peephole) + tf.identity(cs_output, name='cs_output') + tf.identity(h_output, name='h_output') + tf.compat.v1.global_variables_initializer() + tf_net = sess.graph_def + + ref_net = None + return tf_net, ref_net + + @pytest.mark.parametrize('time_len', [2, 5]) + @pytest.mark.parametrize('seq_len_max', [1, 2]) + @pytest.mark.parametrize('input_size', [1, 2, 5]) + @pytest.mark.parametrize('hidden_size', [1, 3]) + @pytest.mark.parametrize('batch_size', [1, 2, 3]) + @pytest.mark.parametrize('forget_bias', [-2.0, 0.0, 1.0]) + @pytest.mark.parametrize('cell_clip', [-1.0]) + @pytest.mark.parametrize("use_peephole", [False]) + @pytest.mark.nightly + @pytest.mark.precommit + def test_block_lstm(self, time_len, seq_len_max, input_size, hidden_size, batch_size, + forget_bias, cell_clip, use_peephole, + ie_device, precision, ir_version, temp_dir, + use_legacy_frontend): + if ie_device == 'GPU': + pytest.skip("Skip BlockLSTM test on GPU") + self._test(*self.create_block_lstm(time_len, seq_len_max, input_size, hidden_size, batch_size, + forget_bias, cell_clip, use_peephole), + ie_device, precision, temp_dir=temp_dir, ir_version=ir_version, + use_legacy_frontend=use_legacy_frontend, custom_eps=3 * 1e-3)