diff --git a/.gitmodules b/.gitmodules index c8bf9375572..5f6560797c6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -81,3 +81,6 @@ [submodule "thirdparty/telemetry"] path = thirdparty/telemetry url = https://github.com/openvinotoolkit/telemetry.git +[submodule "src/plugins/intel_cpu/thirdparty/libxsmm"] + path = src/plugins/intel_cpu/thirdparty/libxsmm + url = https://github.com/libxsmm/libxsmm.git diff --git a/cmake/features.cmake b/cmake/features.cmake index 3cc86bc99f9..edface9cc7b 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -58,6 +58,8 @@ ov_dependent_option (ENABLE_GPU_DEBUG_CAPS "enable GPU debug capabilities at run ov_dependent_option (ENABLE_CPU_DEBUG_CAPS "enable CPU debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS;ENABLE_INTEL_CPU" OFF) ov_dependent_option (ENABLE_SNIPPETS_DEBUG_CAPS "enable Snippets debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS" OFF) +ov_dependent_option (ENABLE_SNIPPETS_LIBXSMM_TPP "allow Snippets to use LIBXSMM Tensor Processing Primitives" OFF "ENABLE_INTEL_CPU AND X86_64" OFF) + ov_option (ENABLE_PROFILING_ITT "Build with ITT tracing. Optionally configure pre-built ittnotify library though INTEL_VTUNE_DIR variable." OFF) ov_option_enum(ENABLE_PROFILING_FILTER "Enable or disable ITT counter groups.\ diff --git a/src/common/snippets/include/snippets/emitter.hpp b/src/common/snippets/include/snippets/emitter.hpp index 6ae41e3cc3a..4ae178810d5 100644 --- a/src/common/snippets/include/snippets/emitter.hpp +++ b/src/common/snippets/include/snippets/emitter.hpp @@ -16,7 +16,7 @@ namespace snippets { * @interface RegType * @brief Register type of input and output operations */ -enum class RegType { gpr, vec }; +enum class RegType { gpr, vec, undefined }; /** * @interface Reg * @brief Register representation: type of register and index diff --git a/src/common/snippets/include/snippets/lowered/pass/init_loops.hpp b/src/common/snippets/include/snippets/lowered/pass/init_loops.hpp index 034587c23c4..c0ad3052d56 100644 --- a/src/common/snippets/include/snippets/lowered/pass/init_loops.hpp +++ b/src/common/snippets/include/snippets/lowered/pass/init_loops.hpp @@ -24,7 +24,7 @@ public: InitLoops() = default; bool run(LinearIR& linear_ir) override; - static void init_loop_info(const LinearIR::LoopManager::LoopInfoPtr& loop_info, bool only_runtime_args = false); + static void init_loop_info(const LinearIR::LoopManager::LoopInfoPtr& loop_info, size_t loop_id, bool only_runtime_args = false); }; } // namespace pass diff --git a/src/common/snippets/include/snippets/lowered/pass/insert_buffers.hpp b/src/common/snippets/include/snippets/lowered/pass/insert_buffers.hpp index 37a03a364e8..06890bc4420 100644 --- a/src/common/snippets/include/snippets/lowered/pass/insert_buffers.hpp +++ b/src/common/snippets/include/snippets/lowered/pass/insert_buffers.hpp @@ -33,12 +33,12 @@ private: const LinearIR::constExprIt& end_it, const LinearIR::LoopManagerPtr& loop_manager, const std::vector& loop_entries, - const std::vector& loop_exits); + const std::vector& loop_exits) const; - LinearIR::constExprIt insertion_position(const LinearIR& linear_ir, - const LinearIR::LoopManagerPtr& loop_manager, - const ExpressionPtr& expr, - const ExpressionPtr& down_expr); + static LinearIR::constExprIt insertion_position(const LinearIR& linear_ir, + const LinearIR::LoopManagerPtr& loop_manager, + const ExpressionPtr& expr, + const ExpressionPtr& down_expr); int32_t m_buffer_allocation_rank; }; diff --git a/src/common/snippets/include/snippets/lowered/port_descriptor.hpp b/src/common/snippets/include/snippets/lowered/port_descriptor.hpp index dfdebbcafed..378f652f063 100644 --- a/src/common/snippets/include/snippets/lowered/port_descriptor.hpp +++ b/src/common/snippets/include/snippets/lowered/port_descriptor.hpp @@ -113,6 +113,17 @@ public: std::vector outputs{}; }; +template +void set_port_desc(const T& port, std::vector subtensor) { + const auto& shape = port.get_shape(); + for (size_t i = 1; i <= std::min(subtensor.size(), shape.size()); i++) { + auto& dim = subtensor[subtensor.size() - i]; + if (dim != PortDescriptor::ServiceDimensions::FULL_DIM) + dim = std::min(dim, shape[shape.size() - i]); + } + PortDescriptorUtils::set_port_descriptor_ptr(port, std::make_shared(shape, subtensor)); +} + } // namespace lowered } // namespace snippets } // namespace ov diff --git a/src/common/snippets/include/snippets/op/brgemm.hpp b/src/common/snippets/include/snippets/op/brgemm.hpp index cfdfbadd561..a170b02b153 100644 --- a/src/common/snippets/include/snippets/op/brgemm.hpp +++ b/src/common/snippets/include/snippets/op/brgemm.hpp @@ -17,33 +17,51 @@ namespace op { * @brief Brgemm is a batch-reduced matrix multiplication with the support of arbitrary strides between matrices rows * @ingroup snippets */ -class Brgemm : public MemoryAccess { +class Brgemm : virtual public modifier::MemoryAccess, public ov::op::Op { public: - OPENVINO_OP("Brgemm", "SnippetsOpset", MemoryAccess); + OPENVINO_OP("Brgemm", "SnippetsOpset"); Brgemm(const Output& A, const Output& B, const size_t offset_a = 0lu, const size_t offset_b = 0lu, const size_t offset_c = 0lu, - std::vector layout_a = {}, std::vector layout_b = {}, std::vector layout_c = {}); + std::vector layout_a = {}, std::vector layout_b = {}, std::vector layout_c = {}, + size_t blk_size_m = 0, size_t blk_size_k = 0, size_t blk_size_n = 0); Brgemm(const Output& A, const Output& B, const PortDescriptor& desc_a, const PortDescriptor& desc_b, const PortDescriptor& desc_c, - std::vector layout_a = {}, std::vector layout_b = {}, std::vector layout_c = {}); + std::vector layout_a = {}, std::vector layout_b = {}, std::vector layout_c = {}, + size_t blk_size_m = 0, size_t blk_size_k = 0, size_t blk_size_n = 0); Brgemm() = default; size_t get_offset_a() const { return get_input_offset(0); } size_t get_offset_b() const { return get_input_offset(1); } size_t get_offset_c() const { return get_output_offset(0); } + size_t get_m_block_size() const { return m_M_blk; } + size_t get_k_block_size() const { return m_K_blk; } + size_t get_n_block_size() const { return m_N_blk; } + float get_beta() const { return m_beta; } + + void set_m_block_size(size_t block_size) { m_M_blk = block_size; } + void set_k_block_size(size_t block_size) { m_K_blk = block_size; } + void set_n_block_size(size_t block_size) { m_N_blk = block_size; } + void set_beta(float beta) { m_beta = beta; } + static ov::element::Type get_output_type(const ov::element::Type& in_type0, const ov::element::Type& in_type1); void validate_and_infer_types() override; std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; bool has_evaluate() const override { return false; } + bool visit_attributes(AttributeVisitor& visitor) override; protected: ov::element::Type get_output_type() const; std::vector get_planar_input_shapes(const std::vector>& inputs) const; ov::PartialShape get_output_partial_shape(const std::vector& input_shapes) const; ov::PartialShape get_planar_output_shape(const ov::PartialShape& output_shape) const; + void compute_block_size_values(size_t blk_size_m, size_t blk_size_k, size_t blk_size_n); + size_t m_M_blk = 0; + size_t m_K_blk = 0; + size_t m_N_blk = 0; + float m_beta = 0.f; private: void custom_constructor_validate_and_infer_types(std::vector layout_a, std::vector layout_b, std::vector layout_c); diff --git a/src/common/snippets/include/snippets/op/broadcastload.hpp b/src/common/snippets/include/snippets/op/broadcastload.hpp index 749eef16119..e9a0fc01131 100644 --- a/src/common/snippets/include/snippets/op/broadcastload.hpp +++ b/src/common/snippets/include/snippets/op/broadcastload.hpp @@ -17,9 +17,9 @@ namespace op { * @brief Is generated for broadcasting by least varying dimension for non-blocked cases and the second varying dimension for blocked * @ingroup snippets */ -class BroadcastLoad : public MemoryAccess { +class BroadcastLoad : public modifier::MemoryAccess, public ov::op::Op { public: - OPENVINO_OP("BroadcastLoad", "SnippetsOpset", ov::snippets::op::MemoryAccess); + OPENVINO_OP("BroadcastLoad", "SnippetsOpset"); BroadcastLoad(const Output& x, ov::Dimension bcast_dimension, size_t offset = 0lu); BroadcastLoad() = default; diff --git a/src/common/snippets/include/snippets/op/load.hpp b/src/common/snippets/include/snippets/op/load.hpp index bc4b00f82db..bca4b09fabd 100644 --- a/src/common/snippets/include/snippets/op/load.hpp +++ b/src/common/snippets/include/snippets/op/load.hpp @@ -19,9 +19,9 @@ namespace op { * and memory offset for loading is determined by "offset" (Default value is "0" - to load starting from the first element) * @ingroup snippets */ -class Load : public MemoryAccess { +class Load : public modifier::MemoryAccess, public ov::op::Op { public: - OPENVINO_OP("Load", "SnippetsOpset", MemoryAccess); + OPENVINO_OP("Load", "SnippetsOpset"); Load(const Output& x, const size_t count = 1lu, const size_t offset = 0lu); Load() = default; @@ -34,6 +34,7 @@ public: void validate_and_infer_types() override; std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; protected: void validate_memory_access_params() const; diff --git a/src/common/snippets/include/snippets/op/memory_access.hpp b/src/common/snippets/include/snippets/op/memory_access.hpp index ea41acb4a0d..701e443f1ce 100644 --- a/src/common/snippets/include/snippets/op/memory_access.hpp +++ b/src/common/snippets/include/snippets/op/memory_access.hpp @@ -8,7 +8,7 @@ namespace ov { namespace snippets { -namespace op { +namespace modifier { /** * @interface MemoryAccess @@ -19,10 +19,8 @@ namespace op { * @ingroup snippets */ -class MemoryAccess : public ov::op::Op { +class MemoryAccess { public: - OPENVINO_OP("MemoryAccess", "SnippetsOpset"); - /** * @interface PortDescriptor * @brief This class describes port of MemoryAccess operation @@ -34,13 +32,16 @@ public: struct PortDescriptor { PortDescriptor(size_t count, size_t offset) : count(count), offset(offset) {} PortDescriptor() = default; - + // TODO: should we deprecate count in favor of subtensors, ticket: 130004 size_t count = 0lu; size_t offset = 0lu; + // Note: stride is interpreted as leading dimension for 2D subtensor ops + size_t stride = 0lu; size_t index = 0lu; private: - PortDescriptor(size_t count, size_t offset, size_t index) : count(count), offset(offset), index(index) {} + PortDescriptor(size_t count, size_t offset, size_t stride, size_t index) : + count(count), offset(offset), stride(stride), index(index) {} friend class MemoryAccess; }; @@ -50,11 +51,15 @@ public: void set_output_count(size_t count, size_t idx = 0); void set_input_offset(size_t offset, size_t idx = 0); void set_output_offset(size_t offset, size_t idx = 0); + void set_input_stride(size_t stride, size_t idx = 0); + void set_output_stride(size_t stride, size_t idx = 0); size_t get_input_count(size_t idx = 0) const; size_t get_output_count(size_t idx = 0) const; size_t get_input_offset(size_t idx = 0) const; size_t get_output_offset(size_t idx = 0) const; + size_t get_input_stride(size_t idx = 0) const; + size_t get_output_stride(size_t idx = 0) const; PortMap get_memory_access_input_ports() const { return m_input_ports; } PortMap get_memory_access_output_ports() const { return m_output_ports; } @@ -62,15 +67,17 @@ public: bool is_memory_access_input_port(size_t idx) const; bool is_memory_access_output_port(size_t idx) const; - // All input and output ports are MemoryAccess - bool is_full_memory_access_op() const; + /** + * @brief Checks if the provided operation memory access on all ports + */ + bool is_full_memory_access_op(const std::shared_ptr& op) const; - bool visit_attributes(AttributeVisitor& visitor) override; + bool visit_attributes(AttributeVisitor& visitor); protected: - explicit MemoryAccess(const OutputVector& arguments, size_t input_count = 0, size_t output_count = 0); - explicit MemoryAccess(const OutputVector& arguments, const std::set& input_ports, const std::set& output_ports); - explicit MemoryAccess(const OutputVector& arguments, const PortMap& input_ports, const PortMap& output_ports); + explicit MemoryAccess(size_t input_count, size_t output_count = 0); + explicit MemoryAccess(const std::set& input_ports, const std::set& output_ports); + explicit MemoryAccess(const PortMap& input_ports, const PortMap& output_ports); MemoryAccess() = default; // This method can be called only in ctors @@ -80,12 +87,14 @@ protected: void set_output_port_descriptor(const PortDescriptor& desc, const size_t i); const PortDescriptor& get_input_port_descriptor(const size_t i) const; const PortDescriptor& get_output_port_descriptor(const size_t i) const; + PortDescriptor& get_input_port_descriptor(const size_t i); + PortDescriptor& get_output_port_descriptor(const size_t i); // [port_num, port_desc] PortMap m_input_ports; PortMap m_output_ports; }; -} // namespace op +} // namespace modifier } // namespace snippets } // namespace ov diff --git a/src/common/snippets/include/snippets/op/store.hpp b/src/common/snippets/include/snippets/op/store.hpp index adb5842b7c9..968f4d73156 100644 --- a/src/common/snippets/include/snippets/op/store.hpp +++ b/src/common/snippets/include/snippets/op/store.hpp @@ -18,9 +18,9 @@ namespace op { * and memory offset for storing is determined by "offset" (Default value is "0" - to store starting at start memory ptr) * @ingroup snippets */ -class Store : public MemoryAccess { +class Store : public modifier::MemoryAccess, public ov::op::Op { public: - OPENVINO_OP("Store", "SnippetsOpset", MemoryAccess); + OPENVINO_OP("Store", "SnippetsOpset"); Store(const Output& x, const size_t count = 1lu, const size_t offset = 0lu); Store() = default; @@ -33,6 +33,7 @@ public: void validate_and_infer_types() override; std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; }; } // namespace op diff --git a/src/common/snippets/src/generator.cpp b/src/common/snippets/src/generator.cpp index 027314d5ad4..c5435c1ad9d 100644 --- a/src/common/snippets/src/generator.cpp +++ b/src/common/snippets/src/generator.cpp @@ -13,6 +13,7 @@ #include "snippets/lowered/pass/optimize_loop_single_evaluation.hpp" #include "snippets/lowered/pass/pass.hpp" #include "snippets/op/kernel.hpp" +#include "snippets/op/memory_access.hpp" namespace ov { namespace snippets { @@ -72,6 +73,9 @@ std::shared_ptr Generator::get_target_machine() const { } RegType Generator::get_op_out_reg_type(const ov::Output& out) const { + auto reg_type = get_specific_op_out_reg_type(out); + if (reg_type != RegType::undefined) + return reg_type; const auto op = out.get_node_shared_ptr(); if (std::dynamic_pointer_cast(op) || std::dynamic_pointer_cast(op) || @@ -107,7 +111,8 @@ RegType Generator::get_op_out_reg_type(const ov::Output& out) const { std::dynamic_pointer_cast(op)) return RegType::vec; else - return get_specific_op_out_reg_type(op); + OPENVINO_THROW("Register type of the operation " + std::string(op->get_type_name()) + " isn't determined!"); + return reg_type; } RegType Generator::get_specific_op_out_reg_type(const ov::Output& out) const { diff --git a/src/common/snippets/src/lowered/pass/allocate_buffers.cpp b/src/common/snippets/src/lowered/pass/allocate_buffers.cpp index 027a5b9d5a4..1499d9f59fa 100644 --- a/src/common/snippets/src/lowered/pass/allocate_buffers.cpp +++ b/src/common/snippets/src/lowered/pass/allocate_buffers.cpp @@ -38,7 +38,7 @@ void AllocateBuffers::set_buffer_offset(const ExpressionPtr& buffer_expr, const const auto& parent_expr = parent_output.get_expr(); const auto port = parent_output.get_index(); const auto& parent_node = parent_expr->get_node(); - auto memory_access = ov::as_type_ptr(parent_node); + auto memory_access = std::dynamic_pointer_cast(parent_node); if (memory_access && memory_access->is_memory_access_output_port(port)) { memory_access->set_output_offset(offset, port); } else { @@ -54,7 +54,7 @@ void AllocateBuffers::set_buffer_offset(const ExpressionPtr& buffer_expr, const const auto& child_expr = child_expr_input.get_expr(); const auto port = child_expr_input.get_index(); const auto& child_node = child_expr->get_node(); - auto memory_access = ov::as_type_ptr(child_node); + auto memory_access = std::dynamic_pointer_cast(child_node); if (memory_access && memory_access->is_memory_access_input_port(port)) { memory_access->set_input_offset(offset, port); } else if (ov::is_type(child_node)) { diff --git a/src/common/snippets/src/lowered/pass/define_buffer_clusters.cpp b/src/common/snippets/src/lowered/pass/define_buffer_clusters.cpp index 5fbf7ae1035..22bfe21c338 100644 --- a/src/common/snippets/src/lowered/pass/define_buffer_clusters.cpp +++ b/src/common/snippets/src/lowered/pass/define_buffer_clusters.cpp @@ -301,8 +301,8 @@ bool DefineBufferClusters::are_buffer_neighbours(const ExpressionPtr& up, const } void DefineBufferClusters::parse_memory_access_op(const ExpressionPtr& expr) { - const auto ma = ov::as_type_ptr(expr->get_node()); - if (!ma->is_full_memory_access_op()) + const auto ma = std::dynamic_pointer_cast(expr->get_node()); + if (!ma->is_full_memory_access_op(expr->get_node())) return; // TODO: Some full MemoryAccess ops can have inplace inputs and outputs in general. // Need to add mechanism of inplace ports using MemoryAccess::PortDescriptor::inplace @@ -331,7 +331,7 @@ bool DefineBufferClusters::run(lowered::LinearIR& linear_ir, lowered::LinearIR:: continue; } - if (ov::is_type(op)) { + if (std::dynamic_pointer_cast(op)) { parse_memory_access_op(expr); continue; } diff --git a/src/common/snippets/src/lowered/pass/init_loops.cpp b/src/common/snippets/src/lowered/pass/init_loops.cpp index 4b258b6967e..c97c93a6ecf 100644 --- a/src/common/snippets/src/lowered/pass/init_loops.cpp +++ b/src/common/snippets/src/lowered/pass/init_loops.cpp @@ -16,6 +16,7 @@ namespace lowered { namespace pass { using LoopPort = LinearIR::LoopManager::LoopPort; +using MemoryAccess = ov::snippets::modifier::MemoryAccess; namespace { inline int64_t get_stride(size_t dim, const VectorDims& shape) { @@ -29,9 +30,47 @@ inline int64_t get_stride(size_t dim, const VectorDims& shape) { return stride; } -inline void init_is_incremented(LoopPort& loop_port) { - if (!ov::is_type(loop_port.expr_port->get_expr()->get_node())) - loop_port.is_incremented = false; +inline void init_is_incremented(LoopPort& port, size_t loop_id) { + const auto& expr = port.expr_port->get_expr(); + const auto& expr_loops = expr->get_loop_ids(); + if (!std::dynamic_pointer_cast(expr->get_node())) { + port.is_incremented = false; + } else if (expr_loops.back() != loop_id) { + // Note: LoopPort connected to Buffer between two loops should not be incremented in the outermost loop + // Consider the example below: + // Store; Loop ids [0,1,2,3] + // IntermediateMemoryBuffer; Loop ids [0,1] + // Load; Loop ids [0,1,4,5] + // Store is exit port of Loop-1, but it should be incremented only in Loop-2 and Loop-3. Similar with Load. + auto is_ignored = [=](const ExpressionPtr& target_expr) { + if (ov::is_type(target_expr->get_node())) { + const auto& target_loops = target_expr->get_loop_ids(); + const auto i_max = std::min(expr_loops.size(), target_loops.size()); + for (size_t i = 0; i < i_max && expr_loops[i] == target_loops[i]; i++) { + if (target_loops[i] == loop_id) + return true; + } + } + return false; + }; + if (port.expr_port->get_type() == ExpressionPort::Type::Output) { + const auto& out_connector = expr->get_output_port_connector(port.expr_port->get_index()); + for (const auto& consumer : out_connector->get_consumers()) { + if (is_ignored(consumer.get_expr())) { + port.is_incremented = false; + return; + } + } + } else if (port.expr_port->get_type() == ExpressionPort::Type::Input) { + const auto& in_connector = expr->get_input_port_connector(port.expr_port->get_index()); + if (is_ignored(in_connector->get_source().get_expr())) { + port.is_incremented = false; + return; + } + } else { + OPENVINO_THROW("Unexpected LoopPort type"); + } + } } inline void init_ptr_increment(LoopPort& loop_port, size_t work_amount) { @@ -97,7 +136,7 @@ inline void init_work_amount(const LinearIR::LoopManager::LoopInfoPtr& loop_info } } // namespace -void InitLoops::init_loop_info(const LinearIR::LoopManager::LoopInfoPtr& loop_info, bool only_runtime_args) { +void InitLoops::init_loop_info(const LinearIR::LoopManager::LoopInfoPtr& loop_info, const size_t loop_id, bool only_runtime_args) { if (utils::is_dynamic_value(loop_info->get_work_amount())) init_work_amount(loop_info); @@ -108,8 +147,8 @@ void InitLoops::init_loop_info(const LinearIR::LoopManager::LoopInfoPtr& loop_in init_finalization_offset(loop_port, work_amount); }; - auto init_all_parameters = [&init_runtime_parameters](LoopPort& loop_port) { - init_is_incremented(loop_port); + auto init_all_parameters = [loop_id, &init_runtime_parameters](LoopPort& loop_port) { + init_is_incremented(loop_port, loop_id); init_data_size(loop_port); init_runtime_parameters(loop_port); }; @@ -131,7 +170,7 @@ bool InitLoops::run(LinearIR& linear_ir) { const auto& loop_manager = linear_ir.get_loop_manager(); const auto& loops = loop_manager->get_map(); for (const auto& loop : loops) { - init_loop_info(loop.second); + init_loop_info(loop.second, loop.first); } return true; diff --git a/src/common/snippets/src/lowered/pass/insert_broadcastmove.cpp b/src/common/snippets/src/lowered/pass/insert_broadcastmove.cpp index 997c9fc0143..bc911b88ecf 100644 --- a/src/common/snippets/src/lowered/pass/insert_broadcastmove.cpp +++ b/src/common/snippets/src/lowered/pass/insert_broadcastmove.cpp @@ -7,7 +7,6 @@ #include "snippets/lowered/linear_ir.hpp" #include "snippets/lowered/loop_manager.hpp" #include "snippets/snippets_isa.hpp" -#include "snippets/utils.hpp" #include "snippets/itt.hpp" namespace ov { @@ -16,9 +15,10 @@ namespace lowered { namespace pass { bool InsertBroadcastMove::is_broadcasting_supported(const std::shared_ptr& n) { - return ov::op::util::supports_auto_broadcast(n) || + return !std::dynamic_pointer_cast(n) && + (ov::op::util::supports_auto_broadcast(n) || n->get_autob().m_type == ov::op::AutoBroadcastType::NUMPY || - is_type(n); + is_type(n)); } bool InsertBroadcastMove::is_broadcasting_needed(const std::shared_ptr& n) { diff --git a/src/common/snippets/src/lowered/pass/insert_buffers.cpp b/src/common/snippets/src/lowered/pass/insert_buffers.cpp index 8e3bafd5b49..6d3cd779351 100644 --- a/src/common/snippets/src/lowered/pass/insert_buffers.cpp +++ b/src/common/snippets/src/lowered/pass/insert_buffers.cpp @@ -34,12 +34,14 @@ std::vector get_buffer_loop_ids(const std::vector& lhs, const st // TODO: This logic covers only several specific cases so it should be generalized. ov::Shape compute_allocation_shape(const LinearIR::LoopManagerPtr& loop_manager, const std::vector& buffer_loop_ids, - const std::vector& parent_loop_ids, - const ExpressionPort& expr_port, + const ExpressionPort& parent_expr_output, const int allocation_rank) { - const auto planar_shape = utils::get_preordered_vdims(expr_port); + const auto& parent_expr = parent_expr_output.get_expr(); + const auto& parent_loop_ids = parent_expr->get_loop_ids(); + const auto planar_shape = utils::get_preordered_vdims(parent_expr_output); - const size_t rank = allocation_rank >= 0 ? std::min(static_cast(allocation_rank), planar_shape.size()) : planar_shape.size(); + const size_t rank = allocation_rank >= 0 ? std::min(static_cast(allocation_rank), planar_shape.size()) + : planar_shape.size(); ov::Shape allocation_shape(rank); for (size_t i = 0; i < rank; ++i) { *(allocation_shape.rbegin() + i) = *(planar_shape.rbegin() + i); @@ -62,7 +64,7 @@ ov::Shape compute_allocation_shape(const LinearIR::LoopManagerPtr& loop_manager, // 2. Buffer is inside M_loop_idx loop => allocation shape is not changed // 3. Buffer is outside N_loop_idx loop => the corresponding allocation shape value is replaced with N loop work amount // So the result allocation shape is [M_blk, N_loop_work_amount] - const auto& subtensor = expr_port.get_descriptor_ptr()->get_subtensor(); + const auto& subtensor = parent_expr_output.get_descriptor_ptr()->get_subtensor(); if (!subtensor.empty()) { for (size_t i = 0; i < std::min(rank, subtensor.size()); ++i) { auto& cur_val = *(allocation_shape.rbegin() + i); @@ -75,8 +77,8 @@ ov::Shape compute_allocation_shape(const LinearIR::LoopManagerPtr& loop_manager, const auto& exit_points = loop_info->get_exit_points(); auto it = std::find_if(exit_points.begin(), exit_points.end(), - [&expr_port](const LinearIR::LoopManager::LoopPort& port) { - return *port.expr_port == expr_port; + [&parent_expr_output](const LinearIR::LoopManager::LoopPort& port) { + return *port.expr_port == parent_expr_output; }); OPENVINO_ASSERT(it != exit_points.end(), "compute_allocation_shape: exit point of parent loop can not be found"); const auto& loop_port = *it; @@ -133,6 +135,10 @@ LinearIR::constExprIt InsertBuffers::insertion_position(const LinearIR& linear_i const auto down_loop_id = down_loops[loop_idx]; return loop_manager->get_loop_bounds(linear_ir, down_loop_id).first; } + // If upper and lower expressions are in the same loop, we should insert Buffer between them + if (loop_idx == up_loop_count && loop_idx == down_loop_count) { + return linear_ir.find(down_expr); + } OPENVINO_THROW("Incorrect configuration for Buffer insertion!"); } @@ -141,7 +147,7 @@ void InsertBuffers::insertion(LinearIR& linear_ir, const LinearIR::constExprIt& end_it, const LinearIR::LoopManagerPtr& loop_manager, const std::vector& loop_entries, - const std::vector& loop_exits) { + const std::vector& loop_exits) const { for (const auto& entry_point : loop_entries) { const auto& entry_port = entry_point.expr_port; const auto& expr = entry_port->get_expr(); @@ -164,12 +170,13 @@ void InsertBuffers::insertion(LinearIR& linear_ir, if (ov::is_type(parent) || ov::is_type(parent) || ov::is_type(parent) || - ov::is_type(parent)) + ov::is_type(parent) || + is_type(parent)) continue; // Each MemoryAccess op needs Buffer - const auto parent_ma = ov::as_type_ptr(parent); - const auto node_ma = ov::as_type_ptr(node); + const auto parent_ma = std::dynamic_pointer_cast(parent); + const auto node_ma = std::dynamic_pointer_cast(node); bool is_buffer_needed = (parent_ma && parent_ma->is_memory_access_output_port(parent_port)) || (node_ma && node_ma->is_memory_access_input_port(port_idx)); const auto& current_loops = expr->get_loop_ids(); @@ -184,7 +191,6 @@ void InsertBuffers::insertion(LinearIR& linear_ir, const auto pos = insertion_position(linear_ir, loop_manager, parent_expr, expr); const auto allocation_shape = compute_allocation_shape(loop_manager, buffer_loop_ids, - parent_loops, parent_expr_output, m_buffer_allocation_rank); const auto buffer = std::make_shared(parent->output(parent_port), allocation_shape); @@ -223,8 +229,8 @@ void InsertBuffers::insertion(LinearIR& linear_ir, continue; } // Each MemoryAccess op needs Buffer - const auto child_ma = ov::as_type_ptr(child); - const auto node_ma = ov::as_type_ptr(node); + const auto child_ma = std::dynamic_pointer_cast(child); + const auto node_ma = std::dynamic_pointer_cast(node); bool is_buffer_needed = (child_ma && child_ma->is_memory_access_input_port(child_port)) || (node_ma && node_ma->is_memory_access_output_port(port_idx)); const auto local_buffer_loop_ids = get_buffer_loop_ids(current_loops, child_expr->get_loop_ids(), is_buffer_needed); @@ -272,7 +278,6 @@ void InsertBuffers::insertion(LinearIR& linear_ir, const auto allocation_shape = compute_allocation_shape(loop_manager, buffer_loop_ids, - current_loops, *exit_port, m_buffer_allocation_rank); auto buffer = std::make_shared(node->output(port_idx), allocation_shape); @@ -304,7 +309,7 @@ bool InsertBuffers::run(LinearIR& linear_ir, lowered::LinearIR::constExprIt begi for (auto expr_it = begin; expr_it != end; expr_it++) { const auto expr = *expr_it; const auto node = (*expr_it)->get_node(); - const auto ma = ov::as_type_ptr(node); + const auto ma = std::dynamic_pointer_cast(node); if (!ma) continue; diff --git a/src/common/snippets/src/lowered/pass/insert_load_store.cpp b/src/common/snippets/src/lowered/pass/insert_load_store.cpp index c8466d85747..caeaa42924b 100644 --- a/src/common/snippets/src/lowered/pass/insert_load_store.cpp +++ b/src/common/snippets/src/lowered/pass/insert_load_store.cpp @@ -42,7 +42,8 @@ bool InsertLoadStore::insert_load(LinearIR& linear_ir, const LinearIR::constExpr const auto& data_out = data_expr->get_output_port_connector(0); for (const auto& consumer_input : data_out->get_consumers()) { const auto& consumer_expr = consumer_input.get_expr(); - const auto ma = ov::as_type_ptr(consumer_expr->get_node()); + const auto& consumer = consumer_expr->get_node(); + const auto ma = std::dynamic_pointer_cast(consumer); if (ma && ma->is_memory_access_input_port(consumer_input.get_index())) return false; @@ -63,7 +64,7 @@ bool InsertLoadStore::insert_store(LinearIR& linear_ir, const LinearIR::constExp const auto& parent_expr = parent_output.get_expr(); const auto port = parent_output.get_index(); const auto& parent = parent_expr->get_node(); - const auto ma = ov::as_type_ptr(parent); + const auto ma = std::dynamic_pointer_cast(parent); if (ma && ma->is_memory_access_output_port(port)) return false; diff --git a/src/common/snippets/src/lowered/pass/iter_handler.cpp b/src/common/snippets/src/lowered/pass/iter_handler.cpp index df815a17f0d..e322f5fbb6b 100644 --- a/src/common/snippets/src/lowered/pass/iter_handler.cpp +++ b/src/common/snippets/src/lowered/pass/iter_handler.cpp @@ -29,7 +29,7 @@ bool UpdateMemoryAccessCounts::run(LinearIR& linear_ir, LinearIR::constExprIt be } const auto& node = expr_it->get()->get_node(); - if (const auto memory_access = ov::as_type_ptr(node)) { + if (const auto memory_access = std::dynamic_pointer_cast(node)) { for (const auto p : memory_access->get_memory_access_input_ports()) { const auto port = p.first; if (memory_access->get_input_count(port) > 1) { diff --git a/src/common/snippets/src/lowered/pass/reduce_decomposition.cpp b/src/common/snippets/src/lowered/pass/reduce_decomposition.cpp index a0657a10500..a7a22b79a5f 100644 --- a/src/common/snippets/src/lowered/pass/reduce_decomposition.cpp +++ b/src/common/snippets/src/lowered/pass/reduce_decomposition.cpp @@ -63,7 +63,7 @@ bool ReduceDecomposition::run(LinearIR& linear_ir, LinearIR::constExprIt begin, for (auto expr_it = begin; expr_it != end; expr_it++) { const auto& reduce_expr = *expr_it; const auto& reduce = ov::as_type_ptr(reduce_expr->get_node()); - if (!reduce) + if (!reduce || std::dynamic_pointer_cast(reduce_expr->get_node())) continue; const auto& reduce_type_info = reduce->get_type_info(); diff --git a/src/common/snippets/src/lowered/pass/validate.cpp b/src/common/snippets/src/lowered/pass/validate.cpp index c890811efb4..7cb58cb1a23 100644 --- a/src/common/snippets/src/lowered/pass/validate.cpp +++ b/src/common/snippets/src/lowered/pass/validate.cpp @@ -38,7 +38,7 @@ void validate_parameter(const ExpressionPtr& expr, const LinearIR& linear_ir) { std::set> layouts; for (const auto& consumer_input : consumer_inputs) { const auto& node = consumer_input.get_expr()->get_node(); - if (const auto ma = ov::as_type_ptr(node)) { + if (const auto ma = std::dynamic_pointer_cast(node)) { OPENVINO_ASSERT(ma->is_memory_access_input_port(consumer_input.get_index()), "Parameter expects MemoryAccess on output"); layouts.insert(consumer_input.get_descriptor_ptr()->get_layout()); @@ -55,7 +55,7 @@ void validate_result(const ExpressionPtr& expr, const LinearIR& linear_ir) { const auto& shape_infer_seq = utils::get_first_parent_shape_infer_expr_seq(expr); const auto& expr_val = shape_infer_seq.empty() ? expr : shape_infer_seq.back(); const auto source = expr_val->get_input_port_connector(0)->get_source(); - const auto ma = ov::as_type_ptr(source.get_expr()->get_node()); + const auto ma = std::dynamic_pointer_cast(source.get_expr()->get_node()); OPENVINO_ASSERT(ma && ma->is_memory_access_output_port(source.get_index()), "Result expects MemoryAccess parent"); } @@ -65,7 +65,7 @@ void validate_buffer(const ExpressionPtr& expr, const LinearIR& linear_ir) { "Buffer validation expects Buffer op"); const auto& in = expr->get_input_port_connector(0); const auto& source = in->get_source(); - const auto ma = ov::as_type_ptr(source.get_expr()->get_node()); + const auto ma = std::dynamic_pointer_cast(source.get_expr()->get_node()); OPENVINO_ASSERT(ma && ma->is_memory_access_input_port(source.get_index()), "Buffer expects MemoryAccess parent"); const auto& shape_infer_seq = utils::get_first_child_shape_infer_expr_seq(expr); @@ -74,7 +74,7 @@ void validate_buffer(const ExpressionPtr& expr, const LinearIR& linear_ir) { const auto consumers = out->get_consumers(); for (const auto& consumer_input : consumers) { const auto& node = consumer_input.get_expr()->get_node(); - if (const auto ma = ov::as_type_ptr(node)) { + if (const auto ma = std::dynamic_pointer_cast(node)) { OPENVINO_ASSERT(ma->is_memory_access_input_port(consumer_input.get_index()), "Buffer expects MemoryAccess on output"); } else { diff --git a/src/common/snippets/src/op/brgemm.cpp b/src/common/snippets/src/op/brgemm.cpp index 36cf9147ee4..12dc34601d6 100644 --- a/src/common/snippets/src/op/brgemm.cpp +++ b/src/common/snippets/src/op/brgemm.cpp @@ -32,23 +32,35 @@ std::vector get_output_layout(const std::shared_ptr& n) Brgemm::Brgemm(const Output& A, const Output& B, const size_t offset_a, const size_t offset_b, const size_t offset_c, - std::vector layout_a, std::vector layout_b, std::vector layout_c) - : MemoryAccess({A, B}, std::set{0, 1}, std::set{0}) { + std::vector layout_a, std::vector layout_b, std::vector layout_c, + const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n) + : MemoryAccess(std::set{0, 1}, std::set{0}), Op({A, B}) { set_output_size(1); set_input_offset(offset_a, 0); set_input_offset(offset_b, 1); set_output_offset(offset_c, 0); + compute_block_size_values(blk_size_m, blk_size_k, blk_size_n); custom_constructor_validate_and_infer_types(std::move(layout_a), std::move(layout_b), std::move(layout_c)); } Brgemm::Brgemm(const Output& A, const Output& B, const PortDescriptor& desc_a, const PortDescriptor& desc_b, const PortDescriptor& desc_c, - std::vector layout_a, std::vector layout_b, std::vector layout_c) - : MemoryAccess({A, B}, PortMap{{0, desc_a}, {1, desc_b}}, PortMap{{0, desc_c}}) { + std::vector layout_a, std::vector layout_b, std::vector layout_c, + const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n) + : MemoryAccess(PortMap{{0, desc_a}, {1, desc_b}}, PortMap{{0, desc_c}}), Op({A, B}) { set_output_size(1); + compute_block_size_values(blk_size_m, blk_size_k, blk_size_n); custom_constructor_validate_and_infer_types(std::move(layout_a), std::move(layout_b), std::move(layout_c)); } +void Brgemm::compute_block_size_values(const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n) { + const auto input_shape_0 = snippets::utils::get_planar_pshape(input(0)).get_shape(); + const auto input_shape_1 = snippets::utils::get_planar_pshape(input(1)).get_shape(); + m_M_blk = blk_size_m != 0 ? blk_size_m : *++input_shape_0.rbegin(); + m_K_blk = blk_size_k != 0 ? blk_size_k : *input_shape_0.rbegin(); + m_N_blk = blk_size_n != 0 ? blk_size_n : *input_shape_1.rbegin(); +} + void Brgemm::custom_constructor_validate_and_infer_types(std::vector layout_a, std::vector layout_b, std::vector layout_c) { INTERNAL_OP_SCOPE(BrgemmCPU_constructor_validate_and_infer_types); validate_inputs(); @@ -87,6 +99,14 @@ std::shared_ptr Brgemm::clone_with_new_inputs(const OutputVector& new_args lowered::PortDescriptorUtils::get_port_descriptor_ptr(output(0))->get_layout()); } +bool Brgemm::visit_attributes(AttributeVisitor& visitor) { + visitor.on_attribute("blk_M", m_M_blk); + visitor.on_attribute("blk_K", m_K_blk); + visitor.on_attribute("blk_N", m_N_blk); + visitor.on_attribute("beta", m_beta); + return MemoryAccess::visit_attributes(visitor); +} + ov::element::Type Brgemm::get_output_type(const ov::element::Type& in_type0, const ov::element::Type& in_type1) { const bool is_f32 = utils::everyone_is(element::f32, in_type0, in_type1); const bool is_int8 = utils::one_of(in_type0, element::i8, element::u8) && in_type1 == element::i8; diff --git a/src/common/snippets/src/op/broadcastload.cpp b/src/common/snippets/src/op/broadcastload.cpp index 152d68006e0..166d7100c37 100644 --- a/src/common/snippets/src/op/broadcastload.cpp +++ b/src/common/snippets/src/op/broadcastload.cpp @@ -12,7 +12,7 @@ namespace snippets { namespace op { BroadcastLoad::BroadcastLoad(const Output& x, ov::Dimension bcast_dimension, size_t offset) - : MemoryAccess({x}, std::set{0}, std::set{}), bcast_dimension(std::move(bcast_dimension)) { + : MemoryAccess(std::set{0}, std::set{}), Op({x}), bcast_dimension(std::move(bcast_dimension)) { set_input_port_descriptor({1, offset}, 0); constructor_validate_and_infer_types(); } diff --git a/src/common/snippets/src/op/load.cpp b/src/common/snippets/src/op/load.cpp index b009ecd2e9e..d48641c17cc 100644 --- a/src/common/snippets/src/op/load.cpp +++ b/src/common/snippets/src/op/load.cpp @@ -13,7 +13,7 @@ namespace snippets { namespace op { Load::Load(const Output& x, const size_t count, const size_t offset) - : MemoryAccess({x}, std::set{0}, std::set{}) { + : MemoryAccess(std::set{0}, std::set{}), Op({x}) { set_input_port_descriptor({count, offset}, 0); constructor_validate_and_infer_types(); } @@ -31,6 +31,10 @@ void Load::validate_and_infer_types() { set_output_type(0, get_input_element_type(0), get_input_partial_shape(0)); } +bool Load::visit_attributes(AttributeVisitor& visitor) { + return MemoryAccess::visit_attributes(visitor); +} + std::shared_ptr Load::clone_with_new_inputs(const OutputVector& new_args) const { INTERNAL_OP_SCOPE(Load); check_new_args_count(this, new_args); @@ -60,7 +64,7 @@ void LoadReshape::validate_and_infer_types() { } bool LoadReshape::visit_attributes(AttributeVisitor& visitor) { - Load::visit_attributes(visitor); + MemoryAccess::visit_attributes(visitor); visitor.on_attribute("order", m_order); return true; } diff --git a/src/common/snippets/src/op/memory_access.cpp b/src/common/snippets/src/op/memory_access.cpp index b721c444709..f3fa1f90c20 100644 --- a/src/common/snippets/src/op/memory_access.cpp +++ b/src/common/snippets/src/op/memory_access.cpp @@ -7,9 +7,9 @@ namespace ov { namespace snippets { -namespace op { +namespace modifier { -MemoryAccess::MemoryAccess(const OutputVector& arguments, size_t input_count, size_t output_count) : Op(arguments) { +MemoryAccess::MemoryAccess(size_t input_count, size_t output_count) { auto init_iota_set = [](size_t num) { if (num == 0) return std::set{}; @@ -20,28 +20,28 @@ MemoryAccess::MemoryAccess(const OutputVector& arguments, size_t input_count, si ctor_initialize(init_iota_set(input_count), init_iota_set(output_count)); } -MemoryAccess::MemoryAccess(const OutputVector& arguments, const std::set& input_ports, const std::set& output_ports) : Op(arguments) { +MemoryAccess::MemoryAccess(const std::set& input_ports, const std::set& output_ports) { ctor_initialize(input_ports, output_ports); } -MemoryAccess::MemoryAccess(const OutputVector& arguments, const PortMap& input_ports, const PortMap& output_ports) - : Op(arguments), m_input_ports(input_ports), m_output_ports(output_ports) {} +MemoryAccess::MemoryAccess(const PortMap& input_ports, const PortMap& output_ports) + : m_input_ports(input_ports), m_output_ports(output_ports) {} void MemoryAccess::ctor_initialize(const std::set& input_ports, const std::set& output_ports) { for (auto port : input_ports) { - m_input_ports[port] = {0, 0, port}; + m_input_ports[port] = {0, 0, 0, port}; } for (auto port : output_ports) { - m_output_ports[port] = {0, 0, port}; + m_output_ports[port] = {0, 0, 0, port}; } } -bool MemoryAccess::is_full_memory_access_op() const { - for (size_t i = 0; i < get_input_size(); ++i) { +bool MemoryAccess::is_full_memory_access_op(const std::shared_ptr& op) const { + for (size_t i = 0; i < op->get_input_size(); ++i) { if (!is_memory_access_input_port(i)) return false; } - for (size_t i = 0; i < get_output_size(); ++i) { + for (size_t i = 0; i < op->get_output_size(); ++i) { if (!is_memory_access_output_port(i)) return false; } @@ -54,12 +54,14 @@ bool MemoryAccess::visit_attributes(AttributeVisitor& visitor) { auto port = p.second; visitor.on_attribute("count_in_" + std::to_string(idx), port.count); visitor.on_attribute("offset_in_" + std::to_string(idx), port.offset); + visitor.on_attribute("stride_in_" + std::to_string(idx), port.stride); } for (const auto& p : m_output_ports) { auto idx = p.first; auto port = p.second; visitor.on_attribute("count_out_" + std::to_string(idx), port.count); visitor.on_attribute("offset_out_" + std::to_string(idx), port.offset); + visitor.on_attribute("stride_out_" + std::to_string(idx), port.stride); } return true; } @@ -74,38 +76,54 @@ bool MemoryAccess::is_memory_access_output_port(size_t idx) const { void MemoryAccess::set_input_port_descriptor(const PortDescriptor& desc, const size_t i) { const auto it = m_input_ports.find(i); OPENVINO_ASSERT(it != m_input_ports.end(), "Index of input port descriptor should be less than count of input ports"); - (*it).second = { desc.count, desc.offset, i}; + it->second = desc; + it->second.index = i; } void MemoryAccess::set_output_port_descriptor(const PortDescriptor& desc, const size_t i) { const auto it = m_output_ports.find(i); OPENVINO_ASSERT(it != m_output_ports.end(), "Index of output port descriptor should be less than count of output ports"); - (*it).second = { desc.count, desc.offset, i}; + it->second = desc; + it->second.index = i; } const MemoryAccess::PortDescriptor& MemoryAccess::get_input_port_descriptor(const size_t i) const { const auto it = m_input_ports.find(i); OPENVINO_ASSERT(it != m_input_ports.end(), "Index of input port descriptor should be less than count of input ports"); - return (*it).second; + return it->second; +} + +MemoryAccess::PortDescriptor& MemoryAccess::get_input_port_descriptor(const size_t i) { + return const_cast(const_cast(this)->get_input_port_descriptor(i)); } const MemoryAccess::PortDescriptor& MemoryAccess::get_output_port_descriptor(const size_t i) const { const auto it = m_output_ports.find(i); OPENVINO_ASSERT(it != m_output_ports.end(), "Index of output port descriptor should be less than count of output ports"); - return (*it).second; + return it->second; +} + +MemoryAccess::PortDescriptor& MemoryAccess::get_output_port_descriptor(const size_t i) { + return const_cast(const_cast(this)->get_output_port_descriptor(i)); } void MemoryAccess::set_input_count(size_t count, size_t idx) { - set_input_port_descriptor({count, get_input_port_descriptor(idx).offset, idx}, idx); + get_input_port_descriptor(idx).count = count; } void MemoryAccess::set_output_count(size_t count, size_t idx) { - set_output_port_descriptor({count, get_output_port_descriptor(idx).offset, idx}, idx); + get_output_port_descriptor(idx).count = count; } void MemoryAccess::set_input_offset(size_t offset, size_t idx) { - set_input_port_descriptor({get_input_port_descriptor(idx).count, offset, idx}, idx); + get_input_port_descriptor(idx).offset = offset; } void MemoryAccess::set_output_offset(size_t offset, size_t idx) { - set_output_port_descriptor({get_output_port_descriptor(idx).count, offset, idx}, idx); + get_output_port_descriptor(idx).offset = offset; +} +void MemoryAccess::set_input_stride(size_t stride, size_t idx) { + get_input_port_descriptor(idx).stride = stride; +} +void MemoryAccess::set_output_stride(size_t stride, size_t idx) { + get_output_port_descriptor(idx).stride = stride; } size_t MemoryAccess::get_input_count(size_t idx) const { return get_input_port_descriptor(idx).count; @@ -119,7 +137,13 @@ size_t MemoryAccess::get_input_offset(size_t idx) const { size_t MemoryAccess::get_output_offset(size_t idx) const { return get_output_port_descriptor(idx).offset; } +size_t MemoryAccess::get_input_stride(size_t idx) const { + return get_input_port_descriptor(idx).stride; +} +size_t MemoryAccess::get_output_stride(size_t idx) const { + return get_output_port_descriptor(idx).stride; +} -} // namespace op +} // namespace modifier } // namespace snippets } // namespace ov diff --git a/src/common/snippets/src/op/store.cpp b/src/common/snippets/src/op/store.cpp index d4f0087791e..ed02accbc75 100644 --- a/src/common/snippets/src/op/store.cpp +++ b/src/common/snippets/src/op/store.cpp @@ -12,7 +12,7 @@ namespace snippets { namespace op { snippets::op::Store::Store(const Output& x, const size_t count, const size_t offset) - : MemoryAccess({x}, std::set{}, std::set{0}) { + : MemoryAccess(std::set{}, std::set{0}), Op({x}) { set_output_port_descriptor({count, offset}, 0); constructor_validate_and_infer_types(); } @@ -31,7 +31,9 @@ std::shared_ptr snippets::op::Store::clone_with_new_inputs(const OutputVec check_new_args_count(this, new_args); return std::make_shared(new_args.at(0), get_count(), get_offset()); } - +bool snippets::op::Store::visit_attributes(AttributeVisitor& visitor) { + return MemoryAccess::visit_attributes(visitor); +} } // namespace op } // namespace snippets } // namespace ov diff --git a/src/common/snippets/src/op/subgraph.cpp b/src/common/snippets/src/op/subgraph.cpp index 73dc0595114..6b78a28476f 100644 --- a/src/common/snippets/src/op/subgraph.cpp +++ b/src/common/snippets/src/op/subgraph.cpp @@ -422,12 +422,13 @@ void Subgraph::control_flow_transformations(lowered::LinearIR& linear_ir, OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "Snippets::op::control_flow_transformations") // Domain optimization must be the first pass, because all other transformations may depend on PortDescriptor shapes - size_t loop_depth = 1; - lowered::pass::OptimizeDomain(loop_depth).run(linear_ir); - linear_ir.set_loop_depth(loop_depth); + size_t loop_depth = linear_ir.get_config().m_loop_depth; + if (!lowered_pass_config->is_disabled()) { + lowered::pass::OptimizeDomain(loop_depth).run(linear_ir); + linear_ir.set_loop_depth(loop_depth); + } const size_t vector_size = get_generator()->get_target_machine()->get_lanes(); - const int32_t buffer_allocation_rank = static_cast(linear_ir.get_config().m_loop_depth); lowered::pass::PassPipeline pipeline(lowered_pass_config); pipeline.register_pass(vector_size); @@ -435,7 +436,7 @@ void Subgraph::control_flow_transformations(lowered::LinearIR& linear_ir, pipeline.register_pass(); pipeline.register_pass(); pipeline.register_pass(); - pipeline.register_pass(buffer_allocation_rank); + pipeline.register_pass(static_cast(loop_depth)); pipeline.register_pass(vector_size); pipeline.register_pass(); pipeline.register_pass(); diff --git a/src/plugins/intel_cpu/CMakeLists.txt b/src/plugins/intel_cpu/CMakeLists.txt index acad485c267..4c24c486671 100644 --- a/src/plugins/intel_cpu/CMakeLists.txt +++ b/src/plugins/intel_cpu/CMakeLists.txt @@ -97,6 +97,11 @@ if(ENABLE_CPU_DEBUG_CAPS) add_definitions(-DCPU_DEBUG_CAPS) endif() +if (ENABLE_SNIPPETS_LIBXSMM_TPP) + # Note: LIBXSMM_DEFAULT_CONFIG needed so libxsmm_config can be included without issues + add_definitions(-DSNIPPETS_LIBXSMM_TPP -DLIBXSMM_DEFAULT_CONFIG) +endif() + set(OV_CPU_WITH_DNNL ON) if(OV_CPU_WITH_DNNL) add_definitions(-DOV_CPU_WITH_DNNL) @@ -139,6 +144,11 @@ if (NOT ENABLE_MLAS_FOR_CPU) list(APPEND EXCLUDE_PATHS ${CMAKE_CURRENT_SOURCE_DIR}/src/mlas/*) endif() +if (NOT ENABLE_SNIPPETS_LIBXSMM_TPP) + list(APPEND EXCLUDE_PATHS ${CMAKE_CURRENT_SOURCE_DIR}/src/emitters/tpp/* + ${CMAKE_CURRENT_SOURCE_DIR}/src/transformations/tpp/*) +endif () + file(GLOB_RECURSE FILES_TO_REMOVE ${EXCLUDE_PATHS}) list(REMOVE_ITEM SOURCES ${FILES_TO_REMOVE}) list(REMOVE_ITEM HEADERS ${FILES_TO_REMOVE}) @@ -170,6 +180,11 @@ if (ENABLE_MLAS_FOR_CPU) target_include_directories(${TARGET_NAME} SYSTEM PRIVATE $) add_definitions(-DOV_CPU_WITH_MLAS) endif() +if (ENABLE_SNIPPETS_LIBXSMM_TPP) + target_compile_definitions(xsmm PRIVATE __BLAS=0) + target_link_libraries(${TARGET_NAME} PRIVATE xsmm) + target_include_directories(${TARGET_NAME} SYSTEM PRIVATE $) +endif () target_include_directories(${TARGET_NAME} SYSTEM PRIVATE $) # Cross compiled function # TODO: The same for proposal, proposalONNX, topk @@ -239,6 +254,9 @@ if(BUILD_SHARED_LIBS) $) target_include_directories(${TARGET_NAME}_obj SYSTEM PUBLIC $) + if(ENABLE_SNIPPETS_LIBXSMM_TPP) + target_include_directories(${TARGET_NAME}_obj SYSTEM PUBLIC $) + endif() if(ENABLE_MLAS_FOR_CPU) target_include_directories(${TARGET_NAME}_obj SYSTEM PUBLIC $) diff --git a/src/plugins/intel_cpu/src/emitters/snippets/x64/cpu_generator.cpp b/src/plugins/intel_cpu/src/emitters/snippets/x64/cpu_generator.cpp index 3a23ce5de6e..53c811a55e1 100644 --- a/src/plugins/intel_cpu/src/emitters/snippets/x64/cpu_generator.cpp +++ b/src/plugins/intel_cpu/src/emitters/snippets/x64/cpu_generator.cpp @@ -39,6 +39,20 @@ #include "emitters/snippets/x64/verbose.hpp" #endif +#ifdef SNIPPETS_LIBXSMM_TPP +#include "transformations/tpp/x64/op/brgemm.hpp" +#include "transformations/tpp/x64/op/eltwise.hpp" +#include "transformations/tpp/x64/op/reduce.hpp" +#include "transformations/tpp/x64/op/modifiers.hpp" +#include "transformations/tpp/x64/op/scalar.hpp" +#include "emitters/tpp/x64/jit_eltwise_emitters.hpp" +#include "emitters/tpp/x64/jit_brgemm_emitter.hpp" +#include "emitters/tpp/x64/jit_scalar_emitter.hpp" +#include "emitters/tpp/x64/jit_debug_emitter.hpp" +// Note: for reference implementations +#include +#endif + namespace ov { #ifdef SNIPPETS_DEBUG_CAPS @@ -90,9 +104,9 @@ static bool is_segfault_detector_emitter(const intel_cpu::jit_emitter *emitter) } \ } #else -#define CREATE_SNIPPETS_EMITTER(e_type) { \ +#define CREATE_SNIPPETS_EMITTER(e_type, ...) { \ [this](const snippets::lowered::ExpressionPtr& expr) -> std::shared_ptr { \ - return std::make_shared(h.get(), isa, expr); \ + return std::make_shared(h.get(), isa, expr, ##__VA_ARGS__); \ }, \ [](const std::shared_ptr& n) -> std::set> { \ return e_type::get_supported_precisions(n); \ @@ -100,6 +114,16 @@ static bool is_segfault_detector_emitter(const intel_cpu::jit_emitter *emitter) } #endif +#define CREATE_DEBUG_TPP_EMITTER(e_type) { \ + [this](const snippets::lowered::ExpressionPtr& expr) -> std::shared_ptr { \ + return std::make_shared(expr, std::make_shared(h.get(), isa, expr)); \ + }, \ + [](const std::shared_ptr& n) -> std::set> { \ + return e_type::get_supported_precisions(n); \ + } \ +} + + #define CREATE_CPU_EMITTER(e_type) { \ [this](const snippets::lowered::ExpressionPtr& expr) -> std::shared_ptr { \ return std::make_shared(h.get(), isa, expr->get_node()); \ @@ -226,6 +250,32 @@ intel_cpu::CPUTargetMachine::CPUTargetMachine(dnnl::impl::cpu::x64::cpu_isa_t ho jitters[ov::intel_cpu::PerfCountRdtscBegin::get_type_info_static()] = CREATE_CPU_EMITTER(ov::intel_cpu::jit_perf_count_rdtsc_start_emitter); jitters[ov::intel_cpu::PerfCountRdtscEnd::get_type_info_static()] = CREATE_CPU_EMITTER(ov::intel_cpu::jit_perf_count_rdtsc_end_emitter); #endif + +#ifdef SNIPPETS_LIBXSMM_TPP + jitters[intel_cpu::tpp::op::BrgemmTPP::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(BrgemmTppEmitter); + jitters[intel_cpu::tpp::op::Add::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(BinaryEltwiseTppEmitter); + jitters[intel_cpu::tpp::op::Subtract::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(BinaryEltwiseTppEmitter); + jitters[intel_cpu::tpp::op::Multiply::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(BinaryEltwiseTppEmitter); + jitters[intel_cpu::tpp::op::Divide::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(BinaryEltwiseTppEmitter); + + jitters[intel_cpu::tpp::op::Exp::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(UnaryEltwiseTppEmitter); + // Note: you can register Debug emitter for Unary/Binary operations as shown below: + // jitters[intel_cpu::tpp::op::Add::get_type_info_static()] = CREATE_DEBUG_TPP_EMITTER(UnaryEltwiseTppEmitter); + // + // Note: you can register Reference emitter for Unary operations using std::function or lambda function as shown below: + // jitters[intel_cpu::tpp::op::Exp::get_type_info_static()] = + // CREATE_SNIPPETS_EMITTER(ReferenceUnaryEltwiseTppEmitter, static_cast(std::exp)); + // jitters[intel_cpu::tpp::op::Reciprocal::get_type_info_static()] = + // CREATE_SNIPPETS_EMITTER(ReferenceUnaryEltwiseTppEmitter, [](float x){ return 1.f/x; }); + jitters[intel_cpu::tpp::op::Reciprocal::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(UnaryEltwiseTppEmitter); + + jitters[intel_cpu::tpp::op::Relu::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(UnaryEltwiseTppEmitter); + jitters[intel_cpu::tpp::op::Square::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(UnaryEltwiseTppEmitter); + jitters[intel_cpu::tpp::op::SquareRoot::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(UnaryEltwiseTppEmitter); + jitters[intel_cpu::tpp::op::ReduceMax::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(ReduceTppEmitter); + jitters[intel_cpu::tpp::op::ReduceSum::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(ReduceTppEmitter); + jitters[intel_cpu::tpp::op::Scalar::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(ScalarTppEmitter); +#endif } size_t intel_cpu::CPUTargetMachine::get_lanes() const { @@ -283,6 +333,10 @@ std::shared_ptr intel_cpu::CPUGenerator::clone() const { ov::snippets::RegType intel_cpu::CPUGenerator::get_specific_op_out_reg_type(const ov::Output& out) const { const auto op = out.get_node_shared_ptr(); if (std::dynamic_pointer_cast(op) || +#ifdef SNIPPETS_LIBXSMM_TPP + std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op) || +#endif std::dynamic_pointer_cast(op)) return ov::snippets::RegType::gpr; else if ( @@ -290,7 +344,7 @@ ov::snippets::RegType intel_cpu::CPUGenerator::get_specific_op_out_reg_type(cons std::dynamic_pointer_cast(op)) return ov::snippets::RegType::vec; else - OPENVINO_THROW("Register type of the operation " + std::string(op->get_type_name()) + " isn't determined!"); + return ov::snippets::RegType::undefined; } bool intel_cpu::CPUGenerator::uses_precompiled_kernel(const std::shared_ptr& e) const { @@ -303,6 +357,10 @@ bool intel_cpu::CPUGenerator::uses_precompiled_kernel(const std::shared_ptr(e) || std::dynamic_pointer_cast(e) || std::dynamic_pointer_cast(e); +#endif +#ifdef SNIPPETS_LIBXSMM_TPP + need |= std::dynamic_pointer_cast(e) || + std::dynamic_pointer_cast(e); #endif return need; } diff --git a/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_kernel_emitter.cpp b/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_kernel_emitter.cpp index 355c29b5451..8914ef95fee 100644 --- a/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_kernel_emitter.cpp +++ b/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_kernel_emitter.cpp @@ -206,7 +206,7 @@ jit_kernel_static_emitter::jit_kernel_static_emitter(dnnl::impl::cpu::x64::jit_g const auto& mem_desc_expr = shape_infer_seq.empty() ? expr : shape_infer_seq.back(); auto consumer_inputs = mem_desc_expr->get_output_port_connector(0)->get_consumers(); for (const auto& child_input : consumer_inputs) { - const auto ma = ov::as_type_ptr(child_input.get_expr()->get_node()); + const auto ma = std::dynamic_pointer_cast(child_input.get_expr()->get_node()); if (ma && ma->is_memory_access_input_port(child_input.get_index())) { desc = child_input.get_descriptor_ptr(); break; @@ -322,7 +322,7 @@ void jit_kernel_static_emitter::init_data_pointers(const std::vectormov(data_ptr_regs[i], h->ptr[reg_runtime_params + GET_OFF(dst_ptrs) + (i - num_inputs) * sizeof(void*)]); reg_tmp = reg_runtime_params; diff --git a/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_snippets_emitters.cpp b/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_snippets_emitters.cpp index fd9d25aca8c..d8066f9a126 100644 --- a/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_snippets_emitters.cpp +++ b/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_snippets_emitters.cpp @@ -15,8 +15,9 @@ using jit_generator = dnnl::impl::cpu::x64::jit_generator; using cpu_isa_t = dnnl::impl::cpu::x64::cpu_isa_t; using ExpressionPtr = ov::snippets::lowered::ExpressionPtr; -jit_nop_emitter::jit_nop_emitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr) : jit_emitter(h, isa) { - in_out_type_ = emitter_in_out_map::gpr_to_gpr; +jit_nop_emitter::jit_nop_emitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr, emitter_in_out_map emitter_type) + : jit_emitter(h, isa) { + in_out_type_ = emitter_type; } jit_parameter_emitter::jit_parameter_emitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr) @@ -67,35 +68,31 @@ void jit_broadcast_move_emitter::emit_isa(const std::vector &in, const s } } -jit_scalar_emitter::jit_scalar_emitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr) : jit_emitter(h, isa) { - const auto n = expr->get_node(); +int32_t jit_scalar_emitter::read_value(const ov::snippets::lowered::ExpressionPtr& expr) { + const auto n = ov::as_type_ptr(expr->get_node()); + OV_CPU_JIT_EMITTER_ASSERT(n, "Invalid node, expected op::v0::Constant"); const auto& precision = n->get_output_element_type(0); + int32_t res = INT_MIN; switch (precision) { - case element::i32: { - value = ov::as_type_ptr(n)->cast_vector()[0]; - break; - } - case element::f32: { - value = dnnl::impl::cpu::x64::float2int(ov::as_type_ptr(n)->cast_vector()[0]); - break; - } - default: { - OV_CPU_JIT_EMITTER_THROW("doesn't support ", precision); - } + case element::i32: res = n->cast_vector(1)[0]; break; + case element::f32: res = dnnl::impl::cpu::x64::float2int(n->cast_vector(1)[0]); break; + default: OV_CPU_JIT_EMITTER_THROW("doesn't support ", precision); } - push_arg_entry_of("scalar", value, true); + return res; +} + +jit_scalar_emitter::jit_scalar_emitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr) : jit_emitter(h, isa) { + push_arg_entry_of("scalar", read_value(expr), true); prepare_table(); } void jit_scalar_emitter::emit_impl(const std::vector& in, const std::vector& out) const { - if (host_isa_ == dnnl::impl::cpu::x64::sse41) { - emit_isa(in, out); - } else if (host_isa_ == dnnl::impl::cpu::x64::avx2) { - emit_isa(in, out); - } else if (host_isa_ == dnnl::impl::cpu::x64::avx512_core) { - emit_isa(in, out); - } else { - OV_CPU_JIT_EMITTER_THROW("Unsupported isa ", host_isa_); + using isa = cpu_isa_t; + switch (host_isa_) { + case isa::sse41: emit_isa(in, out); break; + case isa::avx2: emit_isa(in, out); break; + case isa::avx512_core: emit_isa(in, out); break; + default: OV_CPU_JIT_EMITTER_THROW("Unsupported isa ", host_isa_); } } diff --git a/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_snippets_emitters.hpp b/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_snippets_emitters.hpp index 6526072a1a2..c75f071c4ec 100644 --- a/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_snippets_emitters.hpp +++ b/src/plugins/intel_cpu/src/emitters/snippets/x64/jit_snippets_emitters.hpp @@ -13,7 +13,7 @@ namespace intel_cpu { class jit_nop_emitter : public jit_emitter { public: jit_nop_emitter(dnnl::impl::cpu::x64::jit_generator* h, dnnl::impl::cpu::x64::cpu_isa_t isa, - const ov::snippets::lowered::ExpressionPtr& expr); + const ov::snippets::lowered::ExpressionPtr& expr, emitter_in_out_map emitter_type = gpr_to_gpr); size_t get_inputs_num() const override {return 0;} @@ -59,18 +59,13 @@ public: const ov::snippets::lowered::ExpressionPtr& expr); size_t get_inputs_num() const override {return 0;} - -protected: size_t aux_gprs_count() const override {return 1;} + static int32_t read_value(const ov::snippets::lowered::ExpressionPtr& expr); private: void emit_impl(const std::vector& in, const std::vector& out) const override; - template void emit_isa(const std::vector &in, const std::vector &out) const; - -private: - int32_t value; }; } // namespace intel_cpu diff --git a/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_brgemm_emitter.cpp b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_brgemm_emitter.cpp new file mode 100644 index 00000000000..0fcb394a8a5 --- /dev/null +++ b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_brgemm_emitter.cpp @@ -0,0 +1,113 @@ +// Copyright (C) 2020-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "jit_brgemm_emitter.hpp" +#include "emitters/snippets/x64/jit_snippets_emitters.hpp" +#include "transformations/tpp/x64/op/brgemm.hpp" + +using jit_generator = dnnl::impl::cpu::x64::jit_generator; +using cpu_isa_t = dnnl::impl::cpu::x64::cpu_isa_t; +using ExpressionPtr = ov::snippets::lowered::ExpressionPtr; + +namespace ov { +namespace intel_cpu { + +void BrgemmTppEmitter::validate_subtensors(const VectorDims& in_0, const VectorDims& in_1, const VectorDims& out_0) { + bool subtensors_compatible = in_0.size() == in_1.size() && in_0.size() == out_0.size() && in_0.size() == 2 && + in_0[1] == in_1[0] && in_0[0] == out_0[0] && in_1[1] == out_0[1]; + OV_CPU_JIT_EMITTER_ASSERT(subtensors_compatible, "Incompatible subtensors"); +} + +BrgemmTppEmitter::BrgemmTppEmitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr) + : TppEmitter(h, isa, expr) { + const auto& brgemm_node = as_type_ptr(expr->get_node()); + OV_CPU_JIT_EMITTER_ASSERT(brgemm_node && !brgemm_node->is_dynamic(), "Invoked with invalid node type"); + + const auto& input_0_desc = expr->get_input_port_descriptor(0); + const auto& input_1_desc = expr->get_input_port_descriptor(1); + const auto& output_desc = expr->get_output_port_descriptor(0); + + std::vector leading_dimensions {brgemm_node->get_input_stride(0), + brgemm_node->get_input_stride(1), + brgemm_node->get_output_stride(0)}; + + auto in_0_prec = ov_to_xsmm_dtype(brgemm_node->get_input_element_type(0)); + auto in_1_prec = ov_to_xsmm_dtype(brgemm_node->get_input_element_type(1)); + exec_dtype = in_0_prec == LIBXSMM_DATATYPE_I8 || in_0_prec == LIBXSMM_DATATYPE_U8 ? + LIBXSMM_DATATYPE_I32 : + LIBXSMM_DATATYPE_F32; + auto out_0_prec = exec_dtype == LIBXSMM_DATATYPE_I32 ? + LIBXSMM_DATATYPE_I32 : + LIBXSMM_DATATYPE_F32; + + const auto beta = brgemm_node->get_beta(); + OV_CPU_JIT_EMITTER_ASSERT(beta == 0 || beta == 1, "Detected unsupported beta value: " + std::to_string(beta)); + + const auto& subtensor_in0 = input_0_desc->get_subtensor(); + const auto& subtensor_in1 = input_1_desc->get_subtensor(); + const auto& subtensor_out0 = output_desc->get_subtensor(); + validate_subtensors(subtensor_in0, subtensor_in1, subtensor_out0); + + const auto K = static_cast(*subtensor_in0.rbegin()); + const auto M = static_cast(*++subtensor_in0.rbegin()); + const auto N = static_cast(*subtensor_in1.rbegin()); + + const bool is_f32_gemm = in_0_prec == in_1_prec && in_0_prec == LIBXSMM_DATATYPE_F32; + const bool is_bf16_gemm = in_0_prec == in_1_prec && in_0_prec == LIBXSMM_DATATYPE_BF16; + const bool is_i8_gemm = in_0_prec == LIBXSMM_DATATYPE_U8 || in_0_prec == LIBXSMM_DATATYPE_I8; + OV_CPU_JIT_EMITTER_ASSERT(is_f32_gemm || + (is_bf16_gemm && K % 2 == 0) || + (is_i8_gemm && K % 4 == 0), + "Unsupported parameter combination for kernel configuration"); + + m_compile_flags = is_f32_gemm ? + LIBXSMM_GEMM_FLAGS('N', 'N') : + LIBXSMM_GEMM_VNNI_FLAGS('N', 'N', 'V', 'N') | + LIBXSMM_GEMM_FLAG_NO_SETUP_TILECONFIG | + LIBXSMM_GEMM_FLAG_NO_RESET_TILECONFIG; + + if (beta == 0) + m_compile_flags |= LIBXSMM_GEMM_FLAG_BETA_0; + + if (in_0_prec == LIBXSMM_DATATYPE_U8) { + in_0_prec = LIBXSMM_DATATYPE_I8; + m_compile_flags |= LIBXSMM_GEMM_FLAG_A_UNSIGNED; + } + if (in_1_prec == LIBXSMM_DATATYPE_U8) { + in_1_prec = LIBXSMM_DATATYPE_I8; + m_compile_flags |= LIBXSMM_GEMM_FLAG_B_UNSIGNED; + } + + m_shape = libxsmm_create_gemm_shape(N, M, K, + io_strides[1], io_strides[0], io_strides[2], + in_1_prec, in_0_prec, out_0_prec, + exec_dtype); + m_prefetching_flags = LIBXSMM_GEMM_PREFETCH_NONE; +} + +std::set> BrgemmTppEmitter::get_supported_precisions(const std::shared_ptr& node) { + // Note: BrgemmTpp currently supports only fp32 + return {{element::f32, element::f32}}; +} + +void BrgemmTppEmitter::validate_arguments(const std::vector &in, const std::vector &out) const { + OV_CPU_JIT_EMITTER_ASSERT(in.size() == 2, "Expects 2 input regs, got" + std::to_string(in.size())); + OV_CPU_JIT_EMITTER_ASSERT(out.size() == 1, "Expects 1 output reg, got" + std::to_string(out.size())); +} + +const uintptr_t BrgemmTppEmitter::get_compiled_kernel_ptr() const { + return COMPILE_TPP_KERNEL(libxsmm_dispatch_gemm(m_shape, m_compile_flags, m_prefetching_flags)); +} + +void BrgemmTppEmitter::execute_brgemm_kernel(libxsmm_gemmfunction brg_kernel, void *in0, void *in1, void *out0) { + libxsmm_gemm_param gemm_p; + gemm_p.a.primary = in1; + gemm_p.b.primary = in0; + gemm_p.c.primary = out0; + OV_CPU_JIT_EMITTER_ASSERT(brg_kernel, "Invalid brgemm kernel pointer"); + brg_kernel(&gemm_p); +} + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_brgemm_emitter.hpp b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_brgemm_emitter.hpp new file mode 100644 index 00000000000..b8e13ab10b5 --- /dev/null +++ b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_brgemm_emitter.hpp @@ -0,0 +1,33 @@ +// Copyright (C) 2020-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include "jit_tpp_emitter.hpp" + +namespace ov { +namespace intel_cpu { + +class BrgemmTppEmitter : public TppEmitter { +public: + BrgemmTppEmitter(dnnl::impl::cpu::x64::jit_generator* h, + dnnl::impl::cpu::x64::cpu_isa_t isa, + const ov::snippets::lowered::ExpressionPtr& expr); + + size_t get_inputs_num() const override { return 2; } + static std::set> get_supported_precisions(const std::shared_ptr& node = nullptr); + + static void execute_brgemm_kernel(libxsmm_gemmfunction brgemm_kernel, void *in0, void *in1, void *out0); + + const uintptr_t get_execute_function_ptr() const override { return reinterpret_cast(execute_brgemm_kernel); } + const uintptr_t get_compiled_kernel_ptr() const override; + +protected: + void validate_arguments(const std::vector &in, const std::vector &out) const override; + static void validate_subtensors(const VectorDims& in_0, const VectorDims& in_1, const VectorDims& out_0); + libxsmm_gemm_shape m_shape; + libxsmm_bitfield m_prefetching_flags {0}; +}; + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_debug_emitter.hpp b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_debug_emitter.hpp new file mode 100644 index 00000000000..4afc6ee1d7c --- /dev/null +++ b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_debug_emitter.hpp @@ -0,0 +1,74 @@ +// Copyright (C) 2020-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include "jit_tpp_emitter.hpp" + +namespace ov { +namespace intel_cpu { + +/** + * @interface DebugTppEmitter + * @brief The purpose of this emitter is to facilitate debugging of TPP emitters. It allows to access attributes of the + * owned Tpp emitter at runtime, inspect source ov::Node, in/out memory before and after execution, etc. + */ +class DebugTppEmitter : public TppEmitter { +public: + DebugTppEmitter(const ov::snippets::lowered::ExpressionPtr& expr, const std::shared_ptr& original) + : TppEmitter(*original), + m_original(original), + m_compiled_kernel(m_original->get_compiled_kernel_ptr()), + m_execute_function(m_original->get_execute_function_ptr()), + m_source_expr(expr) { + } + + void validate_arguments(const std::vector &in, const std::vector &out) const override { + m_original->validate_arguments(in, out); + }; + + size_t get_inputs_num() const override { return num_kernel_args - 1; } + +protected: + static void execute_kernel_unary(const DebugTppEmitter* emitter, void *in0, void *out0) { + OV_CPU_JIT_EMITTER_ASSERT(emitter && emitter->m_execute_function && emitter->m_compiled_kernel, + "Unable to execute unary kernel"); + // Note: put a breakpoint here and analyze all the necessary debug info in runtime + std::cout << "Running unary DebugTPPEmitter for node with name " + << emitter->m_source_expr->get_node()->get_friendly_name() << std::endl; + auto f = reinterpret_cast(emitter->m_execute_function); + f(emitter->m_compiled_kernel, in0, out0); + } + + static void execute_kernel_binary(const DebugTppEmitter* emitter, void* in0, void* in1, void* out0) { + OV_CPU_JIT_EMITTER_ASSERT(emitter && emitter->m_execute_function && emitter->m_compiled_kernel, + "Unable to execute binary kernel"); + // Note: put a breakpoint here and analyze all the necessary debug info in runtime + std::cout << "Running binary DebugTPPEmitter for node with name " + << emitter->m_source_expr->get_node()->get_friendly_name() << std::endl; + auto f = reinterpret_cast(emitter->m_execute_function); + f(emitter->m_compiled_kernel, in0, in1, out0); + } + + const uintptr_t get_execute_function_ptr() const override { + // Note: num_kernel_args accounts for both input and output args + switch (num_kernel_args) { + case 2: return reinterpret_cast(execute_kernel_unary); + case 3: return reinterpret_cast(execute_kernel_binary); + default: OV_CPU_JIT_EMITTER_THROW("More than two arguments are not supported"); + } + } + + const uintptr_t get_compiled_kernel_ptr() const override { + return reinterpret_cast(this); + } + +private: + std::shared_ptr m_original {nullptr}; + uintptr_t m_compiled_kernel {0}; + uintptr_t m_execute_function {0}; + snippets::lowered::ExpressionPtr m_source_expr {nullptr}; +}; + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_eltwise_emitters.cpp b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_eltwise_emitters.cpp new file mode 100644 index 00000000000..d91182ada2b --- /dev/null +++ b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_eltwise_emitters.cpp @@ -0,0 +1,136 @@ +// Copyright (C) 2020-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "jit_eltwise_emitters.hpp" +#include "transformations/tpp/x64/op/eltwise.hpp" + +namespace ov { +namespace intel_cpu { +using jit_generator = dnnl::impl::cpu::x64::jit_generator; +using cpu_isa_t = dnnl::impl::cpu::x64::cpu_isa_t; +using ExpressionPtr = ov::snippets::lowered::ExpressionPtr; + +BinaryEltwiseTppEmitter::BinaryEltwiseTppEmitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr) : + TppEmitter(h, isa, expr) { + const auto& subtensor_in0 = get_projected_subtensor(io_port_descriptors[0]); + const auto& subtensor_in1 = get_projected_subtensor(io_port_descriptors[1]); + + const auto N_in0 = static_cast(*subtensor_in0.rbegin()); + const auto M_in0 = static_cast(*++subtensor_in0.rbegin()); + const auto N_in1 = static_cast(*subtensor_in1.rbegin()); + const auto M_in1 = static_cast(*++subtensor_in1.rbegin()); + + std::pair n_bcast_flags, m_bcast_flags; + const auto N = get_broadcasted_dim(N_in0, N_in1, n_bcast_flags); + const auto M = get_broadcasted_dim(M_in0, M_in1, m_bcast_flags); + + m_compile_flags = LIBXSMM_MELTW_FLAG_BINARY_NONE; + if (m_bcast_flags.first && n_bcast_flags.first) { + m_compile_flags |= LIBXSMM_MELTW_FLAG_BINARY_BCAST_SCALAR_IN_0; + } else if (m_bcast_flags.first) { + m_compile_flags |= LIBXSMM_MELTW_FLAG_BINARY_BCAST_COL_IN_0; + } else if (n_bcast_flags.first) { + m_compile_flags |= LIBXSMM_MELTW_FLAG_BINARY_BCAST_ROW_IN_0; + } + if (m_bcast_flags.second && n_bcast_flags.second) { + m_compile_flags |= LIBXSMM_MELTW_FLAG_BINARY_BCAST_SCALAR_IN_1; + } else if (m_bcast_flags.second) { + m_compile_flags |= LIBXSMM_MELTW_FLAG_BINARY_BCAST_COL_IN_1; + } else if (n_bcast_flags.second) { + m_compile_flags |= LIBXSMM_MELTW_FLAG_BINARY_BCAST_ROW_IN_1; + } + const auto& binary_eltw_tpp = std::dynamic_pointer_cast(expr->get_node()); + OV_CPU_JIT_EMITTER_ASSERT(binary_eltw_tpp, "Invalid TPP node type detected"); + m_op_type = binary_eltw_tpp->get_op_type(); + // Note: libxsmm implies column-major layout, so we have to swap M and N here + m_shape = libxsmm_create_meltw_binary_shape(N, M, + io_strides[0], io_strides[1], io_strides[2], + io_dtypes[0], io_dtypes[1], io_dtypes[2], + exec_dtype); +} + +const uintptr_t BinaryEltwiseTppEmitter::get_compiled_kernel_ptr() const { + // Note: libxsmm hides memory management from the user, so we don't have to store pointer to compiled kernel to keep it alive. + // libxsmm will keep the pointer alive until the end of program execution (it doesn't matter whether we save the pointer in the emitter or not) + return COMPILE_TPP_KERNEL(libxsmm_dispatch_meltw_binary(m_op_type, m_shape, m_compile_flags)); +} + +std::set> BinaryEltwiseTppEmitter::get_supported_precisions(const std::shared_ptr& node) { + return {{element::f32, element::f32}}; +} + + +void BinaryEltwiseTppEmitter::validate_arguments(const std::vector &in, const std::vector &out) const { + OV_CPU_JIT_EMITTER_ASSERT(in.size() == 2, "Expects 2 input registers, got " + std::to_string(in.size())); + OV_CPU_JIT_EMITTER_ASSERT(out.size() == 1, "Expects 1 output register, got " + std::to_string(out.size())); +} + +void BinaryEltwiseTppEmitter::execute_kernel(libxsmm_meltwfunction_binary eltwise_kernel, void *in0, void *in1, void *out0) { + libxsmm_meltw_binary_param param; + param.op.primary = nullptr; + param.in0.primary = in0; + param.in1.primary = in1; + param.out.primary = out0; + eltwise_kernel(¶m); +} + +libxsmm_blasint BinaryEltwiseTppEmitter::get_broadcasted_dim(libxsmm_blasint dim0, libxsmm_blasint dim1, std::pair& bcast_flags) { + if (dim0 == dim1) { + bcast_flags = {false, false}; + return dim0; + } else if (dim1 == 1) { + bcast_flags = {false, true}; + return dim0; + } else if (dim0 == 1) { + bcast_flags = {true, false}; + return dim1; + } + OV_CPU_JIT_EMITTER_THROW("Invalid dimensions passed to get_broadcast_flags"); + return -1; +} + +UnaryEltwiseTppEmitter::UnaryEltwiseTppEmitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr) : + TppEmitter(h, isa, expr) { + const auto& subtensor_in0 = get_projected_subtensor(io_port_descriptors[0]); + + const auto N = static_cast(*subtensor_in0.rbegin()); + const auto M = static_cast(*++subtensor_in0.rbegin()); + + const auto& unary_eltw_tpp = std::dynamic_pointer_cast(expr->get_node()); + OV_CPU_JIT_EMITTER_ASSERT(unary_eltw_tpp, "Invalid TPP node type detected"); + m_op_type = unary_eltw_tpp->get_op_type(); + // Note: libxsmm implies column-major layout, so we have to swap M and N here + m_shape = libxsmm_create_meltw_unary_shape(N, M, + io_strides[0], io_strides[1], + io_dtypes[0], io_dtypes[1], + exec_dtype); + m_compile_flags = LIBXSMM_MELTW_FLAG_UNARY_NONE; +} + +void UnaryEltwiseTppEmitter::execute_kernel(libxsmm_meltwfunction_unary eltwise_kernel, void *in0, void *out0) { + libxsmm_meltw_unary_param param; + param.op.primary = nullptr; + param.in.primary = in0; + param.out.primary = out0; + eltwise_kernel(¶m); +} + +std::set> UnaryEltwiseTppEmitter::get_supported_precisions(const std::shared_ptr& node) { + return {{element::f32}}; +} + +void UnaryEltwiseTppEmitter::validate_arguments(const std::vector &in, const std::vector &out) const { + OV_CPU_JIT_EMITTER_ASSERT(in.size() == 1, "Expects 1 input registers, got " + std::to_string(in.size())); + OV_CPU_JIT_EMITTER_ASSERT(out.size() == 1, "Expects 1 output register, got " + std::to_string(out.size())); +} + +ReduceTppEmitter::ReduceTppEmitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr) : + UnaryEltwiseTppEmitter(h, isa, expr) { + m_compile_flags = LIBXSMM_MELTW_FLAG_UNARY_REDUCE_ROWS; + // No need to set ldo for reduce, it is always assumed = 1 inside the kernel + // m_shape.ldo = 1; +} + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_eltwise_emitters.hpp b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_eltwise_emitters.hpp new file mode 100644 index 00000000000..d2e4d6febe4 --- /dev/null +++ b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_eltwise_emitters.hpp @@ -0,0 +1,109 @@ +// Copyright (C) 2020-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "jit_tpp_emitter.hpp" +namespace ov { +namespace intel_cpu { + +class BinaryEltwiseTppEmitter : public TppEmitter { +public: + BinaryEltwiseTppEmitter(dnnl::impl::cpu::x64::jit_generator* h, + dnnl::impl::cpu::x64::cpu_isa_t isa, + const ov::snippets::lowered::ExpressionPtr& expr); + size_t get_inputs_num() const override { return 2; } + static void execute_kernel(libxsmm_meltwfunction_binary eltwise_kernel, void *in0, void *in1, void *out0); + const uintptr_t get_execute_function_ptr() const override { return reinterpret_cast(execute_kernel); } + const uintptr_t get_compiled_kernel_ptr() const override; + static std::set> get_supported_precisions(const std::shared_ptr& node = nullptr); + +protected: + libxsmm_meltw_binary_shape m_shape; + libxsmm_meltw_binary_type m_op_type; + void validate_arguments(const std::vector &in, const std::vector &out) const override; + static libxsmm_blasint get_broadcasted_dim(libxsmm_blasint dim0, libxsmm_blasint dim1, std::pair& bcast_flags); +}; + +class UnaryEltwiseTppEmitter : public TppEmitter { +public: + UnaryEltwiseTppEmitter(dnnl::impl::cpu::x64::jit_generator* h, + dnnl::impl::cpu::x64::cpu_isa_t isa, + const ov::snippets::lowered::ExpressionPtr& expr); + size_t get_inputs_num() const override { return 1; } + + static void execute_kernel(libxsmm_meltwfunction_unary eltwise_kernel, void *in0, void *out0); + const uintptr_t get_compiled_kernel_ptr() const override { + return COMPILE_TPP_KERNEL(libxsmm_dispatch_meltw_unary(m_op_type, + m_shape, + m_compile_flags)); + } + const uintptr_t get_execute_function_ptr() const override { return reinterpret_cast(execute_kernel); } + static std::set> get_supported_precisions(const std::shared_ptr& node = nullptr); + +protected: + libxsmm_meltw_unary_shape m_shape; + libxsmm_meltw_unary_type m_op_type; + void validate_arguments(const std::vector &in, const std::vector &out) const override; +}; + +class ReduceTppEmitter : public UnaryEltwiseTppEmitter { +public: + ReduceTppEmitter(dnnl::impl::cpu::x64::jit_generator* h, + dnnl::impl::cpu::x64::cpu_isa_t isa, + const ov::snippets::lowered::ExpressionPtr& expr); +}; + +class ReferenceUnaryEltwiseTppEmitter : public UnaryEltwiseTppEmitter { +public: + // Note: can create template to suppport different executor signatures + typedef std::function executor_function; + ReferenceUnaryEltwiseTppEmitter(dnnl::impl::cpu::x64::jit_generator* h, + dnnl::impl::cpu::x64::cpu_isa_t isa, + const ov::snippets::lowered::ExpressionPtr& expr, + executor_function executor) : + UnaryEltwiseTppEmitter(h, isa, expr), executor(std::move(executor)) { + } + static void execute_unary_eltw_kernel(ReferenceUnaryEltwiseTppEmitter* ref_emitter, void *in0, void *out0) { + assert(ref_emitter); + // Note: we can instantiate template with different precision combinations here, if we need to + ref_emitter->evaluate_reference_impl(reinterpret_cast(in0), reinterpret_cast(out0)); + } + const uintptr_t get_execute_function_ptr() const override { + return reinterpret_cast(execute_unary_eltw_kernel); + } + const uintptr_t get_compiled_kernel_ptr() const override { + return reinterpret_cast(this); + } + +private: + executor_function executor{nullptr}; + + template::value || !std::is_same::value, bool>::type = true> + void evaluate_reference_impl(Tin* in0, Tout* out0) { + for (int n = 0; n < m_shape.n; n++) { + auto in0_row = in0; + auto out0_row = out0; + for (int m = 0; m < m_shape.m; m++) + out0_row[m] = static_cast(executor(static_cast(in0_row[m]))); + in0 += m_shape.ldi; + out0 += m_shape.ldo; + } + } + void evaluate_reference_impl(float* in0, float* out0) { + for (int n = 0; n < m_shape.n; n++) { + auto in0_row = in0; + auto out0_row = out0; + for (int m = 0; m < m_shape.m; m++) + out0_row[m] = executor(in0_row[m]); + in0 += m_shape.ldi; + out0 += m_shape.ldo; + } + } +}; + + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_scalar_emitter.cpp b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_scalar_emitter.cpp new file mode 100644 index 00000000000..5b156100073 --- /dev/null +++ b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_scalar_emitter.cpp @@ -0,0 +1,30 @@ +// Copyright (C) 2020-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "jit_scalar_emitter.hpp" +#include "emitters/snippets/x64/jit_snippets_emitters.hpp" + +using namespace Xbyak; +using namespace dnnl::impl; +using namespace dnnl::impl::cpu::x64; + +namespace ov { +namespace intel_cpu { + +ScalarTppEmitter::ScalarTppEmitter(jit_generator* h, cpu_isa_t isa, const ov::snippets::lowered::ExpressionPtr& expr) + : jit_emitter(h, isa) { + push_arg_entry_of("scalar_tpp", jit_scalar_emitter::read_value(expr), false); + in_out_type_ = gpr_to_gpr; +} + +void ScalarTppEmitter::emit_impl(const std::vector& in, const std::vector& out) const { + const auto it = entry_map_.find("scalar_tpp"); + OV_CPU_JIT_EMITTER_ASSERT(it != entry_map_.end(), "Value has not been found in the table"); + const auto& out_reg = Reg64(static_cast(out[0])); + h->mov(out_reg, p_table); + h->add(out_reg, (*it).second.off); +} + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_scalar_emitter.hpp b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_scalar_emitter.hpp new file mode 100644 index 00000000000..e8235f6b86e --- /dev/null +++ b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_scalar_emitter.hpp @@ -0,0 +1,23 @@ +// Copyright (C) 2020-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include "snippets/lowered/expression.hpp" +#include "emitters/plugin/x64/jit_emitter.hpp" + +namespace ov { +namespace intel_cpu { +class ScalarTppEmitter : public jit_emitter { +public: + ScalarTppEmitter(dnnl::impl::cpu::x64::jit_generator* h, + dnnl::impl::cpu::x64::cpu_isa_t isa, + const ov::snippets::lowered::ExpressionPtr& expr); + size_t get_inputs_num() const override {return 0;} + size_t aux_gprs_count() const override {return 1;} +private: + void emit_impl(const std::vector& in, const std::vector& out) const override; +}; + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_tpp_emitter.cpp b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_tpp_emitter.cpp new file mode 100644 index 00000000000..24f4bac818e --- /dev/null +++ b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_tpp_emitter.cpp @@ -0,0 +1,138 @@ +// Copyright (C) 2020-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "jit_tpp_emitter.hpp" +#include "snippets/lowered/port_descriptor.hpp" +#include "transformations/tpp/x64/op/eltwise.hpp" + +using namespace Xbyak; +using namespace dnnl::impl; +using namespace dnnl::impl::cpu::x64; + +namespace ov { +namespace intel_cpu { + +VectorDims TppEmitter::get_projected_subtensor(const snippets::lowered::PortDescriptorPtr& desc) { + auto shape = desc->get_shape(); + auto subtensor = desc->get_subtensor(); + // Note: Scalar is a special case, so it's easier to prepend shapes than to handle it explicitly + if (shape.size() == 1) + shape.insert(shape.begin(), 1); + if (subtensor.size() == 1) + subtensor.insert(subtensor.begin(), 1); + OV_CPU_JIT_EMITTER_ASSERT(subtensor.size() <= shape.size() && !subtensor.empty(), + "Invalid subtensor + shape combination"); + auto shape_it = shape.rbegin(); + for (auto sub_it = subtensor.rbegin(); sub_it != subtensor.rend(); sub_it++, shape_it++) { + *sub_it = std::min(*sub_it, *shape_it); + } + return subtensor; +} + +TppEmitter::TppEmitter(dnnl::impl::cpu::x64::jit_generator* h, + dnnl::impl::cpu::x64::cpu_isa_t isa, + const ov::snippets::lowered::ExpressionPtr& expr) + : jit_emitter(h, isa) { + in_out_type_ = emitter_in_out_map::gpr_to_gpr; + const auto& node = expr->get_node(); + const auto& tpp_mod = std::dynamic_pointer_cast(node); + OV_CPU_JIT_EMITTER_ASSERT(tpp_mod, "Invoked with invalid node type"); + + const auto num_ins = node->get_input_size(); + const auto num_outs = node->get_output_size(); + num_kernel_args = static_cast(num_ins + num_outs); + io_dtypes.resize(num_kernel_args); + io_strides.resize(num_kernel_args); + io_offsets.resize(num_kernel_args); + io_port_descriptors.resize(num_kernel_args); + // Note: this is needed mostly for Reduce operations, since they allow the last subternsor dim to be FULL_DIM; + auto replace_full_dim = [](size_t dim, size_t replace_dim) { + if (dim == snippets::lowered::PortDescriptor::ServiceDimensions::FULL_DIM) + return replace_dim; + return dim; + }; + + for (size_t i = 0; i < num_ins; i++) { + io_dtypes[i] = ov_to_xsmm_dtype(node->get_input_element_type(i)); + io_offsets[i] = tpp_mod->get_input_offset(i); + io_strides[i] = replace_full_dim(tpp_mod->get_input_stride(i), + expr->get_input_port_descriptor(i)->get_shape().back()); + io_port_descriptors[i] = expr->get_input_port_descriptor(i); + } + + for (size_t i = 0; i < num_outs; i++) { + const auto i_off = i + num_ins; + io_dtypes[i_off] = ov_to_xsmm_dtype(node->get_output_element_type(i)); + io_offsets[i_off] = tpp_mod->get_output_offset(i); + io_strides[i_off] = replace_full_dim(tpp_mod->get_output_stride(i), + expr->get_output_port_descriptor(i)->get_shape().back()); + io_port_descriptors[i_off] = expr->get_output_port_descriptor(i); + } +} + +void TppEmitter::emit_code(const std::vector &in, const std::vector &out) const { + validate_arguments(in, out); + emit_impl(in, out); +} + +void TppEmitter::emit_impl(const std::vector& in, const std::vector& out) const { + internal_call_preamble(); + // Note: 4 args is currently enough for unary and binary ops. + // To enable ternary ops, we will have to pass extra regs on stack for Windows, + std::array abi_params{abi_param1, abi_param2, abi_param3, abi_param4}; + + // save function address in gpr to pass in call instruction + h->mov(h->rbp, get_execute_function_ptr()); + int aux_xmm_count = 0; + for (auto reg_idx : in) + h->uni_vmovq(Xmm(aux_xmm_count++), Reg64(static_cast(reg_idx))); + for (auto reg_idx : out) + h->uni_vmovq(Xmm(aux_xmm_count++), Reg64(static_cast(reg_idx))); + + OV_CPU_JIT_EMITTER_ASSERT(aux_xmm_count == num_kernel_args, "offsets for some inputs/outputs were not set"); + OV_CPU_JIT_EMITTER_ASSERT(aux_xmm_count < static_cast(abi_params.size()), "too many input/output arguments. More abi params required"); + + const auto data_ptr_reg = [&](Xmm xmm, Xbyak::Reg64 reg, size_t bytes_offset) { + h->uni_vmovq(reg, xmm); + if (bytes_offset) h->add(reg, bytes_offset); + }; + const auto& compiled_kernel = get_compiled_kernel_ptr(); + OV_CPU_JIT_EMITTER_ASSERT(compiled_kernel, "Failed to compile libxsmm_kernel"); + + h->mov(abi_params[0], compiled_kernel); + for (int i = 0; i < num_kernel_args; i++) + data_ptr_reg(Xmm(i), abi_params[i + 1], io_offsets[i]); + +#ifdef _WIN32 + // Before function call we should allocate stack area for + // - register parameters - ABI parameters (shadow space) + // - stack parameters - remaining parameters + // Since we currently have <= 4 args (kernel_ptr, in0, in1, out0), then nothing is passed on stack + h->sub(h->rsp, (num_kernel_args + 1) * gpr_size); +#endif + + internal_call_rsp_align(); + h->call(h->rbp); + internal_call_rsp_restore(); + +#ifdef _WIN32 + h->add(h->rsp, (num_kernel_args + 1) * gpr_size); +#endif + internal_call_postamble(); +} + +libxsmm_datatype TppEmitter::ov_to_xsmm_dtype(ov::element::Type_t elemet_type) { + switch (elemet_type) { + case ov::element::Type_t::f32 : return LIBXSMM_DATATYPE_F32; + case ov::element::Type_t::bf16 : return LIBXSMM_DATATYPE_BF16; + case ov::element::Type_t::i8 : return LIBXSMM_DATATYPE_I8; + case ov::element::Type_t::u8 : return LIBXSMM_DATATYPE_U8; + default: + OV_CPU_JIT_EMITTER_THROW("Attempt to convert unsupported ov data type"); + return LIBXSMM_DATATYPE_IMPLICIT; + } +} + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_tpp_emitter.hpp b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_tpp_emitter.hpp new file mode 100644 index 00000000000..f4795e0fd9c --- /dev/null +++ b/src/plugins/intel_cpu/src/emitters/tpp/x64/jit_tpp_emitter.hpp @@ -0,0 +1,62 @@ +// Copyright (C) 2020-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "snippets/lowered/linear_ir.hpp" +#include "snippets/lowered/expression.hpp" +#include "emitters/plugin/x64/jit_emitter.hpp" +#include "libxsmm.h" + +namespace ov { +namespace intel_cpu { +// Note: The macro allows to automatically set appropriate environment variables for TPP/Libxsmm kernel compilation +// All TPP kernels must be compiled using this macro. +// * LIBXSMM_X86_HINT_USE_HIGH_PREC_ELTWISE_APPROX enables more accurate exp approximation and exact division in TPP +// * LIBXSMM_GEMM_K_A_PF_DIST allows to tweak prefetching for GEMM kernels +#define COMPILE_TPP_KERNEL(...) \ + [&]() { \ + setenv("LIBXSMM_X86_HINT_USE_HIGH_PREC_ELTWISE_APPROX", "1", 1); \ + setenv("LIBXSMM_GEMM_K_A_PF_DIST", "4", 1); \ + auto res = reinterpret_cast(__VA_ARGS__); \ + unsetenv("LIBXSMM_X86_HINT_USE_HIGH_PREC_ELTWISE_APPROX"); \ + unsetenv("LIBXSMM_GEMM_K_A_PF_DIST"); \ + return res; \ + }() +class DebugTppEmitter; +class TppEmitter : public jit_emitter { + friend DebugTppEmitter; + +public: + TppEmitter(dnnl::impl::cpu::x64::jit_generator* h, + dnnl::impl::cpu::x64::cpu_isa_t isa, + const ov::snippets::lowered::ExpressionPtr& expr); + void emit_code(const std::vector &in, + const std::vector &out) const; + static libxsmm_datatype ov_to_xsmm_dtype(ov::element::Type_t elemet_type); + +protected: + void emit_impl(const std::vector& in, + const std::vector& out) const override; + static ov::snippets::VectorDims get_projected_subtensor(const snippets::lowered::PortDescriptorPtr& desc); + + /// Generate function pointer to the thin wrapper over the kernel that is called in runtime on every iteration + virtual const uintptr_t get_execute_function_ptr() const = 0; + /// Generate pointer to compiled kernel + virtual const uintptr_t get_compiled_kernel_ptr() const = 0; + + std::vector io_offsets{}; + std::vector io_dtypes{}; + // Note: almost all emitters use fp32 for internal computations + libxsmm_datatype exec_dtype {LIBXSMM_DATATYPE_F32}; + // aka leading dimensions + std::vector io_strides{}; + std::vector io_port_descriptors{}; + // compile flags has the same type for all eltwises, so we keep them in the base class + libxsmm_bitfield m_compile_flags {0}; + int num_kernel_args = 0; +}; + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/subgraph.cpp b/src/plugins/intel_cpu/src/nodes/subgraph.cpp index f7a50ffa148..8432586367a 100644 --- a/src/plugins/intel_cpu/src/nodes/subgraph.cpp +++ b/src/plugins/intel_cpu/src/nodes/subgraph.cpp @@ -44,6 +44,13 @@ std::mutex err_print_lock; #endif +#ifdef SNIPPETS_LIBXSMM_TPP +#include "transformations/tpp/x64/pass/brgemm_to_brgemm_tpp.hpp" +#include "transformations/tpp/x64/pass/eltwise_to_eltwise_tpp.hpp" +#include "transformations/tpp/x64/pass/scalar_to_scalar_tpp.hpp" +#include "transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.hpp" +#endif + using namespace dnnl::impl::utils; using namespace dnnl::impl::cpu; using namespace dnnl::impl::cpu::x64; @@ -355,6 +362,7 @@ void Snippet::initOptimalPrimitiveDescriptor() { SNIPPETS_REGISTER_PASS_RELATIVE(Place::After, ov::snippets::pass::MatMulToBrgemm, pass::EnforcePrecision, element::f32, element::bf16); } + SNIPPETS_REGISTER_PASS_RELATIVE(Place::Before, ov::snippets::pass::PropagatePrecision, ov::intel_cpu::pass::BrgemmToBrgemmCPU); SNIPPETS_REGISTER_PASS_RELATIVE(Place::After, ov::intel_cpu::pass::BrgemmToBrgemmCPU, @@ -362,6 +370,15 @@ void Snippet::initOptimalPrimitiveDescriptor() { SNIPPETS_REGISTER_PASS_ABSOLUTE(Place::PipelineEnd, ov::intel_cpu::pass::RemoveConverts); SNIPPETS_REGISTER_PASS_ABSOLUTE(Place::PipelineEnd, ov::intel_cpu::pass::MulAddToFMA); +#ifdef SNIPPETS_LIBXSMM_TPP + SNIPPETS_REGISTER_PASS_RELATIVE(Place::Before, ov::intel_cpu::pass::BrgemmToBrgemmCPU, + ov::intel_cpu::tpp::pass::BrgemmToBrgemmTPP); + // Note: There could be several ConvertConstantsToScalars instances in the pipeline + SNIPPETS_REGISTER_PASS_ABSOLUTE(Place::PipelineEnd, ov::intel_cpu::tpp::pass::ScalarToScalarTPP); + SNIPPETS_REGISTER_PASS_RELATIVE(Place::After, ov::intel_cpu::tpp::pass::BrgemmToBrgemmTPP, + ov::intel_cpu::tpp::pass::EltwiseToEltwiseTPP); +#endif + #undef SNIPPETS_REGISTER_PASS std::vector input_precisions; @@ -377,8 +394,14 @@ void Snippet::initOptimalPrimitiveDescriptor() { snippetAttrs.snippet->data_flow_transformations(in_blocked_shapes, input_precisions, output_precisions, backend_passes); // Note: minimal JIT work amount is a predefined value that describes the number of kernel iterations (work amount) // needed to cover kernel call overhead. It is used for balancing between parallel and JIT work amounts in domain optimization. +#ifdef SNIPPETS_LIBXSMM_TPP + const auto& lir = snippetAttrs.snippet->convert_body_to_linear_ir(static_cast(parallel_get_max_threads()), 256, + std::make_shared()); + lir->set_loop_depth(std::min(2ul, lir->get_master_shape().size())); +#else snippetAttrs.snippet->convert_body_to_linear_ir(static_cast(parallel_get_max_threads()), 256, std::make_shared()); +#endif } ov::element::Type Snippet::getRuntimePrecision() const { @@ -641,9 +664,19 @@ void Snippet::SnippetJitExecutor::generate(const jit_snippets_compile_args* jcp) SNIPPETS_REGISTER_PASS_RELATIVE(Place::After, ov::intel_cpu::pass::FuseLoadStoreConvert, ov::intel_cpu::pass::SetBrgemmCopyBBuffersShape); - schedule = snippetAttrs.snippet->generate_from_linear_ir(std::make_shared(), + auto lowering_config = std::make_shared(); +#ifdef SNIPPETS_LIBXSMM_TPP + // Note: temporary disabled. Re-enable after ticket 132833 is resolved + lowering_config->disable(); + SNIPPETS_REGISTER_PASS_RELATIVE(Place::After, ov::intel_cpu::pass::FuseLoadStoreConvert, + ov::intel_cpu::tpp::pass::SetTPPLeadingDim); +#endif + + schedule = snippetAttrs.snippet->generate_from_linear_ir(lowering_config, backend_passes, reinterpret_cast(jcp)); + +#undef SNIPPETS_REGISTER_PASS_RELATIVE } bool Snippet::SnippetJitExecutor::schedule_created() { diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_copy_b.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_copy_b.cpp index 193d2ce808f..9bf6d7a1d62 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_copy_b.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_copy_b.cpp @@ -16,7 +16,7 @@ namespace intel_cpu { intel_cpu::BrgemmCopyB::BrgemmCopyB(const Output& x, const element::Type src_type, const Type type, const size_t offset_in, const size_t offset_out0, const size_t offset_out1, std::vector layout_input, const size_t blk_size_k, const size_t blk_size_n) - : snippets::op::MemoryAccess({x}, 1, type == Type::WithCompensations ? 2 : 1), + : snippets::modifier::MemoryAccess(1, type == Type::WithCompensations ? 2 : 1), op::Op({x}), m_type(type), m_src_type(src_type) { m_brgemmVNNIFactor = 4 / m_src_type.size(); set_output_size(type == Type::WithCompensations ? 2 : 1); @@ -32,7 +32,7 @@ intel_cpu::BrgemmCopyB::BrgemmCopyB(const Output& x, const element::Type s intel_cpu::BrgemmCopyB::BrgemmCopyB(const Output& x, const element::Type src_type, const Type type, const PortDescriptor& desc_in0, const PortDescriptor& desc_out0, const PortDescriptor& desc_out1, std::vector layout_input, const size_t blk_size_k, const size_t blk_size_n) - : snippets::op::MemoryAccess({x}, 1, type == Type::WithCompensations ? 2 : 1), + : snippets::modifier::MemoryAccess(1, type == Type::WithCompensations ? 2 : 1), op::Op({x}), m_type(type), m_src_type(src_type) { m_brgemmVNNIFactor = 4 / m_src_type.size(); set_output_size(type == Type::WithCompensations ? 2 : 1); diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_copy_b.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_copy_b.hpp index f803e5d55fc..3da69e0d889 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_copy_b.hpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_copy_b.hpp @@ -18,9 +18,9 @@ namespace intel_cpu { OneDNN requiers data repacking for second input of Brgemm with input non-fp32 precisions. * @ingroup snippets */ -class BrgemmCopyB : public snippets::op::MemoryAccess { +class BrgemmCopyB : public snippets::modifier::MemoryAccess, public ov::op::Op { public: - OPENVINO_OP("BrgemmCopyB", "SnippetsOpset", snippets::op::MemoryAccess); + OPENVINO_OP("BrgemmCopyB", "SnippetsOpset"); enum Type { OnlyRepacking, // Just data repacking - one output diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_cpu.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_cpu.cpp index e5155a28596..1646e72cb6e 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_cpu.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_cpu.cpp @@ -17,7 +17,7 @@ BrgemmCPU::BrgemmCPU(const Output& A, const Output& B, const Type ty const size_t offset_a, const size_t offset_b, const size_t offset_c, std::vector layout_a, std::vector layout_b, std::vector layout_c, const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n, const float beta) - : Brgemm(), m_type(type), m_beta(beta) { + : Brgemm(), m_type(type) { // We call default ctor of Brgemm class to avoid incorrect shape infer in constructor_validate_and_type_infer() call set_arguments({A, B}); set_output_size(1); @@ -27,13 +27,14 @@ BrgemmCPU::BrgemmCPU(const Output& A, const Output& B, const Type ty set_output_port_descriptor({0, offset_c}, 0); compute_block_size_values(blk_size_m, blk_size_k, blk_size_n); custom_constructor_validate_and_infer_types(std::move(layout_a), std::move(layout_b), std::move(layout_c)); + set_beta(beta); } BrgemmCPU::BrgemmCPU(const Output& A, const Output& B, const Output& scratch, const Type type, const size_t offset_a, const size_t offset_b, const size_t offset_scratch, const size_t offset_c, std::vector layout_a, std::vector layout_b, std::vector layout_c, const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n, const float beta) - : Brgemm(), m_type(type), m_beta(beta) { + : Brgemm(), m_type(type) { set_arguments({A, B, scratch}); set_output_size(1); ctor_initialize(std::set{0, 1, 2}, std::set{0}); @@ -43,32 +44,35 @@ BrgemmCPU::BrgemmCPU(const Output& A, const Output& B, const Output< set_input_port_descriptor({0, offset_scratch}, 2); compute_block_size_values(blk_size_m, blk_size_k, blk_size_n); custom_constructor_validate_and_infer_types(std::move(layout_a), std::move(layout_b), std::move(layout_c)); + set_beta(beta); } BrgemmCPU::BrgemmCPU(const Output& A, const Output& B, const Type type, const PortDescriptor& desc_a, const PortDescriptor& desc_b, const PortDescriptor& desc_c, std::vector layout_a, std::vector layout_b, std::vector layout_c, const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n, const float beta) - : Brgemm(), m_type(type), m_beta(beta) { + : Brgemm(), m_type(type) { set_arguments({A, B}); set_output_size(1); m_input_ports = {{0, desc_a}, {1, desc_b}}; m_output_ports = {{0, desc_c}}; compute_block_size_values(blk_size_m, blk_size_k, blk_size_n); custom_constructor_validate_and_infer_types(std::move(layout_a), std::move(layout_b), std::move(layout_c)); + set_beta(beta); } BrgemmCPU::BrgemmCPU(const Output& A, const Output& B, const Output& scratch, const Type type, const PortDescriptor& desc_a, const PortDescriptor& desc_b, const PortDescriptor& desc_scratch, const PortDescriptor& desc_c, std::vector layout_a, std::vector layout_b, std::vector layout_c, const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n, const float beta) - : Brgemm(), m_type(type), m_beta(beta) { + : Brgemm(), m_type(type) { set_arguments({A, B, scratch}); set_output_size(1); m_input_ports = {{0, desc_a}, {1, desc_b}, {2, desc_scratch}}; m_output_ports = {{0, desc_c}}; compute_block_size_values(blk_size_m, blk_size_k, blk_size_n); custom_constructor_validate_and_infer_types(std::move(layout_a), std::move(layout_b), std::move(layout_c)); + set_beta(beta); } void BrgemmCPU::custom_constructor_validate_and_infer_types(std::vector layout_a, std::vector layout_b, std::vector layout_c) { @@ -89,14 +93,6 @@ void BrgemmCPU::custom_constructor_validate_and_infer_types(std::vector validate_with_scratchpad(planar_input_shapes[1].get_shape()); } -void BrgemmCPU::compute_block_size_values(const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n) { - const auto input_shape_0 = snippets::utils::get_planar_pshape(input(0)).get_shape(); - const auto input_shape_1 = snippets::utils::get_planar_pshape(input(1)).get_shape(); - m_M_blk = blk_size_m != 0 ? blk_size_m : *(input_shape_0.rbegin() + 1); - m_K_blk = blk_size_k != 0 ? blk_size_k : *input_shape_0.rbegin(); - m_N_blk = blk_size_n != 0 ? blk_size_n : *input_shape_1.rbegin(); -} - void BrgemmCPU::validate_and_infer_types() { INTERNAL_OP_SCOPE(BrgemmCPU_validate_and_infer_types); validate_inputs(); @@ -173,10 +169,6 @@ size_t BrgemmCPU::get_offset_scratch() const { bool BrgemmCPU::visit_attributes(AttributeVisitor& visitor) { Brgemm::visit_attributes(visitor); - visitor.on_attribute("blk_M", m_M_blk); - visitor.on_attribute("blk_K", m_K_blk); - visitor.on_attribute("blk_N", m_N_blk); - visitor.on_attribute("beta", m_beta); return true; } diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_cpu.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_cpu.hpp index f5bfa19a7dc..bd3f9e054e1 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_cpu.hpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_cpu.hpp @@ -51,15 +51,6 @@ public: std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; Type get_type() const { return m_type; } - size_t get_m_block_size() const { return m_M_blk; } - size_t get_k_block_size() const { return m_K_blk; } - size_t get_n_block_size() const { return m_N_blk; } - float get_beta() const { return m_beta; } - - void set_m_block_size(size_t block_size) { m_M_blk = block_size; } - void set_k_block_size(size_t block_size) { m_K_blk = block_size; } - void set_n_block_size(size_t block_size) { m_N_blk = block_size; } - void set_beta(float beta) { m_beta = beta; } bool is_with_compensations() const { return m_type == Type::WithCompensations; } bool is_with_data_repacking() const { return m_type != Type::Floating; } @@ -75,15 +66,10 @@ public: private: void custom_constructor_validate_and_infer_types(std::vector layout_a, std::vector layout_b, std::vector layout_c); - void compute_block_size_values(const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n); void validate_with_scratchpad(const ov::Shape& shape_b) const; void validate_inputs() const; Type m_type = Type::Floating; - size_t m_M_blk = 0; - size_t m_K_blk = 0; - size_t m_N_blk = 0; - float m_beta = 1.f; }; } // namespace intel_cpu diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/brgemm_to_brgemm_cpu.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/brgemm_to_brgemm_cpu.cpp index 6d8caae3a85..64cef03e042 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/brgemm_to_brgemm_cpu.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/brgemm_to_brgemm_cpu.cpp @@ -10,6 +10,7 @@ #include "snippets/op/brgemm.hpp" #include "transformations/snippets/x64/op/brgemm_copy_b.hpp" #include "transformations/snippets/x64/op/brgemm_cpu.hpp" +#include "transformations/tpp/x64/op/modifiers.hpp" #include "openvino/core/rt_info.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" @@ -43,8 +44,10 @@ void set_port_desc(const T& port, Args... params) { pass::BrgemmToBrgemmCPU::BrgemmToBrgemmCPU() { MATCHER_SCOPE(BrgemmToBrgemmCPU); - - auto m_brgemm = ov::pass::pattern::wrap_type(); + auto is_not_tpp = [](const Output& out) { + return !std::dynamic_pointer_cast(out.get_node_shared_ptr()); + }; + auto m_brgemm = ov::pass::pattern::wrap_type(is_not_tpp); auto callback = [=](ov::pass::pattern::Matcher& m) { OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "ov::intel_cpu::pass::BrgemmToBrgemmCPU") diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_blocking.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_blocking.cpp index 16e6f897af5..cbc8d1fff9d 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_blocking.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_blocking.cpp @@ -12,6 +12,7 @@ #include "snippets/snippets_isa.hpp" #include "snippets/utils.hpp" #include "transformations/snippets/x64/op/brgemm_cpu.hpp" +#include "transformations/tpp/x64/op/brgemm.hpp" namespace ov { namespace intel_cpu { @@ -39,7 +40,7 @@ void BrgemmBlocking::move_new_memory_buffer(snippets::lowered::LinearIR& linear_ bool BrgemmBlocking::run(LinearIR& linear_ir, LinearIR::constExprIt begin, LinearIR::constExprIt end) { OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "Snippets::BrgemmBlocking") const auto& loop_manager = linear_ir.get_loop_manager(); - auto blocking_loop_exists = [&](const ExpressionPtr& brgemm_expr, const std::shared_ptr& brgemm) { + auto blocking_loop_exists = [&](const ExpressionPtr& brgemm_expr, const std::shared_ptr& brgemm) { auto check_port = [&](const LoopPort& p) { return p.expr_port->get_expr() == brgemm_expr && ov::snippets::utils::one_of(p.dim_idx, 0ul, 1ul); }; @@ -58,9 +59,13 @@ bool BrgemmBlocking::run(LinearIR& linear_ir, LinearIR::constExprIt begin, Linea bool modified = false; for (auto expr_it = begin; expr_it != end; expr_it++) { const auto& brgemm_expr = *expr_it; - const auto brgemm = ov::as_type_ptr(brgemm_expr->get_node()); + const auto& node = brgemm_expr->get_node(); + const auto brgemm = ov::as_type_ptr(node); + const auto brgemm_cpu = ov::as_type_ptr(node); if (!brgemm || blocking_loop_exists(brgemm_expr, brgemm)) continue; + OPENVINO_ASSERT(ov::is_type(node) || ov::is_type(node), + "Detected invalid Brgemm operation: ops must be assigned to a backend when blocking is performed."); const auto& in_0_desc = brgemm_expr->get_input_port_descriptor(0); const auto& in_1_desc = brgemm_expr->get_input_port_descriptor(1); @@ -88,11 +93,13 @@ bool BrgemmBlocking::run(LinearIR& linear_ir, LinearIR::constExprIt begin, Linea auto loop_begin_it = expr_it, loop_end_it = std::next(expr_it); std::vector entries{LoopPort(brgemm_expr->get_input_port(0), true), LoopPort(brgemm_expr->get_input_port(1), false)}; - if (brgemm->is_with_compensations()) { - entries.emplace_back(brgemm_expr->get_input_port(2), false); - } else if (brgemm->is_amx()) { - move_new_memory_buffer(linear_ir, expr_it); - loop_begin_it = std::prev(expr_it); + if (brgemm_cpu) { + if (brgemm_cpu->is_with_compensations()) { + entries.emplace_back(brgemm_expr->get_input_port(2), false); + } else if (brgemm_cpu->is_amx()) { + move_new_memory_buffer(linear_ir, expr_it); + loop_begin_it = std::prev(expr_it); + } } std::vector exits{LoopPort(brgemm_expr->get_output_port(0), true)}; loop_manager->mark_loop(loop_begin_it, loop_end_it, m, block_size_m, 1, entries, exits); @@ -112,11 +119,13 @@ bool BrgemmBlocking::run(LinearIR& linear_ir, LinearIR::constExprIt begin, Linea auto loop_begin_it = expr_it, loop_end_it = std::next(expr_it); std::vector entries{LoopPort(brgemm_expr->get_input_port(0), false), LoopPort(brgemm_expr->get_input_port(1), true)}; - if (brgemm->is_with_compensations()) { - entries.emplace_back(brgemm_expr->get_input_port(2), true); - } else if (brgemm->is_amx()) { - move_new_memory_buffer(linear_ir, expr_it); - loop_begin_it = std::prev(expr_it); + if (brgemm_cpu) { + if (brgemm_cpu->is_with_compensations()) { + entries.emplace_back(brgemm_expr->get_input_port(2), true); + } else if (brgemm_cpu->is_amx()) { + move_new_memory_buffer(linear_ir, expr_it); + loop_begin_it = std::prev(expr_it); + } } std::vector exits{LoopPort(brgemm_expr->get_output_port(0), true)}; loop_manager->mark_loop(loop_begin_it, loop_end_it, n, block_size_n, 0, entries, exits); @@ -138,11 +147,13 @@ bool BrgemmBlocking::run(LinearIR& linear_ir, LinearIR::constExprIt begin, Linea auto loop_begin_it = expr_it, loop_end_it = std::next(expr_it); std::vector entries{LoopPort(brgemm_expr->get_input_port(0), true, 0), LoopPort(brgemm_expr->get_input_port(1), true, 1)}; - if (brgemm->is_with_compensations()) { - entries.emplace_back(brgemm_expr->get_input_port(2), false, 1); - } else if (brgemm->is_amx()) { - move_new_memory_buffer(linear_ir, expr_it); - loop_begin_it = std::prev(expr_it); + if (brgemm_cpu) { + if (brgemm_cpu->is_with_compensations()) { + entries.emplace_back(brgemm_expr->get_input_port(2), false, 1); + } else if (brgemm_cpu->is_amx()) { + move_new_memory_buffer(linear_ir, expr_it); + loop_begin_it = std::prev(expr_it); + } } std::vector exits{LoopPort(brgemm_expr->get_output_port(0), false)}; const auto id = loop_manager->mark_loop(loop_begin_it, loop_end_it, k, block_size_k, entries, exits); diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/cpu_iter_handlers.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/cpu_iter_handlers.cpp index 382ee78d8be..d5e96b2a733 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/cpu_iter_handlers.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/cpu_iter_handlers.cpp @@ -18,7 +18,7 @@ SetBrgemmBeta::SetBrgemmBeta(float beta) : snippets::lowered::pass::RangedPass() bool SetBrgemmBeta::run(LinearIR& linear_ir, LinearIR::constExprIt begin, LinearIR::constExprIt end) { for (auto expr_it = begin; expr_it != end; ++expr_it) { const auto& expr = expr_it->get(); - if (const auto brgemm = ov::as_type_ptr(expr->get_node())) { + if (const auto brgemm = ov::as_type_ptr(expr->get_node())) { brgemm->set_beta(m_beta); } } diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/mul_add_to_fma.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/mul_add_to_fma.cpp index 1d179b1ace1..e10ea0e3d12 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/mul_add_to_fma.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/mul_add_to_fma.cpp @@ -11,15 +11,22 @@ #include "openvino/pass/pattern/op/wrap_type.hpp" #include "openvino/pass/pattern/matcher.hpp" #include "openvino/core/rt_info.hpp" - +#include "snippets/op/memory_access.hpp" ov::intel_cpu::pass::MulAddToFMA::MulAddToFMA() { MATCHER_SCOPE(MulAddToFMA); + auto is_not_memory_access = [](const Output& out) { + return !std::dynamic_pointer_cast(out.get_node_shared_ptr()); + }; auto mul_input_1 = ov::pass::pattern::any_input(); auto mul_input_2 = ov::pass::pattern::any_input(); - auto mul_m = ov::pass::pattern::wrap_type({ mul_input_1, mul_input_2 }, ov::pass::pattern::consumers_count(1)); + auto mul_m = ov::pass::pattern::wrap_type({ mul_input_1, mul_input_2 }, + [=](const Output& out) { + return out.get_target_inputs().size() == 1 && + is_not_memory_access(out); + }); auto add_input_2 = ov::pass::pattern::any_input(); - auto add_m = ov::pass::pattern::wrap_type({ mul_m, add_input_2 }); + auto add_m = ov::pass::pattern::wrap_type({ mul_m, add_input_2 }, is_not_memory_access); matcher_pass_callback callback = [=](ov::pass::pattern::Matcher& m) { OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "Snippets::op::MulAddToFMA_callback") diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/shape_inference.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/shape_inference.cpp index f9ae7711ed1..3be7b60d893 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/shape_inference.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/shape_inference.cpp @@ -11,6 +11,11 @@ #include "op/store_convert.hpp" #include "op/perf_count_rdtsc.hpp" #include "transformations/cpu_opset/common/op/swish_cpu.hpp" +#ifdef SNIPPETS_LIBXSMM_TPP +#include "transformations/tpp/x64/op/brgemm.hpp" +#include "transformations/tpp/x64/op/scalar.hpp" +#include "transformations/tpp/x64/op/reduce.hpp" +#endif namespace ov { namespace snippets { @@ -42,9 +47,14 @@ const CPUShapeInferSnippetsFactory::TRegistry CPUShapeInferSnippetsFactory::spec #ifdef SNIPPETS_DEBUG_CAPS SHAPE_INFER_PREDEFINED(ov::intel_cpu::PerfCountRdtscBegin, EmptyShapeInfer), SHAPE_INFER_PREDEFINED(ov::intel_cpu::PerfCountRdtscEnd, EmptyShapeInfer), +#endif +#ifdef SNIPPETS_LIBXSMM_TPP + SHAPE_INFER_OP_SPECIFIC_EXTERNAL(ov::intel_cpu::tpp::op::BrgemmTPP, BrgemmShapeInfer), + SHAPE_INFER_PREDEFINED(ov::intel_cpu::tpp::op::Scalar, SingleElementShapeInfer), + SHAPE_INFER_OP_SPECIFIC_EXTERNAL(ov::intel_cpu::tpp::op::ReduceMax, ReduceShapeInfer), + SHAPE_INFER_OP_SPECIFIC_EXTERNAL(ov::intel_cpu::tpp::op::ReduceSum, ReduceShapeInfer), #endif SHAPE_INFER_OP_SPECIFIC_EXTERNAL(ov::intel_cpu::BrgemmCPU, BrgemmShapeInfer), - // SHAPE_INFER_OP_SPECIFIC(ov::intel_cpu::BrgemmCopyB), }; #undef SHAPE_INFER_OP_SPECIFIC diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/op/brgemm.cpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/brgemm.cpp new file mode 100644 index 00000000000..4e14ad26246 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/brgemm.cpp @@ -0,0 +1,62 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "brgemm.hpp" +#include "snippets/itt.hpp" +#include "snippets/utils.hpp" +#include "snippets/lowered/port_descriptor.hpp" +#include "utils/general_utils.h" + + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace op { + +BrgemmTPP::BrgemmTPP(const Output& A, const Output& B, + const size_t offset_a, const size_t offset_b, const size_t offset_c, + std::vector layout_a, std::vector layout_b, std::vector layout_c, + const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n, const float beta) + : MemoryAccess(std::set{0, 1}, std::set{0}), + modifier::TensorProcessingPrimitive(), + Brgemm(A, B, + offset_a, offset_b, offset_c, + std::move(layout_a), std::move(layout_b), std::move(layout_c), + blk_size_m, blk_size_k, blk_size_n) { + set_beta(beta); +} + +BrgemmTPP::BrgemmTPP(const Output& A, const Output& B, + const PortDescriptor& desc_a, const PortDescriptor& desc_b, const PortDescriptor& desc_c, + std::vector layout_a, std::vector layout_b, std::vector layout_c, + const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n, const float beta) + : MemoryAccess(PortMap{{0, desc_a}, {1, desc_b}}, PortMap{{0, desc_c}}), + modifier::TensorProcessingPrimitive(), + Brgemm(A, B, + desc_a, desc_b, desc_c, + std::move(layout_a), std::move(layout_b), std::move(layout_c), + blk_size_m, blk_size_n, blk_size_k) { + set_beta(beta); +} + +std::shared_ptr BrgemmTPP::clone_with_new_inputs(const OutputVector& new_args) const { + INTERNAL_OP_SCOPE(BrgemmTPP_clone_with_new_inputs); + check_new_args_count(this, new_args); + return std::make_shared(new_args.at(0), new_args.at(1), + get_input_port_descriptor(0), get_input_port_descriptor(1), get_output_port_descriptor(0), + snippets::lowered::PortDescriptorUtils::get_port_descriptor_ptr(input(0))->get_layout(), + snippets::lowered::PortDescriptorUtils::get_port_descriptor_ptr(input(1))->get_layout(), + snippets::lowered::PortDescriptorUtils::get_port_descriptor_ptr(output(0))->get_layout(), + m_M_blk, m_K_blk, m_N_blk, m_beta); +} + +bool BrgemmTPP::visit_attributes(AttributeVisitor& visitor) { + TensorProcessingPrimitive::visit_attributes(visitor); + return Brgemm::visit_attributes(visitor); +} + +} // namespace op +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/op/brgemm.hpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/brgemm.hpp new file mode 100644 index 00000000000..923980e0978 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/brgemm.hpp @@ -0,0 +1,43 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "transformations/snippets/x64/op/brgemm_cpu.hpp" +#include "modifiers.hpp" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace op { + +/** + * @interface BrgemmTPP + * @brief BrgemmTPP is a batch-reduced matrix multiplication with the support of arbitrary strides between matrices rows + * with support of several precisions on plugin level + * @ingroup snippets + */ +class BrgemmTPP : virtual public modifier::TensorProcessingPrimitive, public snippets::op::Brgemm { +public: + OPENVINO_OP("Brgemm", "TppOpset", snippets::op::Brgemm); + + BrgemmTPP(const Output& A, const Output& B, + size_t offset_a = 0, size_t offset_b = 0, size_t offset_c = 0, + std::vector layout_a = {}, std::vector layout_b = {}, std::vector layout_c = {}, + size_t blk_size_m = 0, size_t blk_size_k = 0, size_t blk_size_n = 0, float beta = 1); + BrgemmTPP(const Output& A, const Output& B, + const PortDescriptor& desc_a, const PortDescriptor& desc_b, const PortDescriptor& desc_c, + std::vector layout_a = {}, std::vector layout_b = {}, std::vector layout_c = {}, + size_t blk_size_m = 0, size_t blk_size_k = 0, size_t blk_size_n = 0, float beta = 1); + BrgemmTPP() = default; + + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + + bool visit_attributes(AttributeVisitor& visitor) override; +}; + +} // namespace op +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/op/eltwise.cpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/eltwise.cpp new file mode 100644 index 00000000000..8b5469543cb --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/eltwise.cpp @@ -0,0 +1,102 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "eltwise.hpp" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace op { + +#define GENERAL_AUX_METHODS(OP, OP_TYPE, ...) \ + std::shared_ptr OP::clone_with_new_inputs(const OutputVector& new_args) const {\ + check_new_args_count(this, new_args);\ + const auto& new_op = std::make_shared(__VA_ARGS__);\ + new_op->clone_memory_access_ports(*this);\ + return new_op;\ +} \ + bool OP::visit_attributes(AttributeVisitor& visitor) {\ + return OP_TYPE::visit_attributes(visitor);\ +} + +#define BINARY_AUX_METHODS(BINARY_OP) GENERAL_AUX_METHODS(BINARY_OP, BinaryEltwiseTPP, new_args.at(0), new_args.at(1), this->get_autob()) +#define UNARY_AUX_METHODS(UNARY_OP) GENERAL_AUX_METHODS(UNARY_OP, UnaryEltwiseTPP, new_args.at(0)) + +bool EltwiseTPP::is_supported(const std::shared_ptr& node) { + return ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node) || + ov::is_type(node); +} + +bool EltwiseTPP::visit_attributes(AttributeVisitor& visitor) { + TensorProcessingPrimitive::visit_attributes(visitor); + return MemoryAccess::visit_attributes(visitor); +} + +BinaryEltwiseTPP::BinaryEltwiseTPP(libxsmm_meltw_binary_type op_type) : EltwiseTPP(), m_op_type(op_type) { + // Initialize input/output ports as memory access ports + ctor_initialize(std::set{0, 1}, std::set{0}); +} + +UnaryEltwiseTPP::UnaryEltwiseTPP(libxsmm_meltw_unary_type op_type) : EltwiseTPP(), m_op_type(op_type) { + ctor_initialize(std::set{0}, std::set{0}); +} + +Add::Add(const Output& arg0, const Output& arg1, const AutoBroadcastSpec& auto_broadcast) +: BinaryEltwiseTPP(LIBXSMM_MELTW_TYPE_BINARY_ADD), ov::op::v1::Add(arg0, arg1, auto_broadcast) { +} + +BINARY_AUX_METHODS(Add) + +Subtract::Subtract(const Output& arg0, const Output& arg1, const AutoBroadcastSpec& auto_broadcast) + : BinaryEltwiseTPP(LIBXSMM_MELTW_TYPE_BINARY_SUB), ov::op::v1::Subtract(arg0, arg1, auto_broadcast) { +} + +BINARY_AUX_METHODS(Subtract) + +Multiply::Multiply(const Output& arg0, const Output& arg1, const AutoBroadcastSpec& auto_broadcast) + : BinaryEltwiseTPP(LIBXSMM_MELTW_TYPE_BINARY_MUL), ov::op::v1::Multiply(arg0, arg1, auto_broadcast) { +} + +BINARY_AUX_METHODS(Multiply) + +Divide::Divide(const Output& arg0, const Output& arg1, const AutoBroadcastSpec& auto_broadcast) + : BinaryEltwiseTPP(LIBXSMM_MELTW_TYPE_BINARY_DIV), ov::op::v1::Divide(arg0, arg1, auto_broadcast) { +} + +BINARY_AUX_METHODS(Divide) + +Exp::Exp(const Output& arg0) : UnaryEltwiseTPP(LIBXSMM_MELTW_TYPE_UNARY_EXP), ov::op::v0::Exp(arg0) { +} + +UNARY_AUX_METHODS(Exp) + +Relu::Relu(const Output& arg0) : UnaryEltwiseTPP(LIBXSMM_MELTW_TYPE_UNARY_RELU), ov::op::v0::Relu(arg0) { +} + +UNARY_AUX_METHODS(Relu) + +Reciprocal::Reciprocal(const Output& arg) : + UnaryEltwiseTPP(LIBXSMM_MELTW_TYPE_UNARY_RECIPROCAL), ov::snippets::op::PowerStatic(arg, -1.f) { +} + +UNARY_AUX_METHODS(Reciprocal) + +Square::Square(const Output& arg) : + UnaryEltwiseTPP(LIBXSMM_MELTW_TYPE_UNARY_X2), ov::snippets::op::PowerStatic(arg, 2.f) { +} + +UNARY_AUX_METHODS(Square) + +SquareRoot::SquareRoot(const Output& arg) : + UnaryEltwiseTPP(LIBXSMM_MELTW_TYPE_UNARY_SQRT), ov::snippets::op::PowerStatic(arg, 0.5f) { +} + +UNARY_AUX_METHODS(SquareRoot) + +} // namespace op +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/op/eltwise.hpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/eltwise.hpp new file mode 100644 index 00000000000..fe8113025f5 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/eltwise.hpp @@ -0,0 +1,123 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "modifiers.hpp" +#include "openvino/op/add.hpp" +#include "openvino/op/subtract.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/divide.hpp" +#include "openvino/op/exp.hpp" +#include "openvino/op/relu.hpp" +#include "snippets/op/powerstatic.hpp" + +#include "libxsmm_typedefs.h" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace op { +using AutoBroadcastSpec = ov::op::AutoBroadcastSpec; +using AutoBroadcastType = ov::op::AutoBroadcastType; + +class EltwiseTPP : public modifier::TensorProcessingPrimitive { +public: + static bool is_supported(const std::shared_ptr& node); + bool visit_attributes(AttributeVisitor& visitor); +}; + +class BinaryEltwiseTPP : public EltwiseTPP { +public: + BinaryEltwiseTPP(libxsmm_meltw_binary_type op_type); + libxsmm_meltw_binary_type get_op_type() const { return m_op_type; } +private: + libxsmm_meltw_binary_type m_op_type; +}; + +class UnaryEltwiseTPP : public EltwiseTPP { +public: + UnaryEltwiseTPP(libxsmm_meltw_unary_type op_type); + libxsmm_meltw_unary_type get_op_type() const { return m_op_type; } +private: + libxsmm_meltw_unary_type m_op_type; +}; + +class Add : public BinaryEltwiseTPP, public ov::op::v1::Add { +public: + OPENVINO_OP("Add", "TppOpset", ov::op::v1::Add); + Add(const Output& arg0, const Output& arg1, const AutoBroadcastSpec& auto_broadcast); + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; +}; + +class Subtract : public BinaryEltwiseTPP, public ov::op::v1::Subtract { +public: + OPENVINO_OP("Subtract", "TppOpset", ov::op::v1::Subtract); + Subtract(const Output& arg0, const Output& arg1, const AutoBroadcastSpec& auto_broadcast); + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; +}; + +class Multiply : public BinaryEltwiseTPP, public ov::op::v1::Multiply { +public: + OPENVINO_OP("Multiply", "TppOpset", ov::op::v1::Multiply); + Multiply(const Output& arg0, const Output& arg1, const AutoBroadcastSpec& auto_broadcast); + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; +}; + +class Divide : public BinaryEltwiseTPP, public ov::op::v1::Divide { +public: + OPENVINO_OP("Divide", "TppOpset", ov::op::v1::Divide); + Divide(const Output& arg0, const Output& arg1, const AutoBroadcastSpec& auto_broadcast); + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; +}; + +class Exp : public UnaryEltwiseTPP, public ov::op::v0::Exp { +public: + OPENVINO_OP("Exp", "TppOpset", ov::op::v0::Exp); + Exp(const Output& arg); + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; +}; + +class Relu : public UnaryEltwiseTPP, public ov::op::v0::Relu { +public: + OPENVINO_OP("Relu", "TppOpset", ov::op::v0::Relu); + Relu(const Output& arg); + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; +}; + +class Reciprocal : public UnaryEltwiseTPP, public ov::snippets::op::PowerStatic { +public: + OPENVINO_OP("Reciprocal", "TppOpset", snippets::op::PowerStatic); + Reciprocal(const Output& arg); + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; +}; + + +class Square : public UnaryEltwiseTPP, public ov::snippets::op::PowerStatic { +public: + OPENVINO_OP("Square", "TppOpset", snippets::op::PowerStatic); + Square(const Output& arg); + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; +}; + +class SquareRoot : public UnaryEltwiseTPP, public ov::snippets::op::PowerStatic { +public: + OPENVINO_OP("SquareRoot", "TppOpset", snippets::op::PowerStatic); + SquareRoot(const Output& arg); + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; +}; + +} // namespace op +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/op/factory.cpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/factory.cpp new file mode 100644 index 00000000000..3fdcd30e7c4 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/factory.cpp @@ -0,0 +1,107 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "factory.hpp" +#include "eltwise.hpp" +#include "reduce.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "ov_ops/type_relaxed.hpp" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace op { +namespace { +struct CustomPowerStaticBuilder : public NodeFactory::TPPCustomBuilder { + CustomPowerStaticBuilder() : NodeFactory::TPPCustomBuilder() { + matcher = [](const std::shared_ptr& n) { + std::set supported_power{-1, 2, 0.5}; + const auto& power_static = ov::as_type_ptr(n); + return power_static && supported_power.count(power_static->get_power()); + }; + builder = [](const std::shared_ptr& n) { + const auto& power_static = ov::as_type_ptr(n); + OPENVINO_ASSERT(power_static, "Attempt to create TPP node from unsupported input in power_static_builder"); + const auto power = power_static->get_power(); + const auto& input = n->input_value(0); + std::shared_ptr tpp_node{nullptr}; + if (power == -1.f) + tpp_node = std::make_shared(input); + else if (power == 2.f) + tpp_node = std::make_shared(input); + else if (power == 0.5f) + tpp_node = std::make_shared(input); + OPENVINO_ASSERT(tpp_node, "Failed to create TPP in power_static_builder"); + return tpp_node; + }; + } +}; + +} // namespace +#define CREATE_UNARY_TPP_NODE(tpp_node_type) \ + [](const std::shared_ptr& node) -> std::shared_ptr { \ + return std::make_shared(node->get_input_source_output(0)); \ + } + +#define CREATE_BINARY_TPP_NODE(tpp_node_type) \ + [](const std::shared_ptr& node) -> std::shared_ptr { \ + return std::make_shared(node->get_input_source_output(0), node->get_input_source_output(1), node->get_autob()); \ + } + +#define CREATE_REDUCE_TPP_NODE(tpp_node_type) \ + [](const std::shared_ptr& node) -> std::shared_ptr { \ + const auto& reduce = ov::as_type_ptr(node); \ + OPENVINO_ASSERT(reduce, "Attempt to create TPP Reduce from invalid node"); \ + return std::make_shared(reduce->get_input_source_output(0), reduce->get_axis()); \ + } + +std::unordered_map NodeFactory::m_direct_mapping { + {ov::op::v1::Add::get_type_info_static(), CREATE_BINARY_TPP_NODE(Add)}, + {ov::op::v1::Subtract::get_type_info_static(), CREATE_BINARY_TPP_NODE(Subtract)}, + {ov::op::v1::Multiply::get_type_info_static(), CREATE_BINARY_TPP_NODE(Multiply)}, + {ov::op::v1::Divide::get_type_info_static(), CREATE_BINARY_TPP_NODE(Divide)}, + {ov::op::v0::Exp::get_type_info_static(), CREATE_UNARY_TPP_NODE(Exp)}, + {ov::op::v0::Relu::get_type_info_static(), CREATE_UNARY_TPP_NODE(Relu)}, + // Note that we don't support conversion from ngraph ops here, since they have a broader semantics (e.g. multiple axis provided at a secont input) + {ov::snippets::op::ReduceMax::get_type_info_static(), CREATE_REDUCE_TPP_NODE(ReduceMax)}, + {ov::snippets::op::ReduceSum::get_type_info_static(), CREATE_REDUCE_TPP_NODE(ReduceSum)}, +}; + + +std::vector NodeFactory::m_custom_mapping{CustomPowerStaticBuilder()}; + +std::shared_ptr NodeFactory::create(const std::shared_ptr& n) { + const auto& found = m_direct_mapping.find(n->get_type_info()); + std::shared_ptr tpp_node{nullptr}; + if (found != m_direct_mapping.end()) { + tpp_node = (found->second)(n); + } else { + for (const auto& custom_builder : m_custom_mapping) { + if (custom_builder.matcher(n)) { + tpp_node = custom_builder.builder(n); + break; + } + } + } + return tpp_node; +} + +bool NodeFactory::is_supported(const std::shared_ptr& n) { + auto matches = [=](const NodeFactory::TPPCustomBuilder& custom_builder) { + return custom_builder.matcher(n); + }; + // TPP currently supports only FP32 precisions (ticket: 130010) + // Note: verify that TypeRelaxed property is maintained (mismatched input precisions) + // after low precisions are enabled (ticket: 132328) + const auto& ins = n->inputs(); + auto is_fp32_input = [](const ov::Input& in){ return in.get_element_type() == element::f32; }; + const bool all_inputs_fp32 = std::all_of(ins.begin(), ins.end(), is_fp32_input); + return (m_direct_mapping.count(n->get_type_info()) || + std::any_of(m_custom_mapping.begin(), m_custom_mapping.end(), matches)) && all_inputs_fp32; +} + +} // namespace op +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/op/factory.hpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/factory.hpp new file mode 100644 index 00000000000..b3090ebdec4 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/factory.hpp @@ -0,0 +1,31 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/op/op.hpp" +#include "openvino/core/type.hpp" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace op { +class NodeFactory { +public: + static std::shared_ptr create(const std::shared_ptr& n); + static bool is_supported(const std::shared_ptr& n); + typedef std::function(const std::shared_ptr&)> tpp_builder; + typedef std::function&)> tpp_matcher; + struct TPPCustomBuilder { + tpp_matcher matcher; + tpp_builder builder; + }; +private: + static std::unordered_map m_direct_mapping; + static std::vector m_custom_mapping; +}; +} // namespace op +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/op/modifiers.hpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/modifiers.hpp new file mode 100644 index 00000000000..507276a1c2c --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/modifiers.hpp @@ -0,0 +1,34 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include "snippets/op/memory_access.hpp" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace modifier { + +/** + * @interface TensorProcessingPrimitive + * @brief TensorProcessingPrimitive a is modifier to mark operations supported with TPP + * @ingroup snippets + */ +class TensorProcessingPrimitive : virtual public snippets::modifier::MemoryAccess { + public: + void clone_memory_access_ports(const TensorProcessingPrimitive& other) { + m_input_ports = other.m_input_ports; + m_output_ports = other.m_output_ports; + } + static bool visit_attributes(AttributeVisitor& visitor) { + std::string modifier{"TPP"}; + visitor.on_attribute("modifier", modifier); + return true; + } +}; + +} // namespace modifier +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/op/reduce.cpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/reduce.cpp new file mode 100644 index 00000000000..63119623856 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/reduce.cpp @@ -0,0 +1,47 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "reduce.hpp" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace op { + +ReduceMax::ReduceMax(const Output& arg, size_t axis) + : UnaryEltwiseTPP(LIBXSMM_MELTW_TYPE_UNARY_REDUCE_X_OP_MAX), ov::snippets::op::ReduceMax(arg, axis) { +} + +std::shared_ptr ReduceMax::clone_with_new_inputs(const OutputVector& new_args) const { + check_new_args_count(this, new_args); + const auto& new_op = std::make_shared(new_args.at(0), m_axis); + new_op->clone_memory_access_ports(*this); + return new_op; +} + +bool ReduceMax::visit_attributes(AttributeVisitor& visitor) { + ReduceBase::visit_attributes(visitor); + return UnaryEltwiseTPP::visit_attributes(visitor); +} + +ReduceSum::ReduceSum(const Output& arg, size_t axis) + : UnaryEltwiseTPP(LIBXSMM_MELTW_TYPE_UNARY_REDUCE_X_OP_ADD), ov::snippets::op::ReduceSum(arg, axis) { +} + +std::shared_ptr ReduceSum::clone_with_new_inputs(const OutputVector& new_args) const { + check_new_args_count(this, new_args); + const auto& new_op = std::make_shared(new_args.at(0), m_axis); + new_op->clone_memory_access_ports(*this); + return new_op; +} + +bool ReduceSum::visit_attributes(AttributeVisitor& visitor) { + ReduceBase::visit_attributes(visitor); + return UnaryEltwiseTPP::visit_attributes(visitor); +} + +} // namespace op +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/op/reduce.hpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/reduce.hpp new file mode 100644 index 00000000000..9542c4ec90b --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/reduce.hpp @@ -0,0 +1,44 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "modifiers.hpp" +#include "eltwise.hpp" +#include "snippets/op/reduce.hpp" + + +#include "libxsmm_typedefs.h" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace op { + +// Note: Reduce ops are implemented as UnaryEltwise in libxsmm, so we inherit this properties here +// Also note that UnaryEltwiseTPP is a modifier, so it won't trigger any flase positive matches in the pipeline +class ReduceMax : public UnaryEltwiseTPP, public ov::snippets::op::ReduceMax { +public: + OPENVINO_OP("ReduceMax", "TppOpset", ov::snippets::op::ReduceMax); + ReduceMax(const Output& arg, size_t axis); + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; +private: + libxsmm_meltw_binary_type m_op_type; +}; + +class ReduceSum : public UnaryEltwiseTPP, public ov::snippets::op::ReduceSum { +public: + OPENVINO_OP("ReduceSum", "TppOpset", ov::snippets::op::ReduceSum); + ReduceSum(const Output& arg, size_t axis); + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; +private: + libxsmm_meltw_binary_type m_op_type; +}; + +} // namespace op +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/op/scalar.cpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/scalar.cpp new file mode 100644 index 00000000000..566a2a5afde --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/scalar.cpp @@ -0,0 +1,31 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "scalar.hpp" +#include "modifiers.hpp" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace op { + +Scalar::Scalar(const snippets::op::Scalar& other) : ov::snippets::op::Scalar(other) { + constructor_validate_and_infer_types(); +} + +std::shared_ptr Scalar::clone_with_new_inputs(const OutputVector& new_args) const { + check_new_args_count(this, new_args); + return std::make_shared(*this); +} + +bool Scalar::visit_attributes(AttributeVisitor& visitor) { + modifier::TensorProcessingPrimitive::visit_attributes(visitor); + return snippets::op::Scalar::visit_attributes(visitor);; +} + + +} // namespace op +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/op/scalar.hpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/scalar.hpp new file mode 100644 index 00000000000..f9578c20fb1 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/op/scalar.hpp @@ -0,0 +1,32 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "modifiers.hpp" +#include "eltwise.hpp" +#include "snippets/op/reduce.hpp" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace op { +// Note that the tpp::op::Scalar is not derived from the TensorProcessingPrimitive modifier. We don't need it because +// the Scalar is not a MemoryAccess operation, since it doesn't need to read from the external +// memory, and hence it is not really a TPP. +class Scalar : public ov::snippets::op::Scalar { +public: + OPENVINO_OP("Scalar", "TppOpset", snippets::op::Scalar); + + Scalar() = default; + explicit Scalar(const snippets::op::Scalar& other); + + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + bool visit_attributes(AttributeVisitor& visitor) override; +}; + +} // namespace op +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/brgemm_to_brgemm_tpp.cpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/brgemm_to_brgemm_tpp.cpp new file mode 100644 index 00000000000..f189e4c5dcc --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/brgemm_to_brgemm_tpp.cpp @@ -0,0 +1,119 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/itt.hpp" + +#include "brgemm_to_brgemm_tpp.hpp" + +#include "snippets/utils.hpp" +#include "snippets/op/brgemm.hpp" +#include "transformations/tpp/x64/op/brgemm.hpp" + +#include "openvino/core/rt_info.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "openvino/pass/pattern/matcher.hpp" + +#include "cpu_shape.h" +#include "utils/general_utils.h" + + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace pass { + +using namespace snippets::lowered; + +namespace { +template +void set_port_desc(const T& port, Args... params) { + PortDescriptorUtils::set_port_descriptor_ptr(port, std::make_shared(params...)); +} +} // namespace + +BrgemmToBrgemmTPP::BrgemmToBrgemmTPP() { + MATCHER_SCOPE(BrgemmToBrgemmTPP); + + auto m_brgemm = ov::pass::pattern::wrap_type(); + + auto callback = [=](ov::pass::pattern::Matcher& m) { + OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "ov::intel_cpu::pass::BrgemmToBrgemmTPP") + const auto node = m.get_match_root(); + const auto brgemm = ov::as_type_ptr(node); + if (!brgemm || ov::as_type_ptr(node)) + OPENVINO_THROW("BrgemmCPU cannot be in body before BrgemmToBrgemmTPP pass"); + + if (brgemm->is_dynamic()) { + return false; + } + + const auto& brgemm_in0_desc = PortDescriptorUtils::get_port_descriptor_ptr(brgemm->input(0)); + const auto& brgemm_in1_desc = PortDescriptorUtils::get_port_descriptor_ptr(brgemm->input(1)); + const auto& brgemm_out_desc = PortDescriptorUtils::get_port_descriptor_ptr(brgemm->output(0)); + + const auto dimsMatMulIn0 = snippets::utils::get_planar_pshape(brgemm->input(0)).get_shape(); + const auto dimsMatMulIn1 = snippets::utils::get_planar_pshape(brgemm->input(1)).get_shape(); + + const auto M = *++dimsMatMulIn0.rbegin(); + const auto K = *dimsMatMulIn0.rbegin(); + const auto N = *dimsMatMulIn1.rbegin(); + + const auto element_type_a = brgemm->get_input_element_type(0); + const auto element_type_b = brgemm->get_input_element_type(1); + + const auto offset_a = brgemm->get_offset_a(); + const auto offset_b = brgemm->get_offset_b(); + const auto offset_c = brgemm->get_offset_c(); + + std::shared_ptr brgemm_tpp = nullptr; + if (element_type_a == ov::element::f32) { + brgemm_tpp = std::make_shared(brgemm->input_value(0), + brgemm->input_value(1), + offset_a, offset_b, offset_c, + brgemm_in0_desc->get_layout(), + brgemm_in1_desc->get_layout(), + brgemm_out_desc->get_layout()); + } + OPENVINO_ASSERT(brgemm_tpp, "Failed to create BrgemmTPP node in the BrgemmToBrgemmTPP pass"); + // Set blocking params + // Ticket: 113745 + // TODO: extend block size selection heuristics + auto get_block_size_m = [](const size_t M) { + return 32; + }; + auto get_block_size_k = [=](const size_t K) { + if (element_type_b != ov::element::f32) + return K; + return K > 1024 ? 1024 : K > 512 ? 512 : K; + }; + auto get_block_size_n = [=](const size_t N) { + return element_type_b != ov::element::f32 ? N : 64; + }; + + brgemm_tpp->set_m_block_size(get_block_size_m(M)); + brgemm_tpp->set_k_block_size(get_block_size_k(K)); + brgemm_tpp->set_n_block_size(get_block_size_n(N)); + + brgemm_tpp->set_friendly_name(brgemm->get_friendly_name()); + ov::replace_node(brgemm, brgemm_tpp); + + // Set FULL_DIM tensors on ports to avoid automatic loop markup (blocked loops will be inserted in a separate transformation) + set_port_desc(brgemm_tpp->input(0), brgemm_in0_desc->get_shape(), brgemm_in0_desc->get_subtensor(), brgemm_in0_desc->get_layout()); + set_port_desc(brgemm_tpp->input(1), brgemm_in1_desc->get_shape(), brgemm_in1_desc->get_subtensor(), brgemm_in1_desc->get_layout()); + set_port_desc(brgemm_tpp->output(0), brgemm_out_desc->get_shape(), brgemm_out_desc->get_subtensor(), brgemm_out_desc->get_layout()); + + // need to run validate_and_infer_types manually: either input shapes were updated or + // output Layout was updated (out shape will be updated in validate_and_infer_types()) + brgemm_tpp->validate_and_infer_types(); + + return true; + }; + + auto m = std::make_shared(m_brgemm, matcher_name); + register_matcher(m, callback); +} +} // namespace pass +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/brgemm_to_brgemm_tpp.hpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/brgemm_to_brgemm_tpp.hpp new file mode 100644 index 00000000000..6d8dc151f23 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/brgemm_to_brgemm_tpp.hpp @@ -0,0 +1,29 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/graph_rewrite.hpp" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace pass { + +/** + * @interface BrgemmToBrgemmTPP + * @brief Converts Snippets Brgemm to BrgemmTPP operation, if possible. Only fp32 Brgemms are currently converted. + * @ingroup snippets + */ +class BrgemmToBrgemmTPP: public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("BrgemmToBrgemmTPP", "0"); + BrgemmToBrgemmTPP(); +}; + + +} // namespace pass +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/eltwise_to_eltwise_tpp.cpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/eltwise_to_eltwise_tpp.cpp new file mode 100644 index 00000000000..b3c04fb7833 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/eltwise_to_eltwise_tpp.cpp @@ -0,0 +1,61 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/itt.hpp" +#include "eltwise_to_eltwise_tpp.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" + +#include "transformations/tpp/x64/op/factory.hpp" + +#include "openvino/op/util/binary_elementwise_arithmetic.hpp" +#include "openvino/op/util/unary_elementwise_arithmetic.hpp" +#include "snippets/lowered/port_descriptor.hpp" +#include "snippets/op/reduce.hpp" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace pass { + +EltwiseToEltwiseTPP::EltwiseToEltwiseTPP() { + MATCHER_SCOPE(EltwiseToEltwiseTPP); + + auto is_supported_by_tpp = [](const Output& out) { + return op::NodeFactory::is_supported(out.get_node_shared_ptr()); + }; + auto supported_eltwise = ov::pass::pattern::wrap_type(is_supported_by_tpp); + + + auto callback = [=](ov::pass::pattern::Matcher& m) { + OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "ov::intel_cpu::pass::EltwiseToEltwiseTPP") + const auto node = m.get_match_root(); + if (node->is_dynamic()) { + return false; + } + + const auto& tpp_eltwise = op::NodeFactory::create(node); + OPENVINO_ASSERT(tpp_eltwise, "Failed to create TPP node"); + + const size_t M_block = 32; + const size_t N_block = ov::is_type(node) ? + snippets::lowered::PortDescriptor::ServiceDimensions::FULL_DIM : + 64; + ov::replace_node_update_name(node, tpp_eltwise); + for (size_t i = 0; i < node->get_input_size(); i++) + snippets::lowered::set_port_desc(tpp_eltwise->input(i), {M_block, N_block}); + + snippets::lowered::set_port_desc(tpp_eltwise->output(0), {M_block, N_block}); + + return true; + }; + + auto m = std::make_shared(supported_eltwise, matcher_name); + register_matcher(m, callback); +} +} // namespace pass +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/eltwise_to_eltwise_tpp.hpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/eltwise_to_eltwise_tpp.hpp new file mode 100644 index 00000000000..189d048e86f --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/eltwise_to_eltwise_tpp.hpp @@ -0,0 +1,29 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/graph_rewrite.hpp" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace pass { + +/** + * @interface EltwiseToEltwiseTPP + * @brief Converts elementwise operations supported by the TPP backend to the dedicated TPP opset + * @ingroup snippets + */ +class EltwiseToEltwiseTPP: public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("EltwiseToEltwiseTPP", "0"); + EltwiseToEltwiseTPP(); +}; + + +} // namespace pass +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.cpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.cpp new file mode 100644 index 00000000000..9fabaf4c3f6 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.cpp @@ -0,0 +1,164 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/itt.hpp" +#include "snippets/op/buffer.hpp" +#include "transformations/tpp/x64/op/modifiers.hpp" +#include "set_tpp_leading_dim.hpp" +#include "snippets/op/brgemm.hpp" +#include "snippets/lowered/loop_manager.hpp" +#include "snippets/utils.hpp" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace pass { +namespace { +using ExpressionPort = snippets::lowered::ExpressionPort; +bool is_planar_layout(const std::vector& layout) { + for (size_t i = 0; i < layout.size(); i++) { + if (layout[i] != i) + return false; + } + return true; +} +// Note: Buffer is directly connected to the port if it remains in the same loops with the port's expression +// Directly connected Buffers store data densely, so strides are defined by subternsor dims +// Indirectly connected Buffers (with loops between the expr and Buffer) store data according +// to their shape and layout +bool has_directly_connected_buffer(const ExpressionPort& port, const snippets::lowered::LinearIR::LoopManagerPtr& loop_mngr) { + auto accepted_loops = [&loop_mngr, &port](const std::vector& orig, const std::vector& connect) { + size_t connect_idx = 0; + auto pred = [&port](const snippets::lowered::LinearIR::LoopManager::LoopPort& loop_port ) { + return *loop_port.expr_port == port; + }; + for (const auto orig_loop : orig) { + if (connect_idx < connect.size() && orig_loop == connect[connect_idx]) { + connect_idx++; + continue; + } + // Note that orig expression can have some extra loops (compared to connect) + // as long as the port is the loop entry/exit, and it is not incremented. + // This is the case for Brgemm K-blocking loops, for example. + const auto loop_info = loop_mngr->get_loop_info(orig_loop); + const auto& border_points = port.get_type() == ExpressionPort::Type::Input ? + loop_info->get_entry_points() : + loop_info->get_exit_points(); + const auto& found = std::find_if(border_points.begin(), border_points.end(), pred); + if (found == border_points.end() || found->is_incremented) + return false; + } + return true; + }; + bool has_buffer = false; + const auto& orig_loop_ids = port.get_expr()->get_loop_ids(); + for (const auto& p : port.get_connected_ports()) { + const auto& connected_expr = p.get_expr(); + if (ov::is_type(connected_expr->get_node()) && + accepted_loops(orig_loop_ids, connected_expr->get_loop_ids())) { + OPENVINO_ASSERT(!has_buffer, "Only one Buffer can be connected to a TPP op"); + has_buffer = true; + } + } + return has_buffer; +} + +size_t get_leading_dim(ExpressionPort port, const snippets::lowered::LinearIR::LoopManagerPtr& loop_mngr) { + const auto& port_desc = port.get_descriptor_ptr(); + auto layout = port_desc->get_layout(); + auto shape = port_desc->get_shape(); + auto subtensor = port_desc->get_subtensor(); + // Some expressions (e.g. ReduceMax/ReduceSum) allow for FULL_DIM values in subtensor. + // Here we should replace them with actual dim values before calculating strides & offsets. + bool full_dim_substituted = false; + for (size_t i = 1; i <= subtensor.size(); i++) { + const auto idx = subtensor.size() - i; + if (subtensor[idx] == snippets::lowered::PortDescriptor::ServiceDimensions::FULL_DIM) { + // the reason that we don't support FULL_DIM substitution for an arbitrary layout is that + // the layout and subtersor can (and usually do) have different ranks + full_dim_substituted = true; + subtensor[idx] = shape[shape.size() - i]; + } + } + OPENVINO_ASSERT(!full_dim_substituted || is_planar_layout(layout), + "Only planar layouts are supported for FULL_DIM substitution"); + + if (has_directly_connected_buffer(port, loop_mngr)) { + shape = port_desc->get_subtensor(); + OPENVINO_ASSERT(is_planar_layout(layout), "Only planar layouts are supported for Buffers"); + const auto rank_diff = static_cast(layout.size()) - static_cast(shape.size()); + if (rank_diff > 0) + layout.erase(layout.end() - rank_diff, layout.end()); + } + + OPENVINO_ASSERT(layout.empty() || (layout.back() == layout.size() - 1 && layout.size() == shape.size()), + "get_leading_dim detected invalid layout values: check shape + layout combination"); + const auto dim = [&]() -> size_t { + switch (port.get_type()) { + // Input shape is original, so we need to correctly read this data by order + // Example: + // Original shape (shape) = [1, 49, 2, 23] + // Layout (transpose order) = [2, 0, 1, 3] + // Transposed shape = [2, 1, 49, 23] + // The leading dimension is equal to stride of shape[layout[3]] = 2 x 23 + case ExpressionPort::Type::Input : + return snippets::utils::get_input_dim_idx(layout, 1); // `1` in example + // Output shape is already transposed, we need to correctly write the data with original shape by the order + // Example: + // Original transposed shape (shape) = [49, 2, 7, 39] + // Layout (transpose order) = [2, 0, 1, 3] + // Before leading dimension with index 3 there is dimension with index 2 in planar layout. + // Since we have non-planar layout, we have to find this before LD dim in transposed order. + // In layout 2nd idx is first element, it means, that the leading dimension is equal to stride of shape[0] + case ExpressionPort::Type::Output : + return snippets::utils::get_output_dim_idx(layout, 1); // 0 in the example: shape[0] = 49 + default: + OPENVINO_THROW("Unsupported Expression port type"); + } + }; + return layout.size() == 1 ? + shape.back() : + std::accumulate(shape.cbegin() + dim() + 1, shape.cend(), 1, std::multiplies()); +} + +} // namespace + +SetTPPLeadingDim::SetTPPLeadingDim() : RangedPass() {} + +bool SetTPPLeadingDim::run(snippets::lowered::LinearIR& linear_ir, + snippets::lowered::LinearIR::constExprIt begin, + snippets::lowered::LinearIR::constExprIt end) { + OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "Snippets::SetTPPLeadingDim") + if (linear_ir.empty()) + return false; + + bool modified = false; + for (auto expr_it = begin; expr_it != end; expr_it++) { + const auto& expr = *expr_it; + const auto& node = expr->get_node(); + auto tpp_expr = std::dynamic_pointer_cast(node); + if (!tpp_expr) + continue; + + OPENVINO_ASSERT(tpp_expr->is_full_memory_access_op(node), "TPP Op is expected to be MemoryAccess on all ports"); + + for (size_t i = 0; i < expr->get_input_count(); i++) { + const auto ld = get_leading_dim(expr->get_input_port(i), linear_ir.get_loop_manager()); + tpp_expr->set_input_stride(ld, i); + } + for (size_t i = 0; i < expr->get_output_count(); i++) { + const auto ld = get_leading_dim(expr->get_output_port(i), linear_ir.get_loop_manager()); + tpp_expr->set_output_stride(ld, i); + } + modified = true; + } + + return modified; +} + + +} // namespace pass +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.hpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.hpp new file mode 100644 index 00000000000..c3f1a3a9957 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.hpp @@ -0,0 +1,33 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "snippets/lowered/pass/pass.hpp" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace pass { + +/** + * @interface SetTPPLeadingDim + * @brief TPP leading dimension depends on the operation it is connected to. If it's a Parameter or Result + * we can compute LD based on shape, if it's a Buffer - we need to consider allocation shape. + * This transformation should be performed before InsertTailLoop because it may change graph connectivity for 1st and last iterations. + * @ingroup snippets + */ +class SetTPPLeadingDim: public snippets::lowered::pass::RangedPass { +public: + OPENVINO_RTTI("SetTPPLeadingDim", "Pass"); + SetTPPLeadingDim(); + bool run(snippets::lowered::LinearIR& linear_ir, + snippets::lowered::LinearIR::constExprIt begin, + snippets::lowered::LinearIR::constExprIt end) override; +}; + +} // namespace pass +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/scalar_to_scalar_tpp.cpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/scalar_to_scalar_tpp.cpp new file mode 100644 index 00000000000..5ea5b135ba5 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/scalar_to_scalar_tpp.cpp @@ -0,0 +1,57 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/itt.hpp" +#include "scalar_to_scalar_tpp.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "snippets/op/scalar.hpp" +#include "transformations/tpp/x64/op/scalar.hpp" +#include "transformations/tpp/x64/op/modifiers.hpp" +#include "snippets/lowered/port_connector.hpp" + + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace pass { + +ScalarToScalarTPP::ScalarToScalarTPP() { + MATCHER_SCOPE(ScalarToScalarTPP); + + auto snippets_scalar = ov::pass::pattern::wrap_type(); + + + auto callback = [=](ov::pass::pattern::Matcher& m) { + OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "ov::intel_cpu::pass::ScalarToScalarTPP") + const auto node = ov::as_type_ptr(m.get_match_root()); + OPENVINO_ASSERT(node, "Failed to obtain a valid Scalar Op in ScalarToScalarTPP"); + size_t num_connected_tpp = 0; + const auto& target_ins = node->get_output_target_inputs(0); + for (const auto& in : target_ins) { + if (dynamic_cast(in.get_node())) + num_connected_tpp++; + } + if (num_connected_tpp == 0) + return false; + // Note: If needed, we can support cases when scalar has TPP and non-TPP consumers if we copy the scalar. + // However, this is rarely needed in practice and the assert is here to flag invalid configurations. + OPENVINO_ASSERT(num_connected_tpp == target_ins.size(), "Either all or none Scalar outputs should be TPP"); + + const auto& tpp_scalar = std::make_shared(*node); + tpp_scalar->set_friendly_name(node->get_friendly_name()); + ov::replace_node_update_name(node, tpp_scalar); + const auto& out = tpp_scalar->output(0); + snippets::lowered::set_port_desc(out, {1}); + for (const auto& in : out.get_target_inputs()) + snippets::lowered::set_port_desc(in, {1}); + return true; + }; + + auto m = std::make_shared(snippets_scalar, matcher_name); + register_matcher(m, callback); +} +} // namespace pass +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/scalar_to_scalar_tpp.hpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/scalar_to_scalar_tpp.hpp new file mode 100644 index 00000000000..8bbfcf80c75 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/scalar_to_scalar_tpp.hpp @@ -0,0 +1,29 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/graph_rewrite.hpp" + +namespace ov { +namespace intel_cpu { +namespace tpp { +namespace pass { + +/** + * @interface ScalarToScalarTPP + * @brief Converts snippets::op::Scalar to tpp::op::Scalar, since TPP operations require a dedicated emitter + * @ingroup snippets + */ +class ScalarToScalarTPP: public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("ScalarToScalarTPP", "0"); + ScalarToScalarTPP(); +}; + + +} // namespace pass +} // namespace tpp +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp index f8491b4c2a6..4bdb488aa46 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp @@ -427,6 +427,29 @@ std::vector disabledTestPatterns() { retVector.emplace_back(R"(.*smoke_Snippets_MHAWOTransposeEnforceBF16.*)"); retVector.emplace_back(R"(.*smoke_Snippets_MHAEnforceBF16.*)"); } +#ifdef SNIPPETS_LIBXSMM_TPP + // TPP performs precision conversion implicitly, it makes all Convert tests irrelevant + retVector.emplace_back(R"(.*smoke_Snippets_Convert.*)"); + // ABS and ROUND operations are needed for TPP support. Disable, since low precisions are not supported by TPP yet. + retVector.emplace_back(R"(.*smoke_Snippets_FQ.*)"); + retVector.emplace_back(R"(.*smoke_Snippets_TransposeMatMulFQ.*)"); + // TPP doesn't support op with 2 outs, when one of them is Result (ticket: 130642) + retVector.emplace_back(R"(.*smoke_Snippets_MaxNumParamsEltwise.*)"); + retVector.emplace_back(R"(.*smoke_Snippets_Eltwise_TwoResults.*)"); + // Accuracy problem with Exp + Reciprocal combination on TPP side (ticket: 130699) + retVector.emplace_back(R"(.*smoke_Snippets_ExpReciprocal.*)"); + retVector.emplace_back(R"(.*smoke_Snippets_AddSoftmax.*)"); + retVector.emplace_back(R"(.*smoke_Snippets_TransposeSoftmaxEltwise.*)"); + // Low-precision Matmuls are not supported by TPP yet + retVector.emplace_back(R"(.*smoke_Snippets_MatMulFQ.*)"); + retVector.emplace_back(R"(.*smoke_Snippets_MatMulBiasQuantized.*)"); + retVector.emplace_back(R"(.*smoke_Snippets_MatMulQuantized.*)"); + retVector.emplace_back(R"(.*smoke_Snippets_MatMulQuantizedSoftmax.*)"); + retVector.emplace_back(R"(.*smoke_Snippets_MHAINT8MatMul.*)"); + retVector.emplace_back(R"(.*smoke_Snippets_MHAQuantMatMul0.*)"); + retVector.emplace_back(R"(.*smoke_Snippets_MHAFQ.*)"); + retVector.emplace_back(R"(.*smoke_Snippets_PrecisionPropagation_Convertion.*)"); +#endif if (ov::with_cpu_x86_avx512_core_amx()) { // Issue: 130463 diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/exp.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/exp.cpp new file mode 100644 index 00000000000..694789fa0ef --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/exp.cpp @@ -0,0 +1,38 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/exp.hpp" +#include "common_test_utils/test_constants.hpp" + +namespace ov { +namespace test { +namespace snippets { + + +namespace { +// ===================================Exp=========================================================// +std::vector inShapesStatic{{{}, {{1, 1, 32, 128}}}}; + +INSTANTIATE_TEST_SUITE_P(smoke_Snippets_Exp, Exp, + ::testing::Combine( + ::testing::ValuesIn(inShapesStatic), + ::testing::Values(ov::element::f32), + ::testing::Values(1), // Exp + ::testing::Values(1), + ::testing::Values(ov::test::utils::DEVICE_CPU)), + Exp::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Snippets_ExpReciprocal, ExpReciprocal, + ::testing::Combine( + ::testing::ValuesIn(inShapesStatic), + ::testing::Values(ov::element::f32), + ::testing::Values(1), // Exp + ::testing::Values(1), + ::testing::Values(ov::test::utils::DEVICE_CPU)), + Exp::getTestCaseName); + +} // namespace +} // namespace snippets +} // namespace test +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/matmul.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/matmul.cpp index abecd3c954e..55c6a2aa095 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/matmul.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/matmul.cpp @@ -40,6 +40,8 @@ static inline std::vector> precisions(bool only_fp32 std::vector> prc = { {element::f32, element::f32}, }; +// Note: TPP doesn't support low precisions yet +#ifndef SNIPPETS_LIBXSMM_TPP if (!only_fp32) { auto quant = quantized_precisions(); std::copy(quant.begin(), quant.end(), std::back_inserter(prc)); @@ -48,6 +50,7 @@ static inline std::vector> precisions(bool only_fp32 prc.emplace_back(std::vector{element::bf16, element::bf16}); } } +#endif return prc; } @@ -71,7 +74,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulFQ, MatMulFQ, INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulBias, MatMulBias, ::testing::Combine( - ::testing::Values(std::vector{{1, 2, 69, 43}, {2, 1, 43, 49}, {1, 1, 69, 49}}), + ::testing::Values(std::vector{{1, 2, 69, 43}, {2, 1, 43, 49}, {1, 1, 69, 49}}, + std::vector{{1, 2, 95, 1023}, {1, 2, 1023, 255}, {1, 2, 95, 255}}), ::testing::ValuesIn(precisions(false)), ::testing::Values(1), // Subgraph; ::testing::Values(1), // Tokenized MatMul+Bias @@ -89,7 +93,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulBiasQuantized, MatMulBiasQuantized ::testing::Values(ov::test::utils::DEVICE_CPU)), MatMul::getTestCaseName); -INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulsQuantized, MatMulsQuantized, +INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulQuantized, MatMulQuantized, ::testing::Combine( ::testing::Values(std::vector{{1, 16, 128, 64}, {1, 16, 64, 128}, {128, 64}}), ::testing::ValuesIn(quantized_precisions()), @@ -98,7 +102,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulsQuantized, MatMulsQuantized, ::testing::Values(ov::test::utils::DEVICE_CPU)), MatMul::getTestCaseName); -INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulsQuantizedSoftmax, MatMulsQuantizedSoftmax, +INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulQuantizedSoftmax, MatMulQuantizedSoftmax, ::testing::Combine( ::testing::Values(std::vector{{1, 16, 128, 64}, {1, 16, 64, 128}, {128, 64}}), ::testing::ValuesIn(quantized_precisions()), diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/max_num_params_eltwise.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/max_num_params_eltwise.cpp index ae9dec5eadc..ef17fdf55b4 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/max_num_params_eltwise.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/max_num_params_eltwise.cpp @@ -23,7 +23,7 @@ std::vector input_shapes {{{}, {{1, 64, 10, 10}}}, {{1, {1, 64}, {10, 20}, -1}, {{1, 64, 10, 10}, {1, 1, 17, 37}, {1, 64, 10, 10}}}, {{1, 1, 1, {1, 128}}, {{1, 1, 1, 1}, {1, 1, 1, 7}, {1, 1, 1, 128}, {1, 1, 1, 14}, {1, 1, 1, 16}, {1, 1, 1, 1}}}}; -INSTANTIATE_TEST_SUITE_P(smoke_Snippets_Eltwise, MaxNumParamsEltwise, +INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MaxNumParamsEltwise, MaxNumParamsEltwise, ::testing::Combine( ::testing::ValuesIn(input_shapes), ::testing::Values(2), // Subgraph + Concat diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/three_inputs_eltwise.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/three_inputs_eltwise.cpp index 6026263ca95..382be73957f 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/three_inputs_eltwise.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/three_inputs_eltwise.cpp @@ -10,7 +10,7 @@ namespace test { namespace snippets { namespace { -INSTANTIATE_TEST_SUITE_P(smoke_Snippets_Eltwise, ThreeInputsEltwise, +INSTANTIATE_TEST_SUITE_P(smoke_Snippets_ThreeInputsEltwise, ThreeInputsEltwise, ::testing::Combine( ::testing::Values(InputShape {{}, {{1, 64, 10, 10}}}), ::testing::Values(InputShape {{}, {{1, 64, 10, 1}}}), diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/transpose_matmul.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/transpose_matmul.cpp index ec97a61647b..9bc85e72cd7 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/transpose_matmul.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/transpose_matmul.cpp @@ -17,6 +17,8 @@ static inline std::vector> precisions(bool only_fp32 std::vector> prc = { {element::f32, element::f32}, }; +// Note: low precisions are not supported by TPP yet (ticker: 130010) +#ifndef SNIPPETS_LIBXSMM_TPP if (!only_fp32) { // In Snippets MatMul INT8 is supported only on VNNI/AMX platforms if (ov::with_cpu_x86_avx512_core_vnni() || ov::with_cpu_x86_avx512_core_amx_int8()) { @@ -28,6 +30,7 @@ static inline std::vector> precisions(bool only_fp32 prc.emplace_back(std::vector{element::bf16, element::bf16}); } } +#endif return prc; } namespace transpose_zero_input { diff --git a/src/plugins/intel_cpu/thirdparty/CMakeLists.txt b/src/plugins/intel_cpu/thirdparty/CMakeLists.txt index 9660b9d7086..98d548dc494 100644 --- a/src/plugins/intel_cpu/thirdparty/CMakeLists.txt +++ b/src/plugins/intel_cpu/thirdparty/CMakeLists.txt @@ -136,6 +136,11 @@ function(ov_add_onednn) endif() endfunction() +if (ENABLE_SNIPPETS_LIBXSMM_TPP) + add_subdirectory(libxsmm) + ov_install_static_lib(libxsmm ${OV_CPACK_COMP_CORE}) +endif() + if(ENABLE_MLAS_FOR_CPU) add_subdirectory(mlas) ov_install_static_lib(mlas ${OV_CPACK_COMP_CORE}) diff --git a/src/plugins/intel_cpu/thirdparty/libxsmm b/src/plugins/intel_cpu/thirdparty/libxsmm new file mode 160000 index 00000000000..13df674c4b7 --- /dev/null +++ b/src/plugins/intel_cpu/thirdparty/libxsmm @@ -0,0 +1 @@ +Subproject commit 13df674c4b73a1b84f6456de8595903ebfbb43e0 diff --git a/src/tests/functional/plugin/shared/include/snippets/exp.hpp b/src/tests/functional/plugin/shared/include/snippets/exp.hpp new file mode 100644 index 00000000000..477377c2154 --- /dev/null +++ b/src/tests/functional/plugin/shared/include/snippets/exp.hpp @@ -0,0 +1,37 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "shared_test_classes/base/snippets_test_utils.hpp" + +namespace ov { +namespace test { +namespace snippets { + +typedef std::tuple< + InputShape, // Input 0 Shape + ov::element::Type, // Element type + size_t, // Expected num nodes + size_t, // Expected num subgraphs + std::string // Target Device +> ExpParams; + +class Exp : public testing::WithParamInterface, + virtual public ov::test::SnippetsTestsCommon { +public: + static std::string getTestCaseName(testing::TestParamInfo obj); + +protected: + void SetUp() override; +}; + +class ExpReciprocal : public Exp { +protected: + void SetUp() override; +}; + +} // namespace snippets +} // namespace test +} // namespace ov \ No newline at end of file diff --git a/src/tests/functional/plugin/shared/include/snippets/matmul.hpp b/src/tests/functional/plugin/shared/include/snippets/matmul.hpp index d4139b11de0..ad9ee8252e3 100644 --- a/src/tests/functional/plugin/shared/include/snippets/matmul.hpp +++ b/src/tests/functional/plugin/shared/include/snippets/matmul.hpp @@ -44,12 +44,12 @@ protected: void init_subgraph(const std::vector& inputShapes, const std::vector& types) override; }; -class MatMulsQuantized : public MatMul { +class MatMulQuantized : public MatMul { protected: void init_subgraph(const std::vector& inputShapes, const std::vector& types) override; }; -class MatMulsQuantizedSoftmax : public MatMul { +class MatMulQuantizedSoftmax : public MatMul { protected: void init_subgraph(const std::vector& inputShapes, const std::vector& types) override; }; diff --git a/src/tests/functional/plugin/shared/src/snippets/exp.cpp b/src/tests/functional/plugin/shared/src/snippets/exp.cpp new file mode 100644 index 00000000000..df8694c4387 --- /dev/null +++ b/src/tests/functional/plugin/shared/src/snippets/exp.cpp @@ -0,0 +1,75 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "common_test_utils/common_utils.hpp" +#include "snippets/exp.hpp" +#include "subgraph_simple.hpp" +#include "functional_test_utils/skip_tests_config.hpp" + +#include "openvino/op/subtract.hpp" + +namespace ov { +namespace test { +namespace snippets { + +std::string Exp::getTestCaseName(testing::TestParamInfo obj) { + ov::test::InputShape inputShapes0; + ov::element::Type type; + std::string targetDevice; + size_t num_nodes, num_subgraphs; + std::tie(inputShapes0, type, num_nodes, num_subgraphs, targetDevice) = obj.param; + + std::ostringstream result; + result << "IS[0]=" << ov::test::utils::partialShape2str({inputShapes0.first}) << "_"; + result << "TS[0]="; + for (const auto& shape : inputShapes0.second) { + result << "(" << ov::test::utils::vec2str(shape) << ")_"; + } + result << "T=" << type << "_"; + result << "#N=" << num_nodes << "_"; + result << "#S=" << num_subgraphs << "_"; + result << "targetDevice=" << targetDevice; + return result.str(); +} + +void Exp::SetUp() { + ov::test::InputShape inputShape0; + ov::element::Type type; + std::tie(inputShape0, type, ref_num_nodes, ref_num_subgraphs, targetDevice) = this->GetParam(); + init_input_shapes({inputShape0}); + auto f = ov::test::snippets::ExpFunction(inputDynamicShapes); + function = f.getOriginal(); + setInferenceType(type); + if (!configuration.count("SNIPPETS_MODE")) { + configuration.insert({"SNIPPETS_MODE", "IGNORE_CALLBACK"}); + } +} + +void ExpReciprocal::SetUp() { + ov::test::InputShape inputShape0; + ov::element::Type type; + std::tie(inputShape0, type, ref_num_nodes, ref_num_subgraphs, targetDevice) = this->GetParam(); + init_input_shapes({inputShape0}); + auto f = ov::test::snippets::ExpReciprocalFunction(inputDynamicShapes); + function = f.getOriginal(); + setInferenceType(type); + if (!configuration.count("SNIPPETS_MODE")) { + configuration.insert({"SNIPPETS_MODE", "IGNORE_CALLBACK"}); + } +} + + +TEST_P(Exp, CompareWithRefImpl) { + run(); + validateNumSubgraphs(); +} + +TEST_P(ExpReciprocal, CompareWithRefImpl) { + run(); + validateNumSubgraphs(); +} + +} // namespace snippets +} // namespace test +} // namespace ov diff --git a/src/tests/functional/plugin/shared/src/snippets/matmul.cpp b/src/tests/functional/plugin/shared/src/snippets/matmul.cpp index 1ddb5332a04..e7bf30f5111 100644 --- a/src/tests/functional/plugin/shared/src/snippets/matmul.cpp +++ b/src/tests/functional/plugin/shared/src/snippets/matmul.cpp @@ -60,12 +60,12 @@ void MatMulBiasQuantized::init_subgraph(const std::vector& inputSh function = f.getOriginal(); } -void MatMulsQuantized::init_subgraph(const std::vector& inputShapes, const std::vector& types) { +void MatMulQuantized::init_subgraph(const std::vector& inputShapes, const std::vector& types) { auto f = ov::test::snippets::MatMulsQuantizedFunction(inputShapes, types); function = f.getOriginal(); } -void MatMulsQuantizedSoftmax::init_subgraph(const std::vector& inputShapes, const std::vector& types) { +void MatMulQuantizedSoftmax::init_subgraph(const std::vector& inputShapes, const std::vector& types) { auto f = ov::test::snippets::MatMulsQuantizedSoftmaxFunction(inputShapes, types); function = f.getOriginal(); } @@ -95,13 +95,13 @@ TEST_P(MatMulBiasQuantized, CompareWithRefImpl) { validateNumSubgraphs(); } -TEST_P(MatMulsQuantized, CompareWithRefImpl) { +TEST_P(MatMulQuantized, CompareWithRefImpl) { SKIP_IF_CURRENT_TEST_IS_DISABLED() run(); validateNumSubgraphs(); } -TEST_P(MatMulsQuantizedSoftmax, CompareWithRefImpl) { +TEST_P(MatMulQuantizedSoftmax, CompareWithRefImpl) { SKIP_IF_CURRENT_TEST_IS_DISABLED() abs_threshold = 4e-6; run(); diff --git a/src/tests/ov_helpers/ov_snippets_models/include/subgraph_simple.hpp b/src/tests/ov_helpers/ov_snippets_models/include/subgraph_simple.hpp index abef899551e..631b060f074 100644 --- a/src/tests/ov_helpers/ov_snippets_models/include/subgraph_simple.hpp +++ b/src/tests/ov_helpers/ov_snippets_models/include/subgraph_simple.hpp @@ -28,6 +28,36 @@ protected: std::shared_ptr initOriginal() const override; std::shared_ptr initReference() const override; }; + +/// One unary operation: Exp +/// Tokenized simply by starting subgraph. +// in1 +// Exp +// Result +class ExpFunction : public SnippetsFunctionBase { +public: + explicit ExpFunction(const std::vector& inputShapes) : SnippetsFunctionBase(inputShapes) { + OPENVINO_ASSERT(input_shapes.size() == 1, "Got invalid number of input shapes"); + } +protected: + std::shared_ptr initOriginal() const override; +}; + +/// Two unary operations: Exp + Power(x, -1) (aka Reciprocal) +/// Tokenized simply by starting subgraph. +// in1 +// Exp +// Power(x, -1) +// Result +class ExpReciprocalFunction : public SnippetsFunctionBase { +public: + explicit ExpReciprocalFunction(const std::vector& inputShapes) : SnippetsFunctionBase(inputShapes) { + OPENVINO_ASSERT(input_shapes.size() == 1, "Got invalid number of input shapes"); + } +protected: + std::shared_ptr initOriginal() const override; +}; + /// Like AddSinh but with a constant second input (and no sinh on in) // in1 in2 // Add @@ -155,8 +185,8 @@ protected: /// Also Output tensors have names to check correct copying output names // in1 in2 // Add -// HSwish Result -// Relu +// Relu Result +// HSwish // Result class EltwiseTwoResultsFunction : public SnippetsFunctionBase { public: diff --git a/src/tests/ov_helpers/ov_snippets_models/src/subgraph_simple.cpp b/src/tests/ov_helpers/ov_snippets_models/src/subgraph_simple.cpp index 91b5c04879c..67a1382b4f6 100644 --- a/src/tests/ov_helpers/ov_snippets_models/src/subgraph_simple.cpp +++ b/src/tests/ov_helpers/ov_snippets_models/src/subgraph_simple.cpp @@ -27,6 +27,18 @@ std::shared_ptr AddFunction::initReference() const { ParameterVector{indata0, indata1})); return std::make_shared(NodeVector{add}, ParameterVector{data0, data1}); } +std::shared_ptr ExpFunction::initOriginal() const { + auto data0 = std::make_shared(precision, input_shapes[0]); + auto exp = std::make_shared(data0); + return std::make_shared(NodeVector{exp}, ParameterVector{data0}); +} +std::shared_ptr ExpReciprocalFunction::initOriginal() const { + auto data0 = std::make_shared(precision, input_shapes[0]); + auto factor = std::make_shared(precision, ov::Shape{1}, std::vector{-1.f}); + auto exp = std::make_shared(data0); + auto reciprocal = std::make_shared(exp, factor); + return std::make_shared(NodeVector{reciprocal}, ParameterVector{data0}); +} std::shared_ptr AddConstFunction::initOriginal() const { auto data0 = std::make_shared(precision, input_shapes[0]); const std::vector const_values = ov::test::utils::generate_float_numbers(shape_size(m_const_shape.get_shape()), -10., 10.);