natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Snippets] TPP FP32 MHA support (#22210) 

### Details:
 - *Enable FP32 MHA lowering using TPP backend*
- *This PR is a productization of the [TPP integration
POC](https://github.com/openvinotoolkit/openvino/pull/20956)*
### Prerequisites:
- https://github.com/openvinotoolkit/openvino/pull/21303
- https://github.com/openvinotoolkit/openvino/pull/21672

Branch to Branch PR in to review the changes before the Prerequisites
are merged: https://github.com/IvanNovoselov/openvino/pull/18

---------

Co-authored-by: egeorgan <evangelos.georganas@intel.com>
This commit is contained in:
Ivan Novoselov 2024-04-22 15:52:42 +01:00 committed by GitHub
parent a19c34a0f7
commit e56310999c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
84 changed files with 2590 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -81,3 +81,6 @@
[submodule "thirdparty/telemetry"]
path = thirdparty/telemetry
url = https://github.com/openvinotoolkit/telemetry.git
[submodule "src/plugins/intel_cpu/thirdparty/libxsmm"]
path = src/plugins/intel_cpu/thirdparty/libxsmm
url = https://github.com/libxsmm/libxsmm.git

2
cmake/features.cmake
View File

@ -58,6 +58,8 @@ ov_dependent_option (ENABLE_GPU_DEBUG_CAPS "enable GPU debug capabilities at run
ov_dependent_option (ENABLE_CPU_DEBUG_CAPS "enable CPU debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS;ENABLE_INTEL_CPU" OFF)
ov_dependent_option (ENABLE_SNIPPETS_DEBUG_CAPS "enable Snippets debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS" OFF)
ov_dependent_option (ENABLE_SNIPPETS_LIBXSMM_TPP "allow Snippets to use LIBXSMM Tensor Processing Primitives" OFF "ENABLE_INTEL_CPU AND X86_64" OFF)
ov_option (ENABLE_PROFILING_ITT "Build with ITT tracing. Optionally configure pre-built ittnotify library though INTEL_VTUNE_DIR variable." OFF)
ov_option_enum(ENABLE_PROFILING_FILTER "Enable or disable ITT counter groups.\

2
src/common/snippets/include/snippets/emitter.hpp
View File

@ -16,7 +16,7 @@ namespace snippets {
* @interface RegType
* @brief Register type of input and output operations
*/
enum class RegType { gpr, vec };
enum class RegType { gpr, vec, undefined };
/**
* @interface Reg
* @brief Register representation: type of register and index

2
src/common/snippets/include/snippets/lowered/pass/init_loops.hpp
View File

@ -24,7 +24,7 @@ public:
InitLoops() = default;
bool run(LinearIR& linear_ir) override;
static void init_loop_info(const LinearIR::LoopManager::LoopInfoPtr& loop_info, bool only_runtime_args = false);
static void init_loop_info(const LinearIR::LoopManager::LoopInfoPtr& loop_info, size_t loop_id, bool only_runtime_args = false);
};
} // namespace pass

4
src/common/snippets/include/snippets/lowered/pass/insert_buffers.hpp
View File

@ -33,9 +33,9 @@ private:
const LinearIR::constExprIt& end_it,
const LinearIR::LoopManagerPtr& loop_manager,
const std::vector<LinearIR::LoopManager::LoopPort>& loop_entries,
const std::vector<LinearIR::LoopManager::LoopPort>& loop_exits);
const std::vector<LinearIR::LoopManager::LoopPort>& loop_exits) const;
LinearIR::constExprIt insertion_position(const LinearIR& linear_ir,
static LinearIR::constExprIt insertion_position(const LinearIR& linear_ir,
const LinearIR::LoopManagerPtr& loop_manager,
const ExpressionPtr& expr,
const ExpressionPtr& down_expr);

11
src/common/snippets/include/snippets/lowered/port_descriptor.hpp
View File

@ -113,6 +113,17 @@ public:
std::vector<PortDescriptorPtr> outputs{};
};
template<typename T>
void set_port_desc(const T& port, std::vector<size_t> subtensor) {
const auto& shape = port.get_shape();
for (size_t i = 1; i <= std::min(subtensor.size(), shape.size()); i++) {
auto& dim = subtensor[subtensor.size() - i];
if (dim != PortDescriptor::ServiceDimensions::FULL_DIM)
dim = std::min(dim, shape[shape.size() - i]);
}
PortDescriptorUtils::set_port_descriptor_ptr(port, std::make_shared<PortDescriptor>(shape, subtensor));
}
} // namespace lowered
} // namespace snippets
} // namespace ov

26
src/common/snippets/include/snippets/op/brgemm.hpp
View File

@ -17,33 +17,51 @@ namespace op {
* @brief Brgemm is a batch-reduced matrix multiplication with the support of arbitrary strides between matrices rows
* @ingroup snippets
*/
class Brgemm : public MemoryAccess {
class Brgemm : virtual public modifier::MemoryAccess, public ov::op::Op {
public:
OPENVINO_OP("Brgemm", "SnippetsOpset", MemoryAccess);
OPENVINO_OP("Brgemm", "SnippetsOpset");
Brgemm(const Output<Node>& A, const Output<Node>& B,
const size_t offset_a = 0lu, const size_t offset_b = 0lu, const size_t offset_c = 0lu,
std::vector<size_t> layout_a = {}, std::vector<size_t> layout_b = {}, std::vector<size_t> layout_c = {});
std::vector<size_t> layout_a = {}, std::vector<size_t> layout_b = {}, std::vector<size_t> layout_c = {},
size_t blk_size_m = 0, size_t blk_size_k = 0, size_t blk_size_n = 0);
Brgemm(const Output<Node>& A, const Output<Node>& B,
const PortDescriptor& desc_a, const PortDescriptor& desc_b, const PortDescriptor& desc_c,
std::vector<size_t> layout_a = {}, std::vector<size_t> layout_b = {}, std::vector<size_t> layout_c = {});
std::vector<size_t> layout_a = {}, std::vector<size_t> layout_b = {}, std::vector<size_t> layout_c = {},
size_t blk_size_m = 0, size_t blk_size_k = 0, size_t blk_size_n = 0);
Brgemm() = default;
size_t get_offset_a() const { return get_input_offset(0); }
size_t get_offset_b() const { return get_input_offset(1); }
size_t get_offset_c() const { return get_output_offset(0); }
size_t get_m_block_size() const { return m_M_blk; }
size_t get_k_block_size() const { return m_K_blk; }
size_t get_n_block_size() const { return m_N_blk; }
float get_beta() const { return m_beta; }
void set_m_block_size(size_t block_size) { m_M_blk = block_size; }
void set_k_block_size(size_t block_size) { m_K_blk = block_size; }
void set_n_block_size(size_t block_size) { m_N_blk = block_size; }
void set_beta(float beta) { m_beta = beta; }
static ov::element::Type get_output_type(const ov::element::Type& in_type0, const ov::element::Type& in_type1);
void validate_and_infer_types() override;
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool has_evaluate() const override { return false; }
bool visit_attributes(AttributeVisitor& visitor) override;
protected:
ov::element::Type get_output_type() const;
std::vector<ov::PartialShape> get_planar_input_shapes(const std::vector<ov::Input<ov::Node>>& inputs) const;
ov::PartialShape get_output_partial_shape(const std::vector<ov::PartialShape>& input_shapes) const;
ov::PartialShape get_planar_output_shape(const ov::PartialShape& output_shape) const;
void compute_block_size_values(size_t blk_size_m, size_t blk_size_k, size_t blk_size_n);
size_t m_M_blk = 0;
size_t m_K_blk = 0;
size_t m_N_blk = 0;
float m_beta = 0.f;
private:
void custom_constructor_validate_and_infer_types(std::vector<size_t> layout_a, std::vector<size_t> layout_b, std::vector<size_t> layout_c);

4
src/common/snippets/include/snippets/op/broadcastload.hpp
View File

@ -17,9 +17,9 @@ namespace op {
* @brief Is generated for broadcasting by least varying dimension for non-blocked cases and the second varying dimension for blocked
* @ingroup snippets
*/
class BroadcastLoad : public MemoryAccess {
class BroadcastLoad : public modifier::MemoryAccess, public ov::op::Op {
public:
OPENVINO_OP("BroadcastLoad", "SnippetsOpset", ov::snippets::op::MemoryAccess);
OPENVINO_OP("BroadcastLoad", "SnippetsOpset");
BroadcastLoad(const Output<Node>& x, ov::Dimension bcast_dimension, size_t offset = 0lu);
BroadcastLoad() = default;

5
src/common/snippets/include/snippets/op/load.hpp
View File

@ -19,9 +19,9 @@ namespace op {
* and memory offset for loading is determined by "offset" (Default value is "0" - to load starting from the first element)
* @ingroup snippets
*/
class Load : public MemoryAccess {
class Load : public modifier::MemoryAccess, public ov::op::Op {
public:
OPENVINO_OP("Load", "SnippetsOpset", MemoryAccess);
OPENVINO_OP("Load", "SnippetsOpset");
Load(const Output<Node>& x, const size_t count = 1lu, const size_t offset = 0lu);
Load() = default;
@ -34,6 +34,7 @@ public:
void validate_and_infer_types() override;
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
protected:
void validate_memory_access_params() const;

35
src/common/snippets/include/snippets/op/memory_access.hpp
View File

@ -8,7 +8,7 @@
namespace ov {
namespace snippets {
namespace op {
namespace modifier {
/**
* @interface MemoryAccess
@ -19,10 +19,8 @@ namespace op {
* @ingroup snippets
*/
class MemoryAccess : public ov::op::Op {
class MemoryAccess {
public:
OPENVINO_OP("MemoryAccess", "SnippetsOpset");
/**
* @interface PortDescriptor
* @brief This class describes port of MemoryAccess operation
@ -34,13 +32,16 @@ public:
struct PortDescriptor {
PortDescriptor(size_t count, size_t offset) : count(count), offset(offset) {}
PortDescriptor() = default;
// TODO: should we deprecate count in favor of subtensors, ticket: 130004
size_t count = 0lu;
size_t offset = 0lu;
// Note: stride is interpreted as leading dimension for 2D subtensor ops
size_t stride = 0lu;
size_t index = 0lu;
private:
PortDescriptor(size_t count, size_t offset, size_t index) : count(count), offset(offset), index(index) {}
PortDescriptor(size_t count, size_t offset, size_t stride, size_t index) :
count(count), offset(offset), stride(stride), index(index) {}
friend class MemoryAccess;
};
@ -50,11 +51,15 @@ public:
void set_output_count(size_t count, size_t idx = 0);
void set_input_offset(size_t offset, size_t idx = 0);
void set_output_offset(size_t offset, size_t idx = 0);
void set_input_stride(size_t stride, size_t idx = 0);
void set_output_stride(size_t stride, size_t idx = 0);
size_t get_input_count(size_t idx = 0) const;
size_t get_output_count(size_t idx = 0) const;
size_t get_input_offset(size_t idx = 0) const;
size_t get_output_offset(size_t idx = 0) const;
size_t get_input_stride(size_t idx = 0) const;
size_t get_output_stride(size_t idx = 0) const;
PortMap get_memory_access_input_ports() const { return m_input_ports; }
PortMap get_memory_access_output_ports() const { return m_output_ports; }
@ -62,15 +67,17 @@ public:
bool is_memory_access_input_port(size_t idx) const;
bool is_memory_access_output_port(size_t idx) const;
// All input and output ports are MemoryAccess
bool is_full_memory_access_op() const;
/**
* @brief Checks if the provided operation memory access on all ports
*/
bool is_full_memory_access_op(const std::shared_ptr<ov::Node>& op) const;
bool visit_attributes(AttributeVisitor& visitor) override;
bool visit_attributes(AttributeVisitor& visitor);
protected:
explicit MemoryAccess(const OutputVector& arguments, size_t input_count = 0, size_t output_count = 0);
explicit MemoryAccess(const OutputVector& arguments, const std::set<size_t>& input_ports, const std::set<size_t>& output_ports);
explicit MemoryAccess(const OutputVector& arguments, const PortMap& input_ports, const PortMap& output_ports);
explicit MemoryAccess(size_t input_count, size_t output_count = 0);
explicit MemoryAccess(const std::set<size_t>& input_ports, const std::set<size_t>& output_ports);
explicit MemoryAccess(const PortMap& input_ports, const PortMap& output_ports);
MemoryAccess() = default;
// This method can be called only in ctors
@ -80,12 +87,14 @@ protected:
void set_output_port_descriptor(const PortDescriptor& desc, const size_t i);
const PortDescriptor& get_input_port_descriptor(const size_t i) const;
const PortDescriptor& get_output_port_descriptor(const size_t i) const;
PortDescriptor& get_input_port_descriptor(const size_t i);
PortDescriptor& get_output_port_descriptor(const size_t i);
// [port_num, port_desc]
PortMap m_input_ports;
PortMap m_output_ports;
};
} // namespace op
} // namespace modifier
} // namespace snippets
} // namespace ov

5
src/common/snippets/include/snippets/op/store.hpp
View File

@ -18,9 +18,9 @@ namespace op {
* and memory offset for storing is determined by "offset" (Default value is "0" - to store starting at start memory ptr)
* @ingroup snippets
*/
class Store : public MemoryAccess {
class Store : public modifier::MemoryAccess, public ov::op::Op {
public:
OPENVINO_OP("Store", "SnippetsOpset", MemoryAccess);
OPENVINO_OP("Store", "SnippetsOpset");
Store(const Output<Node>& x, const size_t count = 1lu, const size_t offset = 0lu);
Store() = default;
@ -33,6 +33,7 @@ public:
void validate_and_infer_types() override;
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
};
} // namespace op

7
src/common/snippets/src/generator.cpp
View File

@ -13,6 +13,7 @@
#include "snippets/lowered/pass/optimize_loop_single_evaluation.hpp"
#include "snippets/lowered/pass/pass.hpp"
#include "snippets/op/kernel.hpp"
#include "snippets/op/memory_access.hpp"
namespace ov {
namespace snippets {
@ -72,6 +73,9 @@ std::shared_ptr<const TargetMachine> Generator::get_target_machine() const {
}
RegType Generator::get_op_out_reg_type(const ov::Output<Node>& out) const {
auto reg_type = get_specific_op_out_reg_type(out);
if (reg_type != RegType::undefined)
return reg_type;
const auto op = out.get_node_shared_ptr();
if (std::dynamic_pointer_cast<ov::op::v0::Parameter>(op) ||
std::dynamic_pointer_cast<ov::op::v0::Result>(op) ||
@ -107,7 +111,8 @@ RegType Generator::get_op_out_reg_type(const ov::Output<Node>& out) const {
std::dynamic_pointer_cast<op::Fill>(op))
return RegType::vec;
else
return get_specific_op_out_reg_type(op);
OPENVINO_THROW("Register type of the operation " + std::string(op->get_type_name()) + " isn't determined!");
return reg_type;
}
RegType Generator::get_specific_op_out_reg_type(const ov::Output<Node>& out) const {

4
src/common/snippets/src/lowered/pass/allocate_buffers.cpp
View File

@ -38,7 +38,7 @@ void AllocateBuffers::set_buffer_offset(const ExpressionPtr& buffer_expr, const
const auto& parent_expr = parent_output.get_expr();
const auto port = parent_output.get_index();
const auto& parent_node = parent_expr->get_node();
auto memory_access = ov::as_type_ptr<ov::snippets::op::MemoryAccess>(parent_node);
auto memory_access = std::dynamic_pointer_cast<modifier::MemoryAccess>(parent_node);
if (memory_access && memory_access->is_memory_access_output_port(port)) {
memory_access->set_output_offset(offset, port);
} else {
@ -54,7 +54,7 @@ void AllocateBuffers::set_buffer_offset(const ExpressionPtr& buffer_expr, const
const auto& child_expr = child_expr_input.get_expr();
const auto port = child_expr_input.get_index();
const auto& child_node = child_expr->get_node();
auto memory_access = ov::as_type_ptr<ov::snippets::op::MemoryAccess>(child_node);
auto memory_access = std::dynamic_pointer_cast<modifier::MemoryAccess>(child_node);
if (memory_access && memory_access->is_memory_access_input_port(port)) {
memory_access->set_input_offset(offset, port);
} else if (ov::is_type<op::LoopEnd>(child_node)) {

6
src/common/snippets/src/lowered/pass/define_buffer_clusters.cpp
View File

@ -301,8 +301,8 @@ bool DefineBufferClusters::are_buffer_neighbours(const ExpressionPtr& up, const
}
void DefineBufferClusters::parse_memory_access_op(const ExpressionPtr& expr) {
const auto ma = ov::as_type_ptr<op::MemoryAccess>(expr->get_node());
if (!ma->is_full_memory_access_op())
const auto ma = std::dynamic_pointer_cast<modifier::MemoryAccess>(expr->get_node());
if (!ma->is_full_memory_access_op(expr->get_node()))
return;
// TODO: Some full MemoryAccess ops can have inplace inputs and outputs in general.
// Need to add mechanism of inplace ports using MemoryAccess::PortDescriptor::inplace
@ -331,7 +331,7 @@ bool DefineBufferClusters::run(lowered::LinearIR& linear_ir, lowered::LinearIR::
continue;
}
if (ov::is_type<op::MemoryAccess>(op)) {
if (std::dynamic_pointer_cast<modifier::MemoryAccess>(op)) {
parse_memory_access_op(expr);
continue;
}

53
src/common/snippets/src/lowered/pass/init_loops.cpp
View File

@ -16,6 +16,7 @@ namespace lowered {
namespace pass {
using LoopPort = LinearIR::LoopManager::LoopPort;
using MemoryAccess = ov::snippets::modifier::MemoryAccess;
namespace {
inline int64_t get_stride(size_t dim, const VectorDims& shape) {
@ -29,9 +30,47 @@ inline int64_t get_stride(size_t dim, const VectorDims& shape) {
return stride;
}
inline void init_is_incremented(LoopPort& loop_port) {
if (!ov::is_type<op::MemoryAccess>(loop_port.expr_port->get_expr()->get_node()))
loop_port.is_incremented = false;
inline void init_is_incremented(LoopPort& port, size_t loop_id) {
const auto& expr = port.expr_port->get_expr();
const auto& expr_loops = expr->get_loop_ids();
if (!std::dynamic_pointer_cast<modifier::MemoryAccess>(expr->get_node())) {
port.is_incremented = false;
} else if (expr_loops.back() != loop_id) {
// Note: LoopPort connected to Buffer between two loops should not be incremented in the outermost loop
// Consider the example below:
// Store; Loop ids [0,1,2,3]
// IntermediateMemoryBuffer; Loop ids [0,1]
// Load; Loop ids [0,1,4,5]
// Store is exit port of Loop-1, but it should be incremented only in Loop-2 and Loop-3. Similar with Load.
auto is_ignored = [=](const ExpressionPtr& target_expr) {
if (ov::is_type<op::IntermediateMemoryBuffer>(target_expr->get_node())) {
const auto& target_loops = target_expr->get_loop_ids();
const auto i_max = std::min(expr_loops.size(), target_loops.size());
for (size_t i = 0; i < i_max && expr_loops[i] == target_loops[i]; i++) {
if (target_loops[i] == loop_id)
return true;
}
}
return false;
};
if (port.expr_port->get_type() == ExpressionPort::Type::Output) {
const auto& out_connector = expr->get_output_port_connector(port.expr_port->get_index());
for (const auto& consumer : out_connector->get_consumers()) {
if (is_ignored(consumer.get_expr())) {
port.is_incremented = false;
return;
}
}
} else if (port.expr_port->get_type() == ExpressionPort::Type::Input) {
const auto& in_connector = expr->get_input_port_connector(port.expr_port->get_index());
if (is_ignored(in_connector->get_source().get_expr())) {
port.is_incremented = false;
return;
}
} else {
OPENVINO_THROW("Unexpected LoopPort type");
}
}
}
inline void init_ptr_increment(LoopPort& loop_port, size_t work_amount) {
@ -97,7 +136,7 @@ inline void init_work_amount(const LinearIR::LoopManager::LoopInfoPtr& loop_info
}
} // namespace
void InitLoops::init_loop_info(const LinearIR::LoopManager::LoopInfoPtr& loop_info, bool only_runtime_args) {
void InitLoops::init_loop_info(const LinearIR::LoopManager::LoopInfoPtr& loop_info, const size_t loop_id, bool only_runtime_args) {
if (utils::is_dynamic_value(loop_info->get_work_amount()))
init_work_amount(loop_info);
@ -108,8 +147,8 @@ void InitLoops::init_loop_info(const LinearIR::LoopManager::LoopInfoPtr& loop_in
init_finalization_offset(loop_port, work_amount);
};
auto init_all_parameters = [&init_runtime_parameters](LoopPort& loop_port) {
init_is_incremented(loop_port);
auto init_all_parameters = [loop_id, &init_runtime_parameters](LoopPort& loop_port) {
init_is_incremented(loop_port, loop_id);
init_data_size(loop_port);
init_runtime_parameters(loop_port);
};
@ -131,7 +170,7 @@ bool InitLoops::run(LinearIR& linear_ir) {
const auto& loop_manager = linear_ir.get_loop_manager();
const auto& loops = loop_manager->get_map();
for (const auto& loop : loops) {
init_loop_info(loop.second);
init_loop_info(loop.second, loop.first);
}
return true;

6
src/common/snippets/src/lowered/pass/insert_broadcastmove.cpp
View File

@ -7,7 +7,6 @@
#include "snippets/lowered/linear_ir.hpp"
#include "snippets/lowered/loop_manager.hpp"
#include "snippets/snippets_isa.hpp"
#include "snippets/utils.hpp"
#include "snippets/itt.hpp"
namespace ov {
@ -16,9 +15,10 @@ namespace lowered {
namespace pass {
bool InsertBroadcastMove::is_broadcasting_supported(const std::shared_ptr<ov::Node>& n) {
return ov::op::util::supports_auto_broadcast(n) ||
return !std::dynamic_pointer_cast<modifier::MemoryAccess>(n) &&
(ov::op::util::supports_auto_broadcast(n) ||
n->get_autob().m_type == ov::op::AutoBroadcastType::NUMPY ||
is_type<ov::op::v0::PRelu>(n);
is_type<ov::op::v0::PRelu>(n));
}
bool InsertBroadcastMove::is_broadcasting_needed(const std::shared_ptr<ov::Node>& n) {

37
src/common/snippets/src/lowered/pass/insert_buffers.cpp
View File

@ -34,12 +34,14 @@ std::vector<size_t> get_buffer_loop_ids(const std::vector<size_t>& lhs, const st
// TODO: This logic covers only several specific cases so it should be generalized.
ov::Shape compute_allocation_shape(const LinearIR::LoopManagerPtr& loop_manager,
const std::vector<size_t>& buffer_loop_ids,
const std::vector<size_t>& parent_loop_ids,
const ExpressionPort& expr_port,
const ExpressionPort& parent_expr_output,
const int allocation_rank) {
const auto planar_shape = utils::get_preordered_vdims(expr_port);
const auto& parent_expr = parent_expr_output.get_expr();
const auto& parent_loop_ids = parent_expr->get_loop_ids();
const auto planar_shape = utils::get_preordered_vdims(parent_expr_output);
const size_t rank = allocation_rank >= 0 ? std::min(static_cast<size_t>(allocation_rank), planar_shape.size()) : planar_shape.size();
const size_t rank = allocation_rank >= 0 ? std::min(static_cast<size_t>(allocation_rank), planar_shape.size())
: planar_shape.size();
ov::Shape allocation_shape(rank);
for (size_t i = 0; i < rank; ++i) {
*(allocation_shape.rbegin() + i) = *(planar_shape.rbegin() + i);
@ -62,7 +64,7 @@ ov::Shape compute_allocation_shape(const LinearIR::LoopManagerPtr& loop_manager,
// 2. Buffer is inside M_loop_idx loop => allocation shape is not changed
// 3. Buffer is outside N_loop_idx loop => the corresponding allocation shape value is replaced with N loop work amount
// So the result allocation shape is [M_blk, N_loop_work_amount]
const auto& subtensor = expr_port.get_descriptor_ptr()->get_subtensor();
const auto& subtensor = parent_expr_output.get_descriptor_ptr()->get_subtensor();
if (!subtensor.empty()) {
for (size_t i = 0; i < std::min(rank, subtensor.size()); ++i) {
auto& cur_val = *(allocation_shape.rbegin() + i);
@ -75,8 +77,8 @@ ov::Shape compute_allocation_shape(const LinearIR::LoopManagerPtr& loop_manager,
const auto& exit_points = loop_info->get_exit_points();
auto it = std::find_if(exit_points.begin(),
exit_points.end(),
[&expr_port](const LinearIR::LoopManager::LoopPort& port) {
return *port.expr_port == expr_port;
[&parent_expr_output](const LinearIR::LoopManager::LoopPort& port) {
return *port.expr_port == parent_expr_output;
});
OPENVINO_ASSERT(it != exit_points.end(), "compute_allocation_shape: exit point of parent loop can not be found");
const auto& loop_port = *it;
@ -133,6 +135,10 @@ LinearIR::constExprIt InsertBuffers::insertion_position(const LinearIR& linear_i
const auto down_loop_id = down_loops[loop_idx];
return loop_manager->get_loop_bounds(linear_ir, down_loop_id).first;
}
// If upper and lower expressions are in the same loop, we should insert Buffer between them
if (loop_idx == up_loop_count && loop_idx == down_loop_count) {
return linear_ir.find(down_expr);
}
OPENVINO_THROW("Incorrect configuration for Buffer insertion!");
}
@ -141,7 +147,7 @@ void InsertBuffers::insertion(LinearIR& linear_ir,
const LinearIR::constExprIt& end_it,
const LinearIR::LoopManagerPtr& loop_manager,
const std::vector<LinearIR::LoopManager::LoopPort>& loop_entries,
const std::vector<LinearIR::LoopManager::LoopPort>& loop_exits) {
const std::vector<LinearIR::LoopManager::LoopPort>& loop_exits) const {
for (const auto& entry_point : loop_entries) {
const auto& entry_port = entry_point.expr_port;
const auto& expr = entry_port->get_expr();
@ -164,12 +170,13 @@ void InsertBuffers::insertion(LinearIR& linear_ir,
if (ov::is_type<op::Buffer>(parent) ||
ov::is_type<op::VectorBuffer>(parent) ||
ov::is_type<ov::op::v0::Parameter>(parent) ||
ov::is_type<ov::op::v0::Constant>(parent))
ov::is_type<ov::op::v0::Constant>(parent) ||
is_type<op::RankNormalization>(parent))
continue;
// Each MemoryAccess op needs Buffer
const auto parent_ma = ov::as_type_ptr<op::MemoryAccess>(parent);
const auto node_ma = ov::as_type_ptr<op::MemoryAccess>(node);
const auto parent_ma = std::dynamic_pointer_cast<modifier::MemoryAccess>(parent);
const auto node_ma = std::dynamic_pointer_cast<modifier::MemoryAccess>(node);
bool is_buffer_needed = (parent_ma && parent_ma->is_memory_access_output_port(parent_port)) ||
(node_ma && node_ma->is_memory_access_input_port(port_idx));
const auto& current_loops = expr->get_loop_ids();
@ -184,7 +191,6 @@ void InsertBuffers::insertion(LinearIR& linear_ir,
const auto pos = insertion_position(linear_ir, loop_manager, parent_expr, expr);
const auto allocation_shape = compute_allocation_shape(loop_manager,
buffer_loop_ids,
parent_loops,
parent_expr_output,
m_buffer_allocation_rank);
const auto buffer = std::make_shared<op::IntermediateMemoryBuffer>(parent->output(parent_port), allocation_shape);
@ -223,8 +229,8 @@ void InsertBuffers::insertion(LinearIR& linear_ir,
continue;
}
// Each MemoryAccess op needs Buffer
const auto child_ma = ov::as_type_ptr<op::MemoryAccess>(child);
const auto node_ma = ov::as_type_ptr<op::MemoryAccess>(node);
const auto child_ma = std::dynamic_pointer_cast<modifier::MemoryAccess>(child);
const auto node_ma = std::dynamic_pointer_cast<modifier::MemoryAccess>(node);
bool is_buffer_needed = (child_ma && child_ma->is_memory_access_input_port(child_port)) ||
(node_ma && node_ma->is_memory_access_output_port(port_idx));
const auto local_buffer_loop_ids = get_buffer_loop_ids(current_loops, child_expr->get_loop_ids(), is_buffer_needed);
@ -272,7 +278,6 @@ void InsertBuffers::insertion(LinearIR& linear_ir,
const auto allocation_shape = compute_allocation_shape(loop_manager,
buffer_loop_ids,
current_loops,
*exit_port,
m_buffer_allocation_rank);
auto buffer = std::make_shared<op::IntermediateMemoryBuffer>(node->output(port_idx), allocation_shape);
@ -304,7 +309,7 @@ bool InsertBuffers::run(LinearIR& linear_ir, lowered::LinearIR::constExprIt begi
for (auto expr_it = begin; expr_it != end; expr_it++) {
const auto expr = *expr_it;
const auto node = (*expr_it)->get_node();
const auto ma = ov::as_type_ptr<op::MemoryAccess>(node);
const auto ma = std::dynamic_pointer_cast<modifier::MemoryAccess>(node);
if (!ma)
continue;

5
src/common/snippets/src/lowered/pass/insert_load_store.cpp
View File

@ -42,7 +42,8 @@ bool InsertLoadStore::insert_load(LinearIR& linear_ir, const LinearIR::constExpr
const auto& data_out = data_expr->get_output_port_connector(0);
for (const auto& consumer_input : data_out->get_consumers()) {
const auto& consumer_expr = consumer_input.get_expr();
const auto ma = ov::as_type_ptr<op::MemoryAccess>(consumer_expr->get_node());
const auto& consumer = consumer_expr->get_node();
const auto ma = std::dynamic_pointer_cast<modifier::MemoryAccess>(consumer);
if (ma && ma->is_memory_access_input_port(consumer_input.get_index()))
return false;
@ -63,7 +64,7 @@ bool InsertLoadStore::insert_store(LinearIR& linear_ir, const LinearIR::constExp
const auto& parent_expr = parent_output.get_expr();
const auto port = parent_output.get_index();
const auto& parent = parent_expr->get_node();
const auto ma = ov::as_type_ptr<op::MemoryAccess>(parent);
const auto ma = std::dynamic_pointer_cast<modifier::MemoryAccess>(parent);
if (ma && ma->is_memory_access_output_port(port))
return false;

2
src/common/snippets/src/lowered/pass/iter_handler.cpp
View File

@ -29,7 +29,7 @@ bool UpdateMemoryAccessCounts::run(LinearIR& linear_ir, LinearIR::constExprIt be
}
const auto& node = expr_it->get()->get_node();
if (const auto memory_access = ov::as_type_ptr<ov::snippets::op::MemoryAccess>(node)) {
if (const auto memory_access = std::dynamic_pointer_cast<ov::snippets::modifier::MemoryAccess>(node)) {
for (const auto p : memory_access->get_memory_access_input_ports()) {
const auto port = p.first;
if (memory_access->get_input_count(port) > 1) {

2
src/common/snippets/src/lowered/pass/reduce_decomposition.cpp
View File

@ -63,7 +63,7 @@ bool ReduceDecomposition::run(LinearIR& linear_ir, LinearIR::constExprIt begin,
for (auto expr_it = begin; expr_it != end; expr_it++) {
const auto& reduce_expr = *expr_it;
const auto& reduce = ov::as_type_ptr<ov::snippets::op::ReduceBase>(reduce_expr->get_node());
if (!reduce)
if (!reduce || std::dynamic_pointer_cast<modifier::MemoryAccess>(reduce_expr->get_node()))
continue;
const auto& reduce_type_info = reduce->get_type_info();

8
src/common/snippets/src/lowered/pass/validate.cpp
View File

@ -38,7 +38,7 @@ void validate_parameter(const ExpressionPtr& expr, const LinearIR& linear_ir) {
std::set<std::vector<size_t>> layouts;
for (const auto& consumer_input : consumer_inputs) {
const auto& node = consumer_input.get_expr()->get_node();
if (const auto ma = ov::as_type_ptr<snippets::op::MemoryAccess>(node)) {
if (const auto ma = std::dynamic_pointer_cast<snippets::modifier::MemoryAccess>(node)) {
OPENVINO_ASSERT(ma->is_memory_access_input_port(consumer_input.get_index()),
"Parameter expects MemoryAccess on output");
layouts.insert(consumer_input.get_descriptor_ptr()->get_layout());
@ -55,7 +55,7 @@ void validate_result(const ExpressionPtr& expr, const LinearIR& linear_ir) {
const auto& shape_infer_seq = utils::get_first_parent_shape_infer_expr_seq(expr);
const auto& expr_val = shape_infer_seq.empty() ? expr : shape_infer_seq.back();
const auto source = expr_val->get_input_port_connector(0)->get_source();
const auto ma = ov::as_type_ptr<snippets::op::MemoryAccess>(source.get_expr()->get_node());
const auto ma = std::dynamic_pointer_cast<snippets::modifier::MemoryAccess>(source.get_expr()->get_node());
OPENVINO_ASSERT(ma && ma->is_memory_access_output_port(source.get_index()),
"Result expects MemoryAccess parent");
}
@ -65,7 +65,7 @@ void validate_buffer(const ExpressionPtr& expr, const LinearIR& linear_ir) {
"Buffer validation expects Buffer op");
const auto& in = expr->get_input_port_connector(0);
const auto& source = in->get_source();
const auto ma = ov::as_type_ptr<snippets::op::MemoryAccess>(source.get_expr()->get_node());
const auto ma = std::dynamic_pointer_cast<snippets::modifier::MemoryAccess>(source.get_expr()->get_node());
OPENVINO_ASSERT(ma && ma->is_memory_access_input_port(source.get_index()),
"Buffer expects MemoryAccess parent");
const auto& shape_infer_seq = utils::get_first_child_shape_infer_expr_seq(expr);
@ -74,7 +74,7 @@ void validate_buffer(const ExpressionPtr& expr, const LinearIR& linear_ir) {
const auto consumers = out->get_consumers();
for (const auto& consumer_input : consumers) {
const auto& node = consumer_input.get_expr()->get_node();
if (const auto ma = ov::as_type_ptr<snippets::op::MemoryAccess>(node)) {
if (const auto ma = std::dynamic_pointer_cast<snippets::modifier::MemoryAccess>(node)) {
OPENVINO_ASSERT(ma->is_memory_access_input_port(consumer_input.get_index()),
"Buffer expects MemoryAccess on output");
} else {

28
src/common/snippets/src/op/brgemm.cpp
View File

@ -32,23 +32,35 @@ std::vector<size_t> get_output_layout(const std::shared_ptr<const ov::Node>& n)
Brgemm::Brgemm(const Output<Node>& A, const Output<Node>& B,
const size_t offset_a, const size_t offset_b, const size_t offset_c,
std::vector<size_t> layout_a, std::vector<size_t> layout_b, std::vector<size_t> layout_c)
: MemoryAccess({A, B}, std::set<size_t>{0, 1}, std::set<size_t>{0}) {
std::vector<size_t> layout_a, std::vector<size_t> layout_b, std::vector<size_t> layout_c,
const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n)
: MemoryAccess(std::set<size_t>{0, 1}, std::set<size_t>{0}), Op({A, B}) {
set_output_size(1);
set_input_offset(offset_a, 0);
set_input_offset(offset_b, 1);
set_output_offset(offset_c, 0);
compute_block_size_values(blk_size_m, blk_size_k, blk_size_n);
custom_constructor_validate_and_infer_types(std::move(layout_a), std::move(layout_b), std::move(layout_c));
}
Brgemm::Brgemm(const Output<Node>& A, const Output<Node>& B,
const PortDescriptor& desc_a, const PortDescriptor& desc_b, const PortDescriptor& desc_c,
std::vector<size_t> layout_a, std::vector<size_t> layout_b, std::vector<size_t> layout_c)
: MemoryAccess({A, B}, PortMap{{0, desc_a}, {1, desc_b}}, PortMap{{0, desc_c}}) {
std::vector<size_t> layout_a, std::vector<size_t> layout_b, std::vector<size_t> layout_c,
const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n)
: MemoryAccess(PortMap{{0, desc_a}, {1, desc_b}}, PortMap{{0, desc_c}}), Op({A, B}) {
set_output_size(1);
compute_block_size_values(blk_size_m, blk_size_k, blk_size_n);
custom_constructor_validate_and_infer_types(std::move(layout_a), std::move(layout_b), std::move(layout_c));
}
void Brgemm::compute_block_size_values(const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n) {
const auto input_shape_0 = snippets::utils::get_planar_pshape(input(0)).get_shape();
const auto input_shape_1 = snippets::utils::get_planar_pshape(input(1)).get_shape();
m_M_blk = blk_size_m != 0 ? blk_size_m : *++input_shape_0.rbegin();
m_K_blk = blk_size_k != 0 ? blk_size_k : *input_shape_0.rbegin();
m_N_blk = blk_size_n != 0 ? blk_size_n : *input_shape_1.rbegin();
}
void Brgemm::custom_constructor_validate_and_infer_types(std::vector<size_t> layout_a, std::vector<size_t> layout_b, std::vector<size_t> layout_c) {
INTERNAL_OP_SCOPE(BrgemmCPU_constructor_validate_and_infer_types);
validate_inputs();
@ -87,6 +99,14 @@ std::shared_ptr<Node> Brgemm::clone_with_new_inputs(const OutputVector& new_args
lowered::PortDescriptorUtils::get_port_descriptor_ptr(output(0))->get_layout());
}
bool Brgemm::visit_attributes(AttributeVisitor& visitor) {
visitor.on_attribute("blk_M", m_M_blk);
visitor.on_attribute("blk_K", m_K_blk);
visitor.on_attribute("blk_N", m_N_blk);
visitor.on_attribute("beta", m_beta);
return MemoryAccess::visit_attributes(visitor);
}
ov::element::Type Brgemm::get_output_type(const ov::element::Type& in_type0, const ov::element::Type& in_type1) {
const bool is_f32 = utils::everyone_is(element::f32, in_type0, in_type1);
const bool is_int8 = utils::one_of(in_type0, element::i8, element::u8) && in_type1 == element::i8;

2
src/common/snippets/src/op/broadcastload.cpp
View File

@ -12,7 +12,7 @@ namespace snippets {
namespace op {
BroadcastLoad::BroadcastLoad(const Output<Node>& x, ov::Dimension bcast_dimension, size_t offset)
: MemoryAccess({x}, std::set<size_t>{0}, std::set<size_t>{}), bcast_dimension(std::move(bcast_dimension)) {
: MemoryAccess(std::set<size_t>{0}, std::set<size_t>{}), Op({x}), bcast_dimension(std::move(bcast_dimension)) {
set_input_port_descriptor({1, offset}, 0);
constructor_validate_and_infer_types();
}

8
src/common/snippets/src/op/load.cpp
View File

@ -13,7 +13,7 @@ namespace snippets {
namespace op {
Load::Load(const Output<Node>& x, const size_t count, const size_t offset)
: MemoryAccess({x}, std::set<size_t>{0}, std::set<size_t>{}) {
: MemoryAccess(std::set<size_t>{0}, std::set<size_t>{}), Op({x}) {
set_input_port_descriptor({count, offset}, 0);
constructor_validate_and_infer_types();
}
@ -31,6 +31,10 @@ void Load::validate_and_infer_types() {
set_output_type(0, get_input_element_type(0), get_input_partial_shape(0));
}
bool Load::visit_attributes(AttributeVisitor& visitor) {
return MemoryAccess::visit_attributes(visitor);
}
std::shared_ptr<Node> Load::clone_with_new_inputs(const OutputVector& new_args) const {
INTERNAL_OP_SCOPE(Load);
check_new_args_count(this, new_args);
@ -60,7 +64,7 @@ void LoadReshape::validate_and_infer_types() {
}
bool LoadReshape::visit_attributes(AttributeVisitor& visitor) {
Load::visit_attributes(visitor);
MemoryAccess::visit_attributes(visitor);
visitor.on_attribute("order", m_order);
return true;
}

62
src/common/snippets/src/op/memory_access.cpp
View File

@ -7,9 +7,9 @@
namespace ov {
namespace snippets {
namespace op {
namespace modifier {
MemoryAccess::MemoryAccess(const OutputVector& arguments, size_t input_count, size_t output_count) : Op(arguments) {
MemoryAccess::MemoryAccess(size_t input_count, size_t output_count) {
auto init_iota_set = [](size_t num) {
if (num == 0)
return std::set<size_t>{};
@ -20,28 +20,28 @@ MemoryAccess::MemoryAccess(const OutputVector& arguments, size_t input_count, si
ctor_initialize(init_iota_set(input_count), init_iota_set(output_count));
}
MemoryAccess::MemoryAccess(const OutputVector& arguments, const std::set<size_t>& input_ports, const std::set<size_t>& output_ports) : Op(arguments) {
MemoryAccess::MemoryAccess(const std::set<size_t>& input_ports, const std::set<size_t>& output_ports) {
ctor_initialize(input_ports, output_ports);
}
MemoryAccess::MemoryAccess(const OutputVector& arguments, const PortMap& input_ports, const PortMap& output_ports)
: Op(arguments), m_input_ports(input_ports), m_output_ports(output_ports) {}
MemoryAccess::MemoryAccess(const PortMap& input_ports, const PortMap& output_ports)
: m_input_ports(input_ports), m_output_ports(output_ports) {}
void MemoryAccess::ctor_initialize(const std::set<size_t>& input_ports, const std::set<size_t>& output_ports) {
for (auto port : input_ports) {
m_input_ports[port] = {0, 0, port};
m_input_ports[port] = {0, 0, 0, port};
}
for (auto port : output_ports) {
m_output_ports[port] = {0, 0, port};
m_output_ports[port] = {0, 0, 0, port};
}
}
bool MemoryAccess::is_full_memory_access_op() const {
for (size_t i = 0; i < get_input_size(); ++i) {
bool MemoryAccess::is_full_memory_access_op(const std::shared_ptr<ov::Node>& op) const {
for (size_t i = 0; i < op->get_input_size(); ++i) {
if (!is_memory_access_input_port(i))
return false;
}
for (size_t i = 0; i < get_output_size(); ++i) {
for (size_t i = 0; i < op->get_output_size(); ++i) {
if (!is_memory_access_output_port(i))
return false;
}
@ -54,12 +54,14 @@ bool MemoryAccess::visit_attributes(AttributeVisitor& visitor) {
auto port = p.second;
visitor.on_attribute("count_in_" + std::to_string(idx), port.count);
visitor.on_attribute("offset_in_" + std::to_string(idx), port.offset);
visitor.on_attribute("stride_in_" + std::to_string(idx), port.stride);
}
for (const auto& p : m_output_ports) {
auto idx = p.first;
auto port = p.second;
visitor.on_attribute("count_out_" + std::to_string(idx), port.count);
visitor.on_attribute("offset_out_" + std::to_string(idx), port.offset);
visitor.on_attribute("stride_out_" + std::to_string(idx), port.stride);
}
return true;
}
@ -74,38 +76,54 @@ bool MemoryAccess::is_memory_access_output_port(size_t idx) const {
void MemoryAccess::set_input_port_descriptor(const PortDescriptor& desc, const size_t i) {
const auto it = m_input_ports.find(i);
OPENVINO_ASSERT(it != m_input_ports.end(), "Index of input port descriptor should be less than count of input ports");
(*it).second = { desc.count, desc.offset, i};
it->second = desc;
it->second.index = i;
}
void MemoryAccess::set_output_port_descriptor(const PortDescriptor& desc, const size_t i) {
const auto it = m_output_ports.find(i);
OPENVINO_ASSERT(it != m_output_ports.end(), "Index of output port descriptor should be less than count of output ports");
(*it).second = { desc.count, desc.offset, i};
it->second = desc;
it->second.index = i;
}
const MemoryAccess::PortDescriptor& MemoryAccess::get_input_port_descriptor(const size_t i) const {
const auto it = m_input_ports.find(i);
OPENVINO_ASSERT(it != m_input_ports.end(), "Index of input port descriptor should be less than count of input ports");
return (*it).second;
return it->second;
}
MemoryAccess::PortDescriptor& MemoryAccess::get_input_port_descriptor(const size_t i) {
return const_cast<MemoryAccess::PortDescriptor&>(const_cast<const MemoryAccess*>(this)->get_input_port_descriptor(i));
}
const MemoryAccess::PortDescriptor& MemoryAccess::get_output_port_descriptor(const size_t i) const {
const auto it = m_output_ports.find(i);
OPENVINO_ASSERT(it != m_output_ports.end(), "Index of output port descriptor should be less than count of output ports");
return (*it).second;
return it->second;
}
MemoryAccess::PortDescriptor& MemoryAccess::get_output_port_descriptor(const size_t i) {
return const_cast<MemoryAccess::PortDescriptor&>(const_cast<const MemoryAccess*>(this)->get_output_port_descriptor(i));
}
void MemoryAccess::set_input_count(size_t count, size_t idx) {
set_input_port_descriptor({count, get_input_port_descriptor(idx).offset, idx}, idx);
get_input_port_descriptor(idx).count = count;
}
void MemoryAccess::set_output_count(size_t count, size_t idx) {
set_output_port_descriptor({count, get_output_port_descriptor(idx).offset, idx}, idx);
get_output_port_descriptor(idx).count = count;
}
void MemoryAccess::set_input_offset(size_t offset, size_t idx) {
set_input_port_descriptor({get_input_port_descriptor(idx).count, offset, idx}, idx);
get_input_port_descriptor(idx).offset = offset;
}
void MemoryAccess::set_output_offset(size_t offset, size_t idx) {
set_output_port_descriptor({get_output_port_descriptor(idx).count, offset, idx}, idx);
get_output_port_descriptor(idx).offset = offset;
}
void MemoryAccess::set_input_stride(size_t stride, size_t idx) {
get_input_port_descriptor(idx).stride = stride;
}
void MemoryAccess::set_output_stride(size_t stride, size_t idx) {
get_output_port_descriptor(idx).stride = stride;
}
size_t MemoryAccess::get_input_count(size_t idx) const {
return get_input_port_descriptor(idx).count;
@ -119,7 +137,13 @@ size_t MemoryAccess::get_input_offset(size_t idx) const {
size_t MemoryAccess::get_output_offset(size_t idx) const {
return get_output_port_descriptor(idx).offset;
}
size_t MemoryAccess::get_input_stride(size_t idx) const {
return get_input_port_descriptor(idx).stride;
}
size_t MemoryAccess::get_output_stride(size_t idx) const {
return get_output_port_descriptor(idx).stride;
}
} // namespace op
} // namespace modifier
} // namespace snippets
} // namespace ov

6
src/common/snippets/src/op/store.cpp
View File

@ -12,7 +12,7 @@ namespace snippets {
namespace op {
snippets::op::Store::Store(const Output<Node>& x, const size_t count, const size_t offset)
: MemoryAccess({x}, std::set<size_t>{}, std::set<size_t>{0}) {
: MemoryAccess(std::set<size_t>{}, std::set<size_t>{0}), Op({x}) {
set_output_port_descriptor({count, offset}, 0);
constructor_validate_and_infer_types();
}
@ -31,7 +31,9 @@ std::shared_ptr<Node> snippets::op::Store::clone_with_new_inputs(const OutputVec
check_new_args_count(this, new_args);
return std::make_shared<Store>(new_args.at(0), get_count(), get_offset());
}
bool snippets::op::Store::visit_attributes(AttributeVisitor& visitor) {
return MemoryAccess::visit_attributes(visitor);
}
} // namespace op
} // namespace snippets
} // namespace ov

7
src/common/snippets/src/op/subgraph.cpp
View File

@ -422,12 +422,13 @@ void Subgraph::control_flow_transformations(lowered::LinearIR& linear_ir,
OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "Snippets::op::control_flow_transformations")
// Domain optimization must be the first pass, because all other transformations may depend on PortDescriptor shapes
size_t loop_depth = 1;
size_t loop_depth = linear_ir.get_config().m_loop_depth;
if (!lowered_pass_config->is_disabled<lowered::pass::OptimizeDomain>()) {
lowered::pass::OptimizeDomain(loop_depth).run(linear_ir);
linear_ir.set_loop_depth(loop_depth);
}
const size_t vector_size = get_generator()->get_target_machine()->get_lanes();
const int32_t buffer_allocation_rank = static_cast<int32_t>(linear_ir.get_config().m_loop_depth);
lowered::pass::PassPipeline pipeline(lowered_pass_config);
pipeline.register_pass<lowered::pass::MarkLoops>(vector_size);
@ -435,7 +436,7 @@ void Subgraph::control_flow_transformations(lowered::LinearIR& linear_ir,
pipeline.register_pass<lowered::pass::FuseLoops>();
pipeline.register_pass<lowered::pass::SplitLoops>();
pipeline.register_pass<lowered::pass::MoveResultOutOfLoop>();
pipeline.register_pass<lowered::pass::InsertBuffers>(buffer_allocation_rank);
pipeline.register_pass<lowered::pass::InsertBuffers>(static_cast<int32_t>(loop_depth));
pipeline.register_pass<lowered::pass::InsertLoadStore>(vector_size);
pipeline.register_pass<lowered::pass::MoveScalarToConsumer>();
pipeline.register_pass<lowered::pass::InsertBroadcastMove>();

18
src/plugins/intel_cpu/CMakeLists.txt
View File

@ -97,6 +97,11 @@ if(ENABLE_CPU_DEBUG_CAPS)
add_definitions(-DCPU_DEBUG_CAPS)
endif()
if (ENABLE_SNIPPETS_LIBXSMM_TPP)
# Note: LIBXSMM_DEFAULT_CONFIG needed so libxsmm_config can be included without issues
add_definitions(-DSNIPPETS_LIBXSMM_TPP -DLIBXSMM_DEFAULT_CONFIG)
endif()
set(OV_CPU_WITH_DNNL ON)
if(OV_CPU_WITH_DNNL)
add_definitions(-DOV_CPU_WITH_DNNL)
@ -139,6 +144,11 @@ if (NOT ENABLE_MLAS_FOR_CPU)
list(APPEND EXCLUDE_PATHS ${CMAKE_CURRENT_SOURCE_DIR}/src/mlas/*)
endif()
if (NOT ENABLE_SNIPPETS_LIBXSMM_TPP)
list(APPEND EXCLUDE_PATHS ${CMAKE_CURRENT_SOURCE_DIR}/src/emitters/tpp/*
${CMAKE_CURRENT_SOURCE_DIR}/src/transformations/tpp/*)
endif ()
file(GLOB_RECURSE FILES_TO_REMOVE ${EXCLUDE_PATHS})
list(REMOVE_ITEM SOURCES ${FILES_TO_REMOVE})
list(REMOVE_ITEM HEADERS ${FILES_TO_REMOVE})
@ -170,6 +180,11 @@ if (ENABLE_MLAS_FOR_CPU)
target_include_directories(${TARGET_NAME} SYSTEM PRIVATE $<TARGET_PROPERTY:mlas,INCLUDE_DIRECTORIES>)
add_definitions(-DOV_CPU_WITH_MLAS)
endif()
if (ENABLE_SNIPPETS_LIBXSMM_TPP)
target_compile_definitions(xsmm PRIVATE __BLAS=0)
target_link_libraries(${TARGET_NAME} PRIVATE xsmm)
target_include_directories(${TARGET_NAME} SYSTEM PRIVATE $<TARGET_PROPERTY:xsmm,INCLUDE_DIRECTORIES>)
endif ()
target_include_directories(${TARGET_NAME} SYSTEM PRIVATE $<TARGET_PROPERTY:dnnl,INCLUDE_DIRECTORIES>)
# Cross compiled function
# TODO: The same for proposal, proposalONNX, topk
@ -239,6 +254,9 @@ if(BUILD_SHARED_LIBS)
$<TARGET_PROPERTY:openvino::conditional_compilation,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(${TARGET_NAME}_obj SYSTEM PUBLIC $<TARGET_PROPERTY:dnnl,INCLUDE_DIRECTORIES>)
if(ENABLE_SNIPPETS_LIBXSMM_TPP)
target_include_directories(${TARGET_NAME}_obj SYSTEM PUBLIC $<TARGET_PROPERTY:xsmm,INCLUDE_DIRECTORIES>)
endif()
if(ENABLE_MLAS_FOR_CPU)
target_include_directories(${TARGET_NAME}_obj SYSTEM PUBLIC $<TARGET_PROPERTY:mlas,INCLUDE_DIRECTORIES>)

64
src/plugins/intel_cpu/src/emitters/snippets/x64/cpu_generator.cpp
View File

@ -39,6 +39,20 @@
#include "emitters/snippets/x64/verbose.hpp"
#endif
#ifdef SNIPPETS_LIBXSMM_TPP
#include "transformations/tpp/x64/op/brgemm.hpp"
#include "transformations/tpp/x64/op/eltwise.hpp"
#include "transformations/tpp/x64/op/reduce.hpp"
#include "transformations/tpp/x64/op/modifiers.hpp"
#include "transformations/tpp/x64/op/scalar.hpp"
#include "emitters/tpp/x64/jit_eltwise_emitters.hpp"
#include "emitters/tpp/x64/jit_brgemm_emitter.hpp"
#include "emitters/tpp/x64/jit_scalar_emitter.hpp"
#include "emitters/tpp/x64/jit_debug_emitter.hpp"
// Note: for reference implementations
#include <cmath>
#endif
namespace ov {
#ifdef SNIPPETS_DEBUG_CAPS
@ -90,9 +104,9 @@ static bool is_segfault_detector_emitter(const intel_cpu::jit_emitter *emitter)
} \
}
#else
#define CREATE_SNIPPETS_EMITTER(e_type) { \
#define CREATE_SNIPPETS_EMITTER(e_type, ...) { \
[this](const snippets::lowered::ExpressionPtr& expr) -> std::shared_ptr<snippets::Emitter> { \
return std::make_shared<e_type>(h.get(), isa, expr); \
return std::make_shared<e_type>(h.get(), isa, expr, ##__VA_ARGS__); \
}, \
[](const std::shared_ptr<ov::Node>& n) -> std::set<std::vector<element::Type>> { \
return e_type::get_supported_precisions(n); \
@ -100,6 +114,16 @@ static bool is_segfault_detector_emitter(const intel_cpu::jit_emitter *emitter)
}
#endif
#define CREATE_DEBUG_TPP_EMITTER(e_type) { \
[this](const snippets::lowered::ExpressionPtr& expr) -> std::shared_ptr<snippets::Emitter> { \
return std::make_shared<DebugTppEmitter>(expr, std::make_shared<e_type>(h.get(), isa, expr)); \
}, \
[](const std::shared_ptr<ov::Node>& n) -> std::set<std::vector<element::Type>> { \
return e_type::get_supported_precisions(n); \
} \
}
#define CREATE_CPU_EMITTER(e_type) { \
[this](const snippets::lowered::ExpressionPtr& expr) -> std::shared_ptr<snippets::Emitter> { \
return std::make_shared<e_type>(h.get(), isa, expr->get_node()); \
@ -226,6 +250,32 @@ intel_cpu::CPUTargetMachine::CPUTargetMachine(dnnl::impl::cpu::x64::cpu_isa_t ho
jitters[ov::intel_cpu::PerfCountRdtscBegin::get_type_info_static()] = CREATE_CPU_EMITTER(ov::intel_cpu::jit_perf_count_rdtsc_start_emitter);
jitters[ov::intel_cpu::PerfCountRdtscEnd::get_type_info_static()] = CREATE_CPU_EMITTER(ov::intel_cpu::jit_perf_count_rdtsc_end_emitter);
#endif
#ifdef SNIPPETS_LIBXSMM_TPP
jitters[intel_cpu::tpp::op::BrgemmTPP::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(BrgemmTppEmitter);
jitters[intel_cpu::tpp::op::Add::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(BinaryEltwiseTppEmitter);
jitters[intel_cpu::tpp::op::Subtract::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(BinaryEltwiseTppEmitter);
jitters[intel_cpu::tpp::op::Multiply::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(BinaryEltwiseTppEmitter);
jitters[intel_cpu::tpp::op::Divide::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(BinaryEltwiseTppEmitter);
jitters[intel_cpu::tpp::op::Exp::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(UnaryEltwiseTppEmitter);
// Note: you can register Debug emitter for Unary/Binary operations as shown below:
// jitters[intel_cpu::tpp::op::Add::get_type_info_static()] = CREATE_DEBUG_TPP_EMITTER(UnaryEltwiseTppEmitter);
//
// Note: you can register Reference emitter for Unary operations using std::function or lambda function as shown below:
// jitters[intel_cpu::tpp::op::Exp::get_type_info_static()] =
// CREATE_SNIPPETS_EMITTER(ReferenceUnaryEltwiseTppEmitter, static_cast<float(*)(float)>(std::exp));
// jitters[intel_cpu::tpp::op::Reciprocal::get_type_info_static()] =
// CREATE_SNIPPETS_EMITTER(ReferenceUnaryEltwiseTppEmitter, [](float x){ return 1.f/x; });
jitters[intel_cpu::tpp::op::Reciprocal::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(UnaryEltwiseTppEmitter);
jitters[intel_cpu::tpp::op::Relu::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(UnaryEltwiseTppEmitter);
jitters[intel_cpu::tpp::op::Square::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(UnaryEltwiseTppEmitter);
jitters[intel_cpu::tpp::op::SquareRoot::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(UnaryEltwiseTppEmitter);
jitters[intel_cpu::tpp::op::ReduceMax::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(ReduceTppEmitter);
jitters[intel_cpu::tpp::op::ReduceSum::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(ReduceTppEmitter);
jitters[intel_cpu::tpp::op::Scalar::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(ScalarTppEmitter);
#endif
}
size_t intel_cpu::CPUTargetMachine::get_lanes() const {
@ -283,6 +333,10 @@ std::shared_ptr<snippets::Generator> intel_cpu::CPUGenerator::clone() const {
ov::snippets::RegType intel_cpu::CPUGenerator::get_specific_op_out_reg_type(const ov::Output<ov::Node>& out) const {
const auto op = out.get_node_shared_ptr();
if (std::dynamic_pointer_cast<intel_cpu::BrgemmCPU>(op) ||
#ifdef SNIPPETS_LIBXSMM_TPP
std::dynamic_pointer_cast<intel_cpu::tpp::modifier::TensorProcessingPrimitive>(op) ||
std::dynamic_pointer_cast<intel_cpu::tpp::op::Scalar>(op) ||
#endif
std::dynamic_pointer_cast<intel_cpu::BrgemmCopyB>(op))
return ov::snippets::RegType::gpr;
else if (
@ -290,7 +344,7 @@ ov::snippets::RegType intel_cpu::CPUGenerator::get_specific_op_out_reg_type(cons
std::dynamic_pointer_cast<intel_cpu::SwishNode>(op))
return ov::snippets::RegType::vec;
else
OPENVINO_THROW("Register type of the operation " + std::string(op->get_type_name()) + " isn't determined!");
return ov::snippets::RegType::undefined;
}
bool intel_cpu::CPUGenerator::uses_precompiled_kernel(const std::shared_ptr<snippets::Emitter>& e) const {
@ -303,6 +357,10 @@ bool intel_cpu::CPUGenerator::uses_precompiled_kernel(const std::shared_ptr<snip
std::dynamic_pointer_cast<intel_cpu::jit_perf_count_chrono_end_emitter>(e) ||
std::dynamic_pointer_cast<intel_cpu::jit_perf_count_rdtsc_start_emitter>(e) ||
std::dynamic_pointer_cast<intel_cpu::jit_perf_count_rdtsc_end_emitter>(e);
#endif
#ifdef SNIPPETS_LIBXSMM_TPP
need |= std::dynamic_pointer_cast<intel_cpu::ReferenceUnaryEltwiseTppEmitter>(e) ||
std::dynamic_pointer_cast<intel_cpu::DebugTppEmitter>(e);
#endif
return need;
}

4
src/plugins/intel_cpu/src/emitters/snippets/x64/jit_kernel_emitter.cpp
View File

@ -206,7 +206,7 @@ jit_kernel_static_emitter::jit_kernel_static_emitter(dnnl::impl::cpu::x64::jit_g
const auto& mem_desc_expr = shape_infer_seq.empty() ? expr : shape_infer_seq.back();
auto consumer_inputs = mem_desc_expr->get_output_port_connector(0)->get_consumers();
for (const auto& child_input : consumer_inputs) {
const auto ma = ov::as_type_ptr<snippets::op::MemoryAccess>(child_input.get_expr()->get_node());
const auto ma = std::dynamic_pointer_cast<snippets::modifier::MemoryAccess>(child_input.get_expr()->get_node());
if (ma && ma->is_memory_access_input_port(child_input.get_index())) {
desc = child_input.get_descriptor_ptr();
break;
@ -322,7 +322,7 @@ void jit_kernel_static_emitter::init_data_pointers(const std::vector<Xbyak::Reg6
// * Static case: we can use reg_runtime_params as the last reg_tmp for the last iteration (and corrupt it), since
// it won't be used anymore
// * Dynamic case: we will need reg_runtime_params to pass runtime args to LoopScheduler, so we have to
// push a reg on the stack, and restore it value afterwards
// push a reg on the stack, and restore it value afterward
if (last_iter_explicitly) {
h->mov(data_ptr_regs[i], h->ptr[reg_runtime_params + GET_OFF(dst_ptrs) + (i - num_inputs) * sizeof(void*)]);
reg_tmp = reg_runtime_params;

43
src/plugins/intel_cpu/src/emitters/snippets/x64/jit_snippets_emitters.cpp
View File

@ -15,8 +15,9 @@ using jit_generator = dnnl::impl::cpu::x64::jit_generator;
using cpu_isa_t = dnnl::impl::cpu::x64::cpu_isa_t;
using ExpressionPtr = ov::snippets::lowered::ExpressionPtr;
jit_nop_emitter::jit_nop_emitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr) : jit_emitter(h, isa) {
in_out_type_ = emitter_in_out_map::gpr_to_gpr;
jit_nop_emitter::jit_nop_emitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr, emitter_in_out_map emitter_type)
: jit_emitter(h, isa) {
in_out_type_ = emitter_type;
}
jit_parameter_emitter::jit_parameter_emitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr)
@ -67,35 +68,31 @@ void jit_broadcast_move_emitter::emit_isa(const std::vector<size_t> &in, const s
}
}
jit_scalar_emitter::jit_scalar_emitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr) : jit_emitter(h, isa) {
const auto n = expr->get_node();
int32_t jit_scalar_emitter::read_value(const ov::snippets::lowered::ExpressionPtr& expr) {
const auto n = ov::as_type_ptr<ov::op::v0::Constant>(expr->get_node());
OV_CPU_JIT_EMITTER_ASSERT(n, "Invalid node, expected op::v0::Constant");
const auto& precision = n->get_output_element_type(0);
int32_t res = INT_MIN;
switch (precision) {
case element::i32: {
value = ov::as_type_ptr<ov::op::v0::Constant>(n)->cast_vector<int32_t>()[0];
break;
case element::i32: res = n->cast_vector<int32_t>(1)[0]; break;
case element::f32: res = dnnl::impl::cpu::x64::float2int(n->cast_vector<float>(1)[0]); break;
default: OV_CPU_JIT_EMITTER_THROW("doesn't support ", precision);
}
case element::f32: {
value = dnnl::impl::cpu::x64::float2int(ov::as_type_ptr<ov::op::v0::Constant>(n)->cast_vector<float>()[0]);
break;
return res;
}
default: {
OV_CPU_JIT_EMITTER_THROW("doesn't support ", precision);
}
}
push_arg_entry_of("scalar", value, true);
jit_scalar_emitter::jit_scalar_emitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr) : jit_emitter(h, isa) {
push_arg_entry_of("scalar", read_value(expr), true);
prepare_table();
}
void jit_scalar_emitter::emit_impl(const std::vector<size_t>& in, const std::vector<size_t>& out) const {
if (host_isa_ == dnnl::impl::cpu::x64::sse41) {
emit_isa<dnnl::impl::cpu::x64::sse41>(in, out);
} else if (host_isa_ == dnnl::impl::cpu::x64::avx2) {
emit_isa<dnnl::impl::cpu::x64::avx2>(in, out);
} else if (host_isa_ == dnnl::impl::cpu::x64::avx512_core) {
emit_isa<dnnl::impl::cpu::x64::avx512_core>(in, out);
} else {
OV_CPU_JIT_EMITTER_THROW("Unsupported isa ", host_isa_);
using isa = cpu_isa_t;
switch (host_isa_) {
case isa::sse41: emit_isa<isa::sse41>(in, out); break;
case isa::avx2: emit_isa<isa::avx2>(in, out); break;
case isa::avx512_core: emit_isa<isa::avx512_core>(in, out); break;
default: OV_CPU_JIT_EMITTER_THROW("Unsupported isa ", host_isa_);
}
}

9
src/plugins/intel_cpu/src/emitters/snippets/x64/jit_snippets_emitters.hpp
View File

@ -13,7 +13,7 @@ namespace intel_cpu {
class jit_nop_emitter : public jit_emitter {
public:
jit_nop_emitter(dnnl::impl::cpu::x64::jit_generator* h, dnnl::impl::cpu::x64::cpu_isa_t isa,
const ov::snippets::lowered::ExpressionPtr& expr);
const ov::snippets::lowered::ExpressionPtr& expr, emitter_in_out_map emitter_type = gpr_to_gpr);
size_t get_inputs_num() const override {return 0;}
@ -59,18 +59,13 @@ public:
const ov::snippets::lowered::ExpressionPtr& expr);
size_t get_inputs_num() const override {return 0;}
protected:
size_t aux_gprs_count() const override {return 1;}
static int32_t read_value(const ov::snippets::lowered::ExpressionPtr& expr);
private:
void emit_impl(const std::vector<size_t>& in, const std::vector<size_t>& out) const override;
template <dnnl::impl::cpu::x64::cpu_isa_t isa>
void emit_isa(const std::vector<size_t> &in, const std::vector<size_t> &out) const;
private:
int32_t value;
};
} // namespace intel_cpu

113
src/plugins/intel_cpu/src/emitters/tpp/x64/jit_brgemm_emitter.cpp Normal file
View File

@ -0,0 +1,113 @@
// Copyright (C) 2020-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "jit_brgemm_emitter.hpp"
#include "emitters/snippets/x64/jit_snippets_emitters.hpp"
#include "transformations/tpp/x64/op/brgemm.hpp"
using jit_generator = dnnl::impl::cpu::x64::jit_generator;
using cpu_isa_t = dnnl::impl::cpu::x64::cpu_isa_t;
using ExpressionPtr = ov::snippets::lowered::ExpressionPtr;
namespace ov {
namespace intel_cpu {
void BrgemmTppEmitter::validate_subtensors(const VectorDims& in_0, const VectorDims& in_1, const VectorDims& out_0) {
bool subtensors_compatible = in_0.size() == in_1.size() && in_0.size() == out_0.size() && in_0.size() == 2 &&
in_0[1] == in_1[0] && in_0[0] == out_0[0] && in_1[1] == out_0[1];
OV_CPU_JIT_EMITTER_ASSERT(subtensors_compatible, "Incompatible subtensors");
}
BrgemmTppEmitter::BrgemmTppEmitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr)
: TppEmitter(h, isa, expr) {
const auto& brgemm_node = as_type_ptr<intel_cpu::tpp::op::BrgemmTPP>(expr->get_node());
OV_CPU_JIT_EMITTER_ASSERT(brgemm_node && !brgemm_node->is_dynamic(), "Invoked with invalid node type");
const auto& input_0_desc = expr->get_input_port_descriptor(0);
const auto& input_1_desc = expr->get_input_port_descriptor(1);
const auto& output_desc = expr->get_output_port_descriptor(0);
std::vector<size_t> leading_dimensions {brgemm_node->get_input_stride(0),
brgemm_node->get_input_stride(1),
brgemm_node->get_output_stride(0)};
auto in_0_prec = ov_to_xsmm_dtype(brgemm_node->get_input_element_type(0));
auto in_1_prec = ov_to_xsmm_dtype(brgemm_node->get_input_element_type(1));
exec_dtype = in_0_prec == LIBXSMM_DATATYPE_I8 || in_0_prec == LIBXSMM_DATATYPE_U8 ?
LIBXSMM_DATATYPE_I32 :
LIBXSMM_DATATYPE_F32;
auto out_0_prec = exec_dtype == LIBXSMM_DATATYPE_I32 ?
LIBXSMM_DATATYPE_I32 :
LIBXSMM_DATATYPE_F32;
const auto beta = brgemm_node->get_beta();
OV_CPU_JIT_EMITTER_ASSERT(beta == 0 || beta == 1, "Detected unsupported beta value: " + std::to_string(beta));
const auto& subtensor_in0 = input_0_desc->get_subtensor();
const auto& subtensor_in1 = input_1_desc->get_subtensor();
const auto& subtensor_out0 = output_desc->get_subtensor();
validate_subtensors(subtensor_in0, subtensor_in1, subtensor_out0);
const auto K = static_cast<libxsmm_blasint>(*subtensor_in0.rbegin());
const auto M = static_cast<libxsmm_blasint>(*++subtensor_in0.rbegin());
const auto N = static_cast<libxsmm_blasint>(*subtensor_in1.rbegin());
const bool is_f32_gemm = in_0_prec == in_1_prec && in_0_prec == LIBXSMM_DATATYPE_F32;
const bool is_bf16_gemm = in_0_prec == in_1_prec && in_0_prec == LIBXSMM_DATATYPE_BF16;
const bool is_i8_gemm = in_0_prec == LIBXSMM_DATATYPE_U8 || in_0_prec == LIBXSMM_DATATYPE_I8;
OV_CPU_JIT_EMITTER_ASSERT(is_f32_gemm ||
(is_bf16_gemm && K % 2 == 0) ||
(is_i8_gemm && K % 4 == 0),
"Unsupported parameter combination for kernel configuration");
m_compile_flags = is_f32_gemm ?
LIBXSMM_GEMM_FLAGS('N', 'N') :
LIBXSMM_GEMM_VNNI_FLAGS('N', 'N', 'V', 'N') |
LIBXSMM_GEMM_FLAG_NO_SETUP_TILECONFIG |
LIBXSMM_GEMM_FLAG_NO_RESET_TILECONFIG;
if (beta == 0)
m_compile_flags |= LIBXSMM_GEMM_FLAG_BETA_0;
if (in_0_prec == LIBXSMM_DATATYPE_U8) {
in_0_prec = LIBXSMM_DATATYPE_I8;
m_compile_flags |= LIBXSMM_GEMM_FLAG_A_UNSIGNED;
}
if (in_1_prec == LIBXSMM_DATATYPE_U8) {
in_1_prec = LIBXSMM_DATATYPE_I8;
m_compile_flags |= LIBXSMM_GEMM_FLAG_B_UNSIGNED;
}
m_shape = libxsmm_create_gemm_shape(N, M, K,
io_strides[1], io_strides[0], io_strides[2],
in_1_prec, in_0_prec, out_0_prec,
exec_dtype);
m_prefetching_flags = LIBXSMM_GEMM_PREFETCH_NONE;
}
std::set<std::vector<element::Type>> BrgemmTppEmitter::get_supported_precisions(const std::shared_ptr<ov::Node>& node) {
// Note: BrgemmTpp currently supports only fp32
return {{element::f32, element::f32}};
}
void BrgemmTppEmitter::validate_arguments(const std::vector<size_t> &in, const std::vector<size_t> &out) const {
OV_CPU_JIT_EMITTER_ASSERT(in.size() == 2, "Expects 2 input regs, got" + std::to_string(in.size()));
OV_CPU_JIT_EMITTER_ASSERT(out.size() == 1, "Expects 1 output reg, got" + std::to_string(out.size()));
}
const uintptr_t BrgemmTppEmitter::get_compiled_kernel_ptr() const {
return COMPILE_TPP_KERNEL(libxsmm_dispatch_gemm(m_shape, m_compile_flags, m_prefetching_flags));
}
void BrgemmTppEmitter::execute_brgemm_kernel(libxsmm_gemmfunction brg_kernel, void *in0, void *in1, void *out0) {
libxsmm_gemm_param gemm_p;
gemm_p.a.primary = in1;
gemm_p.b.primary = in0;
gemm_p.c.primary = out0;
OV_CPU_JIT_EMITTER_ASSERT(brg_kernel, "Invalid brgemm kernel pointer");
brg_kernel(&gemm_p);
}
} // namespace intel_cpu
} // namespace ov

33
src/plugins/intel_cpu/src/emitters/tpp/x64/jit_brgemm_emitter.hpp Normal file
View File

@ -0,0 +1,33 @@
// Copyright (C) 2020-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "jit_tpp_emitter.hpp"
namespace ov {
namespace intel_cpu {
class BrgemmTppEmitter : public TppEmitter {
public:
BrgemmTppEmitter(dnnl::impl::cpu::x64::jit_generator* h,
dnnl::impl::cpu::x64::cpu_isa_t isa,
const ov::snippets::lowered::ExpressionPtr& expr);
size_t get_inputs_num() const override { return 2; }
static std::set<std::vector<element::Type>> get_supported_precisions(const std::shared_ptr<ov::Node>& node = nullptr);
static void execute_brgemm_kernel(libxsmm_gemmfunction brgemm_kernel, void *in0, void *in1, void *out0);
const uintptr_t get_execute_function_ptr() const override { return reinterpret_cast<const uintptr_t>(execute_brgemm_kernel); }
const uintptr_t get_compiled_kernel_ptr() const override;
protected:
void validate_arguments(const std::vector<size_t> &in, const std::vector<size_t> &out) const override;
static void validate_subtensors(const VectorDims& in_0, const VectorDims& in_1, const VectorDims& out_0);
libxsmm_gemm_shape m_shape;
libxsmm_bitfield m_prefetching_flags {0};
};
} // namespace intel_cpu
} // namespace ov

74
src/plugins/intel_cpu/src/emitters/tpp/x64/jit_debug_emitter.hpp Normal file
View File

@ -0,0 +1,74 @@
// Copyright (C) 2020-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "jit_tpp_emitter.hpp"
namespace ov {
namespace intel_cpu {
/**
* @interface DebugTppEmitter
* @brief The purpose of this emitter is to facilitate debugging of TPP emitters. It allows to access attributes of the
* owned Tpp emitter at runtime, inspect source ov::Node, in/out memory before and after execution, etc.
*/
class DebugTppEmitter : public TppEmitter {
public:
DebugTppEmitter(const ov::snippets::lowered::ExpressionPtr& expr, const std::shared_ptr<TppEmitter>& original)
: TppEmitter(*original),
m_original(original),
m_compiled_kernel(m_original->get_compiled_kernel_ptr()),
m_execute_function(m_original->get_execute_function_ptr()),
m_source_expr(expr) {
}
void validate_arguments(const std::vector<size_t> &in, const std::vector<size_t> &out) const override {
m_original->validate_arguments(in, out);
};
size_t get_inputs_num() const override { return num_kernel_args - 1; }
protected:
static void execute_kernel_unary(const DebugTppEmitter* emitter, void *in0, void *out0) {
OV_CPU_JIT_EMITTER_ASSERT(emitter && emitter->m_execute_function && emitter->m_compiled_kernel,
"Unable to execute unary kernel");
// Note: put a breakpoint here and analyze all the necessary debug info in runtime
std::cout << "Running unary DebugTPPEmitter for node with name "
<< emitter->m_source_expr->get_node()->get_friendly_name() << std::endl;
auto f = reinterpret_cast<void(*)(uintptr_t, void*, void*)>(emitter->m_execute_function);
f(emitter->m_compiled_kernel, in0, out0);
}
static void execute_kernel_binary(const DebugTppEmitter* emitter, void* in0, void* in1, void* out0) {
OV_CPU_JIT_EMITTER_ASSERT(emitter && emitter->m_execute_function && emitter->m_compiled_kernel,
"Unable to execute binary kernel");
// Note: put a breakpoint here and analyze all the necessary debug info in runtime
std::cout << "Running binary DebugTPPEmitter for node with name "
<< emitter->m_source_expr->get_node()->get_friendly_name() << std::endl;
auto f = reinterpret_cast<void(*)(uintptr_t, void*, void*, void*)>(emitter->m_execute_function);
f(emitter->m_compiled_kernel, in0, in1, out0);
}
const uintptr_t get_execute_function_ptr() const override {
// Note: num_kernel_args accounts for both input and output args
switch (num_kernel_args) {
case 2: return reinterpret_cast<const uintptr_t>(execute_kernel_unary);
case 3: return reinterpret_cast<const uintptr_t>(execute_kernel_binary);
default: OV_CPU_JIT_EMITTER_THROW("More than two arguments are not supported");
}
}
const uintptr_t get_compiled_kernel_ptr() const override {
return reinterpret_cast<const uintptr_t>(this);
}
private:
std::shared_ptr<TppEmitter> m_original {nullptr};
uintptr_t m_compiled_kernel {0};
uintptr_t m_execute_function {0};
snippets::lowered::ExpressionPtr m_source_expr {nullptr};
};
} // namespace intel_cpu
} // namespace ov

136
src/plugins/intel_cpu/src/emitters/tpp/x64/jit_eltwise_emitters.cpp Normal file
View File

@ -0,0 +1,136 @@
// Copyright (C) 2020-2024 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "jit_eltwise_emitters.hpp"
#include "transformations/tpp/x64/op/eltwise.hpp"
namespace ov {
namespace intel_cpu {
using jit_generator = dnnl::impl::cpu::x64::jit_generator;
using cpu_isa_t = dnnl::impl::cpu::x64::cpu_isa_t;
using ExpressionPtr = ov::snippets::lowered::ExpressionPtr;
BinaryEltwiseTppEmitter::BinaryEltwiseTppEmitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr) :
TppEmitter(h, isa, expr) {
const auto& subtensor_in0 = get_projected_subtensor(io_port_descriptors[0]);
const auto& subtensor_in1 = get_projected_subtensor(io_port_descriptors[1]);
const auto N_in0 = static_cast<libxsmm_blasint>(*subtensor_in0.rbegin());
const auto M_in0 = static_cast<libxsmm_blasint>(*++subtensor_in0.rbegin());
const auto N_in1 = static_cast<libxsmm_blasint>(*subtensor_in1.rbegin());
const auto M_in1 = static_cast<libxsmm_blasint>(*++subtensor_in1.rbegin());
std::pair<bool, bool> n_bcast_flags, m_bcast_flags;
const auto N = get_broadcasted_dim(N_in0, N_in1, n_bcast_flags);
const auto M = get_broadcasted_dim(M_in0, M_in1, m_bcast_flags);
m_compile_flags = LIBXSMM_MELTW_FLAG_BINARY_NONE;
if (m_bcast_flags.first && n_bcast_flags.first) {
m_compile_flags |= LIBXSMM_MELTW_FLAG_BINARY_BCAST_SCALAR_IN_0;
} else if (m_bcast_flags.first) {
m_compile_flags |= LIBXSMM_MELTW_FLAG_BINARY_BCAST_COL_IN_0;
} else if (n_bcast_flags.first) {
m_compile_flags |= LIBXSMM_MELTW_FLAG_BINARY_BCAST_ROW_IN_0;
}
if (m_bcast_flags.second && n_bcast_flags.second) {
m_compile_flags |= LIBXSMM_MELTW_FLAG_BINARY_BCAST_SCALAR_IN_1;
} else if (m_bcast_flags.second) {
m_compile_flags |= LIBXSMM_MELTW_FLAG_BINARY_BCAST_COL_IN_1;
} else if (n_bcast_flags.second) {
m_compile_flags |= LIBXSMM_MELTW_FLAG_BINARY_BCAST_ROW_IN_1;
}
const auto& binary_eltw_tpp = std::dynamic_pointer_cast<tpp::op::BinaryEltwiseTPP>(expr->get_node());
OV_CPU_JIT_EMITTER_ASSERT(binary_eltw_tpp, "Invalid TPP node type detected");
m_op_type = binary_eltw_tpp->get_op_type();
// Note: libxsmm implies column-major layout, so we have to swap M and N here
m_shape = libxsmm_create_meltw_binary_shape(N, M,
io_strides[0], io_strides[1], io_strides[2],
io_dtypes[0], io_dtypes[1], io_dtypes[2],
exec_dtype);
}
const uintptr_t BinaryEltwiseTppEmitter::get_compiled_kernel_ptr() const {
// Note: libxsmm hides memory management from the user, so we don't have to store pointer to compiled kernel to keep it alive.
// libxsmm will keep the pointer alive until the end of program execution (it doesn't matter whether we save the pointer in the emitter or not)
return COMPILE_TPP_KERNEL(libxsmm_dispatch_meltw_binary(m_op_type, m_shape, m_compile_flags));
}
std::set<std::vector<element::Type>> BinaryEltwiseTppEmitter::get_supported_precisions(const std::shared_ptr<ov::Node>& node) {
return {{element::f32, element::f32}};
}
void BinaryEltwiseTppEmitter::validate_arguments(const std::vector<size_t> &in, const std::vector<size_t> &out) const {
OV_CPU_JIT_EMITTER_ASSERT(in.size() == 2, "Expects 2 input registers, got " + std::to_string(in.size()));
OV_CPU_JIT_EMITTER_ASSERT(out.size() == 1, "Expects 1 output register, got " + std::to_string(out.size()));
}
void BinaryEltwiseTppEmitter::execute_kernel(libxsmm_meltwfunction_binary eltwise_kernel, void *in0, void *in1, void *out0) {
libxsmm_meltw_binary_param param;
param.op.primary = nullptr;
param.in0.primary = in0;
param.in1.primary = in1;
param.out.primary = out0;
eltwise_kernel(&param);
}
libxsmm_blasint BinaryEltwiseTppEmitter::get_broadcasted_dim(libxsmm_blasint dim0, libxsmm_blasint dim1, std::pair<bool, bool>& bcast_flags) {
if (dim0 == dim1) {
bcast_flags = {false, false};
return dim0;
} else if (dim1 == 1) {
bcast_flags = {false, true};
return dim0;
} else if (dim0 == 1) {
bcast_flags = {true, false};
return dim1;
}
OV_CPU_JIT_EMITTER_THROW("Invalid dimensions passed to get_broadcast_flags");
return -1;
}
UnaryEltwiseTppEmitter::UnaryEltwiseTppEmitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr) :
TppEmitter(h, isa, expr) {
const auto& subtensor_in0 = get_projected_subtensor(io_port_descriptors[0]);
const auto N = static_cast<libxsmm_blasint>(*subtensor_in0.rbegin());
const auto M = static_cast<libxsmm_blasint>(*++subtensor_in0.rbegin());
const auto& unary_eltw_tpp = std::dynamic_pointer_cast<tpp::op::UnaryEltwiseTPP>(expr->get_node());
OV_CPU_JIT_EMITTER_ASSERT(unary_eltw_tpp, "Invalid TPP node type detected");
m_op_type = unary_eltw_tpp->get_op_type();
// Note: libxsmm implies column-major layout, so we have to swap M and N here
m_shape = libxsmm_create_meltw_unary_shape(N, M,
io_strides[0], io_strides[1],
io_dtypes[0], io_dtypes[1],
exec_dtype);
m_compile_flags = LIBXSMM_MELTW_FLAG_UNARY_NONE;
}
void UnaryEltwiseTppEmitter::execute_kernel(libxsmm_meltwfunction_unary eltwise_kernel, void *in0, void *out0) {
libxsmm_meltw_unary_param param;
param.op.primary = nullptr;
param.in.primary = in0;
param.out.primary = out0;
eltwise_kernel(&param);
}
std::set<std::vector<element::Type>> UnaryEltwiseTppEmitter::get_supported_precisions(const std::shared_ptr<ov::Node>& node) {
return {{element::f32}};
}
void UnaryEltwiseTppEmitter::validate_arguments(const std::vector<size_t> &in, const std::vector<size_t> &out) const {
OV_CPU_JIT_EMITTER_ASSERT(in.size() == 1, "Expects 1 input registers, got " + std::to_string(in.size()));
OV_CPU_JIT_EMITTER_ASSERT(out.size() == 1, "Expects 1 output register, got " + std::to_string(out.size()));
}
ReduceTppEmitter::ReduceTppEmitter(jit_generator* h, cpu_isa_t isa, const ExpressionPtr& expr) :
UnaryEltwiseTppEmitter(h, isa, expr) {
m_compile_flags = LIBXSMM_MELTW_FLAG_UNARY_REDUCE_ROWS;
// No need to set ldo for reduce, it is always assumed = 1 inside the kernel
// m_shape.ldo = 1;
}
} // namespace intel_cpu
} // namespace ov

109
src/plugins/intel_cpu/src/emitters/tpp/x64/jit_eltwise_emitters.hpp Normal file
View File

@ -0,0 +1,109 @@
// Copyright (C) 2020-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "jit_tpp_emitter.hpp"
namespace ov {
namespace intel_cpu {
class BinaryEltwiseTppEmitter : public TppEmitter {
public:
BinaryEltwiseTppEmitter(dnnl::impl::cpu::x64::jit_generator* h,
dnnl::impl::cpu::x64::cpu_isa_t isa,
const ov::snippets::lowered::ExpressionPtr& expr);
size_t get_inputs_num() const override { return 2; }
static void execute_kernel(libxsmm_meltwfunction_binary eltwise_kernel, void *in0, void *in1, void *out0);
const uintptr_t get_execute_function_ptr() const override { return reinterpret_cast<const uintptr_t>(execute_kernel); }
const uintptr_t get_compiled_kernel_ptr() const override;
static std::set<std::vector<element::Type>> get_supported_precisions(const std::shared_ptr<ov::Node>& node = nullptr);
protected:
libxsmm_meltw_binary_shape m_shape;
libxsmm_meltw_binary_type m_op_type;
void validate_arguments(const std::vector<size_t> &in, const std::vector<size_t> &out) const override;
static libxsmm_blasint get_broadcasted_dim(libxsmm_blasint dim0, libxsmm_blasint dim1, std::pair<bool, bool>& bcast_flags);
};
class UnaryEltwiseTppEmitter : public TppEmitter {
public:
UnaryEltwiseTppEmitter(dnnl::impl::cpu::x64::jit_generator* h,
dnnl::impl::cpu::x64::cpu_isa_t isa,
const ov::snippets::lowered::ExpressionPtr& expr);
size_t get_inputs_num() const override { return 1; }
static void execute_kernel(libxsmm_meltwfunction_unary eltwise_kernel, void *in0, void *out0);
const uintptr_t get_compiled_kernel_ptr() const override {
return COMPILE_TPP_KERNEL(libxsmm_dispatch_meltw_unary(m_op_type,
m_shape,
m_compile_flags));
}
const uintptr_t get_execute_function_ptr() const override { return reinterpret_cast<const uintptr_t>(execute_kernel); }
static std::set<std::vector<element::Type>> get_supported_precisions(const std::shared_ptr<ov::Node>& node = nullptr);
protected:
libxsmm_meltw_unary_shape m_shape;
libxsmm_meltw_unary_type m_op_type;
void validate_arguments(const std::vector<size_t> &in, const std::vector<size_t> &out) const override;
};
class ReduceTppEmitter : public UnaryEltwiseTppEmitter {
public:
ReduceTppEmitter(dnnl::impl::cpu::x64::jit_generator* h,
dnnl::impl::cpu::x64::cpu_isa_t isa,
const ov::snippets::lowered::ExpressionPtr& expr);
};
class ReferenceUnaryEltwiseTppEmitter : public UnaryEltwiseTppEmitter {
public:
// Note: can create template to suppport different executor signatures
typedef std::function<float(float)> executor_function;
ReferenceUnaryEltwiseTppEmitter(dnnl::impl::cpu::x64::jit_generator* h,
dnnl::impl::cpu::x64::cpu_isa_t isa,
const ov::snippets::lowered::ExpressionPtr& expr,
executor_function executor) :
UnaryEltwiseTppEmitter(h, isa, expr), executor(std::move(executor)) {
}
static void execute_unary_eltw_kernel(ReferenceUnaryEltwiseTppEmitter* ref_emitter, void *in0, void *out0) {
assert(ref_emitter);
// Note: we can instantiate template with different precision combinations here, if we need to
ref_emitter->evaluate_reference_impl(reinterpret_cast<float*>(in0), reinterpret_cast<float*>(out0));
}
const uintptr_t get_execute_function_ptr() const override {
return reinterpret_cast<const uintptr_t>(execute_unary_eltw_kernel);
}
const uintptr_t get_compiled_kernel_ptr() const override {
return reinterpret_cast<const uintptr_t>(this);
}
private:
executor_function executor{nullptr};
template<class Tin, class Tout,
typename std::enable_if<!std::is_same<Tin, Tout>::value || !std::is_same<Tin, float>::value, bool>::type = true>
void evaluate_reference_impl(Tin* in0, Tout* out0) {
for (int n = 0; n < m_shape.n; n++) {
auto in0_row = in0;
auto out0_row = out0;
for (int m = 0; m < m_shape.m; m++)
out0_row[m] = static_cast<Tout>(executor(static_cast<float>(in0_row[m])));
in0 += m_shape.ldi;
out0 += m_shape.ldo;
}
}
void evaluate_reference_impl(float* in0, float* out0) {
for (int n = 0; n < m_shape.n; n++) {
auto in0_row = in0;
auto out0_row = out0;
for (int m = 0; m < m_shape.m; m++)
out0_row[m] = executor(in0_row[m]);
in0 += m_shape.ldi;
out0 += m_shape.ldo;
}
}
};
} // namespace intel_cpu
} // namespace ov

30
src/plugins/intel_cpu/src/emitters/tpp/x64/jit_scalar_emitter.cpp Normal file
View File

@ -0,0 +1,30 @@
// Copyright (C) 2020-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "jit_scalar_emitter.hpp"
#include "emitters/snippets/x64/jit_snippets_emitters.hpp"
using namespace Xbyak;
using namespace dnnl::impl;
using namespace dnnl::impl::cpu::x64;
namespace ov {
namespace intel_cpu {
ScalarTppEmitter::ScalarTppEmitter(jit_generator* h, cpu_isa_t isa, const ov::snippets::lowered::ExpressionPtr& expr)
: jit_emitter(h, isa) {
push_arg_entry_of("scalar_tpp", jit_scalar_emitter::read_value(expr), false);
in_out_type_ = gpr_to_gpr;
}
void ScalarTppEmitter::emit_impl(const std::vector<size_t>& in, const std::vector<size_t>& out) const {
const auto it = entry_map_.find("scalar_tpp");
OV_CPU_JIT_EMITTER_ASSERT(it != entry_map_.end(), "Value has not been found in the table");
const auto& out_reg = Reg64(static_cast<int>(out[0]));
h->mov(out_reg, p_table);
h->add(out_reg, (*it).second.off);
}
} // namespace intel_cpu
} // namespace ov

23
src/plugins/intel_cpu/src/emitters/tpp/x64/jit_scalar_emitter.hpp Normal file
View File

@ -0,0 +1,23 @@
// Copyright (C) 2020-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "snippets/lowered/expression.hpp"
#include "emitters/plugin/x64/jit_emitter.hpp"
namespace ov {
namespace intel_cpu {
class ScalarTppEmitter : public jit_emitter {
public:
ScalarTppEmitter(dnnl::impl::cpu::x64::jit_generator* h,
dnnl::impl::cpu::x64::cpu_isa_t isa,
const ov::snippets::lowered::ExpressionPtr& expr);
size_t get_inputs_num() const override {return 0;}
size_t aux_gprs_count() const override {return 1;}
private:
void emit_impl(const std::vector<size_t>& in, const std::vector<size_t>& out) const override;
};
} // namespace intel_cpu
} // namespace ov

138
src/plugins/intel_cpu/src/emitters/tpp/x64/jit_tpp_emitter.cpp Normal file
View File

@ -0,0 +1,138 @@
// Copyright (C) 2020-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "jit_tpp_emitter.hpp"
#include "snippets/lowered/port_descriptor.hpp"
#include "transformations/tpp/x64/op/eltwise.hpp"
using namespace Xbyak;
using namespace dnnl::impl;
using namespace dnnl::impl::cpu::x64;
namespace ov {
namespace intel_cpu {
VectorDims TppEmitter::get_projected_subtensor(const snippets::lowered::PortDescriptorPtr& desc) {
auto shape = desc->get_shape();
auto subtensor = desc->get_subtensor();
// Note: Scalar is a special case, so it's easier to prepend shapes than to handle it explicitly
if (shape.size() == 1)
shape.insert(shape.begin(), 1);
if (subtensor.size() == 1)
subtensor.insert(subtensor.begin(), 1);
OV_CPU_JIT_EMITTER_ASSERT(subtensor.size() <= shape.size() && !subtensor.empty(),
"Invalid subtensor + shape combination");
auto shape_it = shape.rbegin();
for (auto sub_it = subtensor.rbegin(); sub_it != subtensor.rend(); sub_it++, shape_it++) {
*sub_it = std::min(*sub_it, *shape_it);
}
return subtensor;
}
TppEmitter::TppEmitter(dnnl::impl::cpu::x64::jit_generator* h,
dnnl::impl::cpu::x64::cpu_isa_t isa,
const ov::snippets::lowered::ExpressionPtr& expr)
: jit_emitter(h, isa) {
in_out_type_ = emitter_in_out_map::gpr_to_gpr;
const auto& node = expr->get_node();
const auto& tpp_mod = std::dynamic_pointer_cast<tpp::modifier::TensorProcessingPrimitive>(node);
OV_CPU_JIT_EMITTER_ASSERT(tpp_mod, "Invoked with invalid node type");
const auto num_ins = node->get_input_size();
const auto num_outs = node->get_output_size();
num_kernel_args = static_cast<int>(num_ins + num_outs);
io_dtypes.resize(num_kernel_args);
io_strides.resize(num_kernel_args);
io_offsets.resize(num_kernel_args);
io_port_descriptors.resize(num_kernel_args);
// Note: this is needed mostly for Reduce operations, since they allow the last subternsor dim to be FULL_DIM;
auto replace_full_dim = [](size_t dim, size_t replace_dim) {
if (dim == snippets::lowered::PortDescriptor::ServiceDimensions::FULL_DIM)
return replace_dim;
return dim;
};
for (size_t i = 0; i < num_ins; i++) {
io_dtypes[i] = ov_to_xsmm_dtype(node->get_input_element_type(i));
io_offsets[i] = tpp_mod->get_input_offset(i);
io_strides[i] = replace_full_dim(tpp_mod->get_input_stride(i),
expr->get_input_port_descriptor(i)->get_shape().back());
io_port_descriptors[i] = expr->get_input_port_descriptor(i);
}
for (size_t i = 0; i < num_outs; i++) {
const auto i_off = i + num_ins;
io_dtypes[i_off] = ov_to_xsmm_dtype(node->get_output_element_type(i));
io_offsets[i_off] = tpp_mod->get_output_offset(i);
io_strides[i_off] = replace_full_dim(tpp_mod->get_output_stride(i),
expr->get_output_port_descriptor(i)->get_shape().back());
io_port_descriptors[i_off] = expr->get_output_port_descriptor(i);
}
}
void TppEmitter::emit_code(const std::vector<size_t> &in, const std::vector<size_t> &out) const {
validate_arguments(in, out);
emit_impl(in, out);
}
void TppEmitter::emit_impl(const std::vector<size_t>& in, const std::vector<size_t>& out) const {
internal_call_preamble();
// Note: 4 args is currently enough for unary and binary ops.
// To enable ternary ops, we will have to pass extra regs on stack for Windows,
std::array<Xbyak::Reg64, 4> abi_params{abi_param1, abi_param2, abi_param3, abi_param4};
// save function address in gpr to pass in call instruction
h->mov(h->rbp, get_execute_function_ptr());
int aux_xmm_count = 0;
for (auto reg_idx : in)
h->uni_vmovq(Xmm(aux_xmm_count++), Reg64(static_cast<int>(reg_idx)));
for (auto reg_idx : out)
h->uni_vmovq(Xmm(aux_xmm_count++), Reg64(static_cast<int>(reg_idx)));
OV_CPU_JIT_EMITTER_ASSERT(aux_xmm_count == num_kernel_args, "offsets for some inputs/outputs were not set");
OV_CPU_JIT_EMITTER_ASSERT(aux_xmm_count < static_cast<int>(abi_params.size()), "too many input/output arguments. More abi params required");
const auto data_ptr_reg = [&](Xmm xmm, Xbyak::Reg64 reg, size_t bytes_offset) {
h->uni_vmovq(reg, xmm);
if (bytes_offset) h->add(reg, bytes_offset);
};
const auto& compiled_kernel = get_compiled_kernel_ptr();
OV_CPU_JIT_EMITTER_ASSERT(compiled_kernel, "Failed to compile libxsmm_kernel");
h->mov(abi_params[0], compiled_kernel);
for (int i = 0; i < num_kernel_args; i++)
data_ptr_reg(Xmm(i), abi_params[i + 1], io_offsets[i]);
#ifdef _WIN32
// Before function call we should allocate stack area for
// - register parameters - ABI parameters (shadow space)
// - stack parameters - remaining parameters
// Since we currently have <= 4 args (kernel_ptr, in0, in1, out0), then nothing is passed on stack
h->sub(h->rsp, (num_kernel_args + 1) * gpr_size);
#endif
internal_call_rsp_align();
h->call(h->rbp);
internal_call_rsp_restore();
#ifdef _WIN32
h->add(h->rsp, (num_kernel_args + 1) * gpr_size);
#endif
internal_call_postamble();
}
libxsmm_datatype TppEmitter::ov_to_xsmm_dtype(ov::element::Type_t elemet_type) {
switch (elemet_type) {
case ov::element::Type_t::f32 : return LIBXSMM_DATATYPE_F32;
case ov::element::Type_t::bf16 : return LIBXSMM_DATATYPE_BF16;
case ov::element::Type_t::i8 : return LIBXSMM_DATATYPE_I8;
case ov::element::Type_t::u8 : return LIBXSMM_DATATYPE_U8;
default:
OV_CPU_JIT_EMITTER_THROW("Attempt to convert unsupported ov data type");
return LIBXSMM_DATATYPE_IMPLICIT;
}
}
} // namespace intel_cpu
} // namespace ov

62
src/plugins/intel_cpu/src/emitters/tpp/x64/jit_tpp_emitter.hpp Normal file
View File

@ -0,0 +1,62 @@
// Copyright (C) 2020-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "snippets/lowered/linear_ir.hpp"
#include "snippets/lowered/expression.hpp"
#include "emitters/plugin/x64/jit_emitter.hpp"
#include "libxsmm.h"
namespace ov {
namespace intel_cpu {
// Note: The macro allows to automatically set appropriate environment variables for TPP/Libxsmm kernel compilation
// All TPP kernels must be compiled using this macro.
// * LIBXSMM_X86_HINT_USE_HIGH_PREC_ELTWISE_APPROX enables more accurate exp approximation and exact division in TPP
// * LIBXSMM_GEMM_K_A_PF_DIST allows to tweak prefetching for GEMM kernels
#define COMPILE_TPP_KERNEL(...) \
[&]() { \
setenv("LIBXSMM_X86_HINT_USE_HIGH_PREC_ELTWISE_APPROX", "1", 1); \
setenv("LIBXSMM_GEMM_K_A_PF_DIST", "4", 1); \
auto res = reinterpret_cast<const uintptr_t>(__VA_ARGS__); \
unsetenv("LIBXSMM_X86_HINT_USE_HIGH_PREC_ELTWISE_APPROX"); \
unsetenv("LIBXSMM_GEMM_K_A_PF_DIST"); \
return res; \
}()
class DebugTppEmitter;
class TppEmitter : public jit_emitter {
friend DebugTppEmitter;
public:
TppEmitter(dnnl::impl::cpu::x64::jit_generator* h,
dnnl::impl::cpu::x64::cpu_isa_t isa,
const ov::snippets::lowered::ExpressionPtr& expr);
void emit_code(const std::vector<size_t> &in,
const std::vector<size_t> &out) const;
static libxsmm_datatype ov_to_xsmm_dtype(ov::element::Type_t elemet_type);
protected:
void emit_impl(const std::vector<size_t>& in,
const std::vector<size_t>& out) const override;
static ov::snippets::VectorDims get_projected_subtensor(const snippets::lowered::PortDescriptorPtr& desc);
/// Generate function pointer to the thin wrapper over the kernel that is called in runtime on every iteration
virtual const uintptr_t get_execute_function_ptr() const = 0;
/// Generate pointer to compiled kernel
virtual const uintptr_t get_compiled_kernel_ptr() const = 0;
std::vector<size_t> io_offsets{};
std::vector<libxsmm_datatype> io_dtypes{};
// Note: almost all emitters use fp32 for internal computations
libxsmm_datatype exec_dtype {LIBXSMM_DATATYPE_F32};
// aka leading dimensions
std::vector<size_t> io_strides{};
std::vector<snippets::lowered::PortDescriptorPtr> io_port_descriptors{};
// compile flags has the same type for all eltwises, so we keep them in the base class
libxsmm_bitfield m_compile_flags {0};
int num_kernel_args = 0;
};
} // namespace intel_cpu
} // namespace ov

35
src/plugins/intel_cpu/src/nodes/subgraph.cpp
View File

@ -44,6 +44,13 @@
std::mutex err_print_lock;
#endif
#ifdef SNIPPETS_LIBXSMM_TPP
#include "transformations/tpp/x64/pass/brgemm_to_brgemm_tpp.hpp"
#include "transformations/tpp/x64/pass/eltwise_to_eltwise_tpp.hpp"
#include "transformations/tpp/x64/pass/scalar_to_scalar_tpp.hpp"
#include "transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.hpp"
#endif
using namespace dnnl::impl::utils;
using namespace dnnl::impl::cpu;
using namespace dnnl::impl::cpu::x64;
@ -355,6 +362,7 @@ void Snippet::initOptimalPrimitiveDescriptor() {
SNIPPETS_REGISTER_PASS_RELATIVE(Place::After, ov::snippets::pass::MatMulToBrgemm,
pass::EnforcePrecision, element::f32, element::bf16);
}
SNIPPETS_REGISTER_PASS_RELATIVE(Place::Before, ov::snippets::pass::PropagatePrecision,
ov::intel_cpu::pass::BrgemmToBrgemmCPU);
SNIPPETS_REGISTER_PASS_RELATIVE(Place::After, ov::intel_cpu::pass::BrgemmToBrgemmCPU,
@ -362,6 +370,15 @@ void Snippet::initOptimalPrimitiveDescriptor() {
SNIPPETS_REGISTER_PASS_ABSOLUTE(Place::PipelineEnd, ov::intel_cpu::pass::RemoveConverts);
SNIPPETS_REGISTER_PASS_ABSOLUTE(Place::PipelineEnd, ov::intel_cpu::pass::MulAddToFMA);
#ifdef SNIPPETS_LIBXSMM_TPP
SNIPPETS_REGISTER_PASS_RELATIVE(Place::Before, ov::intel_cpu::pass::BrgemmToBrgemmCPU,
ov::intel_cpu::tpp::pass::BrgemmToBrgemmTPP);
// Note: There could be several ConvertConstantsToScalars instances in the pipeline
SNIPPETS_REGISTER_PASS_ABSOLUTE(Place::PipelineEnd, ov::intel_cpu::tpp::pass::ScalarToScalarTPP);
SNIPPETS_REGISTER_PASS_RELATIVE(Place::After, ov::intel_cpu::tpp::pass::BrgemmToBrgemmTPP,
ov::intel_cpu::tpp::pass::EltwiseToEltwiseTPP);
#endif
#undef SNIPPETS_REGISTER_PASS
std::vector<ov::element::Type> input_precisions;
@ -377,8 +394,14 @@ void Snippet::initOptimalPrimitiveDescriptor() {
snippetAttrs.snippet->data_flow_transformations(in_blocked_shapes, input_precisions, output_precisions, backend_passes);
// Note: minimal JIT work amount is a predefined value that describes the number of kernel iterations (work amount)
// needed to cover kernel call overhead. It is used for balancing between parallel and JIT work amounts in domain optimization.
#ifdef SNIPPETS_LIBXSMM_TPP
const auto& lir = snippetAttrs.snippet->convert_body_to_linear_ir(static_cast<size_t>(parallel_get_max_threads()), 256,
std::make_shared<snippets::CPUShapeInferSnippetsFactory>());
lir->set_loop_depth(std::min(2ul, lir->get_master_shape().size()));
#else
snippetAttrs.snippet->convert_body_to_linear_ir(static_cast<size_t>(parallel_get_max_threads()), 256,
std::make_shared<snippets::CPUShapeInferSnippetsFactory>());
#endif
}
ov::element::Type Snippet::getRuntimePrecision() const {
@ -641,9 +664,19 @@ void Snippet::SnippetJitExecutor::generate(const jit_snippets_compile_args* jcp)
SNIPPETS_REGISTER_PASS_RELATIVE(Place::After, ov::intel_cpu::pass::FuseLoadStoreConvert,
ov::intel_cpu::pass::SetBrgemmCopyBBuffersShape);
schedule = snippetAttrs.snippet->generate_from_linear_ir(std::make_shared<ov::snippets::lowered::pass::PassConfig>(),
auto lowering_config = std::make_shared<ov::snippets::lowered::pass::PassConfig>();
#ifdef SNIPPETS_LIBXSMM_TPP
// Note: temporary disabled. Re-enable after ticket 132833 is resolved
lowering_config->disable<ov::snippets::lowered::pass::OptimizeDomain>();
SNIPPETS_REGISTER_PASS_RELATIVE(Place::After, ov::intel_cpu::pass::FuseLoadStoreConvert,
ov::intel_cpu::tpp::pass::SetTPPLeadingDim);
#endif
schedule = snippetAttrs.snippet->generate_from_linear_ir(lowering_config,
backend_passes,
reinterpret_cast<const void*>(jcp));
#undef SNIPPETS_REGISTER_PASS_RELATIVE
}
bool Snippet::SnippetJitExecutor::schedule_created() {

4
src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_copy_b.cpp
View File

@ -16,7 +16,7 @@ namespace intel_cpu {
intel_cpu::BrgemmCopyB::BrgemmCopyB(const Output<Node>& x, const element::Type src_type, const Type type,
const size_t offset_in, const size_t offset_out0, const size_t offset_out1,
std::vector<size_t> layout_input, const size_t blk_size_k, const size_t blk_size_n)
: snippets::op::MemoryAccess({x}, 1, type == Type::WithCompensations ? 2 : 1),
: snippets::modifier::MemoryAccess(1, type == Type::WithCompensations ? 2 : 1), op::Op({x}),
m_type(type), m_src_type(src_type) {
m_brgemmVNNIFactor = 4 / m_src_type.size();
set_output_size(type == Type::WithCompensations ? 2 : 1);
@ -32,7 +32,7 @@ intel_cpu::BrgemmCopyB::BrgemmCopyB(const Output<Node>& x, const element::Type s
intel_cpu::BrgemmCopyB::BrgemmCopyB(const Output<Node>& x, const element::Type src_type, const Type type,
const PortDescriptor& desc_in0, const PortDescriptor& desc_out0, const PortDescriptor& desc_out1,
std::vector<size_t> layout_input, const size_t blk_size_k, const size_t blk_size_n)
: snippets::op::MemoryAccess({x}, 1, type == Type::WithCompensations ? 2 : 1),
: snippets::modifier::MemoryAccess(1, type == Type::WithCompensations ? 2 : 1), op::Op({x}),
m_type(type), m_src_type(src_type) {
m_brgemmVNNIFactor = 4 / m_src_type.size();
set_output_size(type == Type::WithCompensations ? 2 : 1);

4
src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_copy_b.hpp
View File

@ -18,9 +18,9 @@ namespace intel_cpu {
OneDNN requiers data repacking for second input of Brgemm with input non-fp32 precisions.
* @ingroup snippets
*/
class BrgemmCopyB : public snippets::op::MemoryAccess {
class BrgemmCopyB : public snippets::modifier::MemoryAccess, public ov::op::Op {
public:
OPENVINO_OP("BrgemmCopyB", "SnippetsOpset", snippets::op::MemoryAccess);
OPENVINO_OP("BrgemmCopyB", "SnippetsOpset");
enum Type {
OnlyRepacking, // Just data repacking - one output

24
src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_cpu.cpp
View File

@ -17,7 +17,7 @@ BrgemmCPU::BrgemmCPU(const Output<Node>& A, const Output<Node>& B, const Type ty
const size_t offset_a, const size_t offset_b, const size_t offset_c,
std::vector<size_t> layout_a, std::vector<size_t> layout_b, std::vector<size_t> layout_c,
const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n, const float beta)
: Brgemm(), m_type(type), m_beta(beta) {
: Brgemm(), m_type(type) {
// We call default ctor of Brgemm class to avoid incorrect shape infer in constructor_validate_and_type_infer() call
set_arguments({A, B});
set_output_size(1);
@ -27,13 +27,14 @@ BrgemmCPU::BrgemmCPU(const Output<Node>& A, const Output<Node>& B, const Type ty
set_output_port_descriptor({0, offset_c}, 0);
compute_block_size_values(blk_size_m, blk_size_k, blk_size_n);
custom_constructor_validate_and_infer_types(std::move(layout_a), std::move(layout_b), std::move(layout_c));
set_beta(beta);
}
BrgemmCPU::BrgemmCPU(const Output<Node>& A, const Output<Node>& B, const Output<Node>& scratch, const Type type,
const size_t offset_a, const size_t offset_b, const size_t offset_scratch, const size_t offset_c,
std::vector<size_t> layout_a, std::vector<size_t> layout_b, std::vector<size_t> layout_c,
const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n, const float beta)
: Brgemm(), m_type(type), m_beta(beta) {
: Brgemm(), m_type(type) {
set_arguments({A, B, scratch});
set_output_size(1);
ctor_initialize(std::set<size_t>{0, 1, 2}, std::set<size_t>{0});
@ -43,32 +44,35 @@ BrgemmCPU::BrgemmCPU(const Output<Node>& A, const Output<Node>& B, const Output<
set_input_port_descriptor({0, offset_scratch}, 2);
compute_block_size_values(blk_size_m, blk_size_k, blk_size_n);
custom_constructor_validate_and_infer_types(std::move(layout_a), std::move(layout_b), std::move(layout_c));
set_beta(beta);
}
BrgemmCPU::BrgemmCPU(const Output<Node>& A, const Output<Node>& B, const Type type,
const PortDescriptor& desc_a, const PortDescriptor& desc_b, const PortDescriptor& desc_c,
std::vector<size_t> layout_a, std::vector<size_t> layout_b, std::vector<size_t> layout_c,
const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n, const float beta)
: Brgemm(), m_type(type), m_beta(beta) {
: Brgemm(), m_type(type) {
set_arguments({A, B});
set_output_size(1);
m_input_ports = {{0, desc_a}, {1, desc_b}};
m_output_ports = {{0, desc_c}};
compute_block_size_values(blk_size_m, blk_size_k, blk_size_n);
custom_constructor_validate_and_infer_types(std::move(layout_a), std::move(layout_b), std::move(layout_c));
set_beta(beta);
}
BrgemmCPU::BrgemmCPU(const Output<Node>& A, const Output<Node>& B, const Output<Node>& scratch, const Type type,
const PortDescriptor& desc_a, const PortDescriptor& desc_b, const PortDescriptor& desc_scratch, const PortDescriptor& desc_c,
std::vector<size_t> layout_a, std::vector<size_t> layout_b, std::vector<size_t> layout_c,
const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n, const float beta)
: Brgemm(), m_type(type), m_beta(beta) {
: Brgemm(), m_type(type) {
set_arguments({A, B, scratch});
set_output_size(1);
m_input_ports = {{0, desc_a}, {1, desc_b}, {2, desc_scratch}};
m_output_ports = {{0, desc_c}};
compute_block_size_values(blk_size_m, blk_size_k, blk_size_n);
custom_constructor_validate_and_infer_types(std::move(layout_a), std::move(layout_b), std::move(layout_c));
set_beta(beta);
}
void BrgemmCPU::custom_constructor_validate_and_infer_types(std::vector<size_t> layout_a, std::vector<size_t> layout_b, std::vector<size_t> layout_c) {
@ -89,14 +93,6 @@ void BrgemmCPU::custom_constructor_validate_and_infer_types(std::vector<size_t>
validate_with_scratchpad(planar_input_shapes[1].get_shape());
}
void BrgemmCPU::compute_block_size_values(const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n) {
const auto input_shape_0 = snippets::utils::get_planar_pshape(input(0)).get_shape();
const auto input_shape_1 = snippets::utils::get_planar_pshape(input(1)).get_shape();
m_M_blk = blk_size_m != 0 ? blk_size_m : *(input_shape_0.rbegin() + 1);
m_K_blk = blk_size_k != 0 ? blk_size_k : *input_shape_0.rbegin();
m_N_blk = blk_size_n != 0 ? blk_size_n : *input_shape_1.rbegin();
}
void BrgemmCPU::validate_and_infer_types() {
INTERNAL_OP_SCOPE(BrgemmCPU_validate_and_infer_types);
validate_inputs();
@ -173,10 +169,6 @@ size_t BrgemmCPU::get_offset_scratch() const {
bool BrgemmCPU::visit_attributes(AttributeVisitor& visitor) {
Brgemm::visit_attributes(visitor);
visitor.on_attribute("blk_M", m_M_blk);
visitor.on_attribute("blk_K", m_K_blk);
visitor.on_attribute("blk_N", m_N_blk);
visitor.on_attribute("beta", m_beta);
return true;
}

14
src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_cpu.hpp
View File

@ -51,15 +51,6 @@ public:
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
Type get_type() const { return m_type; }
size_t get_m_block_size() const { return m_M_blk; }
size_t get_k_block_size() const { return m_K_blk; }
size_t get_n_block_size() const { return m_N_blk; }
float get_beta() const { return m_beta; }
void set_m_block_size(size_t block_size) { m_M_blk = block_size; }
void set_k_block_size(size_t block_size) { m_K_blk = block_size; }
void set_n_block_size(size_t block_size) { m_N_blk = block_size; }
void set_beta(float beta) { m_beta = beta; }
bool is_with_compensations() const { return m_type == Type::WithCompensations; }
bool is_with_data_repacking() const { return m_type != Type::Floating; }
@ -75,15 +66,10 @@ public:
private:
void custom_constructor_validate_and_infer_types(std::vector<size_t> layout_a, std::vector<size_t> layout_b, std::vector<size_t> layout_c);
void compute_block_size_values(const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n);
void validate_with_scratchpad(const ov::Shape& shape_b) const;
void validate_inputs() const;
Type m_type = Type::Floating;
size_t m_M_blk = 0;
size_t m_K_blk = 0;
size_t m_N_blk = 0;
float m_beta = 1.f;
};
} // namespace intel_cpu

7
src/plugins/intel_cpu/src/transformations/snippets/x64/pass/brgemm_to_brgemm_cpu.cpp
View File

@ -10,6 +10,7 @@
#include "snippets/op/brgemm.hpp"
#include "transformations/snippets/x64/op/brgemm_copy_b.hpp"
#include "transformations/snippets/x64/op/brgemm_cpu.hpp"
#include "transformations/tpp/x64/op/modifiers.hpp"
#include "openvino/core/rt_info.hpp"
#include "openvino/pass/pattern/op/wrap_type.hpp"
@ -43,8 +44,10 @@ void set_port_desc(const T& port, Args... params) {
pass::BrgemmToBrgemmCPU::BrgemmToBrgemmCPU() {
MATCHER_SCOPE(BrgemmToBrgemmCPU);
auto m_brgemm = ov::pass::pattern::wrap_type<snippets::op::Brgemm>();
auto is_not_tpp = [](const Output<Node>& out) {
return !std::dynamic_pointer_cast<const intel_cpu::tpp::modifier::TensorProcessingPrimitive>(out.get_node_shared_ptr());
};
auto m_brgemm = ov::pass::pattern::wrap_type<snippets::op::Brgemm>(is_not_tpp);
auto callback = [=](ov::pass::pattern::Matcher& m) {
OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "ov::intel_cpu::pass::BrgemmToBrgemmCPU")

27
src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_blocking.cpp
View File

@ -12,6 +12,7 @@
#include "snippets/snippets_isa.hpp"
#include "snippets/utils.hpp"
#include "transformations/snippets/x64/op/brgemm_cpu.hpp"
#include "transformations/tpp/x64/op/brgemm.hpp"
namespace ov {
namespace intel_cpu {
@ -39,7 +40,7 @@ void BrgemmBlocking::move_new_memory_buffer(snippets::lowered::LinearIR& linear_
bool BrgemmBlocking::run(LinearIR& linear_ir, LinearIR::constExprIt begin, LinearIR::constExprIt end) {
OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "Snippets::BrgemmBlocking")
const auto& loop_manager = linear_ir.get_loop_manager();
auto blocking_loop_exists = [&](const ExpressionPtr& brgemm_expr, const std::shared_ptr<ov::intel_cpu::BrgemmCPU>& brgemm) {
auto blocking_loop_exists = [&](const ExpressionPtr& brgemm_expr, const std::shared_ptr<snippets::op::Brgemm>& brgemm) {
auto check_port = [&](const LoopPort& p) {
return p.expr_port->get_expr() == brgemm_expr && ov::snippets::utils::one_of(p.dim_idx, 0ul, 1ul);
};
@ -58,9 +59,13 @@ bool BrgemmBlocking::run(LinearIR& linear_ir, LinearIR::constExprIt begin, Linea
bool modified = false;
for (auto expr_it = begin; expr_it != end; expr_it++) {
const auto& brgemm_expr = *expr_it;
const auto brgemm = ov::as_type_ptr<ov::intel_cpu::BrgemmCPU>(brgemm_expr->get_node());
const auto& node = brgemm_expr->get_node();
const auto brgemm = ov::as_type_ptr<snippets::op::Brgemm>(node);
const auto brgemm_cpu = ov::as_type_ptr<intel_cpu::BrgemmCPU>(node);
if (!brgemm || blocking_loop_exists(brgemm_expr, brgemm))
continue;
OPENVINO_ASSERT(ov::is_type<intel_cpu::BrgemmCPU>(node) || ov::is_type<intel_cpu::tpp::op::BrgemmTPP>(node),
"Detected invalid Brgemm operation: ops must be assigned to a backend when blocking is performed.");
const auto& in_0_desc = brgemm_expr->get_input_port_descriptor(0);
const auto& in_1_desc = brgemm_expr->get_input_port_descriptor(1);
@ -88,12 +93,14 @@ bool BrgemmBlocking::run(LinearIR& linear_ir, LinearIR::constExprIt begin, Linea
auto loop_begin_it = expr_it, loop_end_it = std::next(expr_it);
std::vector<LoopPort> entries{LoopPort(brgemm_expr->get_input_port(0), true),
LoopPort(brgemm_expr->get_input_port(1), false)};
if (brgemm->is_with_compensations()) {
if (brgemm_cpu) {
if (brgemm_cpu->is_with_compensations()) {
entries.emplace_back(brgemm_expr->get_input_port(2), false);
} else if (brgemm->is_amx()) {
} else if (brgemm_cpu->is_amx()) {
move_new_memory_buffer(linear_ir, expr_it);
loop_begin_it = std::prev(expr_it);
}
}
std::vector<LoopPort> exits{LoopPort(brgemm_expr->get_output_port(0), true)};
loop_manager->mark_loop(loop_begin_it, loop_end_it, m, block_size_m, 1, entries, exits);
};
@ -112,12 +119,14 @@ bool BrgemmBlocking::run(LinearIR& linear_ir, LinearIR::constExprIt begin, Linea
auto loop_begin_it = expr_it, loop_end_it = std::next(expr_it);
std::vector<LoopPort> entries{LoopPort(brgemm_expr->get_input_port(0), false),
LoopPort(brgemm_expr->get_input_port(1), true)};
if (brgemm->is_with_compensations()) {
if (brgemm_cpu) {
if (brgemm_cpu->is_with_compensations()) {
entries.emplace_back(brgemm_expr->get_input_port(2), true);
} else if (brgemm->is_amx()) {
} else if (brgemm_cpu->is_amx()) {
move_new_memory_buffer(linear_ir, expr_it);
loop_begin_it = std::prev(expr_it);
}
}
std::vector<LoopPort> exits{LoopPort(brgemm_expr->get_output_port(0), true)};
loop_manager->mark_loop(loop_begin_it, loop_end_it, n, block_size_n, 0, entries, exits);
};
@ -138,12 +147,14 @@ bool BrgemmBlocking::run(LinearIR& linear_ir, LinearIR::constExprIt begin, Linea
auto loop_begin_it = expr_it, loop_end_it = std::next(expr_it);
std::vector<LoopPort> entries{LoopPort(brgemm_expr->get_input_port(0), true, 0),
LoopPort(brgemm_expr->get_input_port(1), true, 1)};
if (brgemm->is_with_compensations()) {
if (brgemm_cpu) {
if (brgemm_cpu->is_with_compensations()) {
entries.emplace_back(brgemm_expr->get_input_port(2), false, 1);
} else if (brgemm->is_amx()) {
} else if (brgemm_cpu->is_amx()) {
move_new_memory_buffer(linear_ir, expr_it);
loop_begin_it = std::prev(expr_it);
}
}
std::vector<LoopPort> exits{LoopPort(brgemm_expr->get_output_port(0), false)};
const auto id = loop_manager->mark_loop(loop_begin_it, loop_end_it, k, block_size_k, entries, exits);
const auto loop_info = loop_manager->get_loop_info(id);

2
src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/cpu_iter_handlers.cpp
View File

@ -18,7 +18,7 @@ SetBrgemmBeta::SetBrgemmBeta(float beta) : snippets::lowered::pass::RangedPass()
bool SetBrgemmBeta::run(LinearIR& linear_ir, LinearIR::constExprIt begin, LinearIR::constExprIt end) {
for (auto expr_it = begin; expr_it != end; ++expr_it) {
const auto& expr = expr_it->get();
if (const auto brgemm = ov::as_type_ptr<ov::intel_cpu::BrgemmCPU>(expr->get_node())) {
if (const auto brgemm = ov::as_type_ptr<ov::snippets::op::Brgemm>(expr->get_node())) {
brgemm->set_beta(m_beta);
}
}

13
src/plugins/intel_cpu/src/transformations/snippets/x64/pass/mul_add_to_fma.cpp
View File

@ -11,15 +11,22 @@
#include "openvino/pass/pattern/op/wrap_type.hpp"
#include "openvino/pass/pattern/matcher.hpp"
#include "openvino/core/rt_info.hpp"
#include "snippets/op/memory_access.hpp"
ov::intel_cpu::pass::MulAddToFMA::MulAddToFMA() {
MATCHER_SCOPE(MulAddToFMA);
auto is_not_memory_access = [](const Output<Node>& out) {
return !std::dynamic_pointer_cast<const snippets::modifier::MemoryAccess>(out.get_node_shared_ptr());
};
auto mul_input_1 = ov::pass::pattern::any_input();
auto mul_input_2 = ov::pass::pattern::any_input();
auto mul_m = ov::pass::pattern::wrap_type<opset1::Multiply>({ mul_input_1, mul_input_2 }, ov::pass::pattern::consumers_count(1));
auto mul_m = ov::pass::pattern::wrap_type<opset1::Multiply>({ mul_input_1, mul_input_2 },
[=](const Output<Node>& out) {
return out.get_target_inputs().size() == 1 &&
is_not_memory_access(out);
});
auto add_input_2 = ov::pass::pattern::any_input();
auto add_m = ov::pass::pattern::wrap_type<opset1::Add>({ mul_m, add_input_2 });
auto add_m = ov::pass::pattern::wrap_type<opset1::Add>({ mul_m, add_input_2 }, is_not_memory_access);
matcher_pass_callback callback = [=](ov::pass::pattern::Matcher& m) {
OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "Snippets::op::MulAddToFMA_callback")

12
src/plugins/intel_cpu/src/transformations/snippets/x64/shape_inference.cpp
View File

@ -11,6 +11,11 @@
#include "op/store_convert.hpp"
#include "op/perf_count_rdtsc.hpp"
#include "transformations/cpu_opset/common/op/swish_cpu.hpp"
#ifdef SNIPPETS_LIBXSMM_TPP
#include "transformations/tpp/x64/op/brgemm.hpp"
#include "transformations/tpp/x64/op/scalar.hpp"
#include "transformations/tpp/x64/op/reduce.hpp"
#endif
namespace ov {
namespace snippets {
@ -42,9 +47,14 @@ const CPUShapeInferSnippetsFactory::TRegistry CPUShapeInferSnippetsFactory::spec
#ifdef SNIPPETS_DEBUG_CAPS
SHAPE_INFER_PREDEFINED(ov::intel_cpu::PerfCountRdtscBegin, EmptyShapeInfer),
SHAPE_INFER_PREDEFINED(ov::intel_cpu::PerfCountRdtscEnd, EmptyShapeInfer),
#endif
#ifdef SNIPPETS_LIBXSMM_TPP
SHAPE_INFER_OP_SPECIFIC_EXTERNAL(ov::intel_cpu::tpp::op::BrgemmTPP, BrgemmShapeInfer),
SHAPE_INFER_PREDEFINED(ov::intel_cpu::tpp::op::Scalar, SingleElementShapeInfer),
SHAPE_INFER_OP_SPECIFIC_EXTERNAL(ov::intel_cpu::tpp::op::ReduceMax, ReduceShapeInfer),
SHAPE_INFER_OP_SPECIFIC_EXTERNAL(ov::intel_cpu::tpp::op::ReduceSum, ReduceShapeInfer),
#endif
SHAPE_INFER_OP_SPECIFIC_EXTERNAL(ov::intel_cpu::BrgemmCPU, BrgemmShapeInfer),
//
SHAPE_INFER_OP_SPECIFIC(ov::intel_cpu::BrgemmCopyB),
};
#undef SHAPE_INFER_OP_SPECIFIC

62
src/plugins/intel_cpu/src/transformations/tpp/x64/op/brgemm.cpp Normal file
View File

@ -0,0 +1,62 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "brgemm.hpp"
#include "snippets/itt.hpp"
#include "snippets/utils.hpp"
#include "snippets/lowered/port_descriptor.hpp"
#include "utils/general_utils.h"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace op {
BrgemmTPP::BrgemmTPP(const Output<Node>& A, const Output<Node>& B,
const size_t offset_a, const size_t offset_b, const size_t offset_c,
std::vector<size_t> layout_a, std::vector<size_t> layout_b, std::vector<size_t> layout_c,
const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n, const float beta)
: MemoryAccess(std::set<size_t>{0, 1}, std::set<size_t>{0}),
modifier::TensorProcessingPrimitive(),
Brgemm(A, B,
offset_a, offset_b, offset_c,
std::move(layout_a), std::move(layout_b), std::move(layout_c),
blk_size_m, blk_size_k, blk_size_n) {
set_beta(beta);
}
BrgemmTPP::BrgemmTPP(const Output<Node>& A, const Output<Node>& B,
const PortDescriptor& desc_a, const PortDescriptor& desc_b, const PortDescriptor& desc_c,
std::vector<size_t> layout_a, std::vector<size_t> layout_b, std::vector<size_t> layout_c,
const size_t blk_size_m, const size_t blk_size_k, const size_t blk_size_n, const float beta)
: MemoryAccess(PortMap{{0, desc_a}, {1, desc_b}}, PortMap{{0, desc_c}}),
modifier::TensorProcessingPrimitive(),
Brgemm(A, B,
desc_a, desc_b, desc_c,
std::move(layout_a), std::move(layout_b), std::move(layout_c),
blk_size_m, blk_size_n, blk_size_k) {
set_beta(beta);
}
std::shared_ptr<Node> BrgemmTPP::clone_with_new_inputs(const OutputVector& new_args) const {
INTERNAL_OP_SCOPE(BrgemmTPP_clone_with_new_inputs);
check_new_args_count(this, new_args);
return std::make_shared<BrgemmTPP>(new_args.at(0), new_args.at(1),
get_input_port_descriptor(0), get_input_port_descriptor(1), get_output_port_descriptor(0),
snippets::lowered::PortDescriptorUtils::get_port_descriptor_ptr(input(0))->get_layout(),
snippets::lowered::PortDescriptorUtils::get_port_descriptor_ptr(input(1))->get_layout(),
snippets::lowered::PortDescriptorUtils::get_port_descriptor_ptr(output(0))->get_layout(),
m_M_blk, m_K_blk, m_N_blk, m_beta);
}
bool BrgemmTPP::visit_attributes(AttributeVisitor& visitor) {
TensorProcessingPrimitive::visit_attributes(visitor);
return Brgemm::visit_attributes(visitor);
}
} // namespace op
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

43
src/plugins/intel_cpu/src/transformations/tpp/x64/op/brgemm.hpp Normal file
View File

@ -0,0 +1,43 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "transformations/snippets/x64/op/brgemm_cpu.hpp"
#include "modifiers.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace op {
/**
* @interface BrgemmTPP
* @brief BrgemmTPP is a batch-reduced matrix multiplication with the support of arbitrary strides between matrices rows
* with support of several precisions on plugin level
* @ingroup snippets
*/
class BrgemmTPP : virtual public modifier::TensorProcessingPrimitive, public snippets::op::Brgemm {
public:
OPENVINO_OP("Brgemm", "TppOpset", snippets::op::Brgemm);
BrgemmTPP(const Output<Node>& A, const Output<Node>& B,
size_t offset_a = 0, size_t offset_b = 0, size_t offset_c = 0,
std::vector<size_t> layout_a = {}, std::vector<size_t> layout_b = {}, std::vector<size_t> layout_c = {},
size_t blk_size_m = 0, size_t blk_size_k = 0, size_t blk_size_n = 0, float beta = 1);
BrgemmTPP(const Output<Node>& A, const Output<Node>& B,
const PortDescriptor& desc_a, const PortDescriptor& desc_b, const PortDescriptor& desc_c,
std::vector<size_t> layout_a = {}, std::vector<size_t> layout_b = {}, std::vector<size_t> layout_c = {},
size_t blk_size_m = 0, size_t blk_size_k = 0, size_t blk_size_n = 0, float beta = 1);
BrgemmTPP() = default;
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
};
} // namespace op
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

102
src/plugins/intel_cpu/src/transformations/tpp/x64/op/eltwise.cpp Normal file
View File

@ -0,0 +1,102 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "eltwise.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace op {
#define GENERAL_AUX_METHODS(OP, OP_TYPE, ...) \
std::shared_ptr<Node> OP::clone_with_new_inputs(const OutputVector& new_args) const {\
check_new_args_count(this, new_args);\
const auto& new_op = std::make_shared<OP>(__VA_ARGS__);\
new_op->clone_memory_access_ports(*this);\
return new_op;\
} \
bool OP::visit_attributes(AttributeVisitor& visitor) {\
return OP_TYPE::visit_attributes(visitor);\
}
#define BINARY_AUX_METHODS(BINARY_OP) GENERAL_AUX_METHODS(BINARY_OP, BinaryEltwiseTPP, new_args.at(0), new_args.at(1), this->get_autob())
#define UNARY_AUX_METHODS(UNARY_OP) GENERAL_AUX_METHODS(UNARY_OP, UnaryEltwiseTPP, new_args.at(0))
bool EltwiseTPP::is_supported(const std::shared_ptr<ov::Node>& node) {
return ov::is_type<ov::op::v1::Add>(node) ||
ov::is_type<ov::op::v1::Subtract>(node) ||
ov::is_type<ov::op::v1::Multiply>(node) ||
ov::is_type<ov::op::v1::Divide>(node);
}
bool EltwiseTPP::visit_attributes(AttributeVisitor& visitor) {
TensorProcessingPrimitive::visit_attributes(visitor);
return MemoryAccess::visit_attributes(visitor);
}
BinaryEltwiseTPP::BinaryEltwiseTPP(libxsmm_meltw_binary_type op_type) : EltwiseTPP(), m_op_type(op_type) {
// Initialize input/output ports as memory access ports
ctor_initialize(std::set<size_t>{0, 1}, std::set<size_t>{0});
}
UnaryEltwiseTPP::UnaryEltwiseTPP(libxsmm_meltw_unary_type op_type) : EltwiseTPP(), m_op_type(op_type) {
ctor_initialize(std::set<size_t>{0}, std::set<size_t>{0});
}
Add::Add(const Output<Node>& arg0, const Output<Node>& arg1, const AutoBroadcastSpec& auto_broadcast)
: BinaryEltwiseTPP(LIBXSMM_MELTW_TYPE_BINARY_ADD), ov::op::v1::Add(arg0, arg1, auto_broadcast) {
}
BINARY_AUX_METHODS(Add)
Subtract::Subtract(const Output<Node>& arg0, const Output<Node>& arg1, const AutoBroadcastSpec& auto_broadcast)
: BinaryEltwiseTPP(LIBXSMM_MELTW_TYPE_BINARY_SUB), ov::op::v1::Subtract(arg0, arg1, auto_broadcast) {
}
BINARY_AUX_METHODS(Subtract)
Multiply::Multiply(const Output<Node>& arg0, const Output<Node>& arg1, const AutoBroadcastSpec& auto_broadcast)
: BinaryEltwiseTPP(LIBXSMM_MELTW_TYPE_BINARY_MUL), ov::op::v1::Multiply(arg0, arg1, auto_broadcast) {
}
BINARY_AUX_METHODS(Multiply)
Divide::Divide(const Output<Node>& arg0, const Output<Node>& arg1, const AutoBroadcastSpec& auto_broadcast)
: BinaryEltwiseTPP(LIBXSMM_MELTW_TYPE_BINARY_DIV), ov::op::v1::Divide(arg0, arg1, auto_broadcast) {
}
BINARY_AUX_METHODS(Divide)
Exp::Exp(const Output<Node>& arg0) : UnaryEltwiseTPP(LIBXSMM_MELTW_TYPE_UNARY_EXP), ov::op::v0::Exp(arg0) {
}
UNARY_AUX_METHODS(Exp)
Relu::Relu(const Output<Node>& arg0) : UnaryEltwiseTPP(LIBXSMM_MELTW_TYPE_UNARY_RELU), ov::op::v0::Relu(arg0) {
}
UNARY_AUX_METHODS(Relu)
Reciprocal::Reciprocal(const Output<Node>& arg) :
UnaryEltwiseTPP(LIBXSMM_MELTW_TYPE_UNARY_RECIPROCAL), ov::snippets::op::PowerStatic(arg, -1.f) {
}
UNARY_AUX_METHODS(Reciprocal)
Square::Square(const Output<Node>& arg) :
UnaryEltwiseTPP(LIBXSMM_MELTW_TYPE_UNARY_X2), ov::snippets::op::PowerStatic(arg, 2.f) {
}
UNARY_AUX_METHODS(Square)
SquareRoot::SquareRoot(const Output<Node>& arg) :
UnaryEltwiseTPP(LIBXSMM_MELTW_TYPE_UNARY_SQRT), ov::snippets::op::PowerStatic(arg, 0.5f) {
}
UNARY_AUX_METHODS(SquareRoot)
} // namespace op
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

123
src/plugins/intel_cpu/src/transformations/tpp/x64/op/eltwise.hpp Normal file
View File

@ -0,0 +1,123 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "modifiers.hpp"
#include "openvino/op/add.hpp"
#include "openvino/op/subtract.hpp"
#include "openvino/op/multiply.hpp"
#include "openvino/op/divide.hpp"
#include "openvino/op/exp.hpp"
#include "openvino/op/relu.hpp"
#include "snippets/op/powerstatic.hpp"
#include "libxsmm_typedefs.h"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace op {
using AutoBroadcastSpec = ov::op::AutoBroadcastSpec;
using AutoBroadcastType = ov::op::AutoBroadcastType;
class EltwiseTPP : public modifier::TensorProcessingPrimitive {
public:
static bool is_supported(const std::shared_ptr<ov::Node>& node);
bool visit_attributes(AttributeVisitor& visitor);
};
class BinaryEltwiseTPP : public EltwiseTPP {
public:
BinaryEltwiseTPP(libxsmm_meltw_binary_type op_type);
libxsmm_meltw_binary_type get_op_type() const { return m_op_type; }
private:
libxsmm_meltw_binary_type m_op_type;
};
class UnaryEltwiseTPP : public EltwiseTPP {
public:
UnaryEltwiseTPP(libxsmm_meltw_unary_type op_type);
libxsmm_meltw_unary_type get_op_type() const { return m_op_type; }
private:
libxsmm_meltw_unary_type m_op_type;
};
class Add : public BinaryEltwiseTPP, public ov::op::v1::Add {
public:
OPENVINO_OP("Add", "TppOpset", ov::op::v1::Add);
Add(const Output<Node>& arg0, const Output<Node>& arg1, const AutoBroadcastSpec& auto_broadcast);
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
};
class Subtract : public BinaryEltwiseTPP, public ov::op::v1::Subtract {
public:
OPENVINO_OP("Subtract", "TppOpset", ov::op::v1::Subtract);
Subtract(const Output<Node>& arg0, const Output<Node>& arg1, const AutoBroadcastSpec& auto_broadcast);
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
};
class Multiply : public BinaryEltwiseTPP, public ov::op::v1::Multiply {
public:
OPENVINO_OP("Multiply", "TppOpset", ov::op::v1::Multiply);
Multiply(const Output<Node>& arg0, const Output<Node>& arg1, const AutoBroadcastSpec& auto_broadcast);
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
};
class Divide : public BinaryEltwiseTPP, public ov::op::v1::Divide {
public:
OPENVINO_OP("Divide", "TppOpset", ov::op::v1::Divide);
Divide(const Output<Node>& arg0, const Output<Node>& arg1, const AutoBroadcastSpec& auto_broadcast);
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
};
class Exp : public UnaryEltwiseTPP, public ov::op::v0::Exp {
public:
OPENVINO_OP("Exp", "TppOpset", ov::op::v0::Exp);
Exp(const Output<Node>& arg);
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
};
class Relu : public UnaryEltwiseTPP, public ov::op::v0::Relu {
public:
OPENVINO_OP("Relu", "TppOpset", ov::op::v0::Relu);
Relu(const Output<Node>& arg);
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
};
class Reciprocal : public UnaryEltwiseTPP, public ov::snippets::op::PowerStatic {
public:
OPENVINO_OP("Reciprocal", "TppOpset", snippets::op::PowerStatic);
Reciprocal(const Output<Node>& arg);
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
};
class Square : public UnaryEltwiseTPP, public ov::snippets::op::PowerStatic {
public:
OPENVINO_OP("Square", "TppOpset", snippets::op::PowerStatic);
Square(const Output<Node>& arg);
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
};
class SquareRoot : public UnaryEltwiseTPP, public ov::snippets::op::PowerStatic {
public:
OPENVINO_OP("SquareRoot", "TppOpset", snippets::op::PowerStatic);
SquareRoot(const Output<Node>& arg);
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
};
} // namespace op
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

107
src/plugins/intel_cpu/src/transformations/tpp/x64/op/factory.cpp Normal file
View File

@ -0,0 +1,107 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "factory.hpp"
#include "eltwise.hpp"
#include "reduce.hpp"
#include "openvino/pass/pattern/op/wrap_type.hpp"
#include "ov_ops/type_relaxed.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace op {
namespace {
struct CustomPowerStaticBuilder : public NodeFactory::TPPCustomBuilder {
CustomPowerStaticBuilder() : NodeFactory::TPPCustomBuilder() {
matcher = [](const std::shared_ptr<ov::Node>& n) {
std::set<float> supported_power{-1, 2, 0.5};
const auto& power_static = ov::as_type_ptr<const snippets::op::PowerStatic>(n);
return power_static && supported_power.count(power_static->get_power());
};
builder = [](const std::shared_ptr<ov::Node>& n) {
const auto& power_static = ov::as_type_ptr<snippets::op::PowerStatic>(n);
OPENVINO_ASSERT(power_static, "Attempt to create TPP node from unsupported input in power_static_builder");
const auto power = power_static->get_power();
const auto& input = n->input_value(0);
std::shared_ptr<ov::Node> tpp_node{nullptr};
if (power == -1.f)
tpp_node = std::make_shared<Reciprocal>(input);
else if (power == 2.f)
tpp_node = std::make_shared<Square>(input);
else if (power == 0.5f)
tpp_node = std::make_shared<SquareRoot>(input);
OPENVINO_ASSERT(tpp_node, "Failed to create TPP in power_static_builder");
return tpp_node;
};
}
};
} // namespace
#define CREATE_UNARY_TPP_NODE(tpp_node_type) \
[](const std::shared_ptr<ov::Node>& node) -> std::shared_ptr<ov::Node> { \
return std::make_shared<tpp_node_type>(node->get_input_source_output(0)); \
}
#define CREATE_BINARY_TPP_NODE(tpp_node_type) \
[](const std::shared_ptr<ov::Node>& node) -> std::shared_ptr<ov::Node> { \
return std::make_shared<tpp_node_type>(node->get_input_source_output(0), node->get_input_source_output(1), node->get_autob()); \
}
#define CREATE_REDUCE_TPP_NODE(tpp_node_type) \
[](const std::shared_ptr<ov::Node>& node) -> std::shared_ptr<ov::Node> { \
const auto& reduce = ov::as_type_ptr<snippets::op::ReduceBase>(node); \
OPENVINO_ASSERT(reduce, "Attempt to create TPP Reduce from invalid node"); \
return std::make_shared<tpp_node_type>(reduce->get_input_source_output(0), reduce->get_axis()); \
}
std::unordered_map<ov::DiscreteTypeInfo, NodeFactory::tpp_builder> NodeFactory::m_direct_mapping {
{ov::op::v1::Add::get_type_info_static(), CREATE_BINARY_TPP_NODE(Add)},
{ov::op::v1::Subtract::get_type_info_static(), CREATE_BINARY_TPP_NODE(Subtract)},
{ov::op::v1::Multiply::get_type_info_static(), CREATE_BINARY_TPP_NODE(Multiply)},
{ov::op::v1::Divide::get_type_info_static(), CREATE_BINARY_TPP_NODE(Divide)},
{ov::op::v0::Exp::get_type_info_static(), CREATE_UNARY_TPP_NODE(Exp)},
{ov::op::v0::Relu::get_type_info_static(), CREATE_UNARY_TPP_NODE(Relu)},
// Note that we don't support conversion from ngraph ops here, since they have a broader semantics (e.g. multiple axis provided at a secont input)
{ov::snippets::op::ReduceMax::get_type_info_static(), CREATE_REDUCE_TPP_NODE(ReduceMax)},
{ov::snippets::op::ReduceSum::get_type_info_static(), CREATE_REDUCE_TPP_NODE(ReduceSum)},
};
std::vector<NodeFactory::TPPCustomBuilder> NodeFactory::m_custom_mapping{CustomPowerStaticBuilder()};
std::shared_ptr<ov::Node> NodeFactory::create(const std::shared_ptr<ov::Node>& n) {
const auto& found = m_direct_mapping.find(n->get_type_info());
std::shared_ptr<ov::Node> tpp_node{nullptr};
if (found != m_direct_mapping.end()) {
tpp_node = (found->second)(n);
} else {
for (const auto& custom_builder : m_custom_mapping) {
if (custom_builder.matcher(n)) {
tpp_node = custom_builder.builder(n);
break;
}
}
}
return tpp_node;
}
bool NodeFactory::is_supported(const std::shared_ptr<ov::Node>& n) {
auto matches = [=](const NodeFactory::TPPCustomBuilder& custom_builder) {
return custom_builder.matcher(n);
};
// TPP currently supports only FP32 precisions (ticket: 130010)
// Note: verify that TypeRelaxed property is maintained (mismatched input precisions)
// after low precisions are enabled (ticket: 132328)
const auto& ins = n->inputs();
auto is_fp32_input = [](const ov::Input<ov::Node>& in){ return in.get_element_type() == element::f32; };
const bool all_inputs_fp32 = std::all_of(ins.begin(), ins.end(), is_fp32_input);
return (m_direct_mapping.count(n->get_type_info()) ||
std::any_of(m_custom_mapping.begin(), m_custom_mapping.end(), matches)) && all_inputs_fp32;
}
} // namespace op
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

31
src/plugins/intel_cpu/src/transformations/tpp/x64/op/factory.hpp Normal file
View File

@ -0,0 +1,31 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "openvino/op/op.hpp"
#include "openvino/core/type.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace op {
class NodeFactory {
public:
static std::shared_ptr<ov::Node> create(const std::shared_ptr<ov::Node>& n);
static bool is_supported(const std::shared_ptr<ov::Node>& n);
typedef std::function<std::shared_ptr<ov::Node>(const std::shared_ptr<ov::Node>&)> tpp_builder;
typedef std::function<bool(const std::shared_ptr<ov::Node>&)> tpp_matcher;
struct TPPCustomBuilder {
tpp_matcher matcher;
tpp_builder builder;
};
private:
static std::unordered_map<ov::DiscreteTypeInfo, tpp_builder> m_direct_mapping;
static std::vector<TPPCustomBuilder> m_custom_mapping;
};
} // namespace op
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

34
src/plugins/intel_cpu/src/transformations/tpp/x64/op/modifiers.hpp Normal file
View File

@ -0,0 +1,34 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "snippets/op/memory_access.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace modifier {
/**
* @interface TensorProcessingPrimitive
* @brief TensorProcessingPrimitive a is modifier to mark operations supported with TPP
* @ingroup snippets
*/
class TensorProcessingPrimitive : virtual public snippets::modifier::MemoryAccess {
public:
void clone_memory_access_ports(const TensorProcessingPrimitive& other) {
m_input_ports = other.m_input_ports;
m_output_ports = other.m_output_ports;
}
static bool visit_attributes(AttributeVisitor& visitor) {
std::string modifier{"TPP"};
visitor.on_attribute("modifier", modifier);
return true;
}
};
} // namespace modifier
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

47
src/plugins/intel_cpu/src/transformations/tpp/x64/op/reduce.cpp Normal file
View File

@ -0,0 +1,47 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "reduce.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace op {
ReduceMax::ReduceMax(const Output<Node>& arg, size_t axis)
: UnaryEltwiseTPP(LIBXSMM_MELTW_TYPE_UNARY_REDUCE_X_OP_MAX), ov::snippets::op::ReduceMax(arg, axis) {
}
std::shared_ptr<Node> ReduceMax::clone_with_new_inputs(const OutputVector& new_args) const {
check_new_args_count(this, new_args);
const auto& new_op = std::make_shared<ReduceMax>(new_args.at(0), m_axis);
new_op->clone_memory_access_ports(*this);
return new_op;
}
bool ReduceMax::visit_attributes(AttributeVisitor& visitor) {
ReduceBase::visit_attributes(visitor);
return UnaryEltwiseTPP::visit_attributes(visitor);
}
ReduceSum::ReduceSum(const Output<Node>& arg, size_t axis)
: UnaryEltwiseTPP(LIBXSMM_MELTW_TYPE_UNARY_REDUCE_X_OP_ADD), ov::snippets::op::ReduceSum(arg, axis) {
}
std::shared_ptr<Node> ReduceSum::clone_with_new_inputs(const OutputVector& new_args) const {
check_new_args_count(this, new_args);
const auto& new_op = std::make_shared<ReduceSum>(new_args.at(0), m_axis);
new_op->clone_memory_access_ports(*this);
return new_op;
}
bool ReduceSum::visit_attributes(AttributeVisitor& visitor) {
ReduceBase::visit_attributes(visitor);
return UnaryEltwiseTPP::visit_attributes(visitor);
}
} // namespace op
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

44
src/plugins/intel_cpu/src/transformations/tpp/x64/op/reduce.hpp Normal file
View File

@ -0,0 +1,44 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "modifiers.hpp"
#include "eltwise.hpp"
#include "snippets/op/reduce.hpp"
#include "libxsmm_typedefs.h"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace op {
// Note: Reduce ops are implemented as UnaryEltwise in libxsmm, so we inherit this properties here
// Also note that UnaryEltwiseTPP is a modifier, so it won't trigger any flase positive matches in the pipeline
class ReduceMax : public UnaryEltwiseTPP, public ov::snippets::op::ReduceMax {
public:
OPENVINO_OP("ReduceMax", "TppOpset", ov::snippets::op::ReduceMax);
ReduceMax(const Output<Node>& arg, size_t axis);
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
private:
libxsmm_meltw_binary_type m_op_type;
};
class ReduceSum : public UnaryEltwiseTPP, public ov::snippets::op::ReduceSum {
public:
OPENVINO_OP("ReduceSum", "TppOpset", ov::snippets::op::ReduceSum);
ReduceSum(const Output<Node>& arg, size_t axis);
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
private:
libxsmm_meltw_binary_type m_op_type;
};
} // namespace op
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

31
src/plugins/intel_cpu/src/transformations/tpp/x64/op/scalar.cpp Normal file
View File

@ -0,0 +1,31 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "scalar.hpp"
#include "modifiers.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace op {
Scalar::Scalar(const snippets::op::Scalar& other) : ov::snippets::op::Scalar(other) {
constructor_validate_and_infer_types();
}
std::shared_ptr<Node> Scalar::clone_with_new_inputs(const OutputVector& new_args) const {
check_new_args_count(this, new_args);
return std::make_shared<tpp::op::Scalar>(*this);
}
bool Scalar::visit_attributes(AttributeVisitor& visitor) {
modifier::TensorProcessingPrimitive::visit_attributes(visitor);
return snippets::op::Scalar::visit_attributes(visitor);;
}
} // namespace op
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

32
src/plugins/intel_cpu/src/transformations/tpp/x64/op/scalar.hpp Normal file
View File

@ -0,0 +1,32 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "modifiers.hpp"
#include "eltwise.hpp"
#include "snippets/op/reduce.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace op {
// Note that the tpp::op::Scalar is not derived from the TensorProcessingPrimitive modifier. We don't need it because
// the Scalar is not a MemoryAccess operation, since it doesn't need to read from the external
// memory, and hence it is not really a TPP.
class Scalar : public ov::snippets::op::Scalar {
public:
OPENVINO_OP("Scalar", "TppOpset", snippets::op::Scalar);
Scalar() = default;
explicit Scalar(const snippets::op::Scalar& other);
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
bool visit_attributes(AttributeVisitor& visitor) override;
};
} // namespace op
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

119
src/plugins/intel_cpu/src/transformations/tpp/x64/pass/brgemm_to_brgemm_tpp.cpp Normal file
View File

@ -0,0 +1,119 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "snippets/itt.hpp"
#include "brgemm_to_brgemm_tpp.hpp"
#include "snippets/utils.hpp"
#include "snippets/op/brgemm.hpp"
#include "transformations/tpp/x64/op/brgemm.hpp"
#include "openvino/core/rt_info.hpp"
#include "openvino/pass/pattern/op/wrap_type.hpp"
#include "openvino/pass/pattern/matcher.hpp"
#include "cpu_shape.h"
#include "utils/general_utils.h"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace pass {
using namespace snippets::lowered;
namespace {
template<typename T, typename... Args>
void set_port_desc(const T& port, Args... params) {
PortDescriptorUtils::set_port_descriptor_ptr(port, std::make_shared<PortDescriptor>(params...));
}
} // namespace
BrgemmToBrgemmTPP::BrgemmToBrgemmTPP() {
MATCHER_SCOPE(BrgemmToBrgemmTPP);
auto m_brgemm = ov::pass::pattern::wrap_type<snippets::op::Brgemm>();
auto callback = [=](ov::pass::pattern::Matcher& m) {
OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "ov::intel_cpu::pass::BrgemmToBrgemmTPP")
const auto node = m.get_match_root();
const auto brgemm = ov::as_type_ptr<snippets::op::Brgemm>(node);
if (!brgemm || ov::as_type_ptr<tpp::op::BrgemmTPP>(node))
OPENVINO_THROW("BrgemmCPU cannot be in body before BrgemmToBrgemmTPP pass");
if (brgemm->is_dynamic()) {
return false;
}
const auto& brgemm_in0_desc = PortDescriptorUtils::get_port_descriptor_ptr(brgemm->input(0));
const auto& brgemm_in1_desc = PortDescriptorUtils::get_port_descriptor_ptr(brgemm->input(1));
const auto& brgemm_out_desc = PortDescriptorUtils::get_port_descriptor_ptr(brgemm->output(0));
const auto dimsMatMulIn0 = snippets::utils::get_planar_pshape(brgemm->input(0)).get_shape();
const auto dimsMatMulIn1 = snippets::utils::get_planar_pshape(brgemm->input(1)).get_shape();
const auto M = *++dimsMatMulIn0.rbegin();
const auto K = *dimsMatMulIn0.rbegin();
const auto N = *dimsMatMulIn1.rbegin();
const auto element_type_a = brgemm->get_input_element_type(0);
const auto element_type_b = brgemm->get_input_element_type(1);
const auto offset_a = brgemm->get_offset_a();
const auto offset_b = brgemm->get_offset_b();
const auto offset_c = brgemm->get_offset_c();
std::shared_ptr<tpp::op::BrgemmTPP> brgemm_tpp = nullptr;
if (element_type_a == ov::element::f32) {
brgemm_tpp = std::make_shared<tpp::op::BrgemmTPP>(brgemm->input_value(0),
brgemm->input_value(1),
offset_a, offset_b, offset_c,
brgemm_in0_desc->get_layout(),
brgemm_in1_desc->get_layout(),
brgemm_out_desc->get_layout());
}
OPENVINO_ASSERT(brgemm_tpp, "Failed to create BrgemmTPP node in the BrgemmToBrgemmTPP pass");
// Set blocking params
// Ticket: 113745
// TODO: extend block size selection heuristics
auto get_block_size_m = [](const size_t M) {
return 32;
};
auto get_block_size_k = [=](const size_t K) {
if (element_type_b != ov::element::f32)
return K;
return K > 1024 ? 1024 : K > 512 ? 512 : K;
};
auto get_block_size_n = [=](const size_t N) {
return element_type_b != ov::element::f32 ? N : 64;
};
brgemm_tpp->set_m_block_size(get_block_size_m(M));
brgemm_tpp->set_k_block_size(get_block_size_k(K));
brgemm_tpp->set_n_block_size(get_block_size_n(N));
brgemm_tpp->set_friendly_name(brgemm->get_friendly_name());
ov::replace_node(brgemm, brgemm_tpp);
// Set FULL_DIM tensors on ports to avoid automatic loop markup (blocked loops will be inserted in a separate transformation)
set_port_desc(brgemm_tpp->input(0), brgemm_in0_desc->get_shape(), brgemm_in0_desc->get_subtensor(), brgemm_in0_desc->get_layout());
set_port_desc(brgemm_tpp->input(1), brgemm_in1_desc->get_shape(), brgemm_in1_desc->get_subtensor(), brgemm_in1_desc->get_layout());
set_port_desc(brgemm_tpp->output(0), brgemm_out_desc->get_shape(), brgemm_out_desc->get_subtensor(), brgemm_out_desc->get_layout());
// need to run validate_and_infer_types manually: either input shapes were updated or
// output Layout was updated (out shape will be updated in validate_and_infer_types())
brgemm_tpp->validate_and_infer_types();
return true;
};
auto m = std::make_shared<ov::pass::pattern::Matcher>(m_brgemm, matcher_name);
register_matcher(m, callback);
}
} // namespace pass
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

29
src/plugins/intel_cpu/src/transformations/tpp/x64/pass/brgemm_to_brgemm_tpp.hpp Normal file
View File

@ -0,0 +1,29 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "openvino/pass/graph_rewrite.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace pass {
/**
* @interface BrgemmToBrgemmTPP
* @brief Converts Snippets Brgemm to BrgemmTPP operation, if possible. Only fp32 Brgemms are currently converted.
* @ingroup snippets
*/
class BrgemmToBrgemmTPP: public ov::pass::MatcherPass {
public:
OPENVINO_RTTI("BrgemmToBrgemmTPP", "0");
BrgemmToBrgemmTPP();
};
} // namespace pass
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

61
src/plugins/intel_cpu/src/transformations/tpp/x64/pass/eltwise_to_eltwise_tpp.cpp Normal file
View File

@ -0,0 +1,61 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "snippets/itt.hpp"
#include "eltwise_to_eltwise_tpp.hpp"
#include "openvino/pass/pattern/op/wrap_type.hpp"
#include "transformations/tpp/x64/op/factory.hpp"
#include "openvino/op/util/binary_elementwise_arithmetic.hpp"
#include "openvino/op/util/unary_elementwise_arithmetic.hpp"
#include "snippets/lowered/port_descriptor.hpp"
#include "snippets/op/reduce.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace pass {
EltwiseToEltwiseTPP::EltwiseToEltwiseTPP() {
MATCHER_SCOPE(EltwiseToEltwiseTPP);
auto is_supported_by_tpp = [](const Output<Node>& out) {
return op::NodeFactory::is_supported(out.get_node_shared_ptr());
};
auto supported_eltwise = ov::pass::pattern::wrap_type<ov::op::util::UnaryElementwiseArithmetic,
ov::op::util::BinaryElementwiseArithmetic,
ov::snippets::op::ReduceBase>(is_supported_by_tpp);
auto callback = [=](ov::pass::pattern::Matcher& m) {
OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "ov::intel_cpu::pass::EltwiseToEltwiseTPP")
const auto node = m.get_match_root();
if (node->is_dynamic()) {
return false;
}
const auto& tpp_eltwise = op::NodeFactory::create(node);
OPENVINO_ASSERT(tpp_eltwise, "Failed to create TPP node");
const size_t M_block = 32;
const size_t N_block = ov::is_type<ov::snippets::op::ReduceBase>(node) ?
snippets::lowered::PortDescriptor::ServiceDimensions::FULL_DIM :
64;
ov::replace_node_update_name(node, tpp_eltwise);
for (size_t i = 0; i < node->get_input_size(); i++)
snippets::lowered::set_port_desc(tpp_eltwise->input(i), {M_block, N_block});
snippets::lowered::set_port_desc(tpp_eltwise->output(0), {M_block, N_block});
return true;
};
auto m = std::make_shared<ov::pass::pattern::Matcher>(supported_eltwise, matcher_name);
register_matcher(m, callback);
}
} // namespace pass
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

29
src/plugins/intel_cpu/src/transformations/tpp/x64/pass/eltwise_to_eltwise_tpp.hpp Normal file
View File

@ -0,0 +1,29 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "openvino/pass/graph_rewrite.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace pass {
/**
* @interface EltwiseToEltwiseTPP
* @brief Converts elementwise operations supported by the TPP backend to the dedicated TPP opset
* @ingroup snippets
*/
class EltwiseToEltwiseTPP: public ov::pass::MatcherPass {
public:
OPENVINO_RTTI("EltwiseToEltwiseTPP", "0");
EltwiseToEltwiseTPP();
};
} // namespace pass
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

164
src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.cpp Normal file
View File

@ -0,0 +1,164 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "snippets/itt.hpp"
#include "snippets/op/buffer.hpp"
#include "transformations/tpp/x64/op/modifiers.hpp"
#include "set_tpp_leading_dim.hpp"
#include "snippets/op/brgemm.hpp"
#include "snippets/lowered/loop_manager.hpp"
#include "snippets/utils.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace pass {
namespace {
using ExpressionPort = snippets::lowered::ExpressionPort;
bool is_planar_layout(const std::vector<size_t>& layout) {
for (size_t i = 0; i < layout.size(); i++) {
if (layout[i] != i)
return false;
}
return true;
}
// Note: Buffer is directly connected to the port if it remains in the same loops with the port's expression
// Directly connected Buffers store data densely, so strides are defined by subternsor dims
// Indirectly connected Buffers (with loops between the expr and Buffer) store data according
// to their shape and layout
bool has_directly_connected_buffer(const ExpressionPort& port, const snippets::lowered::LinearIR::LoopManagerPtr& loop_mngr) {
auto accepted_loops = [&loop_mngr, &port](const std::vector<size_t>& orig, const std::vector<size_t>& connect) {
size_t connect_idx = 0;
auto pred = [&port](const snippets::lowered::LinearIR::LoopManager::LoopPort& loop_port ) {
return *loop_port.expr_port == port;
};
for (const auto orig_loop : orig) {
if (connect_idx < connect.size() && orig_loop == connect[connect_idx]) {
connect_idx++;
continue;
}
// Note that orig expression can have some extra loops (compared to connect)
// as long as the port is the loop entry/exit, and it is not incremented.
// This is the case for Brgemm K-blocking loops, for example.
const auto loop_info = loop_mngr->get_loop_info(orig_loop);
const auto& border_points = port.get_type() == ExpressionPort::Type::Input ?
loop_info->get_entry_points() :
loop_info->get_exit_points();
const auto& found = std::find_if(border_points.begin(), border_points.end(), pred);
if (found == border_points.end() || found->is_incremented)
return false;
}
return true;
};
bool has_buffer = false;
const auto& orig_loop_ids = port.get_expr()->get_loop_ids();
for (const auto& p : port.get_connected_ports()) {
const auto& connected_expr = p.get_expr();
if (ov::is_type<snippets::op::Buffer>(connected_expr->get_node()) &&
accepted_loops(orig_loop_ids, connected_expr->get_loop_ids())) {
OPENVINO_ASSERT(!has_buffer, "Only one Buffer can be connected to a TPP op");
has_buffer = true;
}
}
return has_buffer;
}
size_t get_leading_dim(ExpressionPort port, const snippets::lowered::LinearIR::LoopManagerPtr& loop_mngr) {
const auto& port_desc = port.get_descriptor_ptr();
auto layout = port_desc->get_layout();
auto shape = port_desc->get_shape();
auto subtensor = port_desc->get_subtensor();
// Some expressions (e.g. ReduceMax/ReduceSum) allow for FULL_DIM values in subtensor.
// Here we should replace them with actual dim values before calculating strides & offsets.
bool full_dim_substituted = false;
for (size_t i = 1; i <= subtensor.size(); i++) {
const auto idx = subtensor.size() - i;
if (subtensor[idx] == snippets::lowered::PortDescriptor::ServiceDimensions::FULL_DIM) {
// the reason that we don't support FULL_DIM substitution for an arbitrary layout is that
// the layout and subtersor can (and usually do) have different ranks
full_dim_substituted = true;
subtensor[idx] = shape[shape.size() - i];
}
}
OPENVINO_ASSERT(!full_dim_substituted || is_planar_layout(layout),
"Only planar layouts are supported for FULL_DIM substitution");
if (has_directly_connected_buffer(port, loop_mngr)) {
shape = port_desc->get_subtensor();
OPENVINO_ASSERT(is_planar_layout(layout), "Only planar layouts are supported for Buffers");
const auto rank_diff = static_cast<int64_t>(layout.size()) - static_cast<int64_t>(shape.size());
if (rank_diff > 0)
layout.erase(layout.end() - rank_diff, layout.end());
}
OPENVINO_ASSERT(layout.empty() || (layout.back() == layout.size() - 1 && layout.size() == shape.size()),
"get_leading_dim detected invalid layout values: check shape + layout combination");
const auto dim = [&]() -> size_t {
switch (port.get_type()) {
// Input shape is original, so we need to correctly read this data by order
// Example:
// Original shape (shape) = [1, 49, 2, 23]
// Layout (transpose order) = [2, 0, 1, 3]
// Transposed shape = [2, 1, 49, 23]
// The leading dimension is equal to stride of shape[layout[3]] = 2 x 23
case ExpressionPort::Type::Input :
return snippets::utils::get_input_dim_idx(layout, 1); // `1` in example
// Output shape is already transposed, we need to correctly write the data with original shape by the order
// Example:
// Original transposed shape (shape) = [49, 2, 7, 39]
// Layout (transpose order) = [2, 0, 1, 3]
// Before leading dimension with index 3 there is dimension with index 2 in planar layout.
// Since we have non-planar layout, we have to find this before LD dim in transposed order.
// In layout 2nd idx is first element, it means, that the leading dimension is equal to stride of shape[0]
case ExpressionPort::Type::Output :
return snippets::utils::get_output_dim_idx(layout, 1); // 0 in the example: shape[0] = 49
default:
OPENVINO_THROW("Unsupported Expression port type");
}
};
return layout.size() == 1 ?
shape.back() :
std::accumulate(shape.cbegin() + dim() + 1, shape.cend(), 1, std::multiplies<size_t>());
}
} // namespace
SetTPPLeadingDim::SetTPPLeadingDim() : RangedPass() {}
bool SetTPPLeadingDim::run(snippets::lowered::LinearIR& linear_ir,
snippets::lowered::LinearIR::constExprIt begin,
snippets::lowered::LinearIR::constExprIt end) {
OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "Snippets::SetTPPLeadingDim")
if (linear_ir.empty())
return false;
bool modified = false;
for (auto expr_it = begin; expr_it != end; expr_it++) {
const auto& expr = *expr_it;
const auto& node = expr->get_node();
auto tpp_expr = std::dynamic_pointer_cast<modifier::TensorProcessingPrimitive>(node);
if (!tpp_expr)
continue;
OPENVINO_ASSERT(tpp_expr->is_full_memory_access_op(node), "TPP Op is expected to be MemoryAccess on all ports");
for (size_t i = 0; i < expr->get_input_count(); i++) {
const auto ld = get_leading_dim(expr->get_input_port(i), linear_ir.get_loop_manager());
tpp_expr->set_input_stride(ld, i);
}
for (size_t i = 0; i < expr->get_output_count(); i++) {
const auto ld = get_leading_dim(expr->get_output_port(i), linear_ir.get_loop_manager());
tpp_expr->set_output_stride(ld, i);
}
modified = true;
}
return modified;
}
} // namespace pass
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

33
src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.hpp Normal file
View File

@ -0,0 +1,33 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "snippets/lowered/pass/pass.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace pass {
/**
* @interface SetTPPLeadingDim
* @brief TPP leading dimension depends on the operation it is connected to. If it's a Parameter or Result
* we can compute LD based on shape, if it's a Buffer - we need to consider allocation shape.
* This transformation should be performed before InsertTailLoop because it may change graph connectivity for 1st and last iterations.
* @ingroup snippets
*/
class SetTPPLeadingDim: public snippets::lowered::pass::RangedPass {
public:
OPENVINO_RTTI("SetTPPLeadingDim", "Pass");
SetTPPLeadingDim();
bool run(snippets::lowered::LinearIR& linear_ir,
snippets::lowered::LinearIR::constExprIt begin,
snippets::lowered::LinearIR::constExprIt end) override;
};
} // namespace pass
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

57
src/plugins/intel_cpu/src/transformations/tpp/x64/pass/scalar_to_scalar_tpp.cpp Normal file
View File

@ -0,0 +1,57 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "snippets/itt.hpp"
#include "scalar_to_scalar_tpp.hpp"
#include "openvino/pass/pattern/op/wrap_type.hpp"
#include "snippets/op/scalar.hpp"
#include "transformations/tpp/x64/op/scalar.hpp"
#include "transformations/tpp/x64/op/modifiers.hpp"
#include "snippets/lowered/port_connector.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace pass {
ScalarToScalarTPP::ScalarToScalarTPP() {
MATCHER_SCOPE(ScalarToScalarTPP);
auto snippets_scalar = ov::pass::pattern::wrap_type<ov::snippets::op::Scalar>();
auto callback = [=](ov::pass::pattern::Matcher& m) {
OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "ov::intel_cpu::pass::ScalarToScalarTPP")
const auto node = ov::as_type_ptr<ov::snippets::op::Scalar>(m.get_match_root());
OPENVINO_ASSERT(node, "Failed to obtain a valid Scalar Op in ScalarToScalarTPP");
size_t num_connected_tpp = 0;
const auto& target_ins = node->get_output_target_inputs(0);
for (const auto& in : target_ins) {
if (dynamic_cast<tpp::modifier::TensorProcessingPrimitive*>(in.get_node()))
num_connected_tpp++;
}
if (num_connected_tpp == 0)
return false;
// Note: If needed, we can support cases when scalar has TPP and non-TPP consumers if we copy the scalar.
// However, this is rarely needed in practice and the assert is here to flag invalid configurations.
OPENVINO_ASSERT(num_connected_tpp == target_ins.size(), "Either all or none Scalar outputs should be TPP");
const auto& tpp_scalar = std::make_shared<tpp::op::Scalar>(*node);
tpp_scalar->set_friendly_name(node->get_friendly_name());
ov::replace_node_update_name(node, tpp_scalar);
const auto& out = tpp_scalar->output(0);
snippets::lowered::set_port_desc(out, {1});
for (const auto& in : out.get_target_inputs())
snippets::lowered::set_port_desc(in, {1});
return true;
};
auto m = std::make_shared<ov::pass::pattern::Matcher>(snippets_scalar, matcher_name);
register_matcher(m, callback);
}
} // namespace pass
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

29
src/plugins/intel_cpu/src/transformations/tpp/x64/pass/scalar_to_scalar_tpp.hpp Normal file
View File

@ -0,0 +1,29 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "openvino/pass/graph_rewrite.hpp"
namespace ov {
namespace intel_cpu {
namespace tpp {
namespace pass {
/**
* @interface ScalarToScalarTPP
* @brief Converts snippets::op::Scalar to tpp::op::Scalar, since TPP operations require a dedicated emitter
* @ingroup snippets
*/
class ScalarToScalarTPP: public ov::pass::MatcherPass {
public:
OPENVINO_RTTI("ScalarToScalarTPP", "0");
ScalarToScalarTPP();
};
} // namespace pass
} // namespace tpp
} // namespace intel_cpu
} // namespace ov

23
src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp
View File

@ -427,6 +427,29 @@ std::vector<std::string> disabledTestPatterns() {
retVector.emplace_back(R"(.*smoke_Snippets_MHAWOTransposeEnforceBF16.*)");
retVector.emplace_back(R"(.*smoke_Snippets_MHAEnforceBF16.*)");
}
#ifdef SNIPPETS_LIBXSMM_TPP
// TPP performs precision conversion implicitly, it makes all Convert tests irrelevant
retVector.emplace_back(R"(.*smoke_Snippets_Convert.*)");
// ABS and ROUND operations are needed for TPP support. Disable, since low precisions are not supported by TPP yet.
retVector.emplace_back(R"(.*smoke_Snippets_FQ.*)");
retVector.emplace_back(R"(.*smoke_Snippets_TransposeMatMulFQ.*)");
// TPP doesn't support op with 2 outs, when one of them is Result (ticket: 130642)
retVector.emplace_back(R"(.*smoke_Snippets_MaxNumParamsEltwise.*)");
retVector.emplace_back(R"(.*smoke_Snippets_Eltwise_TwoResults.*)");
// Accuracy problem with Exp + Reciprocal combination on TPP side (ticket: 130699)
retVector.emplace_back(R"(.*smoke_Snippets_ExpReciprocal.*)");
retVector.emplace_back(R"(.*smoke_Snippets_AddSoftmax.*)");
retVector.emplace_back(R"(.*smoke_Snippets_TransposeSoftmaxEltwise.*)");
// Low-precision Matmuls are not supported by TPP yet
retVector.emplace_back(R"(.*smoke_Snippets_MatMulFQ.*)");
retVector.emplace_back(R"(.*smoke_Snippets_MatMulBiasQuantized.*)");
retVector.emplace_back(R"(.*smoke_Snippets_MatMulQuantized.*)");
retVector.emplace_back(R"(.*smoke_Snippets_MatMulQuantizedSoftmax.*)");
retVector.emplace_back(R"(.*smoke_Snippets_MHAINT8MatMul.*)");
retVector.emplace_back(R"(.*smoke_Snippets_MHAQuantMatMul0.*)");
retVector.emplace_back(R"(.*smoke_Snippets_MHAFQ.*)");
retVector.emplace_back(R"(.*smoke_Snippets_PrecisionPropagation_Convertion.*)");
#endif
if (ov::with_cpu_x86_avx512_core_amx()) {
// Issue: 130463

38
src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/exp.cpp Normal file
View File

@ -0,0 +1,38 @@
// Copyright (C) 2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "snippets/exp.hpp"
#include "common_test_utils/test_constants.hpp"
namespace ov {
namespace test {
namespace snippets {
namespace {
// ===================================Exp=========================================================//
std::vector<ov::test::InputShape> inShapesStatic{{{}, {{1, 1, 32, 128}}}};
INSTANTIATE_TEST_SUITE_P(smoke_Snippets_Exp, Exp,
::testing::Combine(
::testing::ValuesIn(inShapesStatic),
::testing::Values(ov::element::f32),
::testing::Values(1), // Exp
::testing::Values(1),
::testing::Values(ov::test::utils::DEVICE_CPU)),
Exp::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Snippets_ExpReciprocal, ExpReciprocal,
::testing::Combine(
::testing::ValuesIn(inShapesStatic),
::testing::Values(ov::element::f32),
::testing::Values(1), // Exp
::testing::Values(1),
::testing::Values(ov::test::utils::DEVICE_CPU)),
Exp::getTestCaseName);
} // namespace
} // namespace snippets
} // namespace test
} // namespace ov

10
src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/matmul.cpp
View File

@ -40,6 +40,8 @@ static inline std::vector<std::vector<element::Type>> precisions(bool only_fp32
std::vector<std::vector<element::Type>> prc = {
{element::f32, element::f32},
};
// Note: TPP doesn't support low precisions yet
#ifndef SNIPPETS_LIBXSMM_TPP
if (!only_fp32) {
auto quant = quantized_precisions();
std::copy(quant.begin(), quant.end(), std::back_inserter(prc));
@ -48,6 +50,7 @@ static inline std::vector<std::vector<element::Type>> precisions(bool only_fp32
prc.emplace_back(std::vector<element::Type>{element::bf16, element::bf16});
}
}
#endif
return prc;
}
@ -71,7 +74,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulFQ, MatMulFQ,
INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulBias, MatMulBias,
::testing::Combine(
::testing::Values(std::vector<ov::PartialShape>{{1, 2, 69, 43}, {2, 1, 43, 49}, {1, 1, 69, 49}}),
::testing::Values(std::vector<ov::PartialShape>{{1, 2, 69, 43}, {2, 1, 43, 49}, {1, 1, 69, 49}},
std::vector<ov::PartialShape>{{1, 2, 95, 1023}, {1, 2, 1023, 255}, {1, 2, 95, 255}}),
::testing::ValuesIn(precisions(false)),
::testing::Values(1), // Subgraph;
::testing::Values(1), // Tokenized MatMul+Bias
@ -89,7 +93,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulBiasQuantized, MatMulBiasQuantized
::testing::Values(ov::test::utils::DEVICE_CPU)),
MatMul::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulsQuantized, MatMulsQuantized,
INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulQuantized, MatMulQuantized,
::testing::Combine(
::testing::Values(std::vector<ov::PartialShape>{{1, 16, 128, 64}, {1, 16, 64, 128}, {128, 64}}),
::testing::ValuesIn(quantized_precisions()),
@ -98,7 +102,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulsQuantized, MatMulsQuantized,
::testing::Values(ov::test::utils::DEVICE_CPU)),
MatMul::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulsQuantizedSoftmax, MatMulsQuantizedSoftmax,
INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulQuantizedSoftmax, MatMulQuantizedSoftmax,
::testing::Combine(
::testing::Values(std::vector<ov::PartialShape>{{1, 16, 128, 64}, {1, 16, 64, 128}, {128, 64}}),
::testing::ValuesIn(quantized_precisions()),

2
src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/max_num_params_eltwise.cpp
View File

@ -23,7 +23,7 @@ std::vector<InputShape> input_shapes {{{}, {{1, 64, 10, 10}}},
{{1, {1, 64}, {10, 20}, -1}, {{1, 64, 10, 10}, {1, 1, 17, 37}, {1, 64, 10, 10}}},
{{1, 1, 1, {1, 128}}, {{1, 1, 1, 1}, {1, 1, 1, 7}, {1, 1, 1, 128}, {1, 1, 1, 14}, {1, 1, 1, 16}, {1, 1, 1, 1}}}};
INSTANTIATE_TEST_SUITE_P(smoke_Snippets_Eltwise, MaxNumParamsEltwise,
INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MaxNumParamsEltwise, MaxNumParamsEltwise,
::testing::Combine(
::testing::ValuesIn(input_shapes),
::testing::Values(2), // Subgraph + Concat

2
src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/three_inputs_eltwise.cpp
View File

@ -10,7 +10,7 @@ namespace test {
namespace snippets {
namespace {
INSTANTIATE_TEST_SUITE_P(smoke_Snippets_Eltwise, ThreeInputsEltwise,
INSTANTIATE_TEST_SUITE_P(smoke_Snippets_ThreeInputsEltwise, ThreeInputsEltwise,
::testing::Combine(
::testing::Values(InputShape {{}, {{1, 64, 10, 10}}}),
::testing::Values(InputShape {{}, {{1, 64, 10, 1}}}),

3
src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/transpose_matmul.cpp
View File

@ -17,6 +17,8 @@ static inline std::vector<std::vector<element::Type>> precisions(bool only_fp32
std::vector<std::vector<element::Type>> prc = {
{element::f32, element::f32},
};
// Note: low precisions are not supported by TPP yet (ticker: 130010)
#ifndef SNIPPETS_LIBXSMM_TPP
if (!only_fp32) {
// In Snippets MatMul INT8 is supported only on VNNI/AMX platforms
if (ov::with_cpu_x86_avx512_core_vnni() || ov::with_cpu_x86_avx512_core_amx_int8()) {
@ -28,6 +30,7 @@ static inline std::vector<std::vector<element::Type>> precisions(bool only_fp32
prc.emplace_back(std::vector<element::Type>{element::bf16, element::bf16});
}
}
#endif
return prc;
}
namespace transpose_zero_input {

5
src/plugins/intel_cpu/thirdparty/CMakeLists.txt vendored
View File

@ -136,6 +136,11 @@ function(ov_add_onednn)
endif()
endfunction()
if (ENABLE_SNIPPETS_LIBXSMM_TPP)
add_subdirectory(libxsmm)
ov_install_static_lib(libxsmm ${OV_CPACK_COMP_CORE})
endif()
if(ENABLE_MLAS_FOR_CPU)
add_subdirectory(mlas)
ov_install_static_lib(mlas ${OV_CPACK_COMP_CORE})

1
src/plugins/intel_cpu/thirdparty/libxsmm vendored Submodule

@ -0,0 +1 @@
Subproject commit 13df674c4b73a1b84f6456de8595903ebfbb43e0

37
src/tests/functional/plugin/shared/include/snippets/exp.hpp Normal file
View File

@ -0,0 +1,37 @@
// Copyright (C) 2024 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "shared_test_classes/base/snippets_test_utils.hpp"
namespace ov {
namespace test {
namespace snippets {
typedef std::tuple<
InputShape, // Input 0 Shape
ov::element::Type, // Element type
size_t, // Expected num nodes
size_t, // Expected num subgraphs
std::string // Target Device
> ExpParams;
class Exp : public testing::WithParamInterface<ov::test::snippets::ExpParams>,
virtual public ov::test::SnippetsTestsCommon {
public:
static std::string getTestCaseName(testing::TestParamInfo<ov::test::snippets::ExpParams> obj);
protected:
void SetUp() override;
};
class ExpReciprocal : public Exp {
protected:
void SetUp() override;
};
} // namespace snippets
} // namespace test
} // namespace ov

4
src/tests/functional/plugin/shared/include/snippets/matmul.hpp
View File

@ -44,12 +44,12 @@ protected:
void init_subgraph(const std::vector<PartialShape>& inputShapes, const std::vector<ov::element::Type>& types) override;
};
class MatMulsQuantized : public MatMul {
class MatMulQuantized : public MatMul {
protected:
void init_subgraph(const std::vector<PartialShape>& inputShapes, const std::vector<ov::element::Type>& types) override;
};
class MatMulsQuantizedSoftmax : public MatMul {
class MatMulQuantizedSoftmax : public MatMul {
protected:
void init_subgraph(const std::vector<PartialShape>& inputShapes, const std::vector<ov::element::Type>& types) override;
};

75
src/tests/functional/plugin/shared/src/snippets/exp.cpp Normal file
View File

@ -0,0 +1,75 @@
// Copyright (C) 2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "common_test_utils/common_utils.hpp"
#include "snippets/exp.hpp"
#include "subgraph_simple.hpp"
#include "functional_test_utils/skip_tests_config.hpp"
#include "openvino/op/subtract.hpp"
namespace ov {
namespace test {
namespace snippets {
std::string Exp::getTestCaseName(testing::TestParamInfo<ov::test::snippets::ExpParams> obj) {
ov::test::InputShape inputShapes0;
ov::element::Type type;
std::string targetDevice;
size_t num_nodes, num_subgraphs;
std::tie(inputShapes0, type, num_nodes, num_subgraphs, targetDevice) = obj.param;
std::ostringstream result;
result << "IS[0]=" << ov::test::utils::partialShape2str({inputShapes0.first}) << "_";
result << "TS[0]=";
for (const auto& shape : inputShapes0.second) {
result << "(" << ov::test::utils::vec2str(shape) << ")_";
}
result << "T=" << type << "_";
result << "#N=" << num_nodes << "_";
result << "#S=" << num_subgraphs << "_";
result << "targetDevice=" << targetDevice;
return result.str();
}
void Exp::SetUp() {
ov::test::InputShape inputShape0;
ov::element::Type type;
std::tie(inputShape0, type, ref_num_nodes, ref_num_subgraphs, targetDevice) = this->GetParam();
init_input_shapes({inputShape0});
auto f = ov::test::snippets::ExpFunction(inputDynamicShapes);
function = f.getOriginal();
setInferenceType(type);
if (!configuration.count("SNIPPETS_MODE")) {
configuration.insert({"SNIPPETS_MODE", "IGNORE_CALLBACK"});
}
}
void ExpReciprocal::SetUp() {
ov::test::InputShape inputShape0;
ov::element::Type type;
std::tie(inputShape0, type, ref_num_nodes, ref_num_subgraphs, targetDevice) = this->GetParam();
init_input_shapes({inputShape0});
auto f = ov::test::snippets::ExpReciprocalFunction(inputDynamicShapes);
function = f.getOriginal();
setInferenceType(type);
if (!configuration.count("SNIPPETS_MODE")) {
configuration.insert({"SNIPPETS_MODE", "IGNORE_CALLBACK"});
}
}
TEST_P(Exp, CompareWithRefImpl) {
run();
validateNumSubgraphs();
}
TEST_P(ExpReciprocal, CompareWithRefImpl) {
run();
validateNumSubgraphs();
}
} // namespace snippets
} // namespace test
} // namespace ov

8
src/tests/functional/plugin/shared/src/snippets/matmul.cpp
View File

@ -60,12 +60,12 @@ void MatMulBiasQuantized::init_subgraph(const std::vector<PartialShape>& inputSh
function = f.getOriginal();
}
void MatMulsQuantized::init_subgraph(const std::vector<PartialShape>& inputShapes, const std::vector<ov::element::Type>& types) {
void MatMulQuantized::init_subgraph(const std::vector<PartialShape>& inputShapes, const std::vector<ov::element::Type>& types) {
auto f = ov::test::snippets::MatMulsQuantizedFunction(inputShapes, types);
function = f.getOriginal();
}
void MatMulsQuantizedSoftmax::init_subgraph(const std::vector<PartialShape>& inputShapes, const std::vector<ov::element::Type>& types) {
void MatMulQuantizedSoftmax::init_subgraph(const std::vector<PartialShape>& inputShapes, const std::vector<ov::element::Type>& types) {
auto f = ov::test::snippets::MatMulsQuantizedSoftmaxFunction(inputShapes, types);
function = f.getOriginal();
}
@ -95,13 +95,13 @@ TEST_P(MatMulBiasQuantized, CompareWithRefImpl) {
validateNumSubgraphs();
}
TEST_P(MatMulsQuantized, CompareWithRefImpl) {
TEST_P(MatMulQuantized, CompareWithRefImpl) {
SKIP_IF_CURRENT_TEST_IS_DISABLED()
run();
validateNumSubgraphs();
}
TEST_P(MatMulsQuantizedSoftmax, CompareWithRefImpl) {
TEST_P(MatMulQuantizedSoftmax, CompareWithRefImpl) {
SKIP_IF_CURRENT_TEST_IS_DISABLED()
abs_threshold = 4e-6;
run();

34
src/tests/ov_helpers/ov_snippets_models/include/subgraph_simple.hpp
View File

@ -28,6 +28,36 @@ protected:
std::shared_ptr<ov::Model> initOriginal() const override;
std::shared_ptr<ov::Model> initReference() const override;
};
/// One unary operation: Exp
/// Tokenized simply by starting subgraph.
// in1
// Exp
// Result
class ExpFunction : public SnippetsFunctionBase {
public:
explicit ExpFunction(const std::vector<PartialShape>& inputShapes) : SnippetsFunctionBase(inputShapes) {
OPENVINO_ASSERT(input_shapes.size() == 1, "Got invalid number of input shapes");
}
protected:
std::shared_ptr<ov::Model> initOriginal() const override;
};
/// Two unary operations: Exp + Power(x, -1) (aka Reciprocal)
/// Tokenized simply by starting subgraph.
// in1
// Exp
// Power(x, -1)
// Result
class ExpReciprocalFunction : public SnippetsFunctionBase {
public:
explicit ExpReciprocalFunction(const std::vector<PartialShape>& inputShapes) : SnippetsFunctionBase(inputShapes) {
OPENVINO_ASSERT(input_shapes.size() == 1, "Got invalid number of input shapes");
}
protected:
std::shared_ptr<ov::Model> initOriginal() const override;
};
/// Like AddSinh but with a constant second input (and no sinh on in)
// in1 in2
// Add
@ -155,8 +185,8 @@ protected:
/// Also Output tensors have names to check correct copying output names
// in1 in2
// Add
// HSwish Result
// Relu
// Relu Result
// HSwish
// Result
class EltwiseTwoResultsFunction : public SnippetsFunctionBase {
public:

12
src/tests/ov_helpers/ov_snippets_models/src/subgraph_simple.cpp
View File

@ -27,6 +27,18 @@ std::shared_ptr<ov::Model> AddFunction::initReference() const {
ParameterVector{indata0, indata1}));
return std::make_shared<ov::Model>(NodeVector{add}, ParameterVector{data0, data1});
}
std::shared_ptr<ov::Model> ExpFunction::initOriginal() const {
auto data0 = std::make_shared<op::v0::Parameter>(precision, input_shapes[0]);
auto exp = std::make_shared<op::v0::Exp>(data0);
return std::make_shared<ov::Model>(NodeVector{exp}, ParameterVector{data0});
}
std::shared_ptr<ov::Model> ExpReciprocalFunction::initOriginal() const {
auto data0 = std::make_shared<op::v0::Parameter>(precision, input_shapes[0]);
auto factor = std::make_shared<op::v0::Constant>(precision, ov::Shape{1}, std::vector<float>{-1.f});
auto exp = std::make_shared<op::v0::Exp>(data0);
auto reciprocal = std::make_shared<op::v1::Power>(exp, factor);
return std::make_shared<ov::Model>(NodeVector{reciprocal}, ParameterVector{data0});
}
std::shared_ptr<ov::Model> AddConstFunction::initOriginal() const {
auto data0 = std::make_shared<op::v0::Parameter>(precision, input_shapes[0]);
const std::vector<float> const_values = ov::test::utils::generate_float_numbers(shape_size(m_const_shape.get_shape()), -10., 10.);