[SnippetS] Binary size reduce, executor cache disabled if enable perf count (#21494)

cmake/features.cmake

@@ -47,6 +47,7 @@ ov_dependent_option (ENABLE_ONEDNN_FOR_GPU "Enable oneDNN with GPU support" ${EN
 ov_option (ENABLE_DEBUG_CAPS "enable OpenVINO debug capabilities at runtime" OFF)
 ov_dependent_option (ENABLE_GPU_DEBUG_CAPS "enable GPU debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS;ENABLE_INTEL_GPU" OFF)
 ov_dependent_option (ENABLE_CPU_DEBUG_CAPS "enable CPU debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS;ENABLE_INTEL_CPU" OFF)
+ov_dependent_option (ENABLE_SNIPPETS_DEBUG_CAPS "enable Snippets debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS" OFF)
 
 ov_option (ENABLE_PROFILING_ITT "Build with ITT tracing. Optionally configure pre-built ittnotify library though INTEL_VTUNE_DIR variable." OFF)
 
@@ -207,4 +208,8 @@ if (ENABLE_PROFILING_RAW)
     add_definitions(-DENABLE_PROFILING_RAW=1)
 endif()
 
+if (ENABLE_SNIPPETS_DEBUG_CAPS)
+    add_definitions(-DSNIPPETS_DEBUG_CAPS)
+endif()
+
 ov_print_enabled_features()

src/common/snippets/docs/debug_capabilities/README.md

@@ -0,0 +1,10 @@
+# Debug capabilities
+Debug capabilities are the set of useful debug features, most of them are controlled by environment variables.
+
+They can be activated at runtime and might be used to analyze issues, locate faulty code, get more execution details, benchmark execution time, etc.
+
+Use the following cmake option to enable snippets debug capabilities:
+
+`-DENABLE_DEBUG_CAPS=ON`
+
+* [Performance counters](perf_count.md)

src/common/snippets/docs/debug_capabilities/perf_count.md

@@ -0,0 +1,8 @@
+# Performance counters
+
+Subgraph in snippets could be very large. Sometimes developers are interested the detailed performance number of part of the subgraph. This feature help to do it, by inserting a pair of PerfCountBegin and PerfCountEnd operations around a sequence of expression in LIR(linear IR), which developers would like to benchmark. There is an example to insert between last parameter and first result with a [transformation](../../src/lowered/pass/insert_perf_count.cpp). Developers could adjust it to benchmark their interested sequence.
+
+There are two perf count modes.
+ - `Chrono` : Perf count via chrono call. This is a universal method, and support multi-threads scenario to print perf count data for each thread.
+ - `BackendSpecific` : Perf count provided by backend. This is for device specific requirement. For example, for sake of more light overhead and more accurate result, x86 or x86-64 CPU specific mode via reading RDTSC register is implemented. At current this x86 or x86-64 CPU BackendSpecific mode only support single thread.
+ One can select prefered mode by setting `perf_count_mode` default value in [snippets Config](../../include/snippets/lowered/linear_ir.hpp)

src/common/snippets/include/snippets/lowered/linear_ir.hpp

@@ -14,6 +14,7 @@ namespace ov {
 namespace snippets {
 namespace lowered {
 
+#ifdef SNIPPETS_DEBUG_CAPS
 // Snippets performance count mode
 // Disabled - default, w/o perf count for snippets
 // Chrono - perf count with chrono call. This is a universal method, and support multi-thread case to output perf count data for each thread.
@@ -25,6 +26,7 @@ enum PerfCountMode {
     Chrono,
     BackendSpecific,
 };
+#endif
 
 class Config {
 public:
@@ -33,7 +35,9 @@ public:
     // True if we should check runtime info for nodes to call specific needed transformations
     bool m_need_fill_tail_register = false;
     size_t m_loop_depth = 1;
+#ifdef SNIPPETS_DEBUG_CAPS
     PerfCountMode perf_count_mode = PerfCountMode::Disabled;
+#endif
     // Some Subgraphs doesn't support domain optimization due to operations' semantics
     bool m_enable_domain_optimization = false;
     // Minimal advised work amount for parallel execution.

src/common/snippets/include/snippets/lowered/pass/insert_perf_count.hpp

@@ -1,6 +1,7 @@
 // Copyright (C) 2018-2023 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 //
+#ifdef SNIPPETS_DEBUG_CAPS
 
 #pragma once
 
@@ -31,3 +32,4 @@ public:
 } // namespace lowered
 } // namespace snippets
 } // namespace ov
+#endif  // SNIPPETS_DEBUG_CAPS

src/common/snippets/include/snippets/op/perf_count.hpp

@@ -2,6 +2,8 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+#ifdef SNIPPETS_DEBUG_CAPS
+
 #pragma once
 
 #include "openvino/op/op.hpp"
@@ -91,3 +93,4 @@ private:
 } // namespace op
 } // namespace snippets
 } // namespace ov
+#endif  // SNIPPETS_DEBUG_CAPS

src/common/snippets/include/snippets/snippets_isa_tbl.hpp

@@ -24,8 +24,10 @@ OV_OP(Scalar, ov::snippets::op)
 OV_OP(Nop, ov::snippets::op)
 OV_OP(RankNormalization, ov::snippets::op)
 
+#ifdef SNIPPETS_DEBUG_CAPS
 OV_OP(PerfCountBegin, ov::snippets::op)
 OV_OP(PerfCountEnd, ov::snippets::op)
+#endif
 
 // Layout-oblivious from opset1
 

src/common/snippets/src/generator.cpp

@@ -79,9 +79,12 @@ Generator::opRegType Generator::get_op_reg_type(const std::shared_ptr<Node>& op)
         std::dynamic_pointer_cast<op::Brgemm>(op) ||
         std::dynamic_pointer_cast<op::IntermediateMemoryBuffer>(op) ||
         std::dynamic_pointer_cast<op::NewMemoryBuffer>(op) ||
-        std::dynamic_pointer_cast<op::RankNormalization>(op) ||
-        std::dynamic_pointer_cast<op::PerfCountBeginBase>(op) ||
-        std::dynamic_pointer_cast<op::PerfCountEndBase>(op))
+        std::dynamic_pointer_cast<op::RankNormalization>(op)
+#ifdef SNIPPETS_DEBUG_CAPS
+        || std::dynamic_pointer_cast<op::PerfCountBeginBase>(op)
+        || std::dynamic_pointer_cast<op::PerfCountEndBase>(op)
+#endif
+        )
         return gpr2gpr;
     else if (std::dynamic_pointer_cast<snippets::op::Load>(op) ||
              std::dynamic_pointer_cast<snippets::op::BroadcastLoad>(op))

src/common/snippets/src/lowered/pass/insert_perf_count.cpp

@@ -1,6 +1,7 @@
 // Copyright (C) 2018-2023 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 //
+#ifdef SNIPPETS_DEBUG_CAPS
 
 #include "snippets/lowered/pass/insert_perf_count.hpp"
 #include "snippets/lowered/linear_ir.hpp"
@@ -60,3 +61,4 @@ bool InsertPerfCount::run(LinearIR& linear_ir) {
 } // namespace lowered
 } // namespace snippets
 } // namespace ov
+#endif  // SNIPPETS_DEBUG_CAPS

src/common/snippets/src/op/perf_count.cpp

@@ -1,6 +1,7 @@
 // Copyright (C) 2018-2023 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 //
+#ifdef SNIPPETS_DEBUG_CAPS
 
 #include "snippets/op/perf_count.hpp"
 
@@ -113,3 +114,4 @@ void PerfCountEnd::output_perf_count() {
 } // namespace op
 } // namespace snippets
 } // namespace ov
+#endif  // SNIPPETS_DEBUG_CAPS

src/common/snippets/src/op/subgraph.cpp

@@ -349,8 +349,10 @@ VectorDims Subgraph::infer_master_shape() {
 std::shared_ptr<lowered::LinearIR>
 Subgraph::convert_body_to_linear_ir(const std::shared_ptr<IShapeInferSnippetsFactory>& shape_infer_factory) {
     lowered::Config lowering_config;
-    lowering_config.m_save_expressions = config.m_has_domain_sensitive_ops ||
-        (lowering_config.perf_count_mode != lowered::PerfCountMode::Disabled);
+    lowering_config.m_save_expressions = config.m_has_domain_sensitive_ops;
+#ifdef SNIPPETS_DEBUG_CAPS
+    lowering_config.m_save_expressions = lowering_config.m_save_expressions || (lowering_config.perf_count_mode != lowered::PerfCountMode::Disabled);
+#endif
     lowering_config.m_need_fill_tail_register = config.m_has_domain_sensitive_ops;
     lowering_config.m_loop_depth = tileRank;
     lowering_config.m_enable_domain_optimization = !config.m_has_domain_sensitive_ops;
@@ -494,10 +496,12 @@ snippets::Schedule Subgraph::generate_from_linear_ir(const lowered::pass::PassPi
     auto linear_ir {*m_linear_ir->clone()};
     LoweringResult lowering_result;
     control_flow_transformations(linear_ir, lowering_result, backend_passes_pre_common, backend_passes_post_common);
+#ifdef SNIPPETS_DEBUG_CAPS
     if (linear_ir.get_config().perf_count_mode == lowered::PerfCountMode::Chrono) {
         lowered::pass::InsertPerfCount perf_count_pass;
         perf_count_pass.run(linear_ir);
     }
+#endif
     m_generator->generate(linear_ir, lowering_result, compile_params);
 
     VectorDims parallel_exec_domain = linear_ir.get_master_shape();

src/common/snippets/src/shape_inference/shape_inference.cpp

@@ -55,8 +55,10 @@ const IShapeInferSnippetsFactory::TRegistry IShapeInferSnippetsFactory::registry
         SHAPE_INFER_PREDEFINED(op::Scalar, SingleElementShapeInfer),
         SHAPE_INFER_PREDEFINED(op::VectorBuffer, SingleElementShapeInfer),
         SHAPE_INFER_PREDEFINED(op::LoopEnd, EmptyShapeInfer),
+#ifdef SNIPPETS_DEBUG_CAPS
         SHAPE_INFER_PREDEFINED(op::PerfCountBegin, EmptyShapeInfer),
         SHAPE_INFER_PREDEFINED(op::PerfCountEnd, EmptyShapeInfer),
+#endif
         SHAPE_INFER_PREDEFINED(op::Kernel, EmptyShapeInfer),
         SHAPE_INFER_PREDEFINED(op::Nop, EmptyShapeInfer),
         SHAPE_INFER_OP_SPECIFIC_EXTERNAL(opset1::Select, SelectShapeInfer),

src/common/snippets/tests/src/lowering_utils.cpp

@@ -41,8 +41,10 @@ DummyTargetMachine::DummyTargetMachine(const std::vector<ov::Node::type_info_t>&
     jitters[ov::snippets::op::Kernel::get_type_info_static()] = dummy_functor;
     jitters[ov::snippets::op::LoopBegin::get_type_info_static()] = dummy_functor;
     jitters[ov::snippets::op::LoopEnd::get_type_info_static()] = dummy_functor;
+#ifdef SNIPPETS_DEBUG_CAPS
     jitters[ov::snippets::op::PerfCountBegin::get_type_info_static()] = dummy_functor;
     jitters[ov::snippets::op::PerfCountEnd::get_type_info_static()] = dummy_functor;
+#endif
     jitters[ov::snippets::op::Brgemm::get_type_info_static()] = dummy_functor;
     jitters[ov::snippets::op::IntermediateMemoryBuffer::get_type_info_static()] = dummy_functor;
     jitters[ov::snippets::op::NewMemoryBuffer::get_type_info_static()] = dummy_functor;

src/plugins/intel_cpu/src/emitters/x64/cpu_generator.cpp

@@ -14,8 +14,6 @@
 #include "jit_dnnl_emitters.hpp"
 #include "jit_dnnl_ext_emitters.hpp"
 #include "jit_conversion_emitters.hpp"
-#include "jit_perf_count_chrono_emitters.hpp"
-#include "jit_perf_count_rdtsc_emitters.hpp"
 
 #include "transformations/snippets/x64/op/load_convert.hpp"
 #include "transformations/snippets/x64/op/store_convert.hpp"
@@ -28,6 +26,12 @@
 
 #include <openvino/opsets/opset5.hpp>
 
+#ifdef SNIPPETS_DEBUG_CAPS
+#include "jit_perf_count_chrono_emitters.hpp"
+#include "jit_perf_count_rdtsc_emitters.hpp"
+#include "transformations/snippets/x64/op/perf_count_rdtsc.hpp"
+#endif
+
 namespace ov {
 
 #define CREATE_SNIPPETS_EMITTER(e_type) { \
@@ -162,10 +166,12 @@ intel_cpu::CPUTargetMachine::CPUTargetMachine(dnnl::impl::cpu::x64::cpu_isa_t ho
     jitters[intel_cpu::BrgemmCPU::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(BrgemmEmitter);
     jitters[intel_cpu::BrgemmCopyB::get_type_info_static()] = CREATE_SNIPPETS_EMITTER(BrgemmCopyBEmitter);
 
+#ifdef SNIPPETS_DEBUG_CAPS
     jitters[snippets::op::PerfCountBegin::get_type_info_static()] = CREATE_CPU_EMITTER(ov::intel_cpu::jit_perf_count_chrono_start_emitter);
     jitters[snippets::op::PerfCountEnd::get_type_info_static()] = CREATE_CPU_EMITTER(ov::intel_cpu::jit_perf_count_chrono_end_emitter);
     jitters[ov::intel_cpu::PerfCountRdtscBegin::get_type_info_static()] = CREATE_CPU_EMITTER(ov::intel_cpu::jit_perf_count_rdtsc_start_emitter);
     jitters[ov::intel_cpu::PerfCountRdtscEnd::get_type_info_static()] = CREATE_CPU_EMITTER(ov::intel_cpu::jit_perf_count_rdtsc_end_emitter);
+#endif
 }
 
 size_t intel_cpu::CPUTargetMachine::get_lanes() const {
@@ -232,11 +238,15 @@ snippets::Generator::opRegType intel_cpu::CPUGenerator::get_specific_op_reg_type
         OPENVINO_THROW("Register type of the operation " + std::string(op->get_type_name()) + " isn't determined!");
 }
 bool intel_cpu::CPUGenerator::uses_precompiled_kernel(const std::shared_ptr<snippets::Emitter>& e) const {
-    return std::dynamic_pointer_cast<intel_cpu::BrgemmEmitter>(e) ||
-           std::dynamic_pointer_cast<intel_cpu::BrgemmCopyBEmitter>(e) ||
+    bool need = std::dynamic_pointer_cast<intel_cpu::BrgemmEmitter>(e) ||
+                std::dynamic_pointer_cast<intel_cpu::BrgemmCopyBEmitter>(e);
+#ifdef SNIPPETS_DEBUG_CAPS
+    need = need ||
            std::dynamic_pointer_cast<intel_cpu::jit_perf_count_chrono_start_emitter>(e) ||
            std::dynamic_pointer_cast<intel_cpu::jit_perf_count_chrono_end_emitter>(e) ||
            std::dynamic_pointer_cast<intel_cpu::jit_perf_count_rdtsc_start_emitter>(e) ||
            std::dynamic_pointer_cast<intel_cpu::jit_perf_count_rdtsc_end_emitter>(e);
+#endif
+    return need;
 }
 } // namespace ov

src/plugins/intel_cpu/src/emitters/x64/jit_perf_count_chrono_emitters.cpp

@@ -1,6 +1,7 @@
 // Copyright (C) 2018-2023 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 //
+#ifdef SNIPPETS_DEBUG_CAPS
 
 #include "jit_emitter.hpp"
 #include "jit_perf_count_chrono_emitters.hpp"
@@ -71,3 +72,4 @@ void jit_perf_count_chrono_end_emitter::emit_impl(const std::vector<size_t> &in_
 
 }   // namespace intel_cpu
 }   // namespace ov
+#endif // SNIPPETS_DEBUG_CAPS

src/plugins/intel_cpu/src/emitters/x64/jit_perf_count_chrono_emitters.hpp

@@ -1,6 +1,7 @@
 // Copyright (C) 2018-2023 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 //
+#ifdef SNIPPETS_DEBUG_CAPS
 
 #pragma once
 
@@ -38,3 +39,4 @@ private:
 
 }   // namespace intel_cpu
 }   // namespace ov
+#endif // SNIPPETS_DEBUG_CAPS

src/plugins/intel_cpu/src/emitters/x64/jit_perf_count_rdtsc_emitters.cpp

@@ -1,6 +1,7 @@
 // Copyright (C) 2018-2023 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 //
+#ifdef SNIPPETS_DEBUG_CAPS
 
 #include "jit_emitter.hpp"
 #include "jit_perf_count_rdtsc_emitters.hpp"
@@ -84,3 +85,4 @@ void jit_perf_count_rdtsc_end_emitter::emit_impl(const std::vector<size_t> &in_i
 
 }   // namespace intel_cpu
 }   // namespace ov
+#endif // SNIPPETS_DEBUG_CAPS

src/plugins/intel_cpu/src/emitters/x64/jit_perf_count_rdtsc_emitters.hpp

@@ -1,6 +1,7 @@
 // Copyright (C) 2018-2023 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 //
+#ifdef SNIPPETS_DEBUG_CAPS
 
 #pragma once
 
@@ -35,3 +36,4 @@ private:
 
 }   // namespace intel_cpu
 }   // namespace ov
+#endif // SNIPPETS_DEBUG_CAPS

src/plugins/intel_cpu/src/extension.cpp

@@ -161,16 +161,20 @@ std::map<std::string, ngraph::OpSet> Extension::getOpSets() {
         NGRAPH_OP(Subgraph, ov::snippets::op)
         NGRAPH_OP(VectorBuffer, ov::snippets::op)
         NGRAPH_OP(RankNormalization, ov::snippets::op)
+#ifdef SNIPPETS_DEBUG_CAPS
         NGRAPH_OP(PerfCountBegin, ov::snippets::op)
         NGRAPH_OP(PerfCountEnd, ov::snippets::op)
+#endif
         NGRAPH_OP_X64(LoadConvertSaturation, ov::intel_cpu)
         NGRAPH_OP_X64(LoadConvertTruncation, ov::intel_cpu)
         NGRAPH_OP_X64(StoreConvertSaturation, ov::intel_cpu)
         NGRAPH_OP_X64(StoreConvertTruncation, ov::intel_cpu)
         NGRAPH_OP_X64(BrgemmCPU, ov::intel_cpu)
         NGRAPH_OP_X64(BrgemmCopyB, ov::intel_cpu)
+#ifdef SNIPPETS_DEBUG_CAPS
         NGRAPH_OP_X64(PerfCountRdtscBegin, ov::intel_cpu)
         NGRAPH_OP_X64(PerfCountRdtscEnd, ov::intel_cpu)
+#endif
 #undef NGRAPH_OP
 
         return opset;

src/plugins/intel_cpu/src/nodes/subgraph.cpp

@@ -37,6 +37,8 @@
 #include <common/primitive_hashing_utils.hpp>
 #include "snippets/pass/hash.hpp"
 
+#include "snippets/lowered/linear_ir.hpp"
+
 using namespace InferenceEngine;
 using namespace dnnl::impl::utils;
 using namespace dnnl::impl::cpu;
@@ -395,12 +397,26 @@ void Snippet::prepareParams() {
         return executor;
     };
 
-    auto cache = context->getParamsCache();
-    auto result = cache->getOrCreate(key, builder);
-    execPtr = result.first;
-    if (!execPtr) {
-        OPENVINO_THROW("Executor is not created for node ", getName(), ".");
+    auto getOrCreateExecutor = [this, &key, &builder]() {
+        auto cache = context->getParamsCache();
+        auto result = cache->getOrCreate(key, builder);
+        execPtr = result.first;
+        if (!execPtr) {
+            OPENVINO_THROW("Executor is not created for node ", getName(), ".");
+        }
+    };
+
+#ifndef SNIPPETS_DEBUG_CAPS
+    getOrCreateExecutor();
+#else
+    snippets::lowered::Config config;
+    if (config.perf_count_mode == snippets::lowered::PerfCountMode::Disabled) {
+        getOrCreateExecutor();
+    } else {
+        // in case perf count is enabled, disable executor cache by default to not mix up perf counters for different subgraphs.
+        execPtr = std::make_shared<SnippetJitExecutor>(key.attrs, is_dynamic, context->getConfig().inferencePrecision == ov::element::bf16);
     }
+#endif
 }
 
 bool Snippet::needPrepareParams() const {

src/plugins/intel_cpu/src/transformations/snippets/x64/op/perf_count_rdtsc.cpp

@@ -1,6 +1,7 @@
 // Copyright (C) 2018-2023 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 //
+#ifdef SNIPPETS_DEBUG_CAPS
 
 #include "perf_count_rdtsc.hpp"
 
@@ -30,3 +31,4 @@ std::shared_ptr<PerfCountRdtscBegin> PerfCountRdtscEnd::get_pc_begin() {
     OPENVINO_ASSERT(pc_begin != nullptr, "PerfCountRdtscEnd last input is not connected to PerfCountRdtscBegin");
     return  pc_begin;
 }
+#endif // SNIPPETS_DEBUG_CAPS

src/plugins/intel_cpu/src/transformations/snippets/x64/op/perf_count_rdtsc.hpp

@@ -1,6 +1,7 @@
 // Copyright (C) 2018-2023 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 //
+#ifdef SNIPPETS_DEBUG_CAPS
 
 #pragma once
 
@@ -53,3 +54,4 @@ public:
 
 } // namespace intel_cpu
 } // namespace ov
+#endif // SNIPPETS_DEBUG_CAPS

src/plugins/intel_cpu/src/transformations/snippets/x64/shape_inference.cpp

@@ -39,8 +39,10 @@ const CPUShapeInferSnippetsFactory::TRegistry CPUShapeInferSnippetsFactory::spec
         SHAPE_INFER_PREDEFINED(ov::intel_cpu::LoadConvertTruncation, PassThroughShapeInfer),
         SHAPE_INFER_PREDEFINED(ov::intel_cpu::StoreConvertSaturation, PassThroughShapeInfer),
         SHAPE_INFER_PREDEFINED(ov::intel_cpu::StoreConvertTruncation, PassThroughShapeInfer),
+#ifdef SNIPPETS_DEBUG_CAPS
         SHAPE_INFER_PREDEFINED(ov::intel_cpu::PerfCountRdtscBegin, EmptyShapeInfer),
         SHAPE_INFER_PREDEFINED(ov::intel_cpu::PerfCountRdtscEnd, EmptyShapeInfer),
+#endif
         SHAPE_INFER_OP_SPECIFIC_EXTERNAL(ov::intel_cpu::BrgemmCPU, BrgemmShapeInfer),
         //
         SHAPE_INFER_OP_SPECIFIC(ov::intel_cpu::BrgemmCopyB),