diff --git a/.gitmodules b/.gitmodules index 09894aceba7..5e65d4e8abf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -69,3 +69,6 @@ [submodule "thirdparty/snappy"] path = thirdparty/snappy url = https://github.com/google/snappy.git +[submodule "ARMComputeLibrary"] + path = src/plugins/intel_cpu/thirdparty/ComputeLibrary + url = https://github.com/ARM-software/ComputeLibrary.git diff --git a/cmake/features.cmake b/cmake/features.cmake index 402c848a2b2..ea70019396c 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -6,7 +6,7 @@ # Common cmake options # -ie_dependent_option (ENABLE_INTEL_CPU "CPU plugin for OpenVINO Runtime" ON "RISCV64 OR X86 OR X86_64" OFF) +ie_dependent_option (ENABLE_INTEL_CPU "CPU plugin for OpenVINO Runtime" ON "RISCV64 OR X86 OR X86_64 OR AARCH64" OFF) ie_option (ENABLE_TESTS "unit, behavior and functional tests" OFF) diff --git a/src/plugins/intel_cpu/CMakeLists.txt b/src/plugins/intel_cpu/CMakeLists.txt index 85f8b8c5fe8..461e3cc46b4 100644 --- a/src/plugins/intel_cpu/CMakeLists.txt +++ b/src/plugins/intel_cpu/CMakeLists.txt @@ -38,6 +38,43 @@ if(ENABLE_TESTS) add_subdirectory(tests) endif() +add_definitions(-DOV_CPU_WITH_DNNL) +set(OV_CPU_WITH_DNNL ON) + +if(DNNL_AARCH64_USE_ACL) + add_definitions(-DOV_CPU_WITH_ACL) + set(OV_CPU_WITH_ACL ON) +endif() + +if(OV_CPU_WITH_ACL) + set(CMAKE_CXX_STANDARD 14) +endif() + +# remove target specific files from compilation + +if (NOT OV_CPU_WITH_ACL) + set(EXCLUDE_PATHS ${EXCLUDE_PATHS} + ${CMAKE_CURRENT_SOURCE_DIR}/src/nodes/executors/acl/*) +endif() + +if (NOT X86_64) + set(EXCLUDE_PATHS ${EXCLUDE_PATHS} + ${CMAKE_CURRENT_SOURCE_DIR}/src/nodes/executors/x64/* + ${CMAKE_CURRENT_SOURCE_DIR}/src/nodes/kernels/x64/* + ${CMAKE_CURRENT_SOURCE_DIR}/src/emitters/x64/* + ${CMAKE_CURRENT_SOURCE_DIR}/src/transformations/cpu_opset/x64/* + ${CMAKE_CURRENT_SOURCE_DIR}/src/transformations/snippets/x64/*) +endif() + +if (NOT AARCH64) + set(EXCLUDE_PATHS ${EXCLUDE_PATHS} + ${CMAKE_CURRENT_SOURCE_DIR}/src/transformations/cpu_opset/arm/*) +endif() + +file(GLOB_RECURSE FILES_TO_REMOVE ${EXCLUDE_PATHS}) +list(REMOVE_ITEM SOURCES ${FILES_TO_REMOVE}) +list(REMOVE_ITEM HEADERS ${FILES_TO_REMOVE}) + # create plugin ie_add_plugin(NAME ${TARGET_NAME} diff --git a/src/plugins/intel_cpu/README.md b/src/plugins/intel_cpu/README.md index 87530644ebe..d64b0f3284d 100644 --- a/src/plugins/intel_cpu/README.md +++ b/src/plugins/intel_cpu/README.md @@ -27,4 +27,4 @@ CPU Plugin contains the following components: * [OpenVINO™ README](../../../README.md) * [OpenVINO Core Components](../../README.md) * [OpenVINO Plugins](../README.md) - * [Developer documentation](../../../docs/dev/index.md) \ No newline at end of file + * [Developer documentation](../../../docs/dev/index.md) diff --git a/src/plugins/intel_cpu/src/cache/multi_cache.h b/src/plugins/intel_cpu/src/cache/multi_cache.h index f014dad7120..746499bd9b2 100644 --- a/src/plugins/intel_cpu/src/cache/multi_cache.h +++ b/src/plugins/intel_cpu/src/cache/multi_cache.h @@ -79,6 +79,8 @@ MultiCache::EntryPtr MultiCache::getEntry() { return std::static_pointer_cast(itr->second); } +using MultiCacheWeakPtr = std::weak_ptr; +using MultiCacheWeakCPtr = std::weak_ptr; using MultiCachePtr = std::shared_ptr; using MultiCacheCPtr = std::shared_ptr; diff --git a/src/plugins/intel_cpu/src/config.cpp b/src/plugins/intel_cpu/src/config.cpp index 534d835534f..3f51a7ab007 100644 --- a/src/plugins/intel_cpu/src/config.cpp +++ b/src/plugins/intel_cpu/src/config.cpp @@ -263,6 +263,11 @@ void Config::readProperties(const std::map &prop) { if (exclusiveAsyncRequests) // Exclusive request feature disables the streams streamExecutorConfig._streams = 1; +#if defined(OPENVINO_ARCH_ARM) || defined(OPENVINO_ARCH_ARM64) + // TODO: multi-stream execution has functional issues on ARM target + streamExecutorConfig._streams = 1; +#endif + CPU_DEBUG_CAP_ENABLE(applyDebugCapsProperties()); updateProperties(); } diff --git a/src/plugins/intel_cpu/src/config.h b/src/plugins/intel_cpu/src/config.h index 894d21d87c6..09141d36959 100644 --- a/src/plugins/intel_cpu/src/config.h +++ b/src/plugins/intel_cpu/src/config.h @@ -48,7 +48,12 @@ struct Config { std::string device_id = {}; int batchLimit = 0; float fcSparseWeiDecompressionRate = 1.0f; +#if defined(OPENVINO_ARCH_X86_64) size_t rtCacheCapacity = 5000ul; +#else + // TODO: Executor cache may leads to incorrect behavior on oneDNN ACL primitives + size_t rtCacheCapacity = 0ul; +#endif InferenceEngine::IStreamsExecutor::Config streamExecutorConfig; InferenceEngine::PerfHintsConfig perfHintsConfig; bool enableCpuPinning = true; diff --git a/src/plugins/intel_cpu/src/cpu_shape.h b/src/plugins/intel_cpu/src/cpu_shape.h index bd9936dadb4..67fdf377cec 100644 --- a/src/plugins/intel_cpu/src/cpu_shape.h +++ b/src/plugins/intel_cpu/src/cpu_shape.h @@ -194,7 +194,7 @@ public: } enum : Dim { - UNDEFINED_DIM = 0xffffffffffffffff + UNDEFINED_DIM = std::numeric_limits::max() }; private: diff --git a/src/plugins/intel_cpu/src/cpu_types.cpp b/src/plugins/intel_cpu/src/cpu_types.cpp index 818d947311d..9ac98d3dea0 100644 --- a/src/plugins/intel_cpu/src/cpu_types.cpp +++ b/src/plugins/intel_cpu/src/cpu_types.cpp @@ -9,9 +9,6 @@ namespace ov { namespace intel_cpu { -using Dim = std::size_t; -using VectorDims = std::vector; - const InferenceEngine::details::caseless_unordered_map type_to_name_tbl = { { "Constant", Type::Input }, { "Parameter", Type::Input }, @@ -69,6 +66,7 @@ const InferenceEngine::details::caseless_unordered_map type_t { "SoftPlus", Type::Eltwise }, { "SoftSign", Type::Eltwise }, { "Select", Type::Eltwise}, + { "Log", Type::Eltwise }, { "Reshape", Type::Reshape }, { "Squeeze", Type::Reshape }, { "Unsqueeze", Type::Reshape }, @@ -163,7 +161,6 @@ const InferenceEngine::details::caseless_unordered_map type_t { "Floor", Type::Math}, { "HardSigmoid", Type::Math}, { "If", Type::If}, - { "Log", Type::Math}, { "Neg", Type::Math}, { "Reciprocal", Type::Math}, { "Selu", Type::Math}, @@ -448,7 +445,8 @@ std::string algToString(const Algorithm alg) { CASE(EltwiseLogicalXor); CASE(EltwiseLogicalNot); CASE(EltwiseRelu); - CASE(EltwiseGelu); + CASE(EltwiseGeluErf); + CASE(EltwiseGeluTanh); CASE(EltwiseElu); CASE(EltwiseTanh); CASE(EltwiseSelect); @@ -466,10 +464,10 @@ std::string algToString(const Algorithm alg) { CASE(EltwiseRoundHalfToEven); CASE(EltwiseRoundHalfAwayFromZero); CASE(EltwiseErf); + CASE(EltwiseLog); CASE(FQCommon); CASE(FQQuantization); CASE(FQBinarization); - CASE(FQRequantization); CASE(ROIPoolingMax); CASE(ROIPoolingBilinear); CASE(ROIAlignMax); @@ -502,7 +500,6 @@ std::string algToString(const Algorithm alg) { CASE(MathErf); CASE(MathFloor); CASE(MathHardSigmoid); - CASE(MathLog); CASE(MathNegative); CASE(MathReciprocal); CASE(MathSelu); diff --git a/src/plugins/intel_cpu/src/cpu_types.h b/src/plugins/intel_cpu/src/cpu_types.h index 070d08c1371..403ed62d482 100644 --- a/src/plugins/intel_cpu/src/cpu_types.h +++ b/src/plugins/intel_cpu/src/cpu_types.h @@ -160,7 +160,8 @@ enum class Algorithm { EltwiseLogicalXor, EltwiseLogicalNot, EltwiseRelu, - EltwiseGelu, + EltwiseGeluErf, + EltwiseGeluTanh, EltwiseElu, EltwiseTanh, EltwiseSigmoid, @@ -179,12 +180,12 @@ enum class Algorithm { EltwiseRoundHalfAwayFromZero, EltwiseErf, EltwiseSoftSign, + EltwiseLog, // FakeQuantize algorithms FQCommon, FQQuantization, FQBinarization, - FQRequantization, // ROIPooling algorithms ROIPoolingMax, @@ -227,7 +228,6 @@ enum class Algorithm { MathErf, MathFloor, MathHardSigmoid, - MathLog, MathNegative, MathReciprocal, MathSelu, diff --git a/src/plugins/intel_cpu/src/dnnl_extension_utils.cpp b/src/plugins/intel_cpu/src/dnnl_extension_utils.cpp index 8719853e96f..fc1419b92e3 100644 --- a/src/plugins/intel_cpu/src/dnnl_extension_utils.cpp +++ b/src/plugins/intel_cpu/src/dnnl_extension_utils.cpp @@ -203,5 +203,80 @@ const char* DnnlExtensionUtils::query_pd_info(const_dnnl_primitive_desc_t pd) { return pd->info(); } +dnnl::algorithm DnnlExtensionUtils::convertToDnnlAlgorithm(Algorithm alg) { + switch (alg) { + case Algorithm::EltwiseRelu: return dnnl::algorithm::eltwise_relu; + case Algorithm::EltwiseTanh: return dnnl::algorithm::eltwise_tanh; + case Algorithm::EltwiseElu: return dnnl::algorithm::eltwise_elu; + case Algorithm::EltwiseAbs: return dnnl::algorithm::eltwise_abs; + case Algorithm::EltwiseSqrt: return dnnl::algorithm::eltwise_sqrt; + case Algorithm::EltwiseSwish: return dnnl::algorithm::eltwise_swish; + case Algorithm::EltwiseHswish: return dnnl::algorithm::eltwise_hardswish; + case Algorithm::EltwiseSoftRelu: return dnnl::algorithm::eltwise_soft_relu; + case Algorithm::EltwiseMish: return dnnl::algorithm::eltwise_mish; + case Algorithm::EltwiseExp: return dnnl::algorithm::eltwise_exp; + case Algorithm::EltwiseGeluErf: return dnnl::algorithm::eltwise_gelu_erf; + case Algorithm::EltwiseGeluTanh: return dnnl::algorithm::eltwise_gelu_tanh; + case Algorithm::EltwiseSigmoid: return dnnl::algorithm::eltwise_logistic; + case Algorithm::EltwiseClamp: return dnnl::algorithm::eltwise_clip; + case Algorithm::EltwisePowerStatic: return dnnl::algorithm::eltwise_pow; + case Algorithm::EltwiseHsigmoid: return dnnl::algorithm::eltwise_hsigmoid; + case Algorithm::EltwiseRoundHalfToEven: return dnnl::algorithm::eltwise_round_half_to_even; + case Algorithm::EltwiseRoundHalfAwayFromZero: return dnnl::algorithm::eltwise_round_half_away_from_zero; + case Algorithm::EltwiseAdd: return dnnl::algorithm::binary_add; + case Algorithm::EltwiseMultiply: return dnnl::algorithm::binary_mul; + case Algorithm::EltwiseSubtract: return dnnl::algorithm::binary_sub; + case Algorithm::EltwiseDivide: return dnnl::algorithm::binary_div; + case Algorithm::EltwiseMaximum: return dnnl::algorithm::binary_max; + case Algorithm::EltwiseMinimum: return dnnl::algorithm::binary_min; + case Algorithm::EltwiseEqual: return dnnl::algorithm::binary_eq; + case Algorithm::EltwiseNotEqual: return dnnl::algorithm::binary_ne; + case Algorithm::EltwiseGreater: return dnnl::algorithm::binary_gt; + case Algorithm::EltwiseGreaterEqual: return dnnl::algorithm::binary_ge; + case Algorithm::EltwiseLess: return dnnl::algorithm::binary_lt; + case Algorithm::EltwiseLessEqual: return dnnl::algorithm::binary_le; + case Algorithm::EltwisePrelu: return dnnl::algorithm::binary_prelu; + case Algorithm::ReduceMax: return dnnl::algorithm::reduction_max; + case Algorithm::ReduceMin: return dnnl::algorithm::reduction_min; + case Algorithm::ReduceSum: return dnnl::algorithm::reduction_sum; + case Algorithm::ReduceMean: return dnnl::algorithm::reduction_mean; + case Algorithm::FQCommon: return dnnl::algorithm::quantization_quantize_dequantize; + case Algorithm::FQQuantization: return dnnl::algorithm::quantization_quantize; + case Algorithm::FQBinarization: return dnnl::algorithm::binarization_depthwise; + default: return dnnl::algorithm::undef; + } +} + +bool DnnlExtensionUtils::isUnarySupportedAsPostOp(Algorithm alg) { +#if defined(OV_CPU_WITH_ACL) + return one_of(alg, Algorithm::EltwiseRelu, + Algorithm::EltwiseTanh, + Algorithm::EltwiseElu, + Algorithm::EltwiseAbs, + Algorithm::EltwiseSqrt, + Algorithm::EltwiseSoftRelu, + Algorithm::EltwiseSigmoid); +#elif defined(OPENVINO_ARCH_X86_64) + return one_of(alg, Algorithm::EltwiseRelu, + Algorithm::EltwiseGeluErf, + Algorithm::EltwiseGeluTanh, + Algorithm::EltwiseElu, + Algorithm::EltwiseSigmoid, + Algorithm::EltwiseClamp, + Algorithm::EltwiseTanh, + Algorithm::EltwiseSwish, + Algorithm::EltwiseHswish, + Algorithm::EltwiseMish, + Algorithm::EltwiseHsigmoid, + Algorithm::EltwiseRoundHalfToEven, + Algorithm::EltwiseRoundHalfAwayFromZero, + Algorithm::EltwiseAbs, + Algorithm::EltwiseSqrt, + Algorithm::EltwiseSoftRelu); +#else + return false; +#endif +} + } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/src/dnnl_extension_utils.h b/src/plugins/intel_cpu/src/dnnl_extension_utils.h index 8ae9bbac905..c378e305b57 100644 --- a/src/plugins/intel_cpu/src/dnnl_extension_utils.h +++ b/src/plugins/intel_cpu/src/dnnl_extension_utils.h @@ -57,6 +57,8 @@ public: static bool hasProperImplementationType(dnnl::primitive_desc& desc, impl_desc_type implType); static dnnl_memory_desc_t clone_desc(const_dnnl_memory_desc_t cdesc); static const char* query_pd_info(const_dnnl_primitive_desc_t pd); + static dnnl::algorithm convertToDnnlAlgorithm(Algorithm alg); + static bool isUnarySupportedAsPostOp(Algorithm alg); }; } // namespace intel_cpu diff --git a/src/plugins/intel_cpu/src/emitters/cpu_generator.cpp b/src/plugins/intel_cpu/src/emitters/x64/cpu_generator.cpp similarity index 96% rename from src/plugins/intel_cpu/src/emitters/cpu_generator.cpp rename to src/plugins/intel_cpu/src/emitters/x64/cpu_generator.cpp index 3841a768d42..2dad24ae632 100644 --- a/src/plugins/intel_cpu/src/emitters/cpu_generator.cpp +++ b/src/plugins/intel_cpu/src/emitters/x64/cpu_generator.cpp @@ -15,12 +15,13 @@ #include "jit_dnnl_ext_emitters.hpp" #include "jit_conversion_emitters.hpp" -#include "snippets_transformations/op/load_convert.hpp" -#include "snippets_transformations/op/store_convert.hpp" -#include "snippets_transformations/op/fused_mul_add.hpp" -#include "snippets_transformations/op/brgemm_copy_b.hpp" -#include "snippets_transformations/op/brgemm_cpu.hpp" -#include "ngraph_transformations/op/swish_cpu.hpp" +#include "transformations/snippets/x64/op/load_convert.hpp" +#include "transformations/snippets/x64/op/store_convert.hpp" +#include "transformations/snippets/x64/op/fused_mul_add.hpp" +#include "transformations/snippets/x64/op/brgemm_copy_b.hpp" +#include "transformations/snippets/x64/op/brgemm_cpu.hpp" +#include "snippets/op/brgemm.hpp" +#include "transformations/cpu_opset/common/op/swish_cpu.hpp" #include diff --git a/src/plugins/intel_cpu/src/emitters/cpu_generator.hpp b/src/plugins/intel_cpu/src/emitters/x64/cpu_generator.hpp similarity index 100% rename from src/plugins/intel_cpu/src/emitters/cpu_generator.hpp rename to src/plugins/intel_cpu/src/emitters/x64/cpu_generator.hpp diff --git a/src/plugins/intel_cpu/src/emitters/jit_bf16_emitters.hpp b/src/plugins/intel_cpu/src/emitters/x64/jit_bf16_emitters.hpp similarity index 100% rename from src/plugins/intel_cpu/src/emitters/jit_bf16_emitters.hpp rename to src/plugins/intel_cpu/src/emitters/x64/jit_bf16_emitters.hpp diff --git a/src/plugins/intel_cpu/src/emitters/jit_conversion_emitters.cpp b/src/plugins/intel_cpu/src/emitters/x64/jit_conversion_emitters.cpp similarity index 100% rename from src/plugins/intel_cpu/src/emitters/jit_conversion_emitters.cpp rename to src/plugins/intel_cpu/src/emitters/x64/jit_conversion_emitters.cpp diff --git a/src/plugins/intel_cpu/src/emitters/jit_conversion_emitters.hpp b/src/plugins/intel_cpu/src/emitters/x64/jit_conversion_emitters.hpp similarity index 100% rename from src/plugins/intel_cpu/src/emitters/jit_conversion_emitters.hpp rename to src/plugins/intel_cpu/src/emitters/x64/jit_conversion_emitters.hpp diff --git a/src/plugins/intel_cpu/src/emitters/jit_dnnl_emitters.cpp b/src/plugins/intel_cpu/src/emitters/x64/jit_dnnl_emitters.cpp similarity index 100% rename from src/plugins/intel_cpu/src/emitters/jit_dnnl_emitters.cpp rename to src/plugins/intel_cpu/src/emitters/x64/jit_dnnl_emitters.cpp diff --git a/src/plugins/intel_cpu/src/emitters/jit_dnnl_emitters.hpp b/src/plugins/intel_cpu/src/emitters/x64/jit_dnnl_emitters.hpp similarity index 100% rename from src/plugins/intel_cpu/src/emitters/jit_dnnl_emitters.hpp rename to src/plugins/intel_cpu/src/emitters/x64/jit_dnnl_emitters.hpp diff --git a/src/plugins/intel_cpu/src/emitters/jit_dnnl_ext_emitters.hpp b/src/plugins/intel_cpu/src/emitters/x64/jit_dnnl_ext_emitters.hpp similarity index 99% rename from src/plugins/intel_cpu/src/emitters/jit_dnnl_ext_emitters.hpp rename to src/plugins/intel_cpu/src/emitters/x64/jit_dnnl_ext_emitters.hpp index bf326afd2b7..6e31515fa46 100644 --- a/src/plugins/intel_cpu/src/emitters/jit_dnnl_ext_emitters.hpp +++ b/src/plugins/intel_cpu/src/emitters/x64/jit_dnnl_ext_emitters.hpp @@ -5,7 +5,7 @@ #pragma once #include "ngraph/opsets/opset5.hpp" -#include "ngraph_transformations/op/swish_cpu.hpp" +#include "transformations/cpu_opset/common/op/swish_cpu.hpp" #include "jit_dnnl_emitters.hpp" namespace ov { diff --git a/src/plugins/intel_cpu/src/emitters/jit_eltwise_emitters.cpp b/src/plugins/intel_cpu/src/emitters/x64/jit_eltwise_emitters.cpp similarity index 100% rename from src/plugins/intel_cpu/src/emitters/jit_eltwise_emitters.cpp rename to src/plugins/intel_cpu/src/emitters/x64/jit_eltwise_emitters.cpp diff --git a/src/plugins/intel_cpu/src/emitters/jit_eltwise_emitters.hpp b/src/plugins/intel_cpu/src/emitters/x64/jit_eltwise_emitters.hpp similarity index 100% rename from src/plugins/intel_cpu/src/emitters/jit_eltwise_emitters.hpp rename to src/plugins/intel_cpu/src/emitters/x64/jit_eltwise_emitters.hpp diff --git a/src/plugins/intel_cpu/src/emitters/jit_emitter.cpp b/src/plugins/intel_cpu/src/emitters/x64/jit_emitter.cpp similarity index 100% rename from src/plugins/intel_cpu/src/emitters/jit_emitter.cpp rename to src/plugins/intel_cpu/src/emitters/x64/jit_emitter.cpp diff --git a/src/plugins/intel_cpu/src/emitters/jit_emitter.hpp b/src/plugins/intel_cpu/src/emitters/x64/jit_emitter.hpp similarity index 100% rename from src/plugins/intel_cpu/src/emitters/jit_emitter.hpp rename to src/plugins/intel_cpu/src/emitters/x64/jit_emitter.hpp diff --git a/src/plugins/intel_cpu/src/emitters/jit_load_store_emitters.cpp b/src/plugins/intel_cpu/src/emitters/x64/jit_load_store_emitters.cpp similarity index 100% rename from src/plugins/intel_cpu/src/emitters/jit_load_store_emitters.cpp rename to src/plugins/intel_cpu/src/emitters/x64/jit_load_store_emitters.cpp diff --git a/src/plugins/intel_cpu/src/emitters/jit_load_store_emitters.hpp b/src/plugins/intel_cpu/src/emitters/x64/jit_load_store_emitters.hpp similarity index 100% rename from src/plugins/intel_cpu/src/emitters/jit_load_store_emitters.hpp rename to src/plugins/intel_cpu/src/emitters/x64/jit_load_store_emitters.hpp diff --git a/src/plugins/intel_cpu/src/emitters/jit_snippets_emitters.cpp b/src/plugins/intel_cpu/src/emitters/x64/jit_snippets_emitters.cpp similarity index 99% rename from src/plugins/intel_cpu/src/emitters/jit_snippets_emitters.cpp rename to src/plugins/intel_cpu/src/emitters/x64/jit_snippets_emitters.cpp index 338cb62dcec..3612720f13b 100644 --- a/src/plugins/intel_cpu/src/emitters/jit_snippets_emitters.cpp +++ b/src/plugins/intel_cpu/src/emitters/x64/jit_snippets_emitters.cpp @@ -8,8 +8,8 @@ #include "jit_snippets_emitters.hpp" #include "snippets/op/subgraph.hpp" #include "snippets/utils.hpp" -#include "snippets_transformations/op/brgemm_copy_b.hpp" -#include "snippets_transformations/op/brgemm_cpu.hpp" +#include "transformations/snippets/x64/op/brgemm_copy_b.hpp" +#include "transformations/snippets/x64/op//brgemm_cpu.hpp" using namespace InferenceEngine; using ngraph::snippets::op::Subgraph; diff --git a/src/plugins/intel_cpu/src/emitters/jit_snippets_emitters.hpp b/src/plugins/intel_cpu/src/emitters/x64/jit_snippets_emitters.hpp similarity index 99% rename from src/plugins/intel_cpu/src/emitters/jit_snippets_emitters.hpp rename to src/plugins/intel_cpu/src/emitters/x64/jit_snippets_emitters.hpp index 0f00eb6f704..4fc69a1d731 100644 --- a/src/plugins/intel_cpu/src/emitters/jit_snippets_emitters.hpp +++ b/src/plugins/intel_cpu/src/emitters/x64/jit_snippets_emitters.hpp @@ -10,7 +10,7 @@ #include "jit_emitter.hpp" #include "jit_load_store_emitters.hpp" -#include "snippets_transformations/op/store_convert.hpp" +#include "transformations/snippets/x64/op/store_convert.hpp" // Matmul support: #include #include diff --git a/src/plugins/intel_cpu/src/extension.cpp b/src/plugins/intel_cpu/src/extension.cpp index f40e770accc..78096952240 100644 --- a/src/plugins/intel_cpu/src/extension.cpp +++ b/src/plugins/intel_cpu/src/extension.cpp @@ -3,17 +3,17 @@ // #include "extension.h" -#include "ngraph_transformations/op/fully_connected.hpp" -#include "ngraph_transformations/op/interaction.hpp" -#include "ngraph_transformations/op/leaky_relu.hpp" -#include "ngraph_transformations/op/power_static.hpp" -#include "ngraph_transformations/op/swish_cpu.hpp" -#include "ngraph_transformations/op/mha.hpp" -#include "ngraph_transformations/op/ngram.hpp" -#include "snippets_transformations/op/load_convert.hpp" -#include "snippets_transformations/op/store_convert.hpp" -#include "snippets_transformations/op/brgemm_cpu.hpp" -#include "snippets_transformations/op/brgemm_copy_b.hpp" +#include "transformations/cpu_opset/common/op/fully_connected.hpp" +#include "transformations/cpu_opset/common/op/leaky_relu.hpp" +#include "transformations/cpu_opset/common/op/power_static.hpp" +#include "transformations/cpu_opset/common/op/swish_cpu.hpp" +#include "transformations/cpu_opset/common/op/ngram.hpp" +#include "transformations/cpu_opset/x64/op/mha.hpp" +#include "transformations/cpu_opset/x64/op/interaction.hpp" +#include "transformations/snippets/x64/op/load_convert.hpp" +#include "transformations/snippets/x64/op/store_convert.hpp" +#include "transformations/snippets/x64/op/brgemm_cpu.hpp" +#include "transformations/snippets/x64/op/brgemm_copy_b.hpp" #include #include @@ -46,20 +46,20 @@ std::map Extension::getOpSets() { auto cpu_plugin_opset = []() { ngraph::OpSet opset; +#if defined(OPENVINO_ARCH_X86_64) +#define NGRAPH_OP_X64(NAME, NAMESPACE) NGRAPH_OP(NAME, NAMESPACE) +#else +#define NGRAPH_OP_X64(NAME, NAMESPACE) +#endif + #define NGRAPH_OP(NAME, NAMESPACE) opset.insert(); - NGRAPH_OP(InteractionNode, ov::intel_cpu) NGRAPH_OP(FullyConnectedNode, ov::intel_cpu) NGRAPH_OP(LeakyReluNode, ov::intel_cpu) NGRAPH_OP(PowerStaticNode, ov::intel_cpu) NGRAPH_OP(SwishNode, ov::intel_cpu) - NGRAPH_OP(MHANode, ov::intel_cpu) NGRAPH_OP(NgramNode, ov::intel_cpu) - NGRAPH_OP(LoadConvertSaturation, ov::intel_cpu) - NGRAPH_OP(LoadConvertTruncation, ov::intel_cpu) - NGRAPH_OP(StoreConvertSaturation, ov::intel_cpu) - NGRAPH_OP(StoreConvertTruncation, ov::intel_cpu) - NGRAPH_OP(BrgemmCPU, ov::intel_cpu) - NGRAPH_OP(BrgemmCopyB, ov::intel_cpu) + NGRAPH_OP_X64(MHANode, ov::intel_cpu) + NGRAPH_OP_X64(InteractionNode, ov::intel_cpu) #undef NGRAPH_OP return opset; @@ -157,6 +157,12 @@ std::map Extension::getOpSets() { NGRAPH_OP(Store, ngraph::snippets::op) NGRAPH_OP(Subgraph, ngraph::snippets::op) NGRAPH_OP(VectorBuffer, ngraph::snippets::op) + NGRAPH_OP_X64(LoadConvertSaturation, ov::intel_cpu) + NGRAPH_OP_X64(LoadConvertTruncation, ov::intel_cpu) + NGRAPH_OP_X64(StoreConvertSaturation, ov::intel_cpu) + NGRAPH_OP_X64(StoreConvertTruncation, ov::intel_cpu) + NGRAPH_OP_X64(BrgemmCPU, ov::intel_cpu) + NGRAPH_OP_X64(BrgemmCopyB, ov::intel_cpu) #undef NGRAPH_OP return opset; diff --git a/src/plugins/intel_cpu/src/graph_optimizer.cpp b/src/plugins/intel_cpu/src/graph_optimizer.cpp index 949acf7cd6a..18d91debc97 100644 --- a/src/plugins/intel_cpu/src/graph_optimizer.cpp +++ b/src/plugins/intel_cpu/src/graph_optimizer.cpp @@ -988,21 +988,7 @@ void GraphOptimizer::FuseConvolutionAndSimpleOperationThroughMaxPool(Graph &grap continue; } - if (!one_of(fuseCandidate->getAlgorithm(), Algorithm::EltwiseRelu, - Algorithm::EltwiseGelu, - Algorithm::EltwiseElu, - Algorithm::EltwiseSigmoid, - Algorithm::EltwiseClamp, - Algorithm::EltwiseTanh, - Algorithm::EltwiseSwish, - Algorithm::EltwiseHswish, - Algorithm::EltwiseMish, - Algorithm::EltwiseHsigmoid, - Algorithm::EltwiseRoundHalfToEven, - Algorithm::EltwiseRoundHalfAwayFromZero, - Algorithm::EltwiseAbs, - Algorithm::EltwiseSqrt, - Algorithm::EltwiseSoftRelu)) { + if (!DnnlExtensionUtils::isUnarySupportedAsPostOp(fuseCandidate->getAlgorithm())) { parent++; continue; } @@ -1175,17 +1161,7 @@ void GraphOptimizer::FuseConvolutionSumAndConvolutionSumActivation(Graph &graph) auto isFusingSupported = [&](NodePtr conv, NodePtr child) { return child->getType() == Type::Eltwise && - one_of(child->getAlgorithm(), Algorithm::EltwiseRelu, - Algorithm::EltwiseElu, - Algorithm::EltwiseSigmoid, - Algorithm::EltwiseClamp, - Algorithm::EltwiseSwish, - Algorithm::EltwiseHswish, - Algorithm::EltwiseMish, - Algorithm::EltwiseHsigmoid, - Algorithm::EltwiseRoundHalfToEven, - Algorithm::EltwiseRoundHalfAwayFromZero, - Algorithm::EltwiseSoftRelu); + DnnlExtensionUtils::isUnarySupportedAsPostOp(child->getAlgorithm()); }; for (auto &graphNode : graphNodes) { diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_cpu_specific_opset.hpp b/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_cpu_specific_opset.hpp deleted file mode 100644 index ad687888b70..00000000000 --- a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_cpu_specific_opset.hpp +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include -#include "fc_bias_fusion.hpp" -#include "ngraph/op/fake_quantize.hpp" -#include "ngraph/pass/manager.hpp" -#include "reshape_fc_fusion.hpp" -#include "align_matmul_input_ranks.hpp" -#include "transformations/common_optimizations/reshape_prelu.hpp" -#include "convert_broadcast_to_tiles.hpp" -#include "convert_tile_to_seq_tiles.hpp" -#include "convert_matmul_to_fc.hpp" -#include "convert_to_power_static.hpp" -#include "convert_to_leaky_relu.hpp" -#include "convert_to_swish_cpu.hpp" -#include "transformations/convert_precision.hpp" -#include "transformations/utils/utils.hpp" -#include "rnn_sequences_optimization.hpp" -#include "transformations/common_optimizations/reshape_sequence_fusion.hpp" -#include "ngram_fusion.hpp" - -#include "itt.hpp" - -namespace ov { -namespace intel_cpu { - -inline void ConvertToCPUSpecificOpset(std::shared_ptr &nGraphFunc) { - RUN_ON_FUNCTION_SCOPE(ConvertToCPUSpecificOpset); - - ngraph::pass::Manager manager; - manager.set_per_pass_validation(false); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - if (!ov::op::util::has_op_with_type(nGraphFunc)) { - manager.register_pass(); - } - // after transformation "MoveEltwiseUpThroughDataMov" there can be Reshape sequences that should be eliminated or fused - manager.register_pass(); - manager.register_pass(); - manager.register_pass(precisions_map {{ ngraph::element::i64, ngraph::element::i32 }}); - manager.register_pass(); - manager.register_pass(); - - manager.run_passes(nGraphFunc); -} - -} // namespace intel_cpu -} // namespace ov diff --git a/src/plugins/intel_cpu/src/node.cpp b/src/plugins/intel_cpu/src/node.cpp index 67e289aa1cb..2814d3d40d3 100644 --- a/src/plugins/intel_cpu/src/node.cpp +++ b/src/plugins/intel_cpu/src/node.cpp @@ -455,6 +455,7 @@ std::string Node::getPrimitiveDescriptorType() { SEARCH_TYPE(winograd); SEARCH_TYPE(sparse); + SEARCH_TYPE(acl); SEARCH_TYPE(_dw); SEARCH_TYPE(_1x1); @@ -959,6 +960,9 @@ void Node::cleanup() { const std::vector& Node::getPrimitivesPriority() { std::vector priorities = { impl_desc_type::unknown, + // Undef impl type is used to express use-cases there real type is unkown during compilation + // Undef has higher priority than defined types in order to force primitive selection logic to make decision based on other properties + impl_desc_type::undef, impl_desc_type::brgconv_avx512_amx_1x1, impl_desc_type::brgconv_avx512_amx, impl_desc_type::jit_avx512_amx_dw, @@ -988,6 +992,7 @@ const std::vector& Node::getPrimitivesPriority() { impl_desc_type::gemm_avx2, impl_desc_type::gemm_avx, impl_desc_type::gemm_sse42, + impl_desc_type::acl, impl_desc_type::jit_gemm, impl_desc_type::ref_any, impl_desc_type::ref, @@ -1340,6 +1345,7 @@ Node* Node::NodesFactory::create(const std::shared_ptr& op, const } bool Node::canBePerformedAsScaleShift(const Node *parentNode) const { +#if defined(OPENVINO_ARCH_X86_64) IE_ASSERT(parentNode); size_t fusingPort = 0; @@ -1390,6 +1396,10 @@ bool Node::canBePerformedAsScaleShift(const Node *parentNode) const { Algorithm::EltwisePrelu, Algorithm::EltwiseMulAdd) && isBroadcastableToDataInput()) || isConvertablePowerStatic(); +#else + // TODO: provide correct list of operations for other backends + return false; +#endif } // @todo shifts for Subtract and scales for Divide are replaced with @@ -1606,22 +1616,7 @@ bool Node::canFuseSimpleOperation(const NodePtr& node) const { } return ret; } else if (node->getType() == Type::Eltwise) { - return one_of(node->getAlgorithm(), - Algorithm::EltwiseRelu, - Algorithm::EltwiseGelu, - Algorithm::EltwiseElu, - Algorithm::EltwiseSigmoid, - Algorithm::EltwiseClamp, - Algorithm::EltwiseTanh, - Algorithm::EltwiseSwish, - Algorithm::EltwiseHswish, - Algorithm::EltwiseMish, - Algorithm::EltwiseHsigmoid, - Algorithm::EltwiseRoundHalfToEven, - Algorithm::EltwiseRoundHalfAwayFromZero, - Algorithm::EltwiseAbs, - Algorithm::EltwiseSqrt, - Algorithm::EltwiseSoftRelu) || + return DnnlExtensionUtils::isUnarySupportedAsPostOp(node->getAlgorithm()) || node->canBePerformedAsScaleShift(this); } return false; diff --git a/src/plugins/intel_cpu/src/node.h b/src/plugins/intel_cpu/src/node.h index d9f242b353d..bb10731ad2c 100644 --- a/src/plugins/intel_cpu/src/node.h +++ b/src/plugins/intel_cpu/src/node.h @@ -37,6 +37,8 @@ #include "dnnl_postops_composer.h" #include "graph_context.h" +#include "nodes/executors/mvn_list.hpp" +#include "nodes/executors/executor.hpp" namespace ov { namespace intel_cpu { @@ -75,6 +77,12 @@ class NodeDesc { public: NodeDesc(const NodeConfig& conf, impl_desc_type type): config(conf) { implementationType = type; + executorFactory = nullptr; + } + + NodeDesc(const NodeConfig& conf, impl_desc_type type, ExecutorFactoryPtr factory): config(conf) { + implementationType = type; + executorFactory = factory; } const NodeConfig& getConfig() const { @@ -93,9 +101,28 @@ public: implementationType = type; } + ExecutorFactoryPtr getExecutorFactory() const { + return executorFactory; + } + + template ::value && !std::is_reference::value, int>::type = 0, + typename std::enable_if::value, int>::type = 0> + std::shared_ptr getExecutorFactoryAs() { + auto casted = std::dynamic_pointer_cast(executorFactory); + if (!casted) + IE_THROW() << "Cannot dynamically cast ExecutorFactory"; + return casted; + } + + void setExecutorFactory(ExecutorFactoryPtr factory) { + executorFactory = factory; + } + private: NodeConfig config; impl_desc_type implementationType; + ExecutorFactoryPtr executorFactory; }; class Node { diff --git a/src/plugins/intel_cpu/src/nodes/bin_conv.cpp b/src/plugins/intel_cpu/src/nodes/bin_conv.cpp index f2b5a133307..4aa184c9246 100644 --- a/src/plugins/intel_cpu/src/nodes/bin_conv.cpp +++ b/src/plugins/intel_cpu/src/nodes/bin_conv.cpp @@ -42,7 +42,7 @@ using namespace Xbyak; namespace ov { namespace intel_cpu { namespace node { - +#if defined(OPENVINO_ARCH_X86_64) #define GET_OFF(field) offsetof(jit_bin_conv_call_args, field) template @@ -874,7 +874,7 @@ private: } } }; - +#endif bool BinaryConvolution::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { try { if (isDynamicNgraphNode(op)) { @@ -1092,7 +1092,7 @@ void BinaryConvolution::createPrimitive() { IMPLICATION(jcp.kw > 7, (jcp.t_pad == 0 && jcp.l_pad == 0) || (jcp.stride_w == 1 && jcp.stride_h == 1)); if (!args_ok) IE_THROW() << "BinaryConvolution with name '" << getName() << "' has unsupported parameters"; - +#if defined(OPENVINO_ARCH_X86_64) if (implType == impl_desc_type::jit_avx512) { bin_conv_kernel.reset(new jit_uni_bin_conv_kernel_f32(jcp, jcp_dw_conv, *attr.get())); } else if (implType == impl_desc_type::jit_avx2) { @@ -1102,6 +1102,7 @@ void BinaryConvolution::createPrimitive() { } if (bin_conv_kernel) bin_conv_kernel->create_ker(); +#endif } bool BinaryConvolution::canFuse(const NodePtr& node) const { diff --git a/src/plugins/intel_cpu/src/nodes/color_convert.cpp b/src/plugins/intel_cpu/src/nodes/color_convert.cpp index 8bae75d434c..2ccb45791f8 100644 --- a/src/plugins/intel_cpu/src/nodes/color_convert.cpp +++ b/src/plugins/intel_cpu/src/nodes/color_convert.cpp @@ -10,7 +10,7 @@ #include #include #include -#include +#include "kernels/x64/jit_kernel.hpp" using namespace InferenceEngine; using namespace dnnl::impl; @@ -76,6 +76,7 @@ std::tuple Converter::yuv_to_rgb(float y, float u, float v) { return std::make_tuple(r, g, b); } +#if defined(OPENVINO_ARCH_X86_64) struct jit_uni_converter : public jit_kernel { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_converter) @@ -264,6 +265,7 @@ void jit_uni_converter::store_tail(const variable & dst, copy(ptr[dst], s.pointer(), copy_size); } +#endif namespace nv12 { @@ -394,6 +396,7 @@ public: } }; +#if defined(OPENVINO_ARCH_X86_64) template class JitConverter; @@ -611,7 +614,7 @@ public: }); } }; - +#endif } // namespace nv12 namespace i420 { @@ -748,6 +751,7 @@ public: } }; +#if defined(OPENVINO_ARCH_X86_64) template class JitConverter; @@ -964,13 +968,13 @@ public: }); } }; - +#endif } // namespace i420 /** * Implements Color Convert shape inference algorithm. Depending on wether it has only single plain H dimension is * passed through or recalculated as 2/3 of the initial size. - * + * */ class ColorConvertShapeInfer : public ShapeInferEmptyPads { public: @@ -1098,6 +1102,7 @@ void ColorConvert::initSupportedNV12Impls() { impls[Precision::FP32][false] = SUPPORTED_IMPL(TwoPlaneConvert, float, ref); } +#if defined(OPENVINO_ARCH_X86_64) // jit_uni { auto &impls = _supportedImpls[impl_desc_type::jit_uni][algorithm]; @@ -1106,7 +1111,7 @@ void ColorConvert::initSupportedNV12Impls() { impls[Precision::FP32][true] = SUPPORTED_IMPL(SinglePlaneConvert, float, jit_uni); impls[Precision::FP32][false] = SUPPORTED_IMPL(TwoPlaneConvert, float, jit_uni); } - +#endif #undef SUPPORTED_IMPL } @@ -1125,6 +1130,7 @@ void ColorConvert::initSupportedI420Impls() { impls[Precision::FP32][false] = SUPPORTED_IMPL(ThreePlaneConvert, float, ref); } +#if defined(OPENVINO_ARCH_X86_64) // jit_uni { auto &impls = _supportedImpls[impl_desc_type::jit_uni][algorithm]; @@ -1133,7 +1139,7 @@ void ColorConvert::initSupportedI420Impls() { impls[Precision::FP32][true] = SUPPORTED_IMPL(SinglePlaneConvert, float, jit_uni); impls[Precision::FP32][false] = SUPPORTED_IMPL(ThreePlaneConvert, float, jit_uni); } - +#endif #undef SUPPORTED_IMPL } diff --git a/src/plugins/intel_cpu/src/nodes/common/cpu_convert.cpp b/src/plugins/intel_cpu/src/nodes/common/cpu_convert.cpp index 5fef2a6ad08..d8322c709e2 100644 --- a/src/plugins/intel_cpu/src/nodes/common/cpu_convert.cpp +++ b/src/plugins/intel_cpu/src/nodes/common/cpu_convert.cpp @@ -7,25 +7,31 @@ #include #include #include -#include #include #include -#include #include #include #include #include #include +#if defined(OPENVINO_ARCH_X86_64) +#include "nodes/kernels/x64/jit_kernel.hpp" +#include +#endif using namespace InferenceEngine; -using namespace dnnl::impl::utils; -using namespace dnnl::impl::cpu::x64; -using namespace Xbyak; + namespace ov { namespace intel_cpu { namespace { +#if defined(OPENVINO_ARCH_X86_64) + +using namespace dnnl::impl::utils; +using namespace dnnl::impl::cpu::x64; +using namespace Xbyak; + template void convert_vec(jit_generator & gen, const RegExp & src, @@ -156,6 +162,8 @@ void jit_convert(const TI* arg, TO* out, size_t count) { } } +#endif + template struct PrecisionInfo { using value_type = typename PrecisionTrait

::value_type; @@ -356,6 +364,7 @@ struct ConvertPrecision> { } }; +#if defined(OPENVINO_ARCH_X86_64) template struct ConvertPrecision> { void operator()(ConvertContext & ctx) { @@ -462,6 +471,7 @@ struct ConvertPrecision> { ctx.converted = true; } }; +#endif } // namespace diff --git a/src/plugins/intel_cpu/src/nodes/common/permute_kernel.cpp b/src/plugins/intel_cpu/src/nodes/common/permute_kernel.cpp index b7ca0199dec..8fe9dae9297 100644 --- a/src/plugins/intel_cpu/src/nodes/common/permute_kernel.cpp +++ b/src/plugins/intel_cpu/src/nodes/common/permute_kernel.cpp @@ -26,6 +26,8 @@ using namespace Xbyak; namespace ov { namespace intel_cpu { +#if defined(OPENVINO_ARCH_X86_64) + template struct jit_uni_permute_kernel_f32 : public jit_uni_permute_kernel, public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_permute_kernel_f32) @@ -141,6 +143,8 @@ private: Xbyak::Xmm xmm = Xbyak::Xmm(1); }; +#endif // OPENVINO_ARCH_X86_64 + PermuteKernel::PermuteKernel(const PermuteParams& params) : params(params) { prepareParams(); } @@ -257,6 +261,7 @@ void PermuteKernel::prepareParams() { jcp.ndims = sorted_order.size(); jcp.data_size = params.data_size; +#if defined(OPENVINO_ARCH_X86_64) if (mayiuse(cpu::x64::avx512_core)) { permute_kernel.reset(new jit_uni_permute_kernel_f32(jcp)); } else if (mayiuse(cpu::x64::avx2)) { @@ -264,6 +269,7 @@ void PermuteKernel::prepareParams() { } else if (mayiuse(cpu::x64::sse41)) { permute_kernel.reset(new jit_uni_permute_kernel_f32(jcp)); } +#endif // OPENVINO_ARCH_X86_64 if (permute_kernel) permute_kernel->create_ker(); diff --git a/src/plugins/intel_cpu/src/nodes/common/softmax.cpp b/src/plugins/intel_cpu/src/nodes/common/softmax.cpp index b23c78e2345..128e8b89d8b 100644 --- a/src/plugins/intel_cpu/src/nodes/common/softmax.cpp +++ b/src/plugins/intel_cpu/src/nodes/common/softmax.cpp @@ -9,7 +9,7 @@ #include #include #include "utils/bfloat16.hpp" -#include "emitters/jit_bf16_emitters.hpp" +#include "emitters/x64/jit_bf16_emitters.hpp" #include #include @@ -50,7 +50,7 @@ struct jit_uni_softmax_kernel { virtual void create_ker() = 0; }; - +#if defined(OPENVINO_ARCH_X86_64) template struct jit_uni_softmax_kernel_f32 : public jit_uni_softmax_kernel, public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_softmax_kernel_f32) @@ -226,7 +226,7 @@ private: } } }; - +#endif SoftmaxGeneric::SoftmaxGeneric(Precision inpPrc, Precision outPrc) : input_prec(inpPrc), output_prec(outPrc) { if (Precision::BF16 == output_prec) { @@ -236,6 +236,7 @@ SoftmaxGeneric::SoftmaxGeneric(Precision inpPrc, Precision outPrc) } block_size = 1; +#if defined(OPENVINO_ARCH_X86_64) auto jcp = jit_softmax_config_params(); jcp.src_dt = inpPrc; jcp.dst_dt = outPrc; @@ -252,12 +253,14 @@ SoftmaxGeneric::SoftmaxGeneric(Precision inpPrc, Precision outPrc) } if (softmax_kernel) softmax_kernel->create_ker(); +#endif } template void SoftmaxGeneric::calculate(const in_data_t *src_data, out_data_t *dst_data, int B, int C, int H, int W) { for (int b = 0; b < B; b++) { int tail_start = 0; + if (softmax_kernel) { int blocks_num = H*W / block_size; diff --git a/src/plugins/intel_cpu/src/nodes/conv.cpp b/src/plugins/intel_cpu/src/nodes/conv.cpp index 94b6481b8cf..e4b1cd5bfe4 100644 --- a/src/plugins/intel_cpu/src/nodes/conv.cpp +++ b/src/plugins/intel_cpu/src/nodes/conv.cpp @@ -327,6 +327,9 @@ InferenceEngine::Precision Convolution::fusedEltwisePrecision(const NodePtr& fus const std::vector& Convolution::getPrimitivesPriority() { std::vector priorities = { impl_desc_type::unknown, + impl_desc_type::dw_acl, + impl_desc_type::winograd_acl, + impl_desc_type::gemm_acl, impl_desc_type::brgconv_avx512_amx_1x1, impl_desc_type::brgconv_avx512_amx, impl_desc_type::jit_avx512_amx_dw, @@ -556,6 +559,7 @@ void Convolution::getSupportedDescriptors() { auto inputShape = getInputShapeAtPort(0); auto outputShape = getOutputShapeAtPort(0); +#if defined(OPENVINO_ARCH_X86_64) bool acceptedFormat = inputDataType == memory::data_type::bf16; bool nspcAdded = false; acceptedFormat |= (shouldTryBrgconv && inputDataType == memory::data_type::f32); @@ -594,6 +598,15 @@ void Convolution::getSupportedDescriptors() { out_candidate = std::make_shared(outputShape, outputDataType, nspc); createDescriptor({ in_candidate }, { out_candidate }); } +#else + (void)ncsp; + (void)nCsp8c; + (void)nCsp16c; + + in_candidate = std::make_shared(inputShape, inputDataType, nspc); + out_candidate = std::make_shared(outputShape, outputDataType, nspc); + createDescriptor({ in_candidate }, { out_candidate }); +#endif } void Convolution::setPostOps(dnnl::primitive_attr& attr, @@ -899,7 +912,7 @@ void Convolution::createDescriptor(const std::vector& inputDesc, if (isWinograd()) algorithms.push_back(dnnl::algorithm::convolution_winograd); - algorithms.push_back(dnnl::algorithm::convolution_direct); + algorithms.push_back(baseConvAlgorithm); updatePadding(); @@ -1367,7 +1380,8 @@ void Convolution::prepareParams() { getParentEdgeAt(1)->getParent()->isConstant()}; auto engine = getEngine(); - auto builder = [&engine](const ConvKey& key) -> executorPtr { + auto convAlg = baseConvAlgorithm; + auto builder = [&engine, convAlg](const ConvKey& key) -> executorPtr { // remove the requirement on weight memory layout to let primitive // report the best layout for weight to be reordered dynamically at runtime auto wghDescAny = @@ -1405,7 +1419,7 @@ void Convolution::prepareParams() { attr); }; - const auto alg = (key.implType & impl_desc_type::winograd) ? dnnl::algorithm::convolution_winograd : dnnl::algorithm::convolution_direct; + const auto alg = (key.implType & impl_desc_type::winograd) ? dnnl::algorithm::convolution_winograd : convAlg; dnnl::primitive_desc desc = createDnnlConvDesc(engine, key.inp0->getDnnlDesc(), wghDescAny, @@ -1419,6 +1433,7 @@ void Convolution::prepareParams() { key.attr); auto itpd = desc; + executorPtr execPtr = nullptr; while (static_cast(itpd)) { impl_desc_type impl_type = parse_impl_name(itpd.impl_info_str()); @@ -1456,7 +1471,7 @@ void Convolution::prepareParams() { key.dilation, key.paddingL, key.paddingR, - dnnl::algorithm::convolution_direct, + convAlg, key.attr); if (reorderConvDesc) { diff --git a/src/plugins/intel_cpu/src/nodes/conv.h b/src/plugins/intel_cpu/src/nodes/conv.h index d0e4c48c151..48b4b6b7058 100644 --- a/src/plugins/intel_cpu/src/nodes/conv.h +++ b/src/plugins/intel_cpu/src/nodes/conv.h @@ -171,6 +171,13 @@ private: MemoryPtr stockInputZeroPointsMemPtr; dnnl::memory::data_type outputDataType; InferenceEngine::Precision sumPrc = InferenceEngine::Precision::UNSPECIFIED; + + // TODO: migrate on convolution_auto algorithm for x64 +#if defined(OPENVINO_ARCH_X86_64) + const dnnl::algorithm baseConvAlgorithm = dnnl::algorithm::convolution_direct; +#else + const dnnl::algorithm baseConvAlgorithm = dnnl::algorithm::convolution_auto; +#endif }; } // namespace node diff --git a/src/plugins/intel_cpu/src/nodes/def_conv.cpp b/src/plugins/intel_cpu/src/nodes/def_conv.cpp index b7f2b6c6590..6946645961b 100644 --- a/src/plugins/intel_cpu/src/nodes/def_conv.cpp +++ b/src/plugins/intel_cpu/src/nodes/def_conv.cpp @@ -27,7 +27,7 @@ using namespace Xbyak; namespace ov { namespace intel_cpu { namespace node { - +#if defined(OPENVINO_ARCH_X86_64) #define GET_OFF(field) offsetof(jit_def_conv_call_args, field) template @@ -671,7 +671,7 @@ private: pop(reg_sampled_offs); } }; - +#endif bool DeformableConvolution::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { try { if (!one_of(op->get_type_info(), @@ -1033,7 +1033,7 @@ DeformableConvolution::DefConvExecutor::DefConvExecutor(const DefConvAttr &defCo if (withModulation) { modStrides = descVector[MOD_ID]->getStrides(); } - +#if defined(OPENVINO_ARCH_X86_64) const VectorDims srcDims = descVector[DATA_ID]->getShape().getStaticDims(); const VectorDims weiDims = descVector[WEI_ID]->getShape().getStaticDims(); const VectorDims dstDims = descVector[descVector.size() - 1]->getShape().getStaticDims(); @@ -1084,11 +1084,13 @@ DeformableConvolution::DefConvExecutor::DefConvExecutor(const DefConvAttr &defCo jcp.nb_oc_blocking = !mayiuse(cpu::x64::avx2) ? 2 : 4; jcp.nthr = dnnl_get_max_threads(); +#endif } DeformableConvolution::DefConvJitExecutor::DefConvJitExecutor(const DefConvAttr &defConvAttr, const std::vector> &descVector) : DefConvExecutor(defConvAttr, descVector) { +#if defined(OPENVINO_ARCH_X86_64) if (mayiuse(cpu::x64::avx512_core)) { def_conv_kernel.reset(new jit_uni_def_conv_kernel_f32(jcp)); } else if (mayiuse(cpu::x64::avx2)) { @@ -1103,6 +1105,7 @@ DeformableConvolution::DefConvJitExecutor::DefConvJitExecutor(const DefConvAttr } else { IE_THROW() << "Can't compile DefConvJitExecutor"; } +#endif } void DeformableConvolution::DefConvRefExecutor::exec(const float* src, const float* offsets, diff --git a/src/plugins/intel_cpu/src/nodes/dft.cpp b/src/plugins/intel_cpu/src/nodes/dft.cpp index 97a62297cac..463f7001f74 100644 --- a/src/plugins/intel_cpu/src/nodes/dft.cpp +++ b/src/plugins/intel_cpu/src/nodes/dft.cpp @@ -123,7 +123,7 @@ inline float getImaginaryFromComplexProd(float lhsReal, float lhsImag, float rhs /* Returns true while we can iterate - Specified axis is skipped in counters + Specified axis is skipped in counters */ inline bool nextIterationStep(std::vector& counters, const std::vector& iterationRange, size_t axis) { auto itCounter = counters.rbegin(); @@ -535,7 +535,6 @@ void DFT::prepareParams() { hasFFT = true; } } - if (mayiuse(cpu::x64::sse41)) { createJITKernels(hasDFT, hasFFT); } @@ -553,8 +552,8 @@ std::vector DFT::getAxes() const { std::sort(axes.begin(), axes.end()); return axes; } - void DFT::createJITKernels(bool hasDFT, bool hasFFT) { +#if defined(OPENVINO_ARCH_X86_64) if (hasDFT && dftKernel == nullptr) { if (mayiuse(cpu::x64::avx512_core)) { dftKernel.reset(new jit_uni_dft_kernel_f32()); @@ -584,8 +583,8 @@ void DFT::createJITKernels(bool hasDFT, bool hasFFT) { if (fftKernel) fftKernel->create_ker(); } +#endif } - } // namespace node } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/dft.h b/src/plugins/intel_cpu/src/nodes/dft.h index 3d4ba451153..019bed18dab 100644 --- a/src/plugins/intel_cpu/src/nodes/dft.h +++ b/src/plugins/intel_cpu/src/nodes/dft.h @@ -8,7 +8,7 @@ #include #include -#include "kernels/dft_uni_kernel.hpp" +#include "kernels/x64/dft_uni_kernel.hpp" namespace ov { namespace intel_cpu { @@ -31,7 +31,6 @@ public: private: std::vector getAxes() const; void createJITKernels(bool hasDFT, bool hasFFT); - void dftNd(float* output, const VectorDims& outputShape, const VectorDims& outputStrides, diff --git a/src/plugins/intel_cpu/src/nodes/eltwise.cpp b/src/plugins/intel_cpu/src/nodes/eltwise.cpp index a6a9ff7b995..20aef2f09bc 100644 --- a/src/plugins/intel_cpu/src/nodes/eltwise.cpp +++ b/src/plugins/intel_cpu/src/nodes/eltwise.cpp @@ -23,10 +23,10 @@ #include "input.h" #include "common/cpu_convert.h" -#include "emitters/jit_emitter.hpp" -#include "emitters/jit_eltwise_emitters.hpp" -#include "emitters/jit_dnnl_emitters.hpp" -#include "emitters/jit_bf16_emitters.hpp" +#include "emitters/x64/jit_emitter.hpp" +#include "emitters/x64/jit_eltwise_emitters.hpp" +#include "emitters/x64/jit_dnnl_emitters.hpp" +#include "emitters/x64/jit_bf16_emitters.hpp" #include #include "utils/general_utils.h" #include "utils/cpu_utils.hpp" @@ -34,9 +34,9 @@ #include "ngraph/ngraph.hpp" #include -#include "ngraph_transformations/op/power_static.hpp" -#include "ngraph_transformations/op/leaky_relu.hpp" -#include "ngraph_transformations/op/swish_cpu.hpp" +#include "transformations/cpu_opset/common/op/power_static.hpp" +#include "transformations/cpu_opset/common/op/leaky_relu.hpp" +#include "transformations/cpu_opset/common/op/swish_cpu.hpp" #include #include @@ -58,7 +58,8 @@ using namespace Xbyak; namespace ov { namespace intel_cpu { namespace node { -namespace { + +#if defined(OPENVINO_ARCH_X86_64) template struct SupportedPrecisions { @@ -106,61 +107,7 @@ struct EltwiseEmitter { } }; -/** - * Implements Eltwise shape inference algorithm. The algorithm is based on broadcasting all the input shapes - * according to the NUMPY broadcast rule. This implementation is more lightweight than the ngraph one. - * - */ -class EltwiseShapeInfer : public ShapeInferEmptyPads { -public: - Result infer( - const std::vector>& input_shapes, - const std::unordered_map& data_dependency) override { - size_t max_rank = 0; - size_t max_rank_idx = 0; - for (size_t i = 0; i < input_shapes.size(); ++i) { - auto item_rank = input_shapes[i].get().size(); - if (item_rank > max_rank) { - max_rank = item_rank; - max_rank_idx = i; - } - } - auto output_shape = input_shapes[max_rank_idx].get(); - // use NUMPY broadcast rule - for (size_t i = 0; i < input_shapes.size(); i++) { - if (i == max_rank_idx) - continue; - - auto& input_shape = input_shapes[i].get(); - if (input_shape.size() > output_shape.size()) { - IE_THROW() << "Eltwise shape infer input and output shapes rank mismatch"; - } - size_t offset = output_shape.size() - input_shape.size(); - for (size_t j = 0; j < input_shape.size(); ++j) { - if (input_shape[j] != output_shape[offset + j]) { - if (output_shape[offset + j] == 1) { - output_shape[offset + j] = input_shape[j]; - } else { - if (input_shape[j] != 1) IE_THROW() << "Eltwise shape infer input shapes dim index: " << j << " mismatch"; - } - } - } - } - return { { std::move(output_shape) }, ShapeInferStatus::success }; - } - port_mask_t get_port_mask() const override { - return EMPTY_PORT_MASK; - } -}; - -class EltwiseShapeInferFactory : public ShapeInferFactory { -public: - ShapeInferPtr makeShapeInfer() const override { - return std::make_shared(); - } -}; - -void set_intersection(const std::set>& precisions1, +static void set_intersection(const std::set>& precisions1, const std::set>& precisions2, std::set>& intersection) { std::map intersection_types; @@ -181,9 +128,6 @@ void set_intersection(const std::set>& precisions1, } } -} // namespace - - InferenceEngine::Precision eltwise_precision_helper::get_precision(const size_t inputs_number, const InferenceEngine::Precision(&src_prc)[MAX_ELTWISE_INPUTS], const std::vector& eltwise_data) { @@ -261,7 +205,8 @@ std::set> eltwise_precision_helper::get_supported_pre OV_SWITCH(intel_cpu, SupportedPrecisions, precisions, algo, OV_CASE(Algorithm::EltwiseRelu, jit_dnnl_aux_emitter), - OV_CASE(Algorithm::EltwiseGelu, jit_dnnl_aux_emitter), + OV_CASE(Algorithm::EltwiseGeluErf, jit_dnnl_aux_emitter), + OV_CASE(Algorithm::EltwiseGeluTanh, jit_dnnl_aux_emitter), OV_CASE(Algorithm::EltwiseElu, jit_dnnl_aux_emitter), OV_CASE(Algorithm::EltwiseTanh, jit_dnnl_aux_emitter), OV_CASE(Algorithm::EltwiseSigmoid, jit_dnnl_aux_emitter), @@ -633,7 +578,8 @@ private: OV_SWITCH(intel_cpu, EltwiseEmitter, ctx, data.algo, OV_CASE(Algorithm::EltwiseRelu, jit_dnnl_aux_emitter), - OV_CASE(Algorithm::EltwiseGelu, jit_dnnl_aux_emitter), + OV_CASE(Algorithm::EltwiseGeluErf, jit_dnnl_aux_emitter), + OV_CASE(Algorithm::EltwiseGeluTanh, jit_dnnl_aux_emitter), OV_CASE(Algorithm::EltwiseElu, jit_dnnl_aux_emitter), OV_CASE(Algorithm::EltwiseTanh, jit_dnnl_aux_emitter), OV_CASE(Algorithm::EltwiseSigmoid, jit_dnnl_aux_emitter), @@ -972,6 +918,66 @@ private: } }; +#endif // OPENVINO_ARCH_X86_64 + +namespace { + +/** + * Implements Eltwise shape inference algorithm. The algorithm is based on broadcasting all the input shapes + * according to the NUMPY broadcast rule. This implementation is more lightweight than the ngraph one. + * + */ +class EltwiseShapeInfer : public ShapeInferEmptyPads { +public: + Result infer( + const std::vector>& input_shapes, + const std::unordered_map& data_dependency) override { + size_t max_rank = 0; + size_t max_rank_idx = 0; + for (size_t i = 0; i < input_shapes.size(); ++i) { + auto item_rank = input_shapes[i].get().size(); + if (item_rank > max_rank) { + max_rank = item_rank; + max_rank_idx = i; + } + } + auto output_shape = input_shapes[max_rank_idx].get(); + // use NUMPY broadcast rule + for (size_t i = 0; i < input_shapes.size(); i++) { + if (i == max_rank_idx) + continue; + + auto& input_shape = input_shapes[i].get(); + if (input_shape.size() > output_shape.size()) { + IE_THROW() << "Eltwise shape infer input and output shapes rank mismatch"; + } + size_t offset = output_shape.size() - input_shape.size(); + for (size_t j = 0; j < input_shape.size(); ++j) { + if (input_shape[j] != output_shape[offset + j]) { + if (output_shape[offset + j] == 1) { + output_shape[offset + j] = input_shape[j]; + } else { + if (input_shape[j] != 1) IE_THROW() << "Eltwise shape infer input shapes dim index: " << j << " mismatch"; + } + } + } + } + return { { std::move(output_shape) }, ShapeInferStatus::success }; + } + port_mask_t get_port_mask() const override { + return EMPTY_PORT_MASK; + } +}; + +class EltwiseShapeInferFactory : public ShapeInferFactory { +public: + ShapeInferPtr makeShapeInfer() const override { + return std::make_shared(); + } +}; + +} // namespace + Eltwise::BroadcastingPolicy Eltwise::determineBroadcastingPolicy(const std::shared_ptr& op) { const auto const1 = ov::as_type_ptr(op->get_input_node_shared_ptr(0)); const auto const2 = ov::as_type_ptr(op->get_input_node_shared_ptr(1)); @@ -1088,23 +1094,24 @@ const std::map Eltwise::in node.beta = 0.0f; }}, {ngraph::op::v0::Gelu::get_type_info_static(), [](const std::shared_ptr& op, Eltwise& node) { - node.algorithm = Algorithm::EltwiseGelu; + node.algorithm = Algorithm::EltwiseGeluErf; node.onednnAlgorithm = dnnl::algorithm::eltwise_gelu_erf; }}, {ngraph::op::v7::Gelu::get_type_info_static(), [](const std::shared_ptr& op, Eltwise& node) { auto gelu = getNgraphOpAs(op); - node.algorithm = Algorithm::EltwiseGelu; ngraph::op::GeluApproximationMode approximationMode = gelu->get_approximation_mode(); - if (approximationMode == ngraph::op::GeluApproximationMode::ERF) + if (approximationMode == ngraph::op::GeluApproximationMode::ERF) { + node.algorithm = Algorithm::EltwiseGeluErf; node.onednnAlgorithm = dnnl::algorithm::eltwise_gelu_erf; - else if (approximationMode == ngraph::op::GeluApproximationMode::TANH) + } else if (approximationMode == ngraph::op::GeluApproximationMode::TANH) { + node.algorithm = Algorithm::EltwiseGeluTanh; node.onednnAlgorithm = dnnl::algorithm::eltwise_gelu_tanh; - else + } else { IE_THROW(NotImplemented) << "CPU Eltwise node doesn't support ngraph operation Gelu with approximation mode: " << approximationMode; + } }}, {ngraph::op::v0::Elu::get_type_info_static(), [](const std::shared_ptr& op, Eltwise& node) { auto eluOp = getNgraphOpAs(op); - node.alpha = static_cast(eluOp->get_alpha()); node.algorithm = Algorithm::EltwiseElu; node.onednnAlgorithm = dnnl::algorithm::eltwise_elu; @@ -1197,6 +1204,9 @@ const std::map Eltwise::in {ngraph::op::v1::Select::get_type_info_static(), [](const std::shared_ptr& op, Eltwise& node) { node.algorithm = Algorithm::EltwiseSelect; }}, + {ngraph::op::v0::Log::get_type_info_static(), [](const std::shared_ptr& op, Eltwise& node) { + node.algorithm = Algorithm::EltwiseLog; + }}, }; @@ -1503,6 +1513,7 @@ public: std::transform(jep.oc_offsets.begin(), jep.oc_offsets.end(), jep.oc_offsets.begin(), [](size_t& offset) { return offset * sizeof(float);}); +#if defined(OPENVINO_ARCH_X86_64) if (mayiuse(x64::avx512_core)) { _pKernel.reset(new jit_uni_eltwise_generic(jep, eltwise_data, ops_list, post_ops)); } else if (mayiuse(x64::avx2)) { @@ -1512,7 +1523,7 @@ public: } else { IE_THROW() << "Can't create jit eltwise kernel"; } - +#endif // OPENVINO_ARCH_X86_64 if (_pKernel) _pKernel->create_ker(); } @@ -1629,6 +1640,15 @@ public: } void exec(const jit_eltwise_call_args_ptrs &args_ptrs, const VectorDims &dims_out) override { + if (_opData.algo == Algorithm::EltwiseLog) { + const float* src_ptr_f = reinterpret_cast(args_ptrs.src_ptr[0]); + float* dst_ptr_f = reinterpret_cast(args_ptrs.dst_ptr); + parallel_for(_fullWorkAmount, [&](size_t i) { + dst_ptr_f[i] = logf(src_ptr_f[i]); + }); + return; + } + std::shared_ptr ref_eltwise_injector = nullptr; if (_opData.onednnAlgorithm != dnnl::algorithm::undef) { ref_eltwise_injector = std::make_shared( @@ -1671,7 +1691,8 @@ public: switch (_opData.algo) { case Algorithm::EltwiseRelu: - case Algorithm::EltwiseGelu: + case Algorithm::EltwiseGeluErf: + case Algorithm::EltwiseGeluTanh: case Algorithm::EltwiseElu: case Algorithm::EltwiseTanh: case Algorithm::EltwiseSigmoid: @@ -1816,7 +1837,8 @@ size_t Eltwise::getOpInputsNum() const { case Algorithm::EltwiseIsInf: case Algorithm::EltwiseIsNaN: case Algorithm::EltwiseRelu: - case Algorithm::EltwiseGelu: + case Algorithm::EltwiseGeluErf: + case Algorithm::EltwiseGeluTanh: case Algorithm::EltwiseElu: case Algorithm::EltwiseTanh: case Algorithm::EltwiseSigmoid: @@ -1835,6 +1857,7 @@ size_t Eltwise::getOpInputsNum() const { case Algorithm::EltwiseRoundHalfToEven: case Algorithm::EltwiseRoundHalfAwayFromZero: case Algorithm::EltwiseSoftSign: + case Algorithm::EltwiseLog: return 1; case Algorithm::EltwiseAdd: case Algorithm::EltwiseSubtract: @@ -1899,6 +1922,8 @@ void Eltwise::initSupportedPrimitiveDescriptors() { // if dim rank is greater than the maximum possible, we should use the reference execution bool canUseOptimizedImpl = mayiuse(x64::sse41) && getInputShapeAtPort(0).getRank() <= MAX_ELTWISE_DIM_RANK; + // TODO: Add EltwiseLog algorithm support for JIT implementation + canUseOptimizedImpl &= !one_of(getAlgorithm(), Algorithm::EltwiseLog); bool canUseOptimizedShapeAgnosticImpl = isDynamicNode() && canUseOptimizedImpl; if (!canUseOptimizedImpl && !fusedWith.empty()) { @@ -1992,7 +2017,7 @@ void Eltwise::initSupportedPrimitiveDescriptors() { Blocked }; - auto initDesc = [&] (LayoutType lt) -> NodeDesc { + auto initDesc = [&] (LayoutType lt, bool useAclExecutor = false) -> NodeDesc { auto createMemoryDesc = [lt](const Shape &shape, Precision prc, size_t offset) -> std::shared_ptr { const auto &dims = shape.getDims(); if (lt == ChannelsFirst && shape.getRank() != 1) { @@ -2072,18 +2097,36 @@ void Eltwise::initSupportedPrimitiveDescriptors() { config.outConfs.push_back(portConfig); - impl_desc_type impl_type; - if (mayiuse(x64::avx512_core)) { - impl_type = impl_desc_type::jit_avx512; - } else if (mayiuse(x64::avx2)) { - impl_type = impl_desc_type::jit_avx2; - } else if (mayiuse(x64::sse41)) { - impl_type = impl_desc_type::jit_sse42; - } else { - impl_type = impl_desc_type::ref; - } + if (useAclExecutor) { + impl_desc_type impl_type = impl_desc_type::undef; - return {config, impl_type}; + std::vector srcMemoryDescs; + for (int i = 0; i < config.inConfs.size(); i++) { + srcMemoryDescs.push_back(config.inConfs[i].getMemDesc()); + } + std::vector dstMemoryDescs; + for (int i = 0; i < config.outConfs.size(); i++) { + dstMemoryDescs.push_back(config.outConfs[i].getMemDesc()); + } + + auto factory = std::make_shared(eltwiseAttrs, srcMemoryDescs, dstMemoryDescs, + std::make_shared(context, getPrimitivesPriority())); + + return {config, impl_type, !factory->isEmpty() ? factory : nullptr}; + } else { + impl_desc_type impl_type = impl_desc_type::ref; + if (canUseOptimizedImpl) { + if (mayiuse(x64::avx512_core)) { + impl_type = impl_desc_type::jit_avx512; + } else if (mayiuse(x64::avx2)) { + impl_type = impl_desc_type::jit_avx2; + } else if (mayiuse(x64::sse41)) { + impl_type = impl_desc_type::jit_sse42; + } + } + + return {config, impl_type}; + } }; bool isChannelsFirstApplicable = one_of(getOutputShapeAtPort(0).getRank(), 1u, 2u, 3u, 4u, 5u); @@ -2105,14 +2148,31 @@ void Eltwise::initSupportedPrimitiveDescriptors() { isBlockedApplicable = isBlockedApplicable && inShape.getMinDims()[1] != Shape::UNDEFINED_DIM && inShape.getMinDims()[1] > 1; } + inputNum = getParentEdges().size(); + currentInBlkDims.resize(inputNum); + +#if defined (OV_CPU_WITH_ACL) + eltwiseAttrs = {algorithm, alpha, beta, gamma}; + if (isChannelsFirstApplicable) { + auto channelFirstDesc = initDesc(ChannelsFirst, true); + if (channelFirstDesc.getExecutorFactory()) + supportedPrimitiveDescriptors.emplace_back(channelFirstDesc); + } + + auto planarDesc = initDesc(Planar, true); + if (planarDesc.getExecutorFactory()) + supportedPrimitiveDescriptors.emplace_back(planarDesc); + + canUseAclExecutor = !supportedPrimitiveDescriptors.empty(); + if (canUseAclExecutor) + return; +#endif + if (isChannelsFirstApplicable) supportedPrimitiveDescriptors.emplace_back(initDesc(ChannelsFirst)); if (isBlockedApplicable) supportedPrimitiveDescriptors.emplace_back(initDesc(Blocked)); supportedPrimitiveDescriptors.emplace_back(initDesc(Planar)); - - inputNum = getParentEdges().size(); - currentInBlkDims.resize(inputNum); } void Eltwise::createPrimitive() { @@ -2141,6 +2201,21 @@ void Eltwise::createPrimitive() { } void Eltwise::prepareParams() { + if (canUseAclExecutor) { + std::vector srcMemoryDescs; + for (int i = 0; i < getParentEdges().size(); i++) { + srcMemoryDescs.push_back(getParentEdgeAt(i)->getMemoryPtr()->getDescPtr()); + } + std::vector dstMemoryDescs; + dstMemoryDescs.push_back(getChildEdgeAt(0)->getMemoryPtr()->getDescPtr()); + + auto selectedPD = getSelectedPrimitiveDescriptor(); + aclExecPtr = selectedPD->getExecutorFactoryAs()->makeExecutor(eltwiseAttrs, srcMemoryDescs, dstMemoryDescs, {}); + selectedPD->setImplementationType(aclExecPtr->getImplType()); + + return; + } + auto outBlockingDesc = getChildEdgeAt(0)->getMemory().GetDescWithType(); const auto &outOrder = outBlockingDesc->getOrder(); const auto ¤tOutBlkDims = outBlockingDesc->getBlockDims(); @@ -2309,6 +2384,15 @@ void Eltwise::execute(dnnl::stream strm) { } execPtr->exec(args_ptrs, dims_out); + } else if (aclExecPtr) { + std::vector srcMemory; + for (int i = 0; i < getParentEdges().size(); i++) { + srcMemory.push_back(getParentEdgeAt(i)->getMemoryPtr()); + } + std::vector dstMemory; + dstMemory.push_back(getChildEdgeAt(0)->getMemoryPtr()); + + aclExecPtr->exec(srcMemory, dstMemory, fqDataPtrs.data()); } else { IE_THROW() << "Can't execute eltwise node with name: " << getName() << ". Primitive isn't created"; } @@ -2594,6 +2678,9 @@ bool Eltwise::canFuse(const NodePtr& node) const { if (!mayiuse(x64::sse41) || getInputShapeAtPort(0).getRank() > MAX_ELTWISE_DIM_RANK) return false; + // TODO: EltwiseLog is supported only via reference executor + if (getAlgorithm() == Algorithm::EltwiseLog || node->getAlgorithm() == Algorithm::EltwiseLog) + return false; bool isIntegerNode = isIntegerComputeSupported(this); if (isIntegerNode && node->getType() != Type::Eltwise) @@ -2669,4 +2756,4 @@ InferenceEngine::Precision Eltwise::getRuntimePrecision() const { } // namespace node } // namespace intel_cpu -} // namespace ov +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/eltwise.h b/src/plugins/intel_cpu/src/nodes/eltwise.h index 80f53523fea..42249a87d29 100644 --- a/src/plugins/intel_cpu/src/nodes/eltwise.h +++ b/src/plugins/intel_cpu/src/nodes/eltwise.h @@ -10,6 +10,7 @@ #include #include #include +#include "executors/eltwise_list.hpp" namespace ov { namespace intel_cpu { @@ -199,6 +200,10 @@ private: void appendMemory(const std::vector &data, MemoryPtr &memPtr, std::vector& postOpsMem); void appendMemory(const std::vector &data, MemoryPtr &memPtr, std::vector& postOpsMem); + + bool canUseAclExecutor = false; + EltwiseAttrs eltwiseAttrs; + std::shared_ptr aclExecPtr = nullptr; }; class eltwise_precision_helper { @@ -213,4 +218,4 @@ private: } // namespace node } // namespace intel_cpu -} // namespace ov +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_eltwise.cpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_eltwise.cpp new file mode 100644 index 00000000000..25ac0010e0a --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_eltwise.cpp @@ -0,0 +1,390 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "acl_eltwise.hpp" +#include "acl_utils.hpp" + +namespace ov { +namespace intel_cpu { + +using namespace arm_compute; + +inline VectorDims reshape_sizes(VectorDims dims) { + const size_t MAX_NUM_SHAPE = arm_compute::MAX_DIMS; + VectorDims result_dims(MAX_NUM_SHAPE - 1); + if (dims.size() >= MAX_NUM_SHAPE) { + for (int i = 0; i < MAX_NUM_SHAPE - 1; i++) { + result_dims[i] = dims[i]; + } + for (int i = MAX_NUM_SHAPE - 1; i < dims.size(); i++) { + result_dims[MAX_NUM_SHAPE - 2] *= dims[i]; + } + } else { + result_dims = dims; + } + return result_dims; +} + +AclEltwiseExecutor::AclEltwiseExecutor(const ExecutorContext::CPtr context) : EltwiseExecutor(context) {} + +bool AclEltwiseExecutor::init(const EltwiseAttrs &eltwiseAttrs, const std::vector &srcDescs, + const std::vector &dstDescs, + const std::vector &postOps) { + if (!postOps.empty()) { return false; } + aclEltwiseAttrs = eltwiseAttrs; + + std::vector srcVecDims(srcDescs.size()), dstVecDims(dstDescs.size()); + std::vector srcDataLayout(srcDescs.size()), dstDataLayout(dstDescs.size()); + std::vector srcTensorsInfo(srcDescs.size()), dstTensorsInfo(dstDescs.size()); + srcTensors = std::vector(srcDescs.size()); + dstTensors = std::vector(dstDescs.size()); + + for (int i = 0; i < srcVecDims.size(); i++) { + srcVecDims[i] = shapeCast(reshape_sizes(srcDescs[i]->getShape().getDims())); + } + for (int i = 0; i < dstVecDims.size(); i++) { + dstVecDims[i] = shapeCast(reshape_sizes(dstDescs[i]->getShape().getDims())); + } + + for (int i = 0; i < srcDescs.size(); i++) { + srcDataLayout[i] = getAclDataLayoutByMemoryDesc(srcDescs[i]); + if (srcDataLayout[i] == arm_compute::DataLayout::UNKNOWN) { return false; } + } + for (int i = 0; i < dstDescs.size(); i++) { + dstDataLayout[i] = getAclDataLayoutByMemoryDesc(dstDescs[i]); + if (dstDataLayout[i] == arm_compute::DataLayout::UNKNOWN) { return false; } + } + + if (srcDescs.size() == 2 && + srcDescs[0]->hasLayoutType(LayoutType::nspc) && srcDescs[1]->hasLayoutType(LayoutType::nspc) && + srcDescs[0]->getShape().getDims() != srcDescs[1]->getShape().getDims()) { + auto dim_size = srcDescs[0]->getShape().getDims().size(); + auto mover = [&dim_size](TensorShape &_shape) { + if (dim_size == 5) { std::swap(_shape[2], _shape[3]); } + std::swap(_shape[1], _shape[2]); + std::swap(_shape[0], _shape[1]); + }; + if (dim_size < 5) { + srcDataLayout[0] = srcDataLayout[1] = dstDataLayout[0] = DataLayout::NCHW; + } else { + srcDataLayout[0] = srcDataLayout[1] = dstDataLayout[0] = DataLayout::NCDHW; + } + mover(srcVecDims[0]); + mover(srcVecDims[1]); + mover(dstVecDims[0]); + } + + for (int i = 0; i < srcVecDims.size(); i++) { + srcTensorsInfo[i] = TensorInfo(srcVecDims[i], 1, + precisionToAclDataType(srcDescs[i]->getPrecision()), + srcDataLayout[i]); + srcTensors[i].allocator()->init(srcTensorsInfo[i]); + } + + for (int i = 0; i < dstVecDims.size(); i++) { + dstTensorsInfo[i] = TensorInfo(dstVecDims[i], 1, + precisionToAclDataType(dstDescs[i]->getPrecision()), + dstDataLayout[i]); + dstTensors[i].allocator()->init(dstTensorsInfo[i]); + } + + switch (aclEltwiseAttrs.algorithm) { + case Algorithm::EltwiseAdd: + if (!NEArithmeticAddition::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0], ConvertPolicy::SATURATE)) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0], ConvertPolicy::SATURATE); + acl_op->run(); + }; + break; + case Algorithm::EltwiseMultiply: + if (!NEPixelWiseMultiplication::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0], + 1.0f, ConvertPolicy::SATURATE, RoundingPolicy::TO_ZERO)) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0], 1.0f, ConvertPolicy::SATURATE, RoundingPolicy::TO_ZERO); + acl_op->run(); + }; + break; + case Algorithm::EltwiseSubtract: + if (!NEArithmeticSubtraction::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0], ConvertPolicy::SATURATE)) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0], ConvertPolicy::SATURATE); + acl_op->run(); + }; + break; + case Algorithm::EltwiseDivide: + if (!NEElementwiseDivision::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0])) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0]); + acl_op->run(); + }; + break; + case Algorithm::EltwiseMaximum: + if (!NEElementwiseMax::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0])) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0]); + acl_op->run(); + }; + break; + case Algorithm::EltwiseMinimum: + if (!NEElementwiseMin::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0])) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0]); + acl_op->run(); + }; + break; + case Algorithm::EltwiseSquaredDifference: + if (!NEElementwiseSquaredDiff::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0])) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0]); + acl_op->run(); + }; + break; + case Algorithm::EltwisePowerDynamic: + if (!NEElementwisePower::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0])) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0]); + acl_op->run(); + }; + break; + case Algorithm::EltwiseEqual: + if (!NEElementwiseComparison::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0], ComparisonOperation::Equal)) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0], ComparisonOperation::Equal); + acl_op->run(); + }; + break; + case Algorithm::EltwiseNotEqual: + if (!NEElementwiseComparison::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0], ComparisonOperation::NotEqual)) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0], ComparisonOperation::NotEqual); + acl_op->run(); + }; + break; + case Algorithm::EltwiseGreater: + if (!NEElementwiseComparison::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0], ComparisonOperation::Greater)) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0], ComparisonOperation::Greater); + acl_op->run(); + }; + break; + case Algorithm::EltwiseGreaterEqual: + if (!NEElementwiseComparison::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0], ComparisonOperation::GreaterEqual)) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0], ComparisonOperation::GreaterEqual); + acl_op->run(); + }; + break; + case Algorithm::EltwiseLess: + if (!NEElementwiseComparison::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0], ComparisonOperation::Less)) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0], ComparisonOperation::Less); + acl_op->run(); + }; + break; + case Algorithm::EltwiseLessEqual: + if (!NEElementwiseComparison::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0], ComparisonOperation::LessEqual)) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0], ComparisonOperation::LessEqual); + acl_op->run(); + }; + break; + case Algorithm::EltwiseRelu: + if (aclEltwiseAttrs.alpha == 0) { + if (!NEActivationLayer::validate(&srcTensorsInfo[0], &dstTensorsInfo[0], + ActivationLayerInfo::ActivationFunction::RELU)) + return false; + } else { + if (!NEActivationLayer::validate(&srcTensorsInfo[0], &dstTensorsInfo[0], + {ActivationLayerInfo::ActivationFunction::LEAKY_RELU, aclEltwiseAttrs.alpha})) + return false; + } + exec_func = [this]{ + auto acl_op = std::make_unique(); + if (aclEltwiseAttrs.alpha == 0) { + acl_op->configure(&srcTensors[0], &dstTensors[0], ActivationLayerInfo::ActivationFunction::RELU); + } else { + acl_op->configure(&srcTensors[0], &dstTensors[0], + {ActivationLayerInfo::ActivationFunction::LEAKY_RELU, aclEltwiseAttrs.alpha}); + } + acl_op->run(); + }; + break; + case Algorithm::EltwiseGeluErf: + if (!NEActivationLayer::validate(&srcTensorsInfo[0], &dstTensorsInfo[0], ActivationLayerInfo::ActivationFunction::GELU)) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &dstTensors[0], ActivationLayerInfo::ActivationFunction::GELU); + acl_op->run(); + }; + break; + case Algorithm::EltwiseElu: + if (!NEActivationLayer::validate(&srcTensorsInfo[0], &dstTensorsInfo[0], + {ActivationLayerInfo::ActivationFunction::ELU, aclEltwiseAttrs.alpha})) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &dstTensors[0], {ActivationLayerInfo::ActivationFunction::ELU, aclEltwiseAttrs.alpha}); + acl_op->run(); + }; + break; + case Algorithm::EltwiseTanh: + if (!NEActivationLayer::validate(&srcTensorsInfo[0], &dstTensorsInfo[0], + {ActivationLayerInfo::ActivationFunction::TANH, 1.f, 1.f})) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &dstTensors[0], + {ActivationLayerInfo::ActivationFunction::TANH, 1.f, 1.f}); + acl_op->run(); + }; + break; + case Algorithm::EltwiseSigmoid: + if (!NEActivationLayer::validate(&srcTensorsInfo[0], &dstTensorsInfo[0], ActivationLayerInfo::ActivationFunction::LOGISTIC)) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &dstTensors[0], ActivationLayerInfo::ActivationFunction::LOGISTIC); + acl_op->run(); + }; + break; + case Algorithm::EltwiseAbs: + if (!NEAbsLayer::validate(&srcTensorsInfo[0], &dstTensorsInfo[0])) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &dstTensors[0]); + acl_op->run(); + }; + break; + case Algorithm::EltwiseSqrt: + if (!NEActivationLayer::validate(&srcTensorsInfo[0], &dstTensorsInfo[0], ActivationLayerInfo::ActivationFunction::SQRT)) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &dstTensors[0], ActivationLayerInfo::ActivationFunction::SQRT); + acl_op->run(); + }; + break; + case Algorithm::EltwiseSoftRelu: + if (!NEActivationLayer::validate(&srcTensorsInfo[0], &dstTensorsInfo[0], ActivationLayerInfo::ActivationFunction::SOFT_RELU)) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &dstTensors[0], ActivationLayerInfo::ActivationFunction::SOFT_RELU); + acl_op->run(); + }; + break; + case Algorithm::EltwiseExp: + if (!NEExpLayer::validate(&srcTensorsInfo[0], &dstTensorsInfo[0])) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &dstTensors[0]); + acl_op->run(); + }; + break; + case Algorithm::EltwiseClamp: + if (!NEActivationLayer::validate(&srcTensorsInfo[0], &dstTensorsInfo[0], + {ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU, aclEltwiseAttrs.beta, aclEltwiseAttrs.alpha})) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &dstTensors[0], + {ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU, aclEltwiseAttrs.beta, aclEltwiseAttrs.alpha}); + acl_op->run(); + }; + break; + case Algorithm::EltwiseSwish: + if (!NEActivationLayer::validate(&srcTensorsInfo[0], &dstTensorsInfo[0], + {ActivationLayerInfo::ActivationFunction::SWISH, aclEltwiseAttrs.beta})) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &dstTensors[0], + {ActivationLayerInfo::ActivationFunction::SWISH, aclEltwiseAttrs.alpha}); + acl_op->run(); + }; + break; + case Algorithm::EltwisePrelu: + if (!NEPReluLayer::validate(&srcTensorsInfo[0], &srcTensorsInfo[1], &dstTensorsInfo[0])) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &srcTensors[1], &dstTensors[0]); + acl_op->run(); + }; + break; + case Algorithm::EltwiseHswish: + if (!NEActivationLayer::validate(&srcTensorsInfo[0], &dstTensorsInfo[0], ActivationLayerInfo::ActivationFunction::HARD_SWISH)) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &dstTensors[0], ActivationLayerInfo::ActivationFunction::HARD_SWISH); + acl_op->run(); + }; + break; + case Algorithm::EltwiseLog: + if (!NELogLayer::validate(&srcTensorsInfo[0], &dstTensorsInfo[0])) + return false; + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensors[0], &dstTensors[0]); + acl_op->run(); + }; + break; + default: + IE_THROW() << "Unsupported operation type for ACL Eltwise executor: " << static_cast(aclEltwiseAttrs.algorithm); + } + return true; +} + +void AclEltwiseExecutor::exec(const std::vector &src, const std::vector &dst, + const void *post_ops_data_) { + for (int i = 0; i < src.size(); i++) { + srcTensors[i].allocator()->import_memory(src[i]->GetPtr()); + } + for (int i = 0; i < dst.size(); i++) { + dstTensors[i].allocator()->import_memory(dst[i]->GetPtr()); + } + + exec_func(); + + for (int i = 0; i < src.size(); i++) { + srcTensors[i].allocator()->free(); + } + for (int i = 0; i < dst.size(); i++) { + dstTensors[i].allocator()->free(); + } +} +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_eltwise.hpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_eltwise.hpp new file mode 100644 index 00000000000..a0c623eb524 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_eltwise.hpp @@ -0,0 +1,110 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "../eltwise.hpp" +#include "arm_compute/runtime/NEON/NEFunctions.h" +#include "acl_utils.hpp" + +namespace ov { +namespace intel_cpu { + +class AclEltwiseExecutor : public EltwiseExecutor { +public: + AclEltwiseExecutor(const ExecutorContext::CPtr context); + + bool init(const EltwiseAttrs& eltwiseAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const std::vector& postOps) override; + + void exec(const std::vector& src, + const std::vector& dst, + const void *post_ops_data_) override; + + impl_desc_type getImplType() const override { + return implType; + } +private: + EltwiseAttrs aclEltwiseAttrs{}; + impl_desc_type implType = impl_desc_type::acl; + std::vector srcTensors, dstTensors; + std::function exec_func; +}; + +class AclEltwiseExecutorBuilder : public EltwiseExecutorBuilder { +public: + bool isSupported(const EltwiseAttrs& eltwiseAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs) const override { + switch (eltwiseAttrs.algorithm) { + case Algorithm::EltwiseAdd: + case Algorithm::EltwiseMultiply: + case Algorithm::EltwiseSubtract: + case Algorithm::EltwiseDivide: + case Algorithm::EltwiseMaximum: + case Algorithm::EltwiseMinimum: + case Algorithm::EltwiseSquaredDifference: + case Algorithm::EltwisePowerDynamic: + case Algorithm::EltwiseEqual: + case Algorithm::EltwiseNotEqual: + case Algorithm::EltwiseGreater: + case Algorithm::EltwiseGreaterEqual: + case Algorithm::EltwiseLess: + case Algorithm::EltwiseLessEqual: + case Algorithm::EltwiseRelu: + case Algorithm::EltwiseGeluErf: + case Algorithm::EltwiseElu: + case Algorithm::EltwiseTanh: + case Algorithm::EltwiseSigmoid: + case Algorithm::EltwiseAbs: + case Algorithm::EltwiseSqrt: + case Algorithm::EltwiseSoftRelu: + case Algorithm::EltwiseExp: + case Algorithm::EltwiseClamp: + case Algorithm::EltwiseSwish: + case Algorithm::EltwisePrelu: + case Algorithm::EltwiseHswish: + case Algorithm::EltwiseLog: + break; + default: + return false; + } + + // ACL supports only U8 precision on output for comparison operations + if (one_of(eltwiseAttrs.algorithm, Algorithm::EltwiseEqual, Algorithm::EltwiseNotEqual, Algorithm::EltwiseGreater, + Algorithm::EltwiseGreaterEqual, Algorithm::EltwiseLess, Algorithm::EltwiseLessEqual)) { + if (dstDescs[0]->getPrecision() != InferenceEngine::Precision::U8) { + return false; + } + } + for (const auto &srcD : srcDescs) { + for (const auto &dstD : dstDescs) { + if ((srcD->getPrecision() != InferenceEngine::Precision::FP32 && + srcD->getPrecision() != InferenceEngine::Precision::FP16) || + srcD->getPrecision() != dstD->getPrecision()) + return false; + } + } + + for (int i = 0; i < srcDescs.size(); i++) { + if (getAclDataLayoutByMemoryDesc(srcDescs[i]) == arm_compute::DataLayout::UNKNOWN) + return false; + } + for (int i = 0; i < dstDescs.size(); i++) { + if (getAclDataLayoutByMemoryDesc(dstDescs[i]) == arm_compute::DataLayout::UNKNOWN) + return false; + } + + return true; + } + + EltwiseExecutorPtr makeExecutor(const ExecutorContext::CPtr context) const override { + return std::make_shared(context); + } +}; + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_interpolate.cpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_interpolate.cpp new file mode 100644 index 00000000000..47fddc19e6e --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_interpolate.cpp @@ -0,0 +1,185 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "acl_interpolate.hpp" +#include "acl_utils.hpp" + +static arm_compute::TensorShape interpolateShapeCast(const ov::intel_cpu::VectorDims& dims) { + arm_compute::TensorShape tensorShape; + for (std::size_t i = 0; i < dims.size(); ++i) { + tensorShape.set(dims.size() - i - 1, dims[i], false); + } + if (tensorShape.num_dimensions() == 0) { + tensorShape.set(0, 1, false); + tensorShape.set_num_dimensions(1); + } + return tensorShape; +} + +bool ov::intel_cpu::ACLInterpolateExecutor::init(const InterpolateAttrs &interpolateAttrs, + const std::vector &srcDescs, + const std::vector &dstDescs, + const dnnl::primitive_attr &attr) { + InterpolateExecutor::init(interpolateAttrs, srcDescs, dstDescs, attr); + aclInterpolateAttrs = interpolateAttrs; + auto& coord_mode = aclInterpolateAttrs.coordTransMode; + auto& inter_mode = aclInterpolateAttrs.mode; + acl_coord = arm_compute::SamplingPolicy::TOP_LEFT; + auto& out_shape = dstDescs[0]->getShape().getDims(); + + if ((coord_mode == InterpolateCoordTransMode::pytorch_half_pixel && out_shape[2] > 1 && out_shape[3] > 1) || + coord_mode == InterpolateCoordTransMode::half_pixel) { + acl_coord = arm_compute::SamplingPolicy::CENTER; + } + + switch (inter_mode) { + case InterpolateMode::linear: + case InterpolateMode::linear_onnx: + acl_policy = arm_compute::InterpolationPolicy::BILINEAR; + break; + case InterpolateMode::nearest: + acl_policy = arm_compute::InterpolationPolicy::NEAREST_NEIGHBOR; + break; + default: + return false; + } + + auto srcDims = srcDescs[0]->getShape().getStaticDims(); + auto dstDims = dstDescs[0]->getShape().getStaticDims(); + auto srcTensorInfo = arm_compute::TensorInfo(interpolateShapeCast(srcDims), 1, + precisionToAclDataType(srcDescs[0]->getPrecision()), + getAclDataLayoutByMemoryDesc(srcDescs[0])); + auto dstTensorInfo = arm_compute::TensorInfo(interpolateShapeCast(dstDims), 1, + precisionToAclDataType(dstDescs[0]->getPrecision()), + getAclDataLayoutByMemoryDesc(dstDescs[0])); + + if (!arm_compute::NEScale::validate(&srcTensorInfo, + &dstTensorInfo, + arm_compute::ScaleKernelInfo(acl_policy, + arm_compute::BorderMode::REPLICATE, + arm_compute::PixelValue(), + acl_coord, + false, + coord_mode == InterpolateCoordTransMode::align_corners))) + return false; + + srcTensor.allocator()->init(srcTensorInfo); + dstTensor.allocator()->init(dstTensorInfo); + + acl_scale = std::make_unique(); + acl_scale->configure(&srcTensor, &dstTensor, arm_compute::ScaleKernelInfo(acl_policy, + arm_compute::BorderMode::REPLICATE, + arm_compute::PixelValue(), + acl_coord, + false, + aclInterpolateAttrs.coordTransMode == InterpolateCoordTransMode::align_corners)); + return true; +} + +void ov::intel_cpu::ACLInterpolateExecutor::exec(const std::vector& src, const std::vector& dst, const void *post_ops_data_) { + auto in_ptr_ = padPreprocess(src, dst); + srcTensor.allocator()->import_memory(const_cast(reinterpret_cast(in_ptr_))); + dstTensor.allocator()->import_memory(dst[0]->GetPtr()); + + acl_scale->run(); + + srcTensor.allocator()->free(); + dstTensor.allocator()->free(); +} + +bool ov::intel_cpu::ACLInterpolateExecutorBuilder::isSupportedConfiguration( + const ov::intel_cpu::InterpolateAttrs &interpolateAttrs, const std::vector &srcDescs, + const std::vector &dstDescs) { + auto& inp_shape = srcDescs[0]->getShape().getDims(); + auto& out_shape = dstDescs[0]->getShape().getDims(); + + float scale_h = static_cast(out_shape[2]) / inp_shape[2]; + float scale_w = static_cast(out_shape[3]) / inp_shape[3]; + bool is_upsample = scale_h > 1 && scale_w > 1; + + auto& coord_mode = interpolateAttrs.coordTransMode; + auto& nearest_mode = interpolateAttrs.nearestMode; + + if (coord_mode == InterpolateCoordTransMode::asymmetric && + nearest_mode == InterpolateNearestMode::floor) { + return is_upsample; + } + + if (coord_mode == InterpolateCoordTransMode::align_corners && + nearest_mode == InterpolateNearestMode::round_prefer_ceil) { + return true; + } + + if (coord_mode == InterpolateCoordTransMode::half_pixel && + (nearest_mode == InterpolateNearestMode::simple || nearest_mode == InterpolateNearestMode::round_prefer_ceil)) { + return false; + } + + if (coord_mode == InterpolateCoordTransMode::asymmetric && + (nearest_mode == InterpolateNearestMode::simple || nearest_mode == InterpolateNearestMode::floor)) { + return is_upsample; + } + + if (is_upsample) { + bool int_factor = scale_h == static_cast(scale_h) && scale_w == static_cast(scale_w); + if (int_factor && coord_mode != InterpolateCoordTransMode::asymmetric && + (nearest_mode == InterpolateNearestMode::round_prefer_ceil + || nearest_mode == InterpolateNearestMode::round_prefer_floor)) { + return true; + } + } else if (scale_h < 1 && scale_w < 1) { + float down_scale_h = static_cast(inp_shape[2]) / out_shape[2]; + float down_scale_w = static_cast(inp_shape[3]) / out_shape[3]; + bool int_factor = down_scale_h == static_cast(down_scale_h) && down_scale_w == static_cast(down_scale_w); + + if (int_factor && coord_mode != InterpolateCoordTransMode::align_corners && + nearest_mode == InterpolateNearestMode::simple) { + return true; + } + + if (int_factor && nearest_mode == InterpolateNearestMode::round_prefer_ceil && + ((out_shape[2] > 1 && out_shape[3] > 1) || coord_mode != InterpolateCoordTransMode::half_pixel)) { + return true; + } + } + return false; +} + +bool ov::intel_cpu::ACLInterpolateExecutorBuilder::isSupported(const ov::intel_cpu::InterpolateAttrs &interpolateAttrs, + const std::vector &srcDescs, + const std::vector &dstDescs) const { + if (srcDescs[0]->getShape().getDims().size() != 4) { + return false; + } + + auto& pads_begin = interpolateAttrs.padBegin; + auto& pads_end = interpolateAttrs.padEnd; + + if (!std::all_of(pads_begin.begin(), pads_begin.end(), [](int i){return i == 0;}) || + !std::all_of(pads_end.begin(), pads_end.end(), [](int i){return i == 0;})) { + return false; + } + + auto& nearest_mode = interpolateAttrs.nearestMode; + auto& coord_mode = interpolateAttrs.coordTransMode; + if (interpolateAttrs.antialias || + coord_mode == InterpolateCoordTransMode::tf_half_pixel_for_nn || + nearest_mode == InterpolateNearestMode::ceil) { + return false; + } + + if (interpolateAttrs.mode == InterpolateMode::cubic) { + return false; + } + + if (interpolateAttrs.mode == InterpolateMode::nearest && + !isSupportedConfiguration(interpolateAttrs, srcDescs, dstDescs)) { + return false; + } + + if (coord_mode == InterpolateCoordTransMode::pytorch_half_pixel) { + return false; + } + return true; +} diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_interpolate.hpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_interpolate.hpp new file mode 100644 index 00000000000..9850d390d84 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_interpolate.hpp @@ -0,0 +1,52 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "../interpolate.hpp" + +namespace ov { +namespace intel_cpu { + +class ACLInterpolateExecutor : public InterpolateExecutor { +public: + ACLInterpolateExecutor(const ExecutorContext::CPtr context) : InterpolateExecutor(context) {} + + bool init(const InterpolateAttrs& interpolateAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) override; + + void exec(const std::vector& src, const std::vector& dst, const void *post_ops_data_) override; + + impl_desc_type getImplType() const override { + return implType; + } + +private: + impl_desc_type implType = impl_desc_type::acl; + InterpolateAttrs aclInterpolateAttrs; + arm_compute::SamplingPolicy acl_coord; + arm_compute::InterpolationPolicy acl_policy; + bool antialias{}; + arm_compute::Tensor srcTensor, dstTensor; + std::unique_ptr acl_scale; +}; + +class ACLInterpolateExecutorBuilder : public InterpolateExecutorBuilder { +public: + bool isSupported(const InterpolateAttrs& interpolateAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs) const override; + + InterpolateExecutorPtr makeExecutor(const ExecutorContext::CPtr context) const override { + return std::make_shared(context); + } +private: + static bool isSupportedConfiguration(const InterpolateAttrs& interpolateAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs); +}; +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_mvn.cpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_mvn.cpp new file mode 100644 index 00000000000..56d897dd5fb --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_mvn.cpp @@ -0,0 +1,76 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "acl_mvn.hpp" + +namespace ov { +namespace intel_cpu { + +using namespace arm_compute; + +AclMVNExecutor::AclMVNExecutor(const ExecutorContext::CPtr context) : MVNExecutor(context) {} + +bool AclMVNExecutor::init(const MVNAttrs& mvnAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) { + auto srcDims = srcDescs[0]->getShape().getStaticDims(); + auto dstDims = dstDescs[0]->getShape().getStaticDims(); + + size_t X, Y; + if (mvnAttrs.initAcrossChannels_) { + if (srcDims.size() >= 2) { + Y = srcDims[0]; + X = srcDims[1]; + for (int i = 2; i < srcDims.size(); i++) { + X *= srcDims[i]; + } + } else { + Y = srcDims[0]; + X = 1; + } + } else { + if (srcDims.size() > 2) { + Y = srcDims[0] * srcDims[1]; + X = srcDims[2]; + for (int i = 3; i < srcDims.size(); i++) { + X *= srcDims[i]; + } + } else if (srcDims.size() == 2) { + Y = srcDims[0] * srcDims[1]; + X = 1; + } else { + Y = srcDims[0]; + X = 1; + } + } + + TensorInfo srcTensorInfo = TensorInfo(TensorShape(X, Y), 1, precisionToAclDataType(srcDescs[0]->getPrecision()), getAclDataLayoutByMemoryDesc(srcDescs[0])); + TensorInfo dstTensorInfo = TensorInfo(TensorShape(X, Y), 1, precisionToAclDataType(dstDescs[0]->getPrecision()), getAclDataLayoutByMemoryDesc(dstDescs[0])); + + + if (!arm_compute::NEMeanStdDevNormalizationLayer::validate(&srcTensorInfo, &dstTensorInfo, mvnAttrs.epsValue_)) + return false; + + srcTensor.allocator()->init(srcTensorInfo); + dstTensor.allocator()->init(dstTensorInfo); + + mvn = std::make_unique(); + mvn->configure(&srcTensor, &dstTensor, mvnAttrs.epsValue_); + + return true; +} + +void AclMVNExecutor::exec(const std::vector& src, const std::vector& dst, const void *post_ops_data_) { + srcTensor.allocator()->import_memory(src[0]->GetPtr()); + dstTensor.allocator()->import_memory(dst[0]->GetPtr()); + + mvn->run(); + + srcTensor.allocator()->free(); + dstTensor.allocator()->free(); +} + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_mvn.hpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_mvn.hpp new file mode 100644 index 00000000000..478da5357c8 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_mvn.hpp @@ -0,0 +1,73 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "acl_utils.hpp" +#include "nodes/executors/mvn.hpp" +#include "arm_compute/runtime/NEON/NEFunctions.h" + +namespace ov { +namespace intel_cpu { + +class AclMVNExecutor : public MVNExecutor { +public: + AclMVNExecutor(const ExecutorContext::CPtr context); + + bool init(const MVNAttrs& mvnAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) override; + void exec(const std::vector& src, + const std::vector& dst, + const void *post_ops_data_) override; + + impl_desc_type getImplType() const override { + return implType; + } + +private: + impl_desc_type implType = impl_desc_type::acl; + + arm_compute::Tensor srcTensor; + arm_compute::Tensor dstTensor; + std::unique_ptr mvn = nullptr; +}; + +class AclMVNExecutorBuilder : public MVNExecutorBuilder { +public: + bool isSupported(const MVNAttrs& mvnAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs) const override { + if ((srcDescs[0]->getPrecision() != InferenceEngine::Precision::FP32 && + srcDescs[0]->getPrecision() != InferenceEngine::Precision::FP16) || + srcDescs[0]->getPrecision() != dstDescs[0]->getPrecision()) + return false; + + if (!(srcDescs[0]->hasLayoutType(LayoutType::ncsp) && + dstDescs[0]->hasLayoutType(LayoutType::ncsp)) && + !(srcDescs[0]->hasLayoutType(LayoutType::nspc) && + dstDescs[0]->hasLayoutType(LayoutType::nspc))) + return false; + + if (mvnAttrs.epsMode_ == MVNEpsMode::OUTSIDE_SQRT) { + return false; + } + if (!mvnAttrs.normalizeVariance_) { + return false; + } + if (!mvnAttrs.initAcrossChannels_ && getAclDataLayoutByMemoryDesc(srcDescs[0]) == arm_compute::DataLayout::NHWC) { + return false; + } + + return true; + } + + MVNExecutorPtr makeExecutor(const ExecutorContext::CPtr context) const override { + return std::make_shared(context); + } +}; + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_pooling.cpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_pooling.cpp new file mode 100644 index 00000000000..300043519cf --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_pooling.cpp @@ -0,0 +1,183 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "acl_pooling.hpp" +#include "acl_utils.hpp" + +namespace ov { +namespace intel_cpu { + +using namespace arm_compute; + +AclPoolingExecutor::AclPoolingExecutor(const ExecutorContext::CPtr context) : PoolingExecutor(context) {} + +bool AclPoolingExecutor::isSupported(const TensorInfo& srcTensorInfo, + const TensorInfo& dstTensorInfo, + const PoolingAttrs& poolingAttrs, + size_t srcDimsSize, + size_t dstDescsSize, + DataLayout dataLayout, + const VectorDims* indDims, + PoolingLayerInfo* pool_info, + Pooling3dLayerInfo* pool3d_info) { + unsigned int pad_left = (poolingAttrs.data_pad_begin.size() >= 2) ? poolingAttrs.data_pad_begin[1] : poolingAttrs.data_pad_begin[0]; + unsigned int pad_right = (poolingAttrs.data_pad_end.size() >= 2) ? poolingAttrs.data_pad_end[1] : poolingAttrs.data_pad_end[0]; + unsigned int pad_top = (poolingAttrs.data_pad_begin.size() >= 2) ? poolingAttrs.data_pad_begin[0] : 0; + unsigned int pad_bottom = (poolingAttrs.data_pad_end.size() >= 2) ? poolingAttrs.data_pad_end[0] : 0; + unsigned int kernel_w = (poolingAttrs.kernel.size() >= 2) ? poolingAttrs.kernel[1] : poolingAttrs.kernel[0]; + unsigned int kernel_h = (poolingAttrs.kernel.size() >= 2) ? poolingAttrs.kernel[0] : 1; + unsigned int stride_x = (poolingAttrs.stride.size() >= 2) ? poolingAttrs.stride[1] : poolingAttrs.stride[0]; + unsigned int stride_y = (poolingAttrs.stride.size() >= 2) ? poolingAttrs.stride[0] : 1; + + PoolingType pool_type; + bool exclude_padding = false; + if (poolingAttrs.algorithm == Algorithm::PoolingMax) { + pool_type = PoolingType::MAX; + exclude_padding = (poolingAttrs.pad_type != op::PadType::EXPLICIT); + } else if (poolingAttrs.algorithm == Algorithm::PoolingAvg) { + pool_type = PoolingType::AVG; + exclude_padding = poolingAttrs.exclude_pad; + } else { + DEBUG_LOG("Unknown pooling algorithm: ", static_cast(poolingAttrs.algorithm)); + return false; + } + DimensionRoundingType round = (poolingAttrs.rounding == op::RoundingType::CEIL) ? + DimensionRoundingType::CEIL : DimensionRoundingType::FLOOR; + + if (srcDimsSize == 5) { + if (dstDescsSize > 1) { + DEBUG_LOG("NEPooling3dLayer does not support indices"); + return false; + } else { + unsigned int kernel_d = poolingAttrs.kernel[2]; + unsigned int stride_z = poolingAttrs.stride[2]; + unsigned int pad_front = poolingAttrs.data_pad_begin[2]; + unsigned int pad_back = poolingAttrs.data_pad_end[2]; + pool3d_info->pool_type = pool_type; + pool3d_info->exclude_padding = exclude_padding; + pool3d_info->pool_size = arm_compute::Size3D(kernel_w, kernel_h, kernel_d); + pool3d_info->stride = arm_compute::Size3D(stride_x, stride_y, stride_z); + pool3d_info->padding = arm_compute::Padding3D(pad_left, pad_right, pad_top, pad_bottom, pad_front, pad_back); + pool3d_info->round_type = round; + arm_compute::Status s = arm_compute::NEPooling3dLayer::validate(&srcTensorInfo, &dstTensorInfo, *pool3d_info); + if (!s) { + DEBUG_LOG("NEPooling3dLayer validation failed: ", s.error_description()); + return false; + } + } + } else { + pool_info->data_layout = dataLayout; + pool_info->pool_size = arm_compute::Size2D(kernel_w, kernel_h); + pool_info->pad_stride_info = arm_compute::PadStrideInfo(stride_x, stride_y, pad_left, pad_right, pad_top, pad_bottom, round); + pool_info->pool_type = pool_type; + pool_info->exclude_padding = exclude_padding; + if (dstDescsSize > 1) { + TensorInfo indTensorInfo = TensorInfo(shapeCast(*indDims), 1, arm_compute::DataType::U32, dataLayout); + arm_compute::Status s = arm_compute::NEPoolingLayer::validate(&srcTensorInfo, &dstTensorInfo, *pool_info, &indTensorInfo); + if (!s) { + DEBUG_LOG("NEPoolingLayer validation with indices failed: ", s.error_description()); + return false; + } + } else { + arm_compute::Status s = arm_compute::NEPoolingLayer::validate(&srcTensorInfo, &dstTensorInfo, *pool_info); + if (!s) { + DEBUG_LOG("NEPoolingLayer validation without indices failed: ", s.error_description()); + return false; + } + } + } + return true; +} + +bool AclPoolingExecutor::init(const PoolingAttrs& poolingAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) { + auto srcDims = srcDescs[0]->getShape().getStaticDims(); + auto dstDims = dstDescs[0]->getShape().getStaticDims(); + + TensorInfo srcTensorInfo = TensorInfo(shapeCast(srcDims), 1, + precisionToAclDataType(srcDescs[0]->getPrecision()), getAclDataLayoutByMemoryDesc(srcDescs[0])); + TensorInfo dstTensorInfo = TensorInfo(shapeCast(dstDims), 1, + precisionToAclDataType(dstDescs[0]->getPrecision()), getAclDataLayoutByMemoryDesc(dstDescs[0])); + + srcTensor.allocator()->init(srcTensorInfo); + dstTensor.allocator()->init(dstTensorInfo); + + if (srcDims.size() == 5) { + if (dstDescs.size() == 1) { + Pooling3dLayerInfo pool_info; + if (!isSupported(srcTensorInfo, + dstTensorInfo, + poolingAttrs, + srcDims.size(), + dstDescs.size(), + getAclDataLayoutByMemoryDesc(srcDescs[0]), + nullptr, + nullptr, + &pool_info)) + return false; + exec_func = [this, pool_info]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensor, &dstTensor, pool_info); + acl_op->run(); + }; + } + } else { + arm_compute::PoolingLayerInfo pool_info; + if (dstDescs.size() > 1) { + if (!isSupported(srcTensorInfo, + dstTensorInfo, + poolingAttrs, + srcDims.size(), + dstDescs.size(), + getAclDataLayoutByMemoryDesc(srcDescs[0]), + &dstDescs[1]->getShape().getStaticDims(), + &pool_info, + nullptr)) + return false; + auto indDims = dstDescs[1]->getShape().getStaticDims(); + TensorInfo indTensorInfo = TensorInfo(shapeCast(indDims), 1, precisionToAclDataType(dstDescs[1]->getPrecision()), + getAclDataLayoutByMemoryDesc(dstDescs[1])); + indTensor.allocator()->init(indTensorInfo); + exec_func = [this, pool_info]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensor, &dstTensor, pool_info, &indTensor); + acl_op->run(); + }; + } else { + if (!isSupported(srcTensorInfo, + dstTensorInfo, + poolingAttrs, + srcDims.size(), + dstDescs.size(), + getAclDataLayoutByMemoryDesc(srcDescs[0]), + nullptr, + &pool_info, + nullptr)) + return false; + exec_func = [this, pool_info]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensor, &dstTensor, pool_info); + acl_op->run(); + }; + } + } + return true; +} + +void AclPoolingExecutor::exec(const std::vector& src, const std::vector& dst, std::unordered_map postOpsArgs) { + srcTensor.allocator()->import_memory(src[0]->GetPtr()); + dstTensor.allocator()->import_memory(dst[0]->GetPtr()); + if (dst.size() > 1) indTensor.allocator()->import_memory(dst[1]->GetPtr()); + + exec_func(); + + srcTensor.allocator()->free(); + dstTensor.allocator()->free(); + if (dst.size() > 1) indTensor.allocator()->free(); +} + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_pooling.hpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_pooling.hpp new file mode 100644 index 00000000000..d5c96da33d5 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_pooling.hpp @@ -0,0 +1,131 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "nodes/executors/pooling.hpp" +#include "arm_compute/runtime/NEON/NEFunctions.h" +#include "utils/debug_capabilities.h" + +namespace ov { +namespace intel_cpu { + +class AclPoolingExecutor : public PoolingExecutor { +public: + AclPoolingExecutor(const ExecutorContext::CPtr context); + + bool init(const PoolingAttrs& poolingAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) override; + void exec(const std::vector& src, + const std::vector& dst, + std::unordered_map postOpsArgs) override; + + static bool isSupported(const arm_compute::TensorInfo& srcTensorInfo, + const arm_compute::TensorInfo& dstTensorInfo, + const PoolingAttrs& poolingAttrs, + size_t srcDimsSize, + size_t dstDescsSize, + arm_compute::DataLayout dataLayout, + const VectorDims* indDims, + arm_compute::PoolingLayerInfo* pool_info, + arm_compute::Pooling3dLayerInfo* pool3d_info); + + impl_desc_type getImplType() const override { + return implType; + } + +private: + std::function exec_func; + PoolingAttrs poolingAttrs; + impl_desc_type implType = impl_desc_type::acl; + + arm_compute::Tensor srcTensor; + arm_compute::Tensor dstTensor; + arm_compute::Tensor indTensor; + std::unique_ptr pooling = nullptr; +}; + +class AclPoolingExecutorBuilder : public PoolingExecutorBuilder { +public: + bool isSupported(const PoolingAttrs& poolingAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs) const override { + if ((srcDescs[0]->getPrecision() != InferenceEngine::Precision::FP32 && + dstDescs[0]->getPrecision() != InferenceEngine::Precision::FP32) && + (srcDescs[0]->getPrecision() != InferenceEngine::Precision::FP16 && + dstDescs[0]->getPrecision() != InferenceEngine::Precision::FP16)) { + DEBUG_LOG("AclPoolingExecutor does not support precisions:", + " src[0]=", srcDescs[0]->getPrecision(), + " dst[0]=", dstDescs[0]->getPrecision()); + return false; + } + + if (srcDescs.size() == 2 && + (srcDescs[1]->getPrecision() != InferenceEngine::Precision::FP32 && + srcDescs[0]->getPrecision() != InferenceEngine::Precision::FP32 && + dstDescs[0]->getPrecision() != InferenceEngine::Precision::FP32) && + (srcDescs[1]->getPrecision() != InferenceEngine::Precision::FP16 && + srcDescs[0]->getPrecision() != InferenceEngine::Precision::FP16 && + dstDescs[0]->getPrecision() != InferenceEngine::Precision::FP16)) { + DEBUG_LOG("AclPoolingExecutor does not support precisions:", + " src[0]=", srcDescs[0]->getPrecision(), + " src[1]=", srcDescs[1]->getPrecision(), + " dst[0]=", dstDescs[0]->getPrecision()); + return false; + } + + if (dstDescs.size() == 2 && + dstDescs[1]->getPrecision() != InferenceEngine::Precision::U32) { + DEBUG_LOG("AclPoolingExecutor does not support precisions:", + " dst[1]=", dstDescs[1]->getPrecision()); + return false; + } + + if (srcDescs[0]->getShape().getRank() < 5) { + if (!(srcDescs[0]->hasLayoutType(LayoutType::ncsp) && + dstDescs[0]->hasLayoutType(LayoutType::ncsp)) && + !(srcDescs[0]->hasLayoutType(LayoutType::nspc) && + dstDescs[0]->hasLayoutType(LayoutType::nspc))) { + DEBUG_LOG("NEPoolingLayer does not support layouts:", + " src=", srcDescs[0]->serializeFormat(), + " dst=", dstDescs[0]->serializeFormat()); + return false; + } + if (srcDescs.size() == 2 && + !(srcDescs[0]->hasLayoutType(LayoutType::ncsp) && + srcDescs[1]->hasLayoutType(LayoutType::ncsp) && + dstDescs[0]->hasLayoutType(LayoutType::ncsp)) && + !(srcDescs[0]->hasLayoutType(LayoutType::nspc) && + srcDescs[1]->hasLayoutType(LayoutType::nspc) && + dstDescs[0]->hasLayoutType(LayoutType::nspc))) { + DEBUG_LOG("NEPoolingLayer does not support layouts:", + " src[0]=", srcDescs[0]->serializeFormat(), + " src[1]=", srcDescs[1]->serializeFormat(), + " dst=", dstDescs[0]->serializeFormat()); + return false; + } + } else { + if (!(srcDescs[0]->hasLayoutType(LayoutType::nspc) && + dstDescs[0]->hasLayoutType(LayoutType::nspc)) && + !(srcDescs[0]->hasLayoutType(LayoutType::nspc) && + dstDescs[0]->hasLayoutType(LayoutType::nspc))) { + DEBUG_LOG("Pooling3dLayer does not support layouts:", + " src=", srcDescs[0]->serializeFormat(), + " dst=", dstDescs[0]->serializeFormat()); + return false; + } + } + + return true; + } + + PoolingExecutorPtr makeExecutor(const ExecutorContext::CPtr context) const override { + return std::make_shared(context); + } +}; + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_reduce.cpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_reduce.cpp new file mode 100644 index 00000000000..25d6b4d5f0e --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_reduce.cpp @@ -0,0 +1,109 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "acl_utils.hpp" +#include "acl_reduce.hpp" + +namespace ov { +namespace intel_cpu { + +using namespace arm_compute; + +static arm_compute::ReductionOperation getAclReductionOperationByAlgorithm(Algorithm algorithm) { + switch (algorithm) { + case Algorithm::ReduceMax: return arm_compute::ReductionOperation::MAX; + case Algorithm::ReduceMin: return arm_compute::ReductionOperation::MIN; + case Algorithm::ReduceSum: return arm_compute::ReductionOperation::SUM; + case Algorithm::ReduceProd: return arm_compute::ReductionOperation::PROD; + default: IE_THROW() << "Unsupported reduction operation: " << static_cast(algorithm); + } +} + +AclReduceExecutor::AclReduceExecutor(const ExecutorContext::CPtr context) : ReduceExecutor(context) {} + +bool AclReduceExecutor::init(const ReduceAttrs& reduceAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) { + if (reduceAttrs.operation != Algorithm::ReduceMax && + reduceAttrs.operation != Algorithm::ReduceMin && + reduceAttrs.operation != Algorithm::ReduceSum && + reduceAttrs.operation != Algorithm::ReduceProd && + reduceAttrs.operation != Algorithm::ReduceMean) { + DEBUG_LOG("Unknown reduce algorithm passed into AclReduceExecutor: ", static_cast(reduceAttrs.operation)); + return false; + } + + this->reduceAttrs = reduceAttrs; + + auto srcDims = srcDescs[0]->getShape().getStaticDims(); + auto dstDims = dstDescs[0]->getShape().getStaticDims(); + + TensorInfo srcTensorInfo = TensorInfo(shapeCast(srcDims), 1, + precisionToAclDataType(srcDescs[0]->getPrecision()), getAclDataLayoutByMemoryDesc(srcDescs[0])); + TensorInfo dstTensorInfo = TensorInfo(shapeCast(dstDims), 1, + precisionToAclDataType(dstDescs[0]->getPrecision()), getAclDataLayoutByMemoryDesc(dstDescs[0])); + + srcTensor.allocator()->init(srcTensorInfo); + dstTensor.allocator()->init(dstTensorInfo); + + switch (reduceAttrs.operation) { + case Algorithm::ReduceMean: { + for (size_t i = 0; i < reduceAttrs.axes.size(); ++i) { + auto axe = axisCast(reduceAttrs.axes[i], srcDims.size()); + auto pos = axisCast(i, reduceAttrs.axes.size()); + axesMean.set(pos, axe); + } + Status reduceMeanStatus = NEReduceMean::validate(&srcTensorInfo, axesMean, reduceAttrs.keepDims, &dstTensorInfo); + if (!reduceMeanStatus) { + DEBUG_LOG("NEReduceMean validation failed: ", reduceMeanStatus.error_description()); + return false; + } + exec_func = [this]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensor, axesMean, this->reduceAttrs.keepDims, &dstTensor); + acl_op->run(); + }; + break; + } + case Algorithm::ReduceMax: + case Algorithm::ReduceMin: + case Algorithm::ReduceSum: + case Algorithm::ReduceProd: { + if (reduceAttrs.axes.size() != 1) { + return false; + } + Status reductionOperationStatus = NEReductionOperation::validate(&srcTensorInfo, &dstTensorInfo, axisCast(reduceAttrs.axes[0], srcDims.size()), + getAclReductionOperationByAlgorithm(reduceAttrs.operation), reduceAttrs.keepDims); + if (!reductionOperationStatus) { + DEBUG_LOG("NEReductionOperation validation with indices failed: ", reductionOperationStatus.error_description()); + return false; + } + exec_func = [this, srcDims]{ + auto acl_op = std::make_unique(); + acl_op->configure(&srcTensor, &dstTensor, axisCast(this->reduceAttrs.axes[0], srcDims.size()), + getAclReductionOperationByAlgorithm(this->reduceAttrs.operation), this->reduceAttrs.keepDims); + acl_op->run(); + }; + break; + } + default: + IE_THROW() << "Unsupported operation type for ACL Reduce executor: " << static_cast(reduceAttrs.operation); + } + + return true; +} + +void AclReduceExecutor::exec(const std::vector& src, const std::vector& dst, const void *post_ops_data_) { + srcTensor.allocator()->import_memory(src[0]->GetPtr()); + dstTensor.allocator()->import_memory(dst[0]->GetPtr()); + + exec_func(); + + srcTensor.allocator()->free(); + dstTensor.allocator()->free(); +} + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_reduce.hpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_reduce.hpp new file mode 100644 index 00000000000..0a84053b860 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_reduce.hpp @@ -0,0 +1,75 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +// TODO: remove relative path +#include "../reduce.hpp" +#include "arm_compute/runtime/NEON/NEFunctions.h" +#include "utils/debug_capabilities.h" + +namespace ov { +namespace intel_cpu { + +class AclReduceExecutor : public ReduceExecutor { +public: + AclReduceExecutor(const ExecutorContext::CPtr context); + + bool init(const ReduceAttrs& reduceAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) override; + void exec(const std::vector& src, + const std::vector& dst, + const void *post_ops_data_) override; + + impl_desc_type getImplType() const override { + return implType; + } + +private: + std::function exec_func; + ReduceAttrs reduceAttrs; + impl_desc_type implType = impl_desc_type::acl; + + arm_compute::Coordinates axesMean; + arm_compute::Tensor srcTensor; + arm_compute::Tensor dstTensor; +}; + +class AclReduceExecutorBuilder : public ReduceExecutorBuilder { +public: + bool isSupported(const ReduceAttrs& reduceAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs) const override { + if (reduceAttrs.operation == Algorithm::ReduceMean) { + if (srcDescs[0]->getPrecision() != dstDescs[0]->getPrecision() || + (srcDescs[0]->getPrecision() != InferenceEngine::Precision::FP32 && + srcDescs[0]->getPrecision() != InferenceEngine::Precision::FP16)) { + DEBUG_LOG("NEReduceMean does not support precisions:", + " src[0]=", srcDescs[0]->getPrecision(), + " dst[0]=", dstDescs[0]->getPrecision()); + return false; + } + } else { + if (srcDescs[0]->getPrecision() != dstDescs[0]->getPrecision() || + (srcDescs[0]->getPrecision() != InferenceEngine::Precision::FP32 && + srcDescs[0]->getPrecision() != InferenceEngine::Precision::FP16 && + srcDescs[0]->getPrecision() != InferenceEngine::Precision::I32)) { + DEBUG_LOG("NEReductionOperation does not support precisions:", + " src[0]=", srcDescs[0]->getPrecision(), + " dst[0]=", dstDescs[0]->getPrecision()); + return false; + } + } + return true; + } + + ReduceExecutorPtr makeExecutor(const ExecutorContext::CPtr context) const override { + return std::make_shared(context); + } +}; + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_utils.hpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_utils.hpp new file mode 100644 index 00000000000..9a4ccfcdd80 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_utils.hpp @@ -0,0 +1,81 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include "ie_precision.hpp" +#include "memory_desc/cpu_memory_desc.h" +#include "arm_compute/core/Types.h" + +namespace ov { +namespace intel_cpu { + +/** +* @brief Return ComputeLibrary TensorShape with reverted layout schema used in ACL +* @param dims vector of dimensions to convert +* @return ComputeLibrary TensorShape object +*/ +inline arm_compute::TensorShape shapeCast(const VectorDims& dims) { + arm_compute::TensorShape tensorShape; + for (std::size_t i = 0; i < dims.size(); ++i) { + tensorShape.set(dims.size() - i - 1, dims[i], false); + } + if (tensorShape.num_dimensions() == 0) { + tensorShape.set(0, 1, false); + tensorShape.set_num_dimensions(1); + } + return tensorShape; +} + +inline std::size_t axisCast(const std::size_t axis, const std::size_t shapeSize) { + return shapeSize - axis - 1; +} + +inline Dim vectorProduct(const VectorDims& vec, size_t size) { + Dim prod = 1; + for (size_t i = 0; i < size; ++i) + prod *= vec[i]; + return prod; +} + +/** +* @brief Return ComputeLibrary DataType that corresponds to the given precision +* @param precision precision to be converted +* @return ComputeLibrary DataType or UNKNOWN if precision is not mapped to DataType +*/ +inline arm_compute::DataType precisionToAclDataType(InferenceEngine::Precision precision) { + switch (precision) { + case InferenceEngine::Precision::I8: return arm_compute::DataType::S8; + case InferenceEngine::Precision::U8: return arm_compute::DataType::U8; + case InferenceEngine::Precision::I16: return arm_compute::DataType::S16; + case InferenceEngine::Precision::U16: return arm_compute::DataType::U16; + case InferenceEngine::Precision::I32: return arm_compute::DataType::S32; + case InferenceEngine::Precision::U32: return arm_compute::DataType::U32; + case InferenceEngine::Precision::FP16: return arm_compute::DataType::F16; + case InferenceEngine::Precision::FP32: return arm_compute::DataType::F32; + case InferenceEngine::Precision::FP64: return arm_compute::DataType::F64; + case InferenceEngine::Precision::I64: return arm_compute::DataType::S64; + case InferenceEngine::Precision::BF16: return arm_compute::DataType::BFLOAT16; + default: return arm_compute::DataType::UNKNOWN; + } +} + +/** +* @brief Return ComputeLibrary DataLayout that corresponds to MemoryDecs layout +* @param desc MemoryDecs from which layout is retrieved +* @param treatAs4D the flag that treats MemoryDecs as 4D shape +* @return ComputeLibrary DataLayout or UNKNOWN if MemoryDecs layout is not mapped to DataLayout +*/ +inline arm_compute::DataLayout getAclDataLayoutByMemoryDesc(MemoryDescCPtr desc) { + if (desc->hasLayoutType(LayoutType::ncsp)) { + if (desc->getShape().getRank() <= 4) return arm_compute::DataLayout::NCHW; + if (desc->getShape().getRank() == 5) return arm_compute::DataLayout::NCDHW; + } else if (desc->hasLayoutType(LayoutType::nspc)) { + if (desc->getShape().getRank() <= 4) return arm_compute::DataLayout::NHWC; + if (desc->getShape().getRank() == 5) return arm_compute::DataLayout::NDHWC; + } + return arm_compute::DataLayout::UNKNOWN; +} + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/executors/eltwise.cpp b/src/plugins/intel_cpu/src/nodes/executors/eltwise.cpp new file mode 100644 index 00000000000..bd00fae2c1f --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/eltwise.cpp @@ -0,0 +1,15 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0mvn +// + +#include "eltwise.hpp" + +namespace ov { +namespace intel_cpu { + +using namespace InferenceEngine; + +EltwiseExecutor::EltwiseExecutor(const ExecutorContext::CPtr context) : context(context) {} + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/eltwise.hpp b/src/plugins/intel_cpu/src/nodes/executors/eltwise.hpp new file mode 100644 index 00000000000..79eecae4bac --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/eltwise.hpp @@ -0,0 +1,109 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "cpu_memory.h" +#include "onednn/iml_type_mapper.h" +#include "executor.hpp" + +namespace ov { +namespace intel_cpu { + +struct EltwiseAttrs { + Algorithm algorithm; + float alpha; + float beta; + float gamma; + + EltwiseAttrs() : algorithm(Algorithm::Default), alpha(0), beta(0), gamma(0) {} + EltwiseAttrs(Algorithm algorithm, float alpha, float beta, float gamma) : algorithm(algorithm), alpha(alpha), beta(beta), gamma(gamma) {} + + bool operator==(const EltwiseAttrs& rhs) const { + bool retVal = true; + retVal = algorithm == rhs.algorithm && + alpha == rhs.alpha && + beta == rhs.beta && + gamma == rhs.gamma; + + return retVal; + } +}; + +enum class EltwisePostOpType { + Undefined, + Eltwise, + Dnnl +}; + +class EltwisePostOp { +public: + EltwisePostOp(EltwiseAttrs eltwise) { + type = EltwisePostOpType::Eltwise; + this->eltwise = eltwise; + } + + EltwisePostOp(dnnl::post_ops dnnlPostOps) { + type = EltwisePostOpType::Dnnl; + this->dnnlPostOps = dnnlPostOps; + } + + ~EltwisePostOp() = default; + + EltwiseAttrs eltwise; + dnnl::post_ops dnnlPostOps; + + EltwisePostOpType type = EltwisePostOpType::Undefined; + + bool operator==(const EltwisePostOp &rhs) const { + if (type != rhs.type) { return false; } + bool ret = true; + switch (type) { + case EltwisePostOpType::Eltwise: + ret = eltwise == rhs.eltwise; + break; + case EltwisePostOpType::Dnnl: + ret = dnnlPostOps == rhs.dnnlPostOps; + break; + default: assert(!"unsupported eltwise post operation type"); + } + return ret; + } +}; + +class EltwiseExecutor { +public: + EltwiseExecutor(const ExecutorContext::CPtr context); + virtual bool init(const EltwiseAttrs& eltwiseAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const std::vector& postOps) = 0; + + virtual void exec(const std::vector& src, const std::vector& dst, const void *post_ops_data_) = 0; + virtual ~EltwiseExecutor() = default; + + virtual impl_desc_type getImplType() const = 0; + +protected: + EltwiseAttrs eltwiseAttrs; + const ExecutorContext::CPtr context; +}; + +using EltwiseExecutorPtr = std::shared_ptr; +using EltwiseExecutorCPtr = std::shared_ptr; + +class EltwiseExecutorBuilder { +public: + ~EltwiseExecutorBuilder() = default; + virtual bool isSupported(const EltwiseAttrs& eltwiseAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs) const = 0; + virtual EltwiseExecutorPtr makeExecutor(const ExecutorContext::CPtr context) const = 0; +}; + +using EltwiseExecutorBuilderPtr = std::shared_ptr; +using EltwiseExecutorBuilderCPtr = std::shared_ptr; + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/eltwise_list.cpp b/src/plugins/intel_cpu/src/nodes/executors/eltwise_list.cpp new file mode 100644 index 00000000000..4c2fbecb297 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/eltwise_list.cpp @@ -0,0 +1,19 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "eltwise_list.hpp" + +namespace ov { +namespace intel_cpu { + +const std::vector& getEltwiseExecutorsList() { + static std::vector descs = { + OV_CPU_INSTANCE_ACL(ExecutorType::Acl, std::make_shared()) + }; + + return descs; +} + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/executors/eltwise_list.hpp b/src/plugins/intel_cpu/src/nodes/executors/eltwise_list.hpp new file mode 100644 index 00000000000..fc651d0e099 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/eltwise_list.hpp @@ -0,0 +1,84 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "executor.hpp" + +#include "eltwise.hpp" +#if defined(OV_CPU_WITH_ACL) +#include "acl/acl_eltwise.hpp" +#endif + +#include "onednn/iml_type_mapper.h" +#include "common/primitive_cache.hpp" + +namespace ov { +namespace intel_cpu { + +struct EltwiseExecutorDesc { + ExecutorType executorType; + EltwiseExecutorBuilderCPtr builder; +}; + +const std::vector& getEltwiseExecutorsList(); + +class EltwiseExecutorFactory : public ExecutorFactory { +public: + EltwiseExecutorFactory(const EltwiseAttrs& eltwiseAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const ExecutorContext::CPtr context) : ExecutorFactory(context) { + for (auto& desc : getEltwiseExecutorsList()) { + if (desc.builder->isSupported(eltwiseAttrs, srcDescs, dstDescs)) { + supportedDescs.push_back(desc); + } + } + } + + ~EltwiseExecutorFactory() = default; + virtual EltwiseExecutorPtr makeExecutor(const EltwiseAttrs& eltwiseAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const std::vector& postOps) { + auto build = [&](const EltwiseExecutorDesc* desc) { + auto executor = desc->builder->makeExecutor(context); + if (executor->init(eltwiseAttrs, srcDescs, dstDescs, postOps)) { + return executor; + } + + EltwiseExecutorPtr ptr = nullptr; + return ptr; + }; + + if (chosenDesc) { + if (auto executor = build(chosenDesc)) { + return executor; + } + } + + for (const auto& sd : supportedDescs) { + if (auto executor = build(&sd)) { + chosenDesc = &sd; + return executor; + } + } + + IE_THROW() << "Supported Eltwise executor is not found"; + } + + bool isEmpty() { + return supportedDescs.empty(); + } + +private: + std::vector supportedDescs; + const EltwiseExecutorDesc* chosenDesc = nullptr; +}; + +using EltwiseExecutorFactoryPtr = std::shared_ptr; +using EltwiseExecutorFactoryCPtr = std::shared_ptr; + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/executor.hpp b/src/plugins/intel_cpu/src/nodes/executors/executor.hpp new file mode 100644 index 00000000000..4b61f32fa77 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/executor.hpp @@ -0,0 +1,95 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "cache/multi_cache.h" +#include "graph_context.h" +#include "onednn/iml_type_mapper.h" + +namespace ov { +namespace intel_cpu { + +#if defined(OV_CPU_WITH_ACL) +#define OV_CPU_INSTANCE_ACL(...) \ + {__VA_ARGS__}, +#else +#define OV_CPU_INSTANCE_ACL(...) +#endif + +#if defined(OV_CPU_WITH_DNNL) +#define OV_CPU_INSTANCE_DNNL(...) \ + {__VA_ARGS__}, +#else +#define OV_CPU_INSTANCE_DNNL(...) +#endif + +#if defined(OPENVINO_ARCH_X86_64) +#define OV_CPU_INSTANCE_X64(...) \ + {__VA_ARGS__}, +#else +#define OV_CPU_INSTANCE_X64(...) +#endif + +#define OV_CPU_INSTANCE_COMMON(...) \ + {__VA_ARGS__}, + +enum class ExecutorType { + Undefined, + Common, + x64, + Dnnl, + Acl +}; + +class ExecutorContext { +public: + typedef std::shared_ptr Ptr; + typedef std::shared_ptr CPtr; + + ExecutorContext(const GraphContext::CPtr graphContext, const std::vector& implPriorities) { + this->runtimeCache = graphContext->getParamsCache(); + this->scratchPad = graphContext->getScratchPad(); + this->engine = graphContext->getEngine(); + this->implPriorities = implPriorities; + } + + MultiCacheWeakPtr getRuntimeCache() const { + return runtimeCache; + } + + DnnlScratchPadPtr getScratchPad() const { + return scratchPad; + } + + dnnl::engine getEngine() const { + return engine; + } + + const std::vector& getImplPriorities() const { + return implPriorities; + } + +private: + // weak_ptr is required to avoid cycle dependencies with MultiCache + // since ExecutorContext is stored in Executor itself + MultiCacheWeakPtr runtimeCache; + DnnlScratchPadPtr scratchPad = nullptr; + dnnl::engine engine; + std::vector implPriorities = {}; +}; + +class ExecutorFactory { +public: + ExecutorFactory(const ExecutorContext::CPtr context) : context(context) {} + virtual ~ExecutorFactory() = default; + + const ExecutorContext::CPtr context; +}; + +using ExecutorFactoryPtr = std::shared_ptr; +using ExecutorFactoryCPtr = std::shared_ptr; + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/interpolate.cpp b/src/plugins/intel_cpu/src/nodes/executors/interpolate.cpp new file mode 100644 index 00000000000..cb5ecb521ec --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/interpolate.cpp @@ -0,0 +1,528 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "interpolate.hpp" +#include "ie_parallel.hpp" +#include "nodes/common/cpu_memcpy.h" +#include "emitters/x64/jit_load_store_emitters.hpp" + +bool ov::intel_cpu::InterpolateExecutor::init(const InterpolateAttrs& interpolateAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) { + const auto &srcDims = srcDescs[0]->getShape().getStaticDims(); + const auto &dstDims = dstDescs[0]->getShape().getStaticDims(); + interpAttrs = interpolateAttrs; + srcDimPad5d = to5Dim(getPaddedInputShape(srcDims, interpolateAttrs.padBegin, interpolateAttrs.padEnd)); + dstDim5d = to5Dim(dstDims); + srcDataSize = interpolateAttrs.inPrc.size(); + dstDataSize = interpolateAttrs.outPrc.size(); + dataRank = srcDims.size(); + spatialDimSize = getSpatialDimsNum(dataRank); + + switch (interpAttrs.mode) { + case InterpolateMode::nearest: { + buildTblNN(srcDimPad5d, dstDim5d, interpAttrs.dataScales, interpolateAttrs.layout, interpolateAttrs.nearestMode); + break; + } + case InterpolateMode::linear_onnx: { + buildTblLinearOnnx(srcDimPad5d, dstDim5d, interpAttrs.dataScales, interpolateAttrs.layout); + break; + } + case InterpolateMode::linear: { + static constexpr int LINEAR_KERNEL = 2; + buildTblLinear(srcDimPad5d, dstDim5d, interpAttrs.dataScales, LINEAR_KERNEL, interpolateAttrs.antialias); + break; + } + case InterpolateMode::cubic: { + buildTblCubic(srcDimPad5d, dstDim5d, interpAttrs.dataScales, interpolateAttrs.cubeCoeff, interpolateAttrs.layout); + break; + } + default: { + IE_THROW() << "Interpolate executor does not support interpolate mode: " << interpAttrs.mode; + break; + } + } + return true; +} +// ===================================================================================================================== +// index layout: +// d_0............d_OD-1, h_0..............h_OH-1, w_0................w_OW-1 +void ov::intel_cpu::InterpolateExecutor::buildTblNN(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, + const std::vector& dataScales, InterpolateLayoutType layout, InterpolateNearestMode nearestMode) { + const int dimSize = dataRank; + float fz = (dimSize == 5) ? dataScales[dimSize - 3] : 1.f; + float fy = dataScales[dimSize - 2]; + float fx = dataScales[dimSize - 1]; + size_t ID = srcDimPad5d[2], IH = srcDimPad5d[3], IW = srcDimPad5d[4]; + size_t OD = dstDim5d[2], OH = dstDim5d[3], OW = dstDim5d[4]; + + indexTable.resize(OD + OH + OW); + bool isDDownsample = (fz < 1) ? true : false; + bool isHDownsample = (fy < 1) ? true : false; + bool isWDownsample = (fx < 1) ? true : false; + for (int oz = 0; oz < OD; oz++) { + float iz = coordTransToInput(oz, fz, ID, OD); + indexTable[oz] = nearestRound(iz, isDDownsample, nearestMode); + indexTable[oz] = clipCoord(indexTable[oz], ID); + } + for (int oy = 0; oy < OH; oy++) { + float iy = coordTransToInput(oy, fy, IH, OH); + indexTable[OD + oy] = nearestRound(iy, isHDownsample, nearestMode); + indexTable[OD + oy] = clipCoord(indexTable[OD + oy], IH); + } + for (int ox = 0; ox < OW; ox++) { + float ix = coordTransToInput(ox, fx, IW, OW); + indexTable[OD + OH + ox] = nearestRound(ix, isWDownsample, nearestMode); + indexTable[OD + OH + ox] = clipCoord(indexTable[OD + OH + ox], IW); + } +} + +// scale is float(outShape) / float(inShape) +// strictly consistent with onnx calc manner(div scale, not multiply inverse), given this is done offline +// the slight precison diff can produce obvious wrong value due to "nearest round" behavior for NN mode +float ov::intel_cpu::InterpolateExecutor::coordTransToInput(int outCoord, float scale, int inShape, int outShape) const { + if (scale == 1.0f || (inShape == outShape)) { + return outCoord; + } + switch (interpAttrs.coordTransMode) { + case InterpolateCoordTransMode::half_pixel: { + return (outCoord + 0.5f) / scale - 0.5f; + break; + } + case InterpolateCoordTransMode::pytorch_half_pixel: { + if (outShape > 1) + return (outCoord + 0.5f) / scale - 0.5f; + else + return 0; + break; + } + case InterpolateCoordTransMode::asymmetric: { + return static_cast(outCoord) / scale; + break; + } + case InterpolateCoordTransMode::tf_half_pixel_for_nn: { + return (outCoord + 0.5f) / scale; + break; + } + case InterpolateCoordTransMode::align_corners: { + if (outShape > 1) + return outCoord * (static_cast(inShape - 1) / static_cast(outShape - 1)); + else + return 0; + break; + } + default: { + IE_THROW() << "errorPrefix" << " does not support specified coordinate transformation mode"; + break; + } + } +} + +int ov::intel_cpu::InterpolateExecutor::nearestRound(float originCoord, bool isDownsample, InterpolateNearestMode nearestMode) const { + switch (nearestMode) { + case InterpolateNearestMode::round_prefer_floor: { + if (originCoord == (static_cast(originCoord) + 0.5f)) + return static_cast(std::floor(originCoord)); + else + return static_cast(std::round(originCoord)); + break; + } + case InterpolateNearestMode::round_prefer_ceil: { + return static_cast(std::round(originCoord)); + break; + } + case InterpolateNearestMode::floor: { + return static_cast(std::floor(originCoord)); + break; + } + case InterpolateNearestMode::ceil: { + return static_cast(std::ceil(originCoord)); + break; + } + case InterpolateNearestMode::simple: { + if (isDownsample) + return static_cast(std::ceil(originCoord)); + else + return static_cast(originCoord); + } + default: { + IE_THROW() << "errorPrefix" << " does not support specified nearest round mode"; + break; + } + } +} + +void ov::intel_cpu::InterpolateExecutor::linearOnnxCF(int outCoord, float scale, int inShape, int outShape, + int& index0, int& index1, float& weight0, float& weight1) { + float inCoord = coordTransToInput(outCoord, scale, inShape, outShape); + inCoord = std::max(0.0f, std::min(inCoord, static_cast(inShape - 1))); + index0 = std::min(static_cast(inCoord), inShape - 1); + index1 = std::min(index0 + 1, inShape - 1); + + weight1 = std::fabs(inCoord - index0); + weight0 = std::fabs(inCoord - index1); + if (index0 == index1) { + weight0 = 0.5f; + weight1 = 0.5f; + } +} + +void ov::intel_cpu::InterpolateExecutor::buildTblLinearOnnx(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, + const std::vector& dataScales, InterpolateLayoutType layout) { + int dimSize = dataRank; + float fz = (spatialDimSize > 2) ? dataScales[dimSize - 3] : 1.f; + float fy = (spatialDimSize > 1) ? dataScales[dimSize - 2] : 1.f; + float fx = dataScales[dimSize - 1]; + int ID = srcDimPad5d[2], IH = srcDimPad5d[3], IW = srcDimPad5d[4]; + int OD = dstDim5d[2], OH = dstDim5d[3], OW = dstDim5d[4]; + + std::vector indexPtr(MAX_INPUT_INTERPOLATE, 0); + std::vector weightPtr(MAX_INPUT_INTERPOLATE, 0); + if (layout == InterpolateLayoutType::planar) { + // FrontTopLeft:0, FrontTopRight:1, FrontBottomLeft:2, FrontBottomRight:3, + // EndTopLeft:4, EndTopRight:5, EndBottomLeft:6, EndBottomRight:7 + // weight: Left:0, ritht:1, top:2, bottom:3, front:4, end:5 + int eltInGrid = (spatialDimSize > 2) ? MAX_INPUT_INTERPOLATE : ((spatialDimSize > 1) ? 4 : 2); + int idxType = 2; + int scratchLen = rnd_up(eltInGrid * OW * OH * OD, 16); + indexTable.resize(idxType * scratchLen); + + indexPtr[0] = static_cast(&indexTable[0]); + indexPtr[1] = static_cast(&indexTable[OW * OH * OD]); + weightPtr[0] = reinterpret_cast(&indexTable[scratchLen]); + weightPtr[1] = reinterpret_cast(&indexTable[scratchLen + OW * OH * OD]); + if (spatialDimSize > 1) { + indexPtr[2] = static_cast(&indexTable[2 * OW * OH * OD]); + indexPtr[3] = static_cast(&indexTable[3 * OW * OH * OD]); + weightPtr[2] = reinterpret_cast(&indexTable[scratchLen + 2 * OW * OH * OD]); + weightPtr[3] = reinterpret_cast(&indexTable[scratchLen + 3 * OW * OH * OD]); + } + if (spatialDimSize > 2) { + indexPtr[4] = static_cast(&indexTable[4 * OW * OH * OD]); + indexPtr[5] = static_cast(&indexTable[5 * OW * OH * OD]); + indexPtr[6] = static_cast(&indexTable[6 * OW * OH * OD]); + indexPtr[7] = static_cast(&indexTable[7 * OW * OH * OD]); + weightPtr[4] = reinterpret_cast(&indexTable[scratchLen + 4 * OW * OH * OD]); + weightPtr[5] = reinterpret_cast(&indexTable[scratchLen + 5 * OW * OH * OD]); + } + int scale = dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::sse41) ? srcDataSize : 1; + + for (int oz = 0; oz < OD; oz++) { + int izF, izE; + float weightF, weightE; + linearOnnxCF(oz, fz, ID, OD, izF, izE, weightF, weightE); + int idxOz = oz * OH * OW; + for (int oy = 0; oy < OH; oy++) { + int iyT, iyB; + float weightT, weightB; + linearOnnxCF(oy, fy, IH, OH, iyT, iyB, weightT, weightB); + int idxOzOy = idxOz + oy * OW; + for (int ox = 0; ox < OW; ox++) { + int ixL, ixR; + float weightL, weightR; + linearOnnxCF(ox, fx, IW, OW, ixL, ixR, weightL, weightR); + + int idxOzOyOx = idxOzOy + ox; + indexPtr[0][idxOzOyOx] = (izF * IH * IW + iyT * IW + ixL) * scale; + indexPtr[1][idxOzOyOx] = (izF * IH * IW + iyT * IW + ixR) * scale; + weightPtr[0][idxOzOyOx] = weightL; + weightPtr[1][idxOzOyOx] = weightR; + if (spatialDimSize > 1) { + indexPtr[2][idxOzOyOx] = (izF * IH * IW + iyB * IW + ixL) * scale; + indexPtr[3][idxOzOyOx] = (izF * IH * IW + iyB * IW + ixR) * scale; + weightPtr[2][idxOzOyOx] = weightT; + weightPtr[3][idxOzOyOx] = weightB; + } + if (spatialDimSize > 2) { + indexPtr[4][idxOzOyOx] = (izE * IH * IW + iyT * IW + ixL) * scale; + indexPtr[5][idxOzOyOx] = (izE * IH * IW + iyT * IW + ixR) * scale; + indexPtr[6][idxOzOyOx] = (izE * IH * IW + iyB * IW + ixL) * scale; + indexPtr[7][idxOzOyOx] = (izE * IH * IW + iyB * IW + ixR) * scale; + weightPtr[4][idxOzOyOx] = weightF; + weightPtr[5][idxOzOyOx] = weightE; + } + } + } + } + } else { + // index: left:OW right:OW Top:OH Bottom:OH, Front:OD, End:OD + // weight:same as index + size_t scratchLen = rnd_up(OW + OW + OH + OH + OD + OD, 16); + int idxType = 2; + indexTable.resize(idxType * scratchLen); + indexPtr[0] = static_cast(&indexTable[0]); + indexPtr[1] = static_cast(&indexTable[OW]); + indexPtr[2] = static_cast(&indexTable[2 * OW]); + indexPtr[3] = static_cast(&indexTable[2 * OW + OH]); + indexPtr[4] = static_cast(&indexTable[2 * OW + 2 * OH]); + indexPtr[5] = static_cast(&indexTable[2 * OW + 2 * OH + OD]); + + weightPtr[0] = reinterpret_cast(&indexTable[scratchLen]); + weightPtr[1] = reinterpret_cast(&indexTable[scratchLen + OW]); + weightPtr[2] = reinterpret_cast(&indexTable[scratchLen + 2 * OW]); + weightPtr[3] = reinterpret_cast(&indexTable[scratchLen + 2 * OW + OH]); + weightPtr[4] = reinterpret_cast(&indexTable[scratchLen + 2 * OW + 2 * OH]); + weightPtr[5] = reinterpret_cast(&indexTable[scratchLen + 2 * OW + 2 * OH + OD]); + + for (int ox = 0; ox < OW; ox++) { + linearOnnxCF(ox, fx, IW, OW, indexPtr[0][ox], indexPtr[1][ox], weightPtr[0][ox], weightPtr[1][ox]); + } + for (int oy = 0; oy < OH; oy++) { + linearOnnxCF(oy, fy, IH, OH, indexPtr[2][oy], indexPtr[3][oy], weightPtr[2][oy], weightPtr[3][oy]); + } + for (int oz = 0; oz < OD; oz++) { + linearOnnxCF(oz, fz, ID, OD, indexPtr[4][oz], indexPtr[5][oz], weightPtr[4][oz], weightPtr[5][oz]); + } + } +} + +// table layout: +// wd .........wd, wh............wh, ww.............ww, id...........id, ih............ih, iw..............iw +// | | +// wh0.....wh_diameter ih0.....ih_diameter +void ov::intel_cpu::InterpolateExecutor::buildTblLinear(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, + const std::vector& dataScales, int kernel_width, bool antialias) { + int dimSize = dataRank; + float fz = (dimSize == 5) ? dataScales[dimSize - 3] : 1.f; + float fy = dataScales[dimSize - 2]; + float fx = dataScales[dimSize - 1]; + size_t ID = srcDimPad5d[2], IH = srcDimPad5d[3], IW = srcDimPad5d[4]; + size_t OD = dstDim5d[2], OH = dstDim5d[3], OW = dstDim5d[4]; + + if (!(IW == OW && IH == OH && ID == OD)) { + float ax = antialias ? fx : 1.0f; + float ay = antialias ? fy : 1.0f; + float az = antialias ? fz : 1.0f; + + int rx = (fx > 1.0f) ? 2 : static_cast(ceil(static_cast(kernel_width) / ax)); + int ry = (fy > 1.0f) ? 2 : static_cast(ceil(static_cast(kernel_width) / ay)); + int rz = (fz > 1.0f) ? 2 : static_cast(ceil(static_cast(kernel_width) / az)); + + int diaOD = 2 * rz + 1; + int diaOH = 2 * ry + 1; + int diaOW = 2 * rx + 1; + int sizeOD = OD * diaOD; + int sizeOH = OH * diaOH; + int sizeOW = OW * diaOW; + indexTable.resize((sizeOD + sizeOH + sizeOW) * 2); + float *weightTable = reinterpret_cast(&indexTable[0]); + float *weightOD = static_cast(&weightTable[0]); + float *weightOH = static_cast(&weightTable[sizeOD]); + float *weightOW = static_cast(&weightTable[sizeOD + sizeOH]); + + int *idxTable = static_cast(&indexTable[sizeOD + sizeOH + sizeOW]); + int *idxOD = static_cast(&idxTable[0]); + int *idxOH = static_cast(&idxTable[sizeOD]); + int *idxOW = static_cast(&idxTable[sizeOD + sizeOH]); + + for (int oz = 0; oz < OD; oz++) { + float iz = coordTransToInput(oz, fz, ID, OD); + int iz_r = static_cast(std::round(iz)); + for (int r = iz_r - rz, i = 0; r <= iz_r + rz; r++, i++) { + idxOD[oz * diaOD + i] = r; + if (r < 0 || r >= static_cast(ID)) { + weightOD[oz * diaOD + i] = 0.f; + } else { + float dz = iz - r; + weightOD[oz * diaOD + i] = az * triangleCoeff(az * dz); + } + } + } + for (int oy = 0; oy < OH; oy++) { + float iy = coordTransToInput(oy, fy, IH, OH); + int iy_r = static_cast(std::round(iy)); + for (int r = iy_r - ry, i = 0; r <= iy_r + ry; r++, i++) { + idxOH[oy * diaOH + i] = r; + if (r < 0 || r >= static_cast(IH)) { + weightOH[oy * diaOH + i] = 0.f; + } else { + float dy = iy - r; + weightOH[oy * diaOH + i] = ay * triangleCoeff(ay * dy); + } + } + } + for (int ox = 0; ox < OW; ox++) { + float ix = coordTransToInput(ox, fx, IW, OW); + int ix_r = static_cast(std::round(ix)); + for (int r = ix_r - rx, i = 0; r <= ix_r + rx; r++, i++) { + idxOW[ox * diaOW + i] = r; + if (r < 0 || r >= static_cast(IW)) { + weightOW[ox * diaOW + i] = 0.f; + } else { + float dx = ix - r; + weightOW[ox * diaOW + i] = ax * triangleCoeff(ax * dx); + } + } + } + } +} + +std::vector ov::intel_cpu::InterpolateExecutor::getCubicCoeffs(float mantissa, float a) { + float m = std::fabs(mantissa); + std::vector coeffs(4, 0.f); + + coeffs[0] = a * (m - 1.0) * (m - 1.0) * m; + coeffs[1] = ((a + 2.0) * m - (a + 3.0)) * m * m + 1.0; + coeffs[2] = (((-a - 2.0) * m + (2.0 * a + 3.0)) * m - a) * m; + coeffs[3] = -a * m * m * (m - 1.0); + return coeffs; +} + +// table layout: +// OW OW OW OW OW OH OH OH OH OH +// x_idx x_weight0 x_weight1 x_weight2 x_weight3 y_idx y_weight0 y_weight1 y_weight2 y_weight3 +void ov::intel_cpu::InterpolateExecutor::buildTblCubic(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, const std::vector& dataScales, + float cubicCoeff, InterpolateLayoutType layout) { + int dimSize = dataRank; + float fy = dataScales[dimSize - 2]; + float fx = dataScales[dimSize - 1]; + int IH = srcDimPad5d[3], IW = srcDimPad5d[4]; + int OH = dstDim5d[3], OW = dstDim5d[4]; + + // idxNum for index, CUBIC_GRID_LEN for weight + const int idxNum = 1; + size_t idxWeightSize = (CUBIC_GRID_LEN + idxNum) * OW + (CUBIC_GRID_LEN + idxNum) * OH; + if (layout != InterpolateLayoutType::planar) { + indexTable.resize(idxWeightSize); + } else { + size_t sequenceSize = 2 * OH * OW; + indexTable.resize(idxWeightSize + sequenceSize); + } + + int tblAdvance = 0; + int *xOrigin = static_cast(&indexTable[tblAdvance]); + tblAdvance += OW; + float *xFactor = reinterpret_cast(&indexTable[tblAdvance]); + for (int ox = 0; ox < OW; ox++) { + float ix = coordTransToInput(ox, fx, IW, OW); + int ix_r = static_cast(std::floor(ix)); + xOrigin[ox] = ix_r; + float m = ix - ix_r; + std::vector coffes = getCubicCoeffs(m, cubicCoeff); + xFactor[CUBIC_GRID_LEN * ox] = coffes[0]; + xFactor[CUBIC_GRID_LEN * ox + 1] = coffes[1]; + xFactor[CUBIC_GRID_LEN * ox + 2] = coffes[2]; + xFactor[CUBIC_GRID_LEN * ox + 3] = coffes[3]; + } + + tblAdvance += CUBIC_GRID_LEN * OW; + int *yOrigin = static_cast(&indexTable[tblAdvance]); + tblAdvance += OH; + float *yFactor = reinterpret_cast(&indexTable[tblAdvance]); + for (int oy = 0; oy < OH; oy++) { + float iy = coordTransToInput(oy, fy, IH, OH); + int iy_r = static_cast(std::floor(iy)); + yOrigin[oy] = iy_r; + float m = iy - iy_r; + std::vector coffes = getCubicCoeffs(m, cubicCoeff); + yFactor[CUBIC_GRID_LEN * oy] = coffes[0]; + yFactor[CUBIC_GRID_LEN * oy + 1] = coffes[1]; + yFactor[CUBIC_GRID_LEN * oy + 2] = coffes[2]; + yFactor[CUBIC_GRID_LEN * oy + 3] = coffes[3]; + } + + if (layout == InterpolateLayoutType::planar) { + tblAdvance += CUBIC_GRID_LEN * OH; + int *sequenceOH = static_cast(&indexTable[tblAdvance]); + tblAdvance += OH * OW; + int *sequenceOW = static_cast(&indexTable[tblAdvance]); + for (int h = 0; h < OH; ++h) { + int offset = h * OW; + for (int w = 0; w < OW; ++w) { + sequenceOH[offset + w] = h * sizeof(int); + sequenceOW[offset + w] = w * sizeof(int); + } + } + } +} + +// shapeND: n c d h w +// blockND: ncdhw cdhw dhw hw w 1 +// index : 0 1 2 3 4 5 +inline SizeVector getBlockND(const SizeVector& shape) { + int shapeRank = shape.size(); + SizeVector blockND(shapeRank + 1, 1); + for (int i = shapeRank - 1; i >= 0; i--) { + blockND[i] = shape[i] * blockND[i+1]; + } + return blockND; +} + +const uint8_t* ov::intel_cpu::InterpolateExecutor::padPreprocess(const std::vector& src, const std::vector& dst) { + const uint8_t *src_data_origin = reinterpret_cast(src[0]->GetData()); + + const auto &srcDim = src[0]->getStaticDims(); + const auto &dstDim = dst[0]->getStaticDims(); + size_t dimSize = srcDim.size(); + auto srcDimPad = getSrcDimPad5d(); + + const auto srcDim5d = to5Dim(srcDim); + const auto srcDimPad5d = to5Dim(srcDimPad); + const auto dstDim5d = to5Dim(dstDim); + const auto srcDataSize = src[0]->getDesc().getPrecision().size(); + + const uint8_t *src_data = nullptr; + std::vector srcPadded; + if (interpAttrs.hasPad) { + int padB0 = (dimSize > 2) ? interpAttrs.padBegin[0] : 0; + int padB1 = (dimSize > 2) ? interpAttrs.padBegin[1] : 0; + int padB2 = (dimSize == 5) ? interpAttrs.padBegin[dimSize - 3] : 0; + int padB3 = interpAttrs.padBegin[dimSize - 2]; + int padB4 = interpAttrs.padBegin[dimSize - 1]; + + SizeVector inShapeBlock = getBlockND(srcDim5d); + SizeVector inShapePadBlock = getBlockND(srcDimPad5d); + + if (interpAttrs.layout == InterpolateLayoutType::planar) { + srcPadded.resize(inShapePadBlock[0] * srcDataSize, 0); + uint8_t *src_data_pad = static_cast(&srcPadded[0]); + parallel_for4d(srcDim5d[0], srcDim5d[1], srcDim5d[2], srcDim5d[3], [&](int n, int c, int d, int h) { + const uint8_t *src = src_data_origin + (inShapeBlock[1] * n + inShapeBlock[2] * c + inShapeBlock[3] * d + inShapeBlock[4] * h) * srcDataSize; + uint8_t *srcPad = src_data_pad + (inShapePadBlock[1] * (n + padB0) + inShapePadBlock[2] * (c + padB1) + + inShapePadBlock[3] * (d + padB2) + inShapePadBlock[4] * (h + padB3) + padB4) * srcDataSize; + cpu_memcpy(srcPad, src, srcDim5d[4] * srcDataSize); + }); + src_data = src_data_pad; + } else if (interpAttrs.layout == InterpolateLayoutType::by_channel) { + srcPadded.resize(inShapePadBlock[0] * srcDataSize, 0); + uint8_t *src_data_pad = static_cast(&srcPadded[0]); + parallel_for4d(srcDim5d[0], srcDim5d[2], srcDim5d[3], srcDim5d[4], [&](int n, int d, int h, int w) { + const uint8_t *src = src_data_origin + (inShapeBlock[1] * n + + (inShapeBlock[3] * d + inShapeBlock[4] * h + inShapeBlock[5] * w) * srcDim5d[1]) * srcDataSize; + uint8_t *srcPad = src_data_pad + (inShapePadBlock[1] * (n + padB0) + (inShapePadBlock[3] * (d + padB2) + + inShapePadBlock[4] * (h + padB3) + + inShapePadBlock[5] * (w + padB4)) * srcDimPad5d[1] + padB1) * srcDataSize; + cpu_memcpy(srcPad, src, srcDim5d[1] * srcDataSize); + }); + src_data = src_data_pad; + } else if (interpAttrs.layout == InterpolateLayoutType::block) { + size_t blkSize = dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core) ? 16 : 8; + size_t CB = div_up(srcDimPad5d[1], blkSize); + size_t eltsTotal = srcDimPad5d[0] * CB * srcDimPad5d[2] * srcDimPad5d[3] * srcDimPad5d[4] * blkSize; + srcPadded.resize(eltsTotal * srcDataSize, 0x0); + uint8_t *src_data_pad = static_cast(&srcPadded[0]); + if ((srcDim5d[0] != srcDimPad5d[0]) || (srcDim5d[1] != srcDimPad5d[1])) { + IE_THROW() << "Interpolate layer with name does not support padding on batch and channel dimensions"; + } + parallel_for5d(srcDim5d[0], CB, srcDim5d[2], srcDim5d[3], srcDim5d[4], [&](int n, int cb, int d, int h, int w) { + const uint8_t *src = src_data_origin + (n * CB * srcDim5d[2] * srcDim5d[3] * srcDim5d[4] * blkSize) * srcDataSize + + (cb * srcDim5d[2] * srcDim5d[3] * srcDim5d[4] * blkSize) * srcDataSize + + (d * srcDim5d[3] * srcDim5d[4] * blkSize) * srcDataSize + + (h * srcDim5d[4] * blkSize) * srcDataSize + + (w * blkSize) * srcDataSize; + uint8_t *srcPad = src_data_pad + (n * CB * srcDimPad5d[2] * srcDimPad5d[3] * srcDimPad5d[4] * blkSize) * srcDataSize + + (cb * srcDimPad5d[2] * srcDimPad5d[3] * srcDimPad5d[4] * blkSize) * srcDataSize + + ((d + padB2) * srcDimPad5d[3] * srcDimPad5d[4] * blkSize) * srcDataSize + + ((h + padB3) * srcDimPad5d[4] * blkSize) * srcDataSize + + ((w + padB4) * blkSize) * srcDataSize; + cpu_memcpy(srcPad, src, blkSize * srcDataSize); + }); + src_data = src_data_pad; + } + } else { + src_data = src_data_origin; + } + return src_data; +} diff --git a/src/plugins/intel_cpu/src/nodes/executors/interpolate.hpp b/src/plugins/intel_cpu/src/nodes/executors/interpolate.hpp new file mode 100644 index 00000000000..214f67460c5 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/interpolate.hpp @@ -0,0 +1,187 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include + +#define MAX_INPUT_INTERPOLATE 8 + +using namespace InferenceEngine; + +namespace ov { +namespace intel_cpu { + +enum InterpolateLayoutType { + planar, + block, + by_channel +}; + +enum InterpolateMode { + nearest, + linear, + linear_onnx, + cubic +}; + +enum InterpolateCoordTransMode { + half_pixel, + pytorch_half_pixel, + asymmetric, + tf_half_pixel_for_nn, + align_corners +}; + +enum class InterpolateNearestMode { + round_prefer_floor, + round_prefer_ceil, + floor, + ceil, + simple +}; + +enum class InterpolateShapeCalcMode { + sizes, + scales +}; + +struct InterpolateAttrs { + InterpolateMode mode = InterpolateMode::nearest; + InterpolateCoordTransMode coordTransMode = InterpolateCoordTransMode::half_pixel; + InterpolateNearestMode nearestMode = InterpolateNearestMode::round_prefer_floor; + bool antialias = false; + float cubeCoeff = -0.75; + std::vector padBegin; + std::vector padEnd; + InferenceEngine::Precision inPrc; + InferenceEngine::Precision outPrc; + InterpolateLayoutType layout; + std::vector dataScales; + bool hasPad = false; +}; + +inline SizeVector getPaddedInputShape(const VectorDims &srcDims, + const std::vector &padBegin, + const std::vector &padEnd) { + SizeVector paddedShape; + int dataRank = srcDims.size(); + for (int i = 0; i < dataRank; i++) { + paddedShape.push_back(srcDims[i] + padBegin[i] + padEnd[i]); + } + return paddedShape; +} + +inline int clipCoord(int pos, int length) { + return std::max(static_cast(0), std::min(pos, length - 1)); +} + +inline size_t getSpatialDimsNum(const Dim rank) { + switch (rank) { + case 1: + case 3: + return 1; + case 2: + case 4: + return 2; + case 5: + return 3; + default: + IE_THROW() << "Can't define number spatial"; + } +} + +// w/hw/ncw/nchw/ncdhw to ncdhw +inline SizeVector to5Dim(SizeVector casesDim) { + size_t caseSize = casesDim.size(); + SizeVector dim5(5, 1lu); + dim5[4] = casesDim[caseSize - 1]; + if (caseSize > 1) { + dim5[3] = casesDim[caseSize - 2]; + } + if (caseSize > 2) { + dim5[0] = casesDim[0]; + } + if (caseSize > 3) { + dim5[1] = casesDim[1]; + } + if (caseSize > 4) { + dim5[2] = casesDim[2]; + } + if (caseSize == 3) { // nhw -> ncw + dim5[1] = dim5[3]; + dim5[3] = 1lu; + } + return dim5; +} + +static inline float triangleCoeff(float x) { + return (std::max)(0.0f, 1 - std::abs(x)); +} + +class InterpolateExecutor { +public: + static constexpr size_t DATA_ID = 0; + static constexpr size_t TARGET_SHAPE_ID = 1; + static constexpr size_t SCALES_ID = 2; + static constexpr size_t AXES_ID = 3; + static constexpr int CUBIC_GRID_LEN = 4; + InterpolateExecutor(const ExecutorContext::CPtr context) : _context(context) {} + + virtual bool init(const InterpolateAttrs& interpolateAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr); + virtual void exec(const std::vector& src, const std::vector& dst, const void *post_ops_data_) = 0; + virtual impl_desc_type getImplType() const = 0; + + virtual ~InterpolateExecutor() = default; + VectorDims getSrcDimPad5d() const { return srcDimPad5d; } + const uint8_t* padPreprocess(const std::vector& src, const std::vector& dst); + +private: + void buildTblNN(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, const std::vector& dataScales, + InterpolateLayoutType layout, InterpolateNearestMode nearestMode); + void buildTblLinearOnnx(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, const std::vector& dataScales, + InterpolateLayoutType layout); + void buildTblLinear(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, const std::vector& dataScales, int kernel_width, + bool antialias); + void buildTblCubic(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, const std::vector& dataScales, float cubicCoeff, + InterpolateLayoutType layout); + + float coordTransToInput(int outCoord, float scale, int inShape, int outShape) const; + int nearestRound(float origin, bool isDownsample, InterpolateNearestMode nearestMode) const; + void linearOnnxCF(int outCoord, float scale, int inShape, int outShape, int& index0, int& index1, float& weight0, float& weight1); + std::vector getCubicCoeffs(float mantissa, float a); + +protected: + InterpolateAttrs interpAttrs; + VectorDims srcDimPad5d, dstDim5d; + size_t srcDataSize, dstDataSize; + int spatialDimSize; + size_t dataRank; + std::vector indexTable; + const ExecutorContext::CPtr _context; +}; + +using InterpolateExecutorPtr = std::shared_ptr; +using InterpolateExecutorCPtr = std::shared_ptr; + +class InterpolateExecutorBuilder { +public: + ~InterpolateExecutorBuilder() = default; + virtual bool isSupported(const InterpolateAttrs& InterpolateAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs) const = 0; + virtual InterpolateExecutorPtr makeExecutor(const ExecutorContext::CPtr context) const = 0; +}; + +using InterpolateExecutorBuilderPtr = std::shared_ptr; +using InterpolateExecutorBuilderCPtr = std::shared_ptr; +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/interpolate_list.cpp b/src/plugins/intel_cpu/src/nodes/executors/interpolate_list.cpp new file mode 100644 index 00000000000..2362b644583 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/interpolate_list.cpp @@ -0,0 +1,19 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "interpolate_list.hpp" + +namespace ov { +namespace intel_cpu { + +const std::vector& getInterpolateExecutorsList() { + static std::vector descs = { + OV_CPU_INSTANCE_ACL(ExecutorType::Acl, std::make_shared()) + }; + + return descs; +} + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/interpolate_list.hpp b/src/plugins/intel_cpu/src/nodes/executors/interpolate_list.hpp new file mode 100644 index 00000000000..179c09c799b --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/interpolate_list.hpp @@ -0,0 +1,85 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "executor.hpp" + +#include "interpolate.hpp" +#if defined(OV_CPU_WITH_ACL) +#include "acl/acl_interpolate.hpp" +#endif + +#include "onednn/iml_type_mapper.h" +#include "common/primitive_cache.hpp" + +namespace ov { +namespace intel_cpu { + +struct InterpolateExecutorDesc { + ExecutorType executorType; + InterpolateExecutorBuilderCPtr builder; +}; + +const std::vector& getInterpolateExecutorsList(); + +class InterpolateExecutorFactory : public ExecutorFactory { +public: + InterpolateExecutorFactory(const InterpolateAttrs& InterpolateAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const ExecutorContext::CPtr context) : ExecutorFactory(context) { + for (auto& desc : getInterpolateExecutorsList()) { + if (desc.builder->isSupported(InterpolateAttrs, srcDescs, dstDescs)) { + supportedDescs.push_back(desc); + } + } + } + + ~InterpolateExecutorFactory() = default; + virtual InterpolateExecutorPtr makeExecutor(const InterpolateAttrs& interpolateAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) { + auto build = [&](const InterpolateExecutorDesc* desc) { + auto executor = desc->builder->makeExecutor(context); + if (executor->init(interpolateAttrs, srcDescs, dstDescs, attr)) { + return executor; + } + + InterpolateExecutorPtr ptr = nullptr; + return ptr; + }; + + + if (chosenDesc) { + if (auto executor = build(chosenDesc)) { + return executor; + } + } + + for (const auto& sd : supportedDescs) { + if (auto executor = build(&sd)) { + chosenDesc = &sd; + return executor; + } + } + + IE_THROW() << "Supported Interpolate executor is not found"; + } + + bool isEmpty() { + return supportedDescs.empty(); + } + +private: + std::vector supportedDescs; + const InterpolateExecutorDesc* chosenDesc = nullptr; +}; + +using InterpolateExecutorFactoryPtr = std::shared_ptr; +using InterpolateExecutorFactoryCPtr = std::shared_ptr; + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/mvn.cpp b/src/plugins/intel_cpu/src/nodes/executors/mvn.cpp new file mode 100644 index 00000000000..5fc8c5d87b5 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/mvn.cpp @@ -0,0 +1,38 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "mvn.hpp" + +namespace ov { +namespace intel_cpu { + +using namespace InferenceEngine; + +MVNExecutor::MVNExecutor(const ExecutorContext::CPtr context) : context(context) {} + +SizeVector MVNExecutor::transformTo5DCase(const SizeVector& shape, bool initAcrossChannels) { + switch (shape.size()) { + // for 1 and 2 rank, if initAcrossChannels_ is true, adjust shape to fully vectorize under unified 5d procedure. + // otherwise there are not enough data in spatial dimension to process in one kernel. + case 1 : // C + if (initAcrossChannels) { + return SizeVector({1, 1, 1, 1, shape[0]}); + } else { + return SizeVector({1, shape[0], 1, 1, 1}); + } + case 2 : // NC + if (initAcrossChannels) { + return SizeVector({1, shape[0], 1, shape[1], 1}); + } else { + return SizeVector({shape[0], shape[1], 1, 1, 1}); + } + case 3 : { return SizeVector({shape[0], shape[1], 1, shape[2], 1}); } + case 4 : { return SizeVector({shape[0], shape[1], 1, shape[2], shape[3]}); } + case 5 : { return SizeVector({shape[0], shape[1], shape[2], shape[3], shape[4]}); } + default : { IE_THROW() << "MVN executor doesn't support planar layout with rank: " << shape.size(); } + } +} + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/mvn.hpp b/src/plugins/intel_cpu/src/nodes/executors/mvn.hpp new file mode 100644 index 00000000000..aba987ee190 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/mvn.hpp @@ -0,0 +1,72 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "cpu_memory.h" +#include "onednn/iml_type_mapper.h" +#include "executor.hpp" + +namespace ov { +namespace intel_cpu { + +enum MVNLayoutType { + mvn_planar, + mvn_block, + mvn_by_channel +}; + +// Defines way to add epsilon: inside sqrt or outside. +enum MVNEpsMode { + INSIDE_SQRT, + OUTSIDE_SQRT +}; + +struct MVNAttrs { + MVNLayoutType layout; + std::tuple shape5D; + bool initAcrossChannels_; + bool execAcrossChannels_; + bool normalizeVariance_; + float epsValue_; + MVNEpsMode epsMode_; + InferenceEngine::Precision src_prc; + InferenceEngine::Precision dst_prc; +}; + +class MVNExecutor { +public: + MVNExecutor(const ExecutorContext::CPtr context); + virtual bool init(const MVNAttrs& mvnAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) = 0; + + virtual void exec(const std::vector& src, const std::vector& dst, const void *post_ops_data_) = 0; + virtual ~MVNExecutor() = default; + + virtual impl_desc_type getImplType() const = 0; + + static InferenceEngine::SizeVector transformTo5DCase(const InferenceEngine::SizeVector& shape, bool initAcrossChannels); + +protected: + MVNAttrs mvnAttrs; + const ExecutorContext::CPtr context; +}; + +using MVNExecutorPtr = std::shared_ptr; +using MVNExecutorCPtr = std::shared_ptr; + +class MVNExecutorBuilder { +public: + ~MVNExecutorBuilder() = default; + virtual bool isSupported(const MVNAttrs& mvnAttrs, const std::vector& srcDescs, const std::vector& dstDescs) const = 0; + virtual MVNExecutorPtr makeExecutor(const ExecutorContext::CPtr context) const = 0; +}; + +using MVNExecutorBuilderPtr = std::shared_ptr; +using MVNExecutorBuilderCPtr = std::shared_ptr; + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/mvn_list.cpp b/src/plugins/intel_cpu/src/nodes/executors/mvn_list.cpp new file mode 100644 index 00000000000..c27751b7a2d --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/mvn_list.cpp @@ -0,0 +1,19 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "mvn_list.hpp" + +namespace ov { +namespace intel_cpu { + +const std::vector& getMVNExecutorsList() { + static std::vector descs = { + OV_CPU_INSTANCE_ACL(ExecutorType::Acl, std::make_shared()) + }; + + return descs; +} + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/mvn_list.hpp b/src/plugins/intel_cpu/src/nodes/executors/mvn_list.hpp new file mode 100644 index 00000000000..bb12beab800 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/mvn_list.hpp @@ -0,0 +1,84 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "executor.hpp" + +#include "mvn.hpp" +#if defined(OV_CPU_WITH_ACL) +#include "acl/acl_mvn.hpp" +#endif + +#include "onednn/iml_type_mapper.h" +#include "common/primitive_cache.hpp" + +namespace ov { +namespace intel_cpu { + +struct MVNExecutorDesc { + ExecutorType executorType; + MVNExecutorBuilderCPtr builder; +}; + +const std::vector& getMVNExecutorsList(); + +class MVNExecutorFactory : public ExecutorFactory { +public: + MVNExecutorFactory(const MVNAttrs& mvnAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const ExecutorContext::CPtr context) : ExecutorFactory(context) { + for (auto& desc : getMVNExecutorsList()) { + if (desc.builder->isSupported(mvnAttrs, srcDescs, dstDescs)) { + supportedDescs.push_back(desc); + } + } + } + + ~MVNExecutorFactory() = default; + virtual MVNExecutorPtr makeExecutor(const MVNAttrs& mvnAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) { + auto build = [&](const MVNExecutorDesc* desc) { + auto executor = desc->builder->makeExecutor(context); + if (executor->init(mvnAttrs, srcDescs, dstDescs, attr)) { + return executor; + } + + MVNExecutorPtr ptr = nullptr; + return ptr; + }; + + if (chosenDesc) { + if (auto executor = build(chosenDesc)) { + return executor; + } + } + + for (const auto& sd : supportedDescs) { + if (auto executor = build(&sd)) { + chosenDesc = &sd; + return executor; + } + } + + IE_THROW() << "Supported MVN executor is not found"; + } + + bool isEmpty() { + return supportedDescs.empty(); + } + +private: + std::vector supportedDescs; + const MVNExecutorDesc* chosenDesc = nullptr; +}; + +using MVNExecutorFactoryPtr = std::shared_ptr; +using MVNExecutorFactoryCPtr = std::shared_ptr; + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/pooling.cpp b/src/plugins/intel_cpu/src/nodes/executors/pooling.cpp new file mode 100644 index 00000000000..23acf632d0f --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/pooling.cpp @@ -0,0 +1,15 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "pooling.hpp" + +namespace ov { +namespace intel_cpu { + +using namespace InferenceEngine; + +PoolingExecutor::PoolingExecutor(const ExecutorContext::CPtr context) : context(context) {} + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/pooling.hpp b/src/plugins/intel_cpu/src/nodes/executors/pooling.hpp new file mode 100644 index 00000000000..5ea358c68af --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/pooling.hpp @@ -0,0 +1,75 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "cpu_memory.h" +#include "onednn/iml_type_mapper.h" +#include "executor.hpp" + +namespace ov { +namespace intel_cpu { + +struct PoolingAttrs { + bool exclude_pad = false; + bool auto_pad = false; + + op::PadType pad_type; + Algorithm algorithm; + + op::RoundingType rounding; + + std::vector stride; + std::vector kernel; + std::vector dilation; + + std::vector data_pad_begin; + std::vector data_pad_end; + + /// Effective padding. Used to define correct output shape by oneDNN + /// reshape formula: (iw - kernel + pad_l + pad_r) / strides[i - 2] + 1 + /// should be passed into pooling desc constructor. + std::vector effective_pad_begin; + std::vector effective_pad_end; + + /// Effective dilation. Used to define correct dilation for OneDNN. + /// For OneDNN default dilation is vector of zero + std::vector effective_dilation; +}; + +class PoolingExecutor { +public: + PoolingExecutor(const ExecutorContext::CPtr context); + virtual bool init(const PoolingAttrs& poolingAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) = 0; + + virtual void exec(const std::vector& src, const std::vector& dst, std::unordered_map postOpsArgs) = 0; + virtual ~PoolingExecutor() = default; + + virtual impl_desc_type getImplType() const = 0; + +protected: + PoolingAttrs poolingAttrs; + const ExecutorContext::CPtr context; +}; + +using PoolingExecutorPtr = std::shared_ptr; +using PoolingExecutorCPtr = std::shared_ptr; + +class PoolingExecutorBuilder { +public: + ~PoolingExecutorBuilder() = default; + virtual bool isSupported(const PoolingAttrs& poolingAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs) const = 0; + virtual PoolingExecutorPtr makeExecutor(const ExecutorContext::CPtr context) const = 0; +}; + +using PoolingExecutorBuilderPtr = std::shared_ptr; +using PoolingExecutorBuilderCPtr = std::shared_ptr; + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/pooling_list.cpp b/src/plugins/intel_cpu/src/nodes/executors/pooling_list.cpp new file mode 100644 index 00000000000..4b130f37bff --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/pooling_list.cpp @@ -0,0 +1,19 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "pooling_list.hpp" + +namespace ov { +namespace intel_cpu { + +const std::vector& getPoolingExecutorsList() { + static std::vector descs = { + OV_CPU_INSTANCE_ACL(ExecutorType::Acl, std::make_shared()) + }; + + return descs; +} + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/pooling_list.hpp b/src/plugins/intel_cpu/src/nodes/executors/pooling_list.hpp new file mode 100644 index 00000000000..2bd2400e71d --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/pooling_list.hpp @@ -0,0 +1,78 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "executor.hpp" + +#include "pooling.hpp" +#if defined(OV_CPU_WITH_ACL) +#include "acl/acl_pooling.hpp" +#endif + +namespace ov { +namespace intel_cpu { + +struct PoolingExecutorDesc { + ExecutorType executorType; + PoolingExecutorBuilderCPtr builder; +}; + +const std::vector& getPoolingExecutorsList(); + +class PoolingExecutorFactory : public ExecutorFactory { +public: + PoolingExecutorFactory(const PoolingAttrs& poolingAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const ExecutorContext::CPtr context) : ExecutorFactory(context) { + for (auto& desc : getPoolingExecutorsList()) { + if (desc.builder->isSupported(poolingAttrs, srcDescs, dstDescs)) { + supportedDescs.push_back(desc); + } + } + } + + ~PoolingExecutorFactory() = default; + virtual PoolingExecutorPtr makeExecutor(const PoolingAttrs& poolingAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) { + auto build = [&](const PoolingExecutorDesc* desc) { + auto executor = desc->builder->makeExecutor(context); + if (executor->init(poolingAttrs, srcDescs, dstDescs, attr)) { + return executor; + } + + PoolingExecutorPtr ptr = nullptr; + return ptr; + }; + + + if (chosenDesc) { + if (auto executor = build(chosenDesc)) { + return executor; + } + } + + for (const auto& sd : supportedDescs) { + if (auto executor = build(&sd)) { + chosenDesc = &sd; + return executor; + } + } + + IE_THROW() << "Supported Pooling executor is not found"; + } + +private: + std::vector supportedDescs; + const PoolingExecutorDesc* chosenDesc = nullptr; +}; + +using PoolingExecutorFactoryPtr = std::shared_ptr; +using PoolingExecutorFactoryCPtr = std::shared_ptr; + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/reduce.cpp b/src/plugins/intel_cpu/src/nodes/executors/reduce.cpp new file mode 100644 index 00000000000..a7906b6d20b --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/reduce.cpp @@ -0,0 +1,15 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "reduce.hpp" + +namespace ov { +namespace intel_cpu { + +using namespace InferenceEngine; + +ReduceExecutor::ReduceExecutor(const ExecutorContext::CPtr context) : context(context) {} + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/reduce.hpp b/src/plugins/intel_cpu/src/nodes/executors/reduce.hpp new file mode 100644 index 00000000000..8aa6e8f0aaa --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/reduce.hpp @@ -0,0 +1,55 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "cpu_memory.h" +#include "onednn/iml_type_mapper.h" +#include "dnnl_scratch_pad.h" +#include "executor.hpp" + +namespace ov { +namespace intel_cpu { + +struct ReduceAttrs { + std::vector axes; + Algorithm operation; + bool keepDims; +}; + +class ReduceExecutor { +public: + ReduceExecutor(const ExecutorContext::CPtr context); + virtual bool init(const ReduceAttrs& reduceAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) = 0; + + virtual void exec(const std::vector& src, const std::vector& dst, const void *post_ops_data_) = 0; + virtual ~ReduceExecutor() = default; + + virtual impl_desc_type getImplType() const = 0; + +protected: + ReduceAttrs reduceAttrs; + const ExecutorContext::CPtr context; +}; + +using ReduceExecutorPtr = std::shared_ptr; +using ReduceExecutorCPtr = std::shared_ptr; + +class ReduceExecutorBuilder { +public: + ~ReduceExecutorBuilder() = default; + virtual bool isSupported(const ReduceAttrs& reduceAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs) const = 0; + virtual ReduceExecutorPtr makeExecutor(const ExecutorContext::CPtr context) const = 0; +}; + +using ReduceExecutorBuilderPtr = std::shared_ptr; +using ReduceExecutorBuilderCPtr = std::shared_ptr; + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/reduce_list.cpp b/src/plugins/intel_cpu/src/nodes/executors/reduce_list.cpp new file mode 100644 index 00000000000..aec5c7eb905 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/reduce_list.cpp @@ -0,0 +1,19 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "reduce_list.hpp" + +namespace ov { +namespace intel_cpu { + +const std::vector& getReduceExecutorsList() { + static std::vector descs = { + OV_CPU_INSTANCE_ACL(ExecutorType::Acl, std::make_shared()) + }; + + return descs; +} + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/executors/reduce_list.hpp b/src/plugins/intel_cpu/src/nodes/executors/reduce_list.hpp new file mode 100644 index 00000000000..be2b7aa5f59 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/executors/reduce_list.hpp @@ -0,0 +1,85 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "executor.hpp" + +#include "reduce.hpp" +#if defined(OV_CPU_WITH_ACL) +#include "acl/acl_reduce.hpp" +#endif + +#include "onednn/iml_type_mapper.h" +#include "common/primitive_cache.hpp" + +namespace ov { +namespace intel_cpu { + +struct ReduceExecutorDesc { + ExecutorType executorType; + ReduceExecutorBuilderCPtr builder; +}; + +const std::vector& getReduceExecutorsList(); + +class ReduceExecutorFactory : public ExecutorFactory { +public: + ReduceExecutorFactory(const ReduceAttrs& reduceAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const ExecutorContext::CPtr context) : ExecutorFactory(context) { + for (auto& desc : getReduceExecutorsList()) { + if (desc.builder->isSupported(reduceAttrs, srcDescs, dstDescs)) { + supportedDescs.push_back(desc); + } + } + } + + ~ReduceExecutorFactory() = default; + virtual ReduceExecutorPtr makeExecutor(const ReduceAttrs& reduceAttrs, + const std::vector& srcDescs, + const std::vector& dstDescs, + const dnnl::primitive_attr &attr) { + auto build = [&](const ReduceExecutorDesc* desc) { + auto executor = desc->builder->makeExecutor(context); + if (executor->init(reduceAttrs, srcDescs, dstDescs, attr)) { + return executor; + } + + ReduceExecutorPtr ptr = nullptr; + return ptr; + }; + + + if (chosenDesc) { + if (auto executor = build(chosenDesc)) { + return executor; + } + } + + for (const auto& sd : supportedDescs) { + if (auto executor = build(&sd)) { + chosenDesc = &sd; + return executor; + } + } + + IE_THROW() << "Supported Reduce executor is not found"; + } + + bool isEmpty() { + return supportedDescs.empty(); + } + +private: + std::vector supportedDescs; + const ReduceExecutorDesc* chosenDesc = nullptr; +}; + +using ReduceExecutorFactoryPtr = std::shared_ptr; +using ReduceExecutorFactoryCPtr = std::shared_ptr; + +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/extract_image_patches.cpp b/src/plugins/intel_cpu/src/nodes/extract_image_patches.cpp index 27732dea2d3..f5aedf06d9e 100644 --- a/src/plugins/intel_cpu/src/nodes/extract_image_patches.cpp +++ b/src/plugins/intel_cpu/src/nodes/extract_image_patches.cpp @@ -25,7 +25,7 @@ using namespace Xbyak; namespace ov { namespace intel_cpu { namespace node { - +#if defined(OPENVINO_ARCH_X86_64) #define GET_OFF(field) offsetof(jit_extract_image_patches_args, field) template @@ -270,6 +270,7 @@ private: dd(i * jpp.SW * jpp.dtype_size); } }; +#endif // OPENVINO_ARCH_X86_64 bool ExtractImagePatches::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { try { @@ -481,6 +482,7 @@ void ExtractImagePatches::ExtractImagePatchesRefExecutor::executeReference( void ExtractImagePatches::ExtractImagePatchesJitExecutor::executeOptimizedGeneric( void* src, void* dst, const VectorDims& istrides, const VectorDims& ostrides) const { +#if defined(OPENVINO_ARCH_X86_64) const char* src_data = reinterpret_cast(src); char* dst_data = reinterpret_cast(dst); const auto& jpp = pKernel->jpp; @@ -507,6 +509,7 @@ void ExtractImagePatches::ExtractImagePatchesJitExecutor::executeOptimizedGeneri args.w_hi_pad = iw_hpad; (*pKernel)(&args); }); +#endif // OPENVINO_ARCH_X86_64 } jit_extract_image_patches_params ExtractImagePatches::ExtractImagePatchesExecutor::fillJpp( @@ -585,6 +588,7 @@ ExtractImagePatches::ExtractImagePatchesJitExecutor::ExtractImagePatchesJitExecu const VectorDims& rates, const ExtImgPatcherPadType& padType, const size_t prcSize) { +#if defined(OPENVINO_ARCH_X86_64) auto jpp = fillJpp(inDims, outDims, kSizes, strides, rates, padType, prcSize); if (mayiuse(x64::avx512_core)) { pKernel.reset(new jit_extract_image_patches_kernel(jpp)); @@ -598,6 +602,7 @@ ExtractImagePatches::ExtractImagePatchesJitExecutor::ExtractImagePatchesJitExecu if (pKernel) pKernel->create_ker(); +#endif // OPENVINO_ARCH_X86_64 } void ExtractImagePatches::ExtractImagePatchesJitExecutor::exec( diff --git a/src/plugins/intel_cpu/src/nodes/fake_quantize.cpp b/src/plugins/intel_cpu/src/nodes/fake_quantize.cpp index 06203699ba2..a8b23feb59b 100644 --- a/src/plugins/intel_cpu/src/nodes/fake_quantize.cpp +++ b/src/plugins/intel_cpu/src/nodes/fake_quantize.cpp @@ -45,7 +45,7 @@ using namespace Xbyak; namespace ov { namespace intel_cpu { namespace node { - +#if defined(OPENVINO_ARCH_X86_64) #define GET_OFF(field) offsetof(jit_quantize_call_args, field) template @@ -228,7 +228,7 @@ struct jit_uni_quantization_kernel : public jit_uni_quantize_kernel, public jit_ }; void generate() override { - do_dequantization = jqp_.op_type == Algorithm::FQCommon || jqp_.op_type == Algorithm::FQRequantization; + do_dequantization = jqp_.op_type == Algorithm::FQCommon; do_rounding = do_dequantization || jqp_.dst_prc == Precision::FP32; this->preamble(); @@ -863,7 +863,7 @@ private: } } }; - +#endif bool FakeQuantize::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { try { const auto fq = std::dynamic_pointer_cast(op); @@ -1236,8 +1236,7 @@ FakeQuantize::FakeQuantize(const std::shared_ptr& op, const GraphC } } - algorithm = quantizationOnly ? Algorithm::FQQuantization : - (isFakeQuantization || isFakeQuantizationWithScale) ? Algorithm::FQCommon : Algorithm::FQRequantization; + algorithm = quantizationOnly ? Algorithm::FQQuantization : Algorithm::FQCommon; } } else { IE_THROW(NotImplemented) << errorMessage; @@ -1326,7 +1325,6 @@ void FakeQuantize::initSupportedPrimitiveDescriptors() { } else { impl_type = impl_desc_type::ref; } - if (!mayiuse(cpu::x64::sse41) || getAxis() != 1) { impl_type = impl_desc_type::ref; @@ -1597,8 +1595,8 @@ void FakeQuantize::executeReference() { }); } } - void FakeQuantize::executeBinarization(const std::unique_ptr &pKernel) const { +#if defined(OPENVINO_ARCH_X86_64) const auto &srcMemory = getParentEdgeAt(0)->getMemoryPtr(); auto &dstMemory = getChildEdgeAt(0)->getMemoryPtr(); @@ -1636,9 +1634,11 @@ void FakeQuantize::executeBinarization(const std::unique_ptr &pKernel) const { +#if defined(OPENVINO_ARCH_X86_64) auto &srcMemory = getParentEdgeAt(0)->getMemoryPtr(); auto &dstMemory = getChildEdgeAt(0)->getMemoryPtr(); @@ -1761,6 +1761,7 @@ void FakeQuantize::executeQuantization(const std::unique_ptrcreate_ker(); } +#endif } void FakeQuantize::FakeQuantizeJitExecutor::exec(const FakeQuantize& node) { diff --git a/src/plugins/intel_cpu/src/nodes/fake_quantize.h b/src/plugins/intel_cpu/src/nodes/fake_quantize.h index 47a7c13a8b0..d791457bec8 100644 --- a/src/plugins/intel_cpu/src/nodes/fake_quantize.h +++ b/src/plugins/intel_cpu/src/nodes/fake_quantize.h @@ -166,13 +166,11 @@ private: }; using executorPtr = std::shared_ptr; executorPtr execPtr = nullptr; - struct FakeQuantizeJitExecutor : public FakeQuantizeExecutor { FakeQuantizeJitExecutor(const jit_quantize_params &_jqp); void exec(const FakeQuantize& node) override; std::unique_ptr pKernel; }; - void init() override; std::vector getDataFormats() const; void initializePostOpData(const VectorDims &postOpDims, const size_t bufferAlignment, bool doRounding); diff --git a/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp b/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp index 7e2181c444c..bc441ac79fa 100644 --- a/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp +++ b/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp @@ -9,7 +9,7 @@ #include "fake_quantize.h" #include "input.h" #include "reorder.h" -#include "ngraph_transformations/op/fully_connected.hpp" +#include "transformations/cpu_opset/common/op/fully_connected.hpp" #include "ngraph/opsets/opset1.hpp" #include "dnnl_extension_utils.h" #include "onednn/dnnl.h" diff --git a/src/plugins/intel_cpu/src/nodes/gather.cpp b/src/plugins/intel_cpu/src/nodes/gather.cpp index 5afa3c34ff1..04406319dec 100644 --- a/src/plugins/intel_cpu/src/nodes/gather.cpp +++ b/src/plugins/intel_cpu/src/nodes/gather.cpp @@ -10,7 +10,7 @@ #include #include "common/cpu_memcpy.h" #include -#include "kernels/gather_uni_kernel.hpp" +#include "kernels/x64/gather_uni_kernel.hpp" #include "utils/shape_inference/shape_inference_cpu.hpp" using namespace InferenceEngine; @@ -205,6 +205,7 @@ void Gather::initSupportedPrimitiveDescriptors() { } void Gather::createPrimitive() { +#if defined(OPENVINO_ARCH_X86_64) uint64_t idxElPerVec = 1; if (!isDynamicNode()) { idxElPerVec = x64::mayiuse(x64::avx512_core) ? x64::cpu_isa_traits::vlen / idxTypeSize : @@ -269,7 +270,7 @@ void Gather::createPrimitive() { } } } - +#endif Node::createPrimitive(); } @@ -323,6 +324,7 @@ void Gather::prepareParams() { totalWork = beforeBatchSize * betweenBatchAndAxisSize * specIndicesSize * afterAxisSize; } +#if defined(OPENVINO_ARCH_X86_64) const auto& selectedPD = getSelectedPrimitiveDescriptor(); if (jitKernel && jitKernel->isSupportedConfiguration(afterAxisSize)) { if (x64::mayiuse(x64::avx512_core)) { @@ -330,12 +332,12 @@ void Gather::prepareParams() { } else if (x64::mayiuse(x64::avx2)) { selectedPD->setImplementationType(jit_avx2); } - } else { - selectedPD->setImplementationType(ref_any); } +#endif } void Gather::execute(dnnl::stream strm) { +#if defined(OPENVINO_ARCH_X86_64) if (jitKernel && jitKernel->isSupportedConfiguration(afterAxisSize)) { const void* srcIndices = getParentEdgeAt(GATHER_INDICES)->getMemoryPtr()->GetPtr(); const void* srcData = getParentEdgeAt(GATHER_DATA)->getMemoryPtr()->GetPtr(); @@ -383,12 +385,15 @@ void Gather::execute(dnnl::stream strm) { }; parallel_nt(0, threadBody); - } else { - execReference(); + + return; } +#endif + execReference(); } void Gather::executeDynamicImpl(dnnl::stream strm) { +#if defined(OPENVINO_ARCH_X86_64) if (jitKernel && jitKernel->isSupportedConfiguration(afterAxisSize)) { const void* srcIndices = getParentEdgeAt(GATHER_INDICES)->getMemoryPtr()->GetPtr(); const void* srcData = getParentEdgeAt(GATHER_DATA)->getMemoryPtr()->GetPtr(); @@ -442,9 +447,11 @@ void Gather::executeDynamicImpl(dnnl::stream strm) { }; parallel_nt(0, threadBody); - } else { - execReference(); + + return; } +#endif + execReference(); } void Gather::initShortParams(threadExecParams& p, const uint64_t start) { diff --git a/src/plugins/intel_cpu/src/nodes/gather.h b/src/plugins/intel_cpu/src/nodes/gather.h index a1276cfc35d..d89c94f437f 100644 --- a/src/plugins/intel_cpu/src/nodes/gather.h +++ b/src/plugins/intel_cpu/src/nodes/gather.h @@ -5,7 +5,7 @@ #pragma once #include -#include "kernels/gather_uni_kernel.hpp" +#include "kernels/x64/gather_uni_kernel.hpp" #include #include diff --git a/src/plugins/intel_cpu/src/nodes/grid_sample.hpp b/src/plugins/intel_cpu/src/nodes/grid_sample.hpp index a00691e2530..89a1a409764 100644 --- a/src/plugins/intel_cpu/src/nodes/grid_sample.hpp +++ b/src/plugins/intel_cpu/src/nodes/grid_sample.hpp @@ -5,7 +5,7 @@ #pragma once #include -#include "kernels/grid_sample.hpp" +#include "kernels/x64/grid_sample.hpp" #include #include diff --git a/src/plugins/intel_cpu/src/nodes/input.cpp b/src/plugins/intel_cpu/src/nodes/input.cpp index 6a1e6bece30..86f866ca49f 100644 --- a/src/plugins/intel_cpu/src/nodes/input.cpp +++ b/src/plugins/intel_cpu/src/nodes/input.cpp @@ -33,8 +33,9 @@ using namespace Xbyak; namespace ov { namespace intel_cpu { namespace node { -namespace { +#if defined(OPENVINO_ARCH_X86_64) +namespace { struct jit_has_subnormals_base : public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_has_subnormals_base) @@ -229,6 +230,7 @@ jit_has_subnormals_base::fn_t jit_has_subnormals_function() { } } // namespace +#endif Input::Input(const std::shared_ptr& op, const GraphContext::CPtr context) : Node(op, context, PassThroughShapeInferFactory()) { @@ -297,6 +299,7 @@ void Input::cloneBlobIfRequired() { if (!size) return false; +#if defined(OPENVINO_ARCH_X86_64) if (auto fn = jit_has_subnormals_function()) { static const size_t batch_size = 2048; const size_t iterations_num = size / batch_size + 1; @@ -318,11 +321,12 @@ void Input::cloneBlobIfRequired() { }); return has_subnormals; - } else { - for (size_t i = 0; i < size; ++i) { - if (u32data[i] && (u32data[i] & (0xFF << 23)) == 0) { - return true; - } + } +#endif + + for (size_t i = 0; i < size; ++i) { + if (u32data[i] && (u32data[i] & (0xFF << 23)) == 0) { + return true; } } } diff --git a/src/plugins/intel_cpu/src/nodes/interaction.cpp b/src/plugins/intel_cpu/src/nodes/interaction.cpp index 2e6336ea447..c23a524e104 100644 --- a/src/plugins/intel_cpu/src/nodes/interaction.cpp +++ b/src/plugins/intel_cpu/src/nodes/interaction.cpp @@ -6,7 +6,7 @@ #include #include -#include "ngraph_transformations/op/interaction.hpp" +#include "transformations/cpu_opset/x64/op/interaction.hpp" #include "interaction.h" #include #include @@ -18,8 +18,8 @@ #include #include #include -#include "emitters/jit_dnnl_emitters.hpp" -#include "emitters/jit_load_store_emitters.hpp" +#include "emitters/x64/jit_dnnl_emitters.hpp" +#include "emitters/x64/jit_load_store_emitters.hpp" using namespace InferenceEngine; using namespace dnnl::impl::cpu::x64; diff --git a/src/plugins/intel_cpu/src/nodes/interpolate.cpp b/src/plugins/intel_cpu/src/nodes/interpolate.cpp index 2dd610086c1..7517afb0e54 100644 --- a/src/plugins/intel_cpu/src/nodes/interpolate.cpp +++ b/src/plugins/intel_cpu/src/nodes/interpolate.cpp @@ -20,8 +20,8 @@ #include #include "common/cpu_memcpy.h" #include "utils/bfloat16.hpp" -#include "emitters/jit_bf16_emitters.hpp" -#include "emitters/jit_load_store_emitters.hpp" +#include "emitters/x64/jit_bf16_emitters.hpp" +#include "emitters/x64/jit_load_store_emitters.hpp" #include #include @@ -46,6 +46,8 @@ namespace ov { namespace intel_cpu { namespace node { +#if defined(OPENVINO_ARCH_X86_64) + template struct jit_uni_interpolate_kernel_f32 : public jit_uni_interpolate_kernel, public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_interpolate_kernel_f32) @@ -1364,9 +1366,11 @@ private: } }; +#endif // OPENVINO_ARCH_X86_64 + namespace { struct InterpolateKey { - Interpolate::InterpolateAttrs nodeAttrs; + InterpolateAttrs nodeAttrs; VectorDims srcDims; VectorDims dstDims; std::vector dataScales; @@ -1548,7 +1552,7 @@ bool Interpolate::isSupportedOperation(const std::shared_ptr namespace { /** * Interpolate shape inference factory. It defines the input mask depending on the shape calculation mode. - * + * */ class InterpolateShapeInferFactory : public ShapeInferFactory { public: @@ -1769,7 +1773,7 @@ void Interpolate::initSupportedPrimitiveDescriptors() { auto axesType = Precision::I32; auto& creatorsMap = BlockedDescCreator::getCommonCreators(); - auto pushDesc = [&](LayoutType dataFormat, impl_desc_type implDetail) { + auto pushDesc = [&](LayoutType dataFormat, impl_desc_type implDetail, bool useAclExecutor = false) { config.inConfs[DATA_ID].setMemDesc(creatorsMap.at(dataFormat)->createSharedDesc(inputPrecision, getInputShapeAtPort(DATA_ID))); config.inConfs[TARGET_SHAPE_ID].setMemDesc(creatorsMap.at(LayoutType::ncsp)->createSharedDesc(targetShapeType, getInputShapeAtPort(TARGET_SHAPE_ID))); config.inConfs[SCALES_ID].setMemDesc(creatorsMap.at(LayoutType::ncsp)->createSharedDesc(scalesType, getInputShapeAtPort(SCALES_ID))); @@ -1778,12 +1782,39 @@ void Interpolate::initSupportedPrimitiveDescriptors() { config.inConfs[AXES_ID].setMemDesc(creatorsMap.at(LayoutType::ncsp)->createSharedDesc(axesType, getInputShapeAtPort(AXES_ID))); config.outConfs[0].setMemDesc(creatorsMap.at(dataFormat)->createSharedDesc(outputPrecision, getOutputShapeAtPort(0))); - supportedPrimitiveDescriptors.push_back({config, implDetail}); + + if (useAclExecutor) { + std::vector srcMemoryDescs; + for (int i = 0; i < config.inConfs.size(); i++) { + srcMemoryDescs.push_back(config.inConfs[i].getMemDesc()); + } + std::vector dstMemoryDescs; + for (int i = 0; i < config.outConfs.size(); i++) { + dstMemoryDescs.push_back(config.outConfs[i].getMemDesc()); + } + + auto factory = std::make_shared(interpAttrs, srcMemoryDescs, dstMemoryDescs, + std::make_shared(context, getPrimitivesPriority())); + if (!factory->isEmpty()) { + supportedPrimitiveDescriptors.push_back({config, implDetail, factory}); + } + } else { + supportedPrimitiveDescriptors.push_back({config, implDetail}); + } }; const auto &dataMinDims = getInputShapeAtPort(DATA_ID).getMinDims(); bool isBlkApplied = getInputShapeAtPort(DATA_ID).getRank() > 1 && dataMinDims[1] != Shape::UNDEFINED_DIM && dataMinDims[1] > 1; +#if defined (OV_CPU_WITH_ACL) + interpAttrs.hasPad = hasPad; + pushDesc(LayoutType::nspc, undef, true); + pushDesc(LayoutType::ncsp, undef, true); + canUseAclExecutor = !supportedPrimitiveDescriptors.empty(); + if (canUseAclExecutor) + return; +#endif + if (!mayiuse(cpu::x64::sse41) || interpAttrs.mode == InterpolateMode::linear) { pushDesc(LayoutType::ncsp, ref); } else { @@ -1897,11 +1928,28 @@ void Interpolate::prepareParams() { IE_THROW() << "Interpolate layer only supports resize on spatial dimensions(depth, height and width)"; } + if (canUseAclExecutor) { + interpAttrs.dataScales = dataScales; + + std::vector srcMemoryDescs; + for (int i = 0; i < getParentEdges().size(); i++) { + srcMemoryDescs.push_back(getParentEdgeAt(i)->getMemoryPtr()->getDescPtr()); + } + std::vector dstMemoryDescs; + dstMemoryDescs.push_back(getChildEdgeAt(0)->getMemoryPtr()->getDescPtr()); + + auto selectedPD = getSelectedPrimitiveDescriptor(); + aclExecPtr = selectedPD->getExecutorFactoryAs()->makeExecutor(interpAttrs, srcMemoryDescs, dstMemoryDescs, {}); + selectedPD->setImplementationType(aclExecPtr->getImplType()); + + return; + } + InterpolateKey key = {interpAttrs, srcDims, dstDims, dataScales, dnnl::primitive_attr()}; setPostOps(key.attr, dstDims); - auto buildExecutor = [&](const InterpolateKey& key) -> std::shared_ptr { - std::shared_ptr executor; + auto buildExecutor = [&](const InterpolateKey& key) -> std::shared_ptr { + std::shared_ptr executor; if ((key.nodeAttrs.mode == InterpolateMode::nearest || key.nodeAttrs.mode == InterpolateMode::linear_onnx || key.nodeAttrs.mode == InterpolateMode::cubic) && ((key.nodeAttrs.layout != InterpolateLayoutType::planar && mayiuse(cpu::x64::sse41)) || @@ -2013,89 +2061,92 @@ std::vector Interpolate::getScales(const VectorDims &srcDimPad, const Vec } void Interpolate::execute(dnnl::stream strm) { - if (!execPtr) { - IE_THROW() << "Can't execute Interpolate node. Primitive didn't created"; - } - auto &dstMemPtr = getChildEdgeAt(0)->getMemoryPtr(); auto &srcMemPtr = getParentEdgeAt(DATA_ID)->getMemoryPtr(); - uint8_t *dst_data = reinterpret_cast(dstMemPtr->GetPtr()); - const uint8_t *src_data_origin = reinterpret_cast(srcMemPtr->GetData()); + if (execPtr) { + uint8_t *dst_data = reinterpret_cast(dstMemPtr->GetPtr()); + const uint8_t *src_data_origin = reinterpret_cast(srcMemPtr->GetData()); - const auto &srcDim = srcMemPtr->getStaticDims(); - const auto &dstDim = dstMemPtr->getStaticDims(); - size_t dimSize = srcDim.size(); - auto srcDimPad = execPtr->getSrcDimPad5d(); + const auto &srcDim = srcMemPtr->getStaticDims(); + const auto &dstDim = dstMemPtr->getStaticDims(); + size_t dimSize = srcDim.size(); + auto srcDimPad = execPtr->getSrcDimPad5d(); - const auto srcDim5d = to5Dim(srcDim); - const auto srcDimPad5d = to5Dim(srcDimPad); - const auto dstDim5d = to5Dim(dstDim); - const auto srcDataSize = srcMemPtr->getDesc().getPrecision().size(); + const auto srcDim5d = to5Dim(srcDim); + const auto srcDimPad5d = to5Dim(srcDimPad); + const auto dstDim5d = to5Dim(dstDim); + const auto srcDataSize = srcMemPtr->getDesc().getPrecision().size(); - const uint8_t *src_data = nullptr; - std::vector srcPadded; - if (hasPad) { - int padB0 = (dimSize > 2) ? interpAttrs.padBegin[0] : 0; - int padB1 = (dimSize > 2) ? interpAttrs.padBegin[1] : 0; - int padB2 = (dimSize == 5) ? interpAttrs.padBegin[dimSize - 3] : 0; - int padB3 = interpAttrs.padBegin[dimSize - 2]; - int padB4 = interpAttrs.padBegin[dimSize - 1]; + const uint8_t *src_data = nullptr; + std::vector srcPadded; + if (hasPad) { + int padB0 = (dimSize > 2) ? interpAttrs.padBegin[0] : 0; + int padB1 = (dimSize > 2) ? interpAttrs.padBegin[1] : 0; + int padB2 = (dimSize == 5) ? interpAttrs.padBegin[dimSize - 3] : 0; + int padB3 = interpAttrs.padBegin[dimSize - 2]; + int padB4 = interpAttrs.padBegin[dimSize - 1]; - SizeVector inShapeBlock = getBlockND(srcDim5d); - SizeVector inShapePadBlock = getBlockND(srcDimPad5d); + SizeVector inShapeBlock = getBlockND(srcDim5d); + SizeVector inShapePadBlock = getBlockND(srcDimPad5d); - if (interpAttrs.layout == InterpolateLayoutType::planar) { - srcPadded.resize(inShapePadBlock[0] * srcDataSize, 0); - uint8_t *src_data_pad = static_cast(&srcPadded[0]); - parallel_for4d(srcDim5d[0], srcDim5d[1], srcDim5d[2], srcDim5d[3], [&](int n, int c, int d, int h) { - const uint8_t *src = src_data_origin + (inShapeBlock[1] * n + inShapeBlock[2] * c + inShapeBlock[3] * d + inShapeBlock[4] * h) * srcDataSize; - uint8_t *srcPad = src_data_pad + (inShapePadBlock[1] * (n + padB0) + inShapePadBlock[2] * (c + padB1) + - inShapePadBlock[3] * (d + padB2) + inShapePadBlock[4] * (h + padB3) + padB4) * srcDataSize; - cpu_memcpy(srcPad, src, srcDim5d[4] * srcDataSize); - }); - src_data = src_data_pad; - } else if (interpAttrs.layout == InterpolateLayoutType::by_channel) { - srcPadded.resize(inShapePadBlock[0] * srcDataSize, 0); - uint8_t *src_data_pad = static_cast(&srcPadded[0]); - parallel_for4d(srcDim5d[0], srcDim5d[2], srcDim5d[3], srcDim5d[4], [&](int n, int d, int h, int w) { - const uint8_t *src = src_data_origin + (inShapeBlock[1] * n + - (inShapeBlock[3] * d + inShapeBlock[4] * h + inShapeBlock[5] * w) * srcDim5d[1]) * srcDataSize; - uint8_t *srcPad = src_data_pad + (inShapePadBlock[1] * (n + padB0) + (inShapePadBlock[3] * (d + padB2) + - inShapePadBlock[4] * (h + padB3) + inShapePadBlock[5] * (w + padB4)) * srcDimPad5d[1] + padB1) * srcDataSize; - cpu_memcpy(srcPad, src, srcDim5d[1] * srcDataSize); - }); - src_data = src_data_pad; - } else if (interpAttrs.layout == InterpolateLayoutType::block) { - size_t blkSize = mayiuse(cpu::x64::avx512_core) ? 16 : 8; - size_t CB = div_up(srcDimPad5d[1], blkSize); - size_t eltsTotal = srcDimPad5d[0] * CB * srcDimPad5d[2] * srcDimPad5d[3] * srcDimPad5d[4] * blkSize; - srcPadded.resize(eltsTotal * srcDataSize, 0x0); - uint8_t *src_data_pad = static_cast(&srcPadded[0]); - if ((srcDim5d[0] != srcDimPad5d[0]) || (srcDim5d[1] != srcDimPad5d[1])) { - IE_THROW() << "Interpolate layer with name '" << getName() << - "' does not support padding on batch and channel dimensions"; + if (interpAttrs.layout == InterpolateLayoutType::planar) { + srcPadded.resize(inShapePadBlock[0] * srcDataSize, 0); + uint8_t *src_data_pad = static_cast(&srcPadded[0]); + parallel_for4d(srcDim5d[0], srcDim5d[1], srcDim5d[2], srcDim5d[3], [&](int n, int c, int d, int h) { + const uint8_t *src = src_data_origin + + (inShapeBlock[1] * n + inShapeBlock[2] * c + inShapeBlock[3] * d + inShapeBlock[4] * h) * srcDataSize; + uint8_t *srcPad = src_data_pad + (inShapePadBlock[1] * (n + padB0) + inShapePadBlock[2] * (c + padB1) + + inShapePadBlock[3] * (d + padB2) + inShapePadBlock[4] * (h + padB3) + padB4) * srcDataSize; + cpu_memcpy(srcPad, src, srcDim5d[4] * srcDataSize); + }); + src_data = src_data_pad; + } else if (interpAttrs.layout == InterpolateLayoutType::by_channel) { + srcPadded.resize(inShapePadBlock[0] * srcDataSize, 0); + uint8_t *src_data_pad = static_cast(&srcPadded[0]); + parallel_for4d(srcDim5d[0], srcDim5d[2], srcDim5d[3], srcDim5d[4], [&](int n, int d, int h, int w) { + const uint8_t *src = src_data_origin + (inShapeBlock[1] * n + + (inShapeBlock[3] * d + inShapeBlock[4] * h + inShapeBlock[5] * w) * srcDim5d[1]) * srcDataSize; + uint8_t *srcPad = src_data_pad + (inShapePadBlock[1] * (n + padB0) + (inShapePadBlock[3] * (d + padB2) + + inShapePadBlock[4] * (h + padB3) + inShapePadBlock[5] * (w + padB4)) * srcDimPad5d[1] + padB1) * srcDataSize; + cpu_memcpy(srcPad, src, srcDim5d[1] * srcDataSize); + }); + src_data = src_data_pad; + } else if (interpAttrs.layout == InterpolateLayoutType::block) { + size_t blkSize = mayiuse(cpu::x64::avx512_core) ? 16 : 8; + size_t CB = div_up(srcDimPad5d[1], blkSize); + size_t eltsTotal = srcDimPad5d[0] * CB * srcDimPad5d[2] * srcDimPad5d[3] * srcDimPad5d[4] * blkSize; + srcPadded.resize(eltsTotal * srcDataSize, 0x0); + uint8_t *src_data_pad = static_cast(&srcPadded[0]); + if ((srcDim5d[0] != srcDimPad5d[0]) || (srcDim5d[1] != srcDimPad5d[1])) { + IE_THROW() << "Interpolate layer with name '" << getName() << + "' does not support padding on batch and channel dimensions"; + } + parallel_for5d(srcDim5d[0], CB, srcDim5d[2], srcDim5d[3], srcDim5d[4], [&](int n, int cb, int d, int h, int w) { + const uint8_t *src = src_data_origin + (n * CB * srcDim5d[2] * srcDim5d[3] * srcDim5d[4] * blkSize) * srcDataSize + + (cb * srcDim5d[2] * srcDim5d[3] * srcDim5d[4] * blkSize) * srcDataSize + + (d * srcDim5d[3] * srcDim5d[4] * blkSize) * srcDataSize + + (h * srcDim5d[4] * blkSize) * srcDataSize + + (w * blkSize) * srcDataSize; + uint8_t *srcPad = src_data_pad + (n * CB * srcDimPad5d[2] * srcDimPad5d[3] * srcDimPad5d[4] * blkSize) * srcDataSize + + (cb * srcDimPad5d[2] * srcDimPad5d[3] * srcDimPad5d[4] * blkSize) * srcDataSize + + ((d + padB2) * srcDimPad5d[3] * srcDimPad5d[4] * blkSize) * srcDataSize + + ((h + padB3) * srcDimPad5d[4] * blkSize) * srcDataSize + + ((w + padB4) * blkSize) * srcDataSize; + cpu_memcpy(srcPad, src, blkSize * srcDataSize); + }); + src_data = src_data_pad; } - parallel_for5d(srcDim5d[0], CB, srcDim5d[2], srcDim5d[3], srcDim5d[4], [&](int n, int cb, int d, int h, int w) { - const uint8_t *src = src_data_origin + (n * CB * srcDim5d[2] * srcDim5d[3] * srcDim5d[4] * blkSize) * srcDataSize - + (cb * srcDim5d[2] * srcDim5d[3] * srcDim5d[4] * blkSize) * srcDataSize - + (d * srcDim5d[3] * srcDim5d[4] * blkSize) * srcDataSize - + (h * srcDim5d[4] * blkSize) * srcDataSize - + (w * blkSize) * srcDataSize; - uint8_t *srcPad = src_data_pad + (n * CB * srcDimPad5d[2] * srcDimPad5d[3] * srcDimPad5d[4] * blkSize) * srcDataSize - + (cb * srcDimPad5d[2] * srcDimPad5d[3] * srcDimPad5d[4] * blkSize) * srcDataSize - + ((d + padB2) * srcDimPad5d[3] * srcDimPad5d[4] * blkSize) * srcDataSize - + ((h + padB3) * srcDimPad5d[4] * blkSize) * srcDataSize - + ((w + padB4) * blkSize) * srcDataSize; - cpu_memcpy(srcPad, src, blkSize * srcDataSize); - }); - src_data = src_data_pad; + } else { + src_data = src_data_origin; } - } else { - src_data = src_data_origin; - } - execPtr->exec(src_data, dst_data, postOpsDataPtrs.data()); + execPtr->exec(src_data, dst_data, postOpsDataPtrs.data()); + } else if (aclExecPtr) { + aclExecPtr->exec({srcMemPtr}, {dstMemPtr}, postOpsDataPtrs.data()); + } else { + IE_THROW() << "Can't execute Interpolate node. Primitive didn't created"; + } } // for ndhwc and nCdhw8c[16c] @@ -2369,7 +2420,7 @@ void Interpolate::InterpolateJitExecutor::cubicPlanar(const uint8_t *in_ptr_, ui // ===================================================================================================================== // index layout: // d_0............d_OD-1, h_0..............h_OH-1, w_0................w_OW-1 -void Interpolate::InterpolateExecutor::buildTblNN(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, +void Interpolate::InterpolateExecutorBase::buildTblNN(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, const std::vector& dataScales, InterpolateLayoutType layout, InterpolateNearestMode nearestMode) { const int dimSize = dataRank; float fz = (dimSize == 5) ? dataScales[dimSize - 3] : 1.f; @@ -2402,7 +2453,7 @@ void Interpolate::InterpolateExecutor::buildTblNN(const SizeVector& srcDimPad5d, // scale is float(outShape) / float(inShape) // strictly consistent with onnx calc manner(div scale, not multiply inverse), given this is done offline // the slight precison diff can produce obvious wrong value due to "nearest round" behavior for NN mode -float Interpolate::InterpolateExecutor::coordTransToInput(int outCoord, float scale, int inShape, int outShape) const { +float Interpolate::InterpolateExecutorBase::coordTransToInput(int outCoord, float scale, int inShape, int outShape) const { if (scale == 1.0f || (inShape == outShape)) { return outCoord; } @@ -2440,7 +2491,7 @@ float Interpolate::InterpolateExecutor::coordTransToInput(int outCoord, float sc } } -int Interpolate::InterpolateExecutor::nearestRound(float originCoord, bool isDownsample, InterpolateNearestMode nearestMode) const { +int Interpolate::InterpolateExecutorBase::nearestRound(float originCoord, bool isDownsample, InterpolateNearestMode nearestMode) const { switch (nearestMode) { case InterpolateNearestMode::round_prefer_floor: { if (originCoord == (static_cast(originCoord) + 0.5f)) @@ -2474,7 +2525,7 @@ int Interpolate::InterpolateExecutor::nearestRound(float originCoord, bool isDow } } -void Interpolate::InterpolateExecutor::linearOnnxCF(int outCoord, float scale, int inShape, int outShape, +void Interpolate::InterpolateExecutorBase::linearOnnxCF(int outCoord, float scale, int inShape, int outShape, int& index0, int& index1, float& weight0, float& weight1) { float inCoord = coordTransToInput(outCoord, scale, inShape, outShape); inCoord = std::max(0.0f, std::min(inCoord, static_cast(inShape - 1))); @@ -2489,7 +2540,7 @@ void Interpolate::InterpolateExecutor::linearOnnxCF(int outCoord, float scale, i } } -void Interpolate::InterpolateExecutor::buildTblLinearOnnx(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, +void Interpolate::InterpolateExecutorBase::buildTblLinearOnnx(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, const std::vector& dataScales, InterpolateLayoutType layout) { int dimSize = dataRank; float fz = (spatialDimSize > 2) ? dataScales[dimSize - 3] : 1.f; @@ -2602,7 +2653,7 @@ void Interpolate::InterpolateExecutor::buildTblLinearOnnx(const SizeVector& srcD // wd .........wd, wh............wh, ww.............ww, id...........id, ih............ih, iw..............iw // | | // wh0.....wh_diameter ih0.....ih_diameter -void Interpolate::InterpolateExecutor::buildTblLinear(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, +void Interpolate::InterpolateExecutorBase::buildTblLinear(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, const std::vector& dataScales, int kernel_width, bool antialias) { int dimSize = dataRank; float fz = (dimSize == 5) ? dataScales[dimSize - 3] : 1.f; @@ -2679,7 +2730,7 @@ void Interpolate::InterpolateExecutor::buildTblLinear(const SizeVector& srcDimPa } } -std::vector Interpolate::InterpolateExecutor::getCubicCoeffs(float mantissa, float a) { +std::vector Interpolate::InterpolateExecutorBase::getCubicCoeffs(float mantissa, float a) { float m = std::fabs(mantissa); std::vector coeffs(4, 0.f); @@ -2693,7 +2744,7 @@ std::vector Interpolate::InterpolateExecutor::getCubicCoeffs(float mantis // table layout: // OW OW OW OW OW OH OH OH OH OH // x_idx x_weight0 x_weight1 x_weight2 x_weight3 y_idx y_weight0 y_weight1 y_weight2 y_weight3 -void Interpolate::InterpolateExecutor::buildTblCubic(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, const std::vector& dataScales, +void Interpolate::InterpolateExecutorBase::buildTblCubic(const SizeVector& srcDimPad5d, const SizeVector& dstDim5d, const std::vector& dataScales, float cubicCoeff, InterpolateLayoutType layout) { int dimSize = dataRank; float fy = dataScales[dimSize - 2]; @@ -3085,7 +3136,7 @@ void Interpolate::InterpolateRefExecutor::linearInterpolation(const uint8_t *in_ }); } -Interpolate::InterpolateExecutor::InterpolateExecutor(const InterpolateAttrs& interpAttrs, +Interpolate::InterpolateExecutorBase::InterpolateExecutorBase(const InterpolateAttrs& interpAttrs, const VectorDims &srcDims, const VectorDims &dstDims, const std::vector &dataScales) : @@ -3128,7 +3179,7 @@ Interpolate::InterpolateJitExecutor::InterpolateJitExecutor(const InterpolateAtt const VectorDims &dstDims, const std::vector &dataScales, const dnnl::primitive_attr &attr) : - InterpolateExecutor(interpAttrs, srcDims, dstDims, dataScales) { + InterpolateExecutorBase(interpAttrs, srcDims, dstDims, dataScales) { auto jcp = jit_interpolate_config_params(); jcp.mode = mode; jcp.src_prc = interpAttrs.inPrc; @@ -3145,6 +3196,7 @@ Interpolate::InterpolateJitExecutor::InterpolateJitExecutor(const InterpolateAtt jcp.ID = srcDimPad5d[2]; jcp.spatial_dim_size = getSpatialDimsNum(srcDims.size()); jcp.layout = interpAttrs.layout; +#if defined(OPENVINO_ARCH_X86_64) if (jcp.layout != InterpolateLayoutType::planar) { if (mayiuse(cpu::x64::avx512_core)) { interpolateKernel.reset(new jit_uni_interpolate_kernel_f32(jcp, *attr.get())); @@ -3159,6 +3211,7 @@ Interpolate::InterpolateJitExecutor::InterpolateJitExecutor(const InterpolateAtt } else { IE_THROW() << "Can't create InterpolateJitExecutor"; } +#endif // OPENVINO_ARCH_X86_64 if (interpolateKernel) { interpolateKernel->create_ker(); } else { @@ -3266,4 +3319,4 @@ bool Interpolate::created() const { } // namespace node } // namespace intel_cpu -} // namespace ov +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/interpolate.h b/src/plugins/intel_cpu/src/nodes/interpolate.h index 95f07df5c71..9c7179d5435 100644 --- a/src/plugins/intel_cpu/src/nodes/interpolate.h +++ b/src/plugins/intel_cpu/src/nodes/interpolate.h @@ -9,6 +9,8 @@ #include #include #include +#include "executors/interpolate.hpp" +#include "executors/interpolate_list.hpp" #define MAX_INPUT_INTERPOLATE 8 @@ -18,40 +20,6 @@ namespace ov { namespace intel_cpu { namespace node { -enum InterpolateLayoutType { - planar, - block, - by_channel -}; - -enum InterpolateMode { - nearest, - linear, - linear_onnx, - cubic -}; - -enum InterpolateCoordTransMode { - half_pixel, - pytorch_half_pixel, - asymmetric, - tf_half_pixel_for_nn, - align_corners -}; - -enum class InterpolateNearestMode { - round_prefer_floor, - round_prefer_ceil, - floor, - ceil, - simple -}; - -enum class InterpolateShapeCalcMode { - sizes, - scales -}; - struct jit_interpolate_config_params { InterpolateLayoutType layout; InterpolateMode mode; @@ -121,31 +89,18 @@ public: bool needPrepareParams() const override; void prepareParams() override; - struct InterpolateAttrs { - InterpolateMode mode = InterpolateMode::nearest; - InterpolateCoordTransMode coordTransMode = InterpolateCoordTransMode::half_pixel; - InterpolateNearestMode nearestMode = InterpolateNearestMode::round_prefer_floor; - bool antialias = false; - float cubeCoeff = -0.75; - std::vector padBegin; - std::vector padEnd; - InferenceEngine::Precision inPrc; - InferenceEngine::Precision outPrc; - InterpolateLayoutType layout; - }; - private: InterpolateAttrs interpAttrs; - class InterpolateExecutor { + class InterpolateExecutorBase { public: - InterpolateExecutor(const InterpolateAttrs& interpAttrs, + InterpolateExecutorBase(const InterpolateAttrs& interpAttrs, const VectorDims &srcDims, const VectorDims &dstDims, const std::vector &dataScales); virtual void exec(const uint8_t *in_ptr_, uint8_t *out_ptr_, const void *post_ops_data_) = 0; - virtual ~InterpolateExecutor() = default; + virtual ~InterpolateExecutorBase() = default; VectorDims getSrcDimPad5d() const { return srcDimPad5d; } private: @@ -174,9 +129,9 @@ private: size_t dataRank; std::vector indexTable; }; - std::shared_ptr execPtr = nullptr; + std::shared_ptr execPtr = nullptr; - class InterpolateJitExecutor : public InterpolateExecutor { + class InterpolateJitExecutor : public InterpolateExecutorBase { public: InterpolateJitExecutor(const InterpolateAttrs& interpAttrs, const VectorDims &srcDims, @@ -209,13 +164,13 @@ private: std::shared_ptr interpolateKernel = nullptr; }; - class InterpolateRefExecutor : public InterpolateExecutor { + class InterpolateRefExecutor : public InterpolateExecutorBase { public: InterpolateRefExecutor(const InterpolateAttrs& interpAttrs, const VectorDims &srcDims, const VectorDims &dstDims, const std::vector &_dataScales) : - InterpolateExecutor(interpAttrs, srcDims, dstDims, _dataScales), + InterpolateExecutorBase(interpAttrs, srcDims, dstDims, _dataScales), antialias(interpAttrs.antialias), dataScales(_dataScales) {} void exec(const uint8_t *in_ptr_, uint8_t *out_ptr_, const void *post_ops_data_) override; @@ -259,8 +214,11 @@ private: VectorDims lastOutputDims; std::string errorPrefix; + + bool canUseAclExecutor = false; + std::shared_ptr aclExecPtr = nullptr; }; } // namespace node } // namespace intel_cpu -} // namespace ov +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/kernels/dft_uni_kernel.cpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/dft_uni_kernel.cpp similarity index 100% rename from src/plugins/intel_cpu/src/nodes/kernels/dft_uni_kernel.cpp rename to src/plugins/intel_cpu/src/nodes/kernels/x64/dft_uni_kernel.cpp diff --git a/src/plugins/intel_cpu/src/nodes/kernels/dft_uni_kernel.hpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/dft_uni_kernel.hpp similarity index 100% rename from src/plugins/intel_cpu/src/nodes/kernels/dft_uni_kernel.hpp rename to src/plugins/intel_cpu/src/nodes/kernels/x64/dft_uni_kernel.hpp diff --git a/src/plugins/intel_cpu/src/nodes/kernels/gather_uni_kernel.cpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/gather_uni_kernel.cpp similarity index 100% rename from src/plugins/intel_cpu/src/nodes/kernels/gather_uni_kernel.cpp rename to src/plugins/intel_cpu/src/nodes/kernels/x64/gather_uni_kernel.cpp diff --git a/src/plugins/intel_cpu/src/nodes/kernels/gather_uni_kernel.hpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/gather_uni_kernel.hpp similarity index 100% rename from src/plugins/intel_cpu/src/nodes/kernels/gather_uni_kernel.hpp rename to src/plugins/intel_cpu/src/nodes/kernels/x64/gather_uni_kernel.hpp diff --git a/src/plugins/intel_cpu/src/nodes/kernels/grid_sample.cpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/grid_sample.cpp similarity index 100% rename from src/plugins/intel_cpu/src/nodes/kernels/grid_sample.cpp rename to src/plugins/intel_cpu/src/nodes/kernels/x64/grid_sample.cpp diff --git a/src/plugins/intel_cpu/src/nodes/kernels/grid_sample.hpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/grid_sample.hpp similarity index 100% rename from src/plugins/intel_cpu/src/nodes/kernels/grid_sample.hpp rename to src/plugins/intel_cpu/src/nodes/kernels/x64/grid_sample.hpp diff --git a/src/plugins/intel_cpu/src/utils/jit_kernel.cpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel.cpp similarity index 100% rename from src/plugins/intel_cpu/src/utils/jit_kernel.cpp rename to src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel.cpp diff --git a/src/plugins/intel_cpu/src/utils/jit_kernel.hpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel.hpp similarity index 99% rename from src/plugins/intel_cpu/src/utils/jit_kernel.hpp rename to src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel.hpp index 01694771390..8c6aa2695c1 100644 --- a/src/plugins/intel_cpu/src/utils/jit_kernel.hpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel.hpp @@ -4,7 +4,7 @@ #pragma once #include -#include +#include "emitters/x64/jit_load_store_emitters.hpp" #include #include #include diff --git a/src/plugins/intel_cpu/src/nodes/kernels/jit_kernel_base.cpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel_base.cpp similarity index 100% rename from src/plugins/intel_cpu/src/nodes/kernels/jit_kernel_base.cpp rename to src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel_base.cpp diff --git a/src/plugins/intel_cpu/src/nodes/kernels/jit_kernel_base.hpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel_base.hpp similarity index 100% rename from src/plugins/intel_cpu/src/nodes/kernels/jit_kernel_base.hpp rename to src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel_base.hpp diff --git a/src/plugins/intel_cpu/src/nodes/kernels/rdft_kernel.cpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/rdft_kernel.cpp similarity index 100% rename from src/plugins/intel_cpu/src/nodes/kernels/rdft_kernel.cpp rename to src/plugins/intel_cpu/src/nodes/kernels/x64/rdft_kernel.cpp diff --git a/src/plugins/intel_cpu/src/nodes/kernels/rdft_kernel.hpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/rdft_kernel.hpp similarity index 100% rename from src/plugins/intel_cpu/src/nodes/kernels/rdft_kernel.hpp rename to src/plugins/intel_cpu/src/nodes/kernels/x64/rdft_kernel.hpp diff --git a/src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/registers_pool.hpp similarity index 100% rename from src/plugins/intel_cpu/src/nodes/kernels/registers_pool.hpp rename to src/plugins/intel_cpu/src/nodes/kernels/x64/registers_pool.hpp diff --git a/src/plugins/intel_cpu/src/nodes/mathematics.cpp b/src/plugins/intel_cpu/src/nodes/mathematics.cpp index 30e92a48957..42fc88166a8 100644 --- a/src/plugins/intel_cpu/src/nodes/mathematics.cpp +++ b/src/plugins/intel_cpu/src/nodes/mathematics.cpp @@ -138,11 +138,6 @@ void Math::execute(dnnl::stream strm) { dst_data[i] = (std::max)(0.f, (std::min)(1.f, alpha * src_data[i] + beta)); }); break; - case Algorithm::MathLog: - parallel_for(dataSize, [&](size_t i) { - dst_data[i] = logf(src_data[i]); - }); - break; case Algorithm::MathNegative: parallel_for(dataSize, [&](size_t i) { dst_data[i] = -src_data[i]; @@ -242,9 +237,6 @@ std::map(op->get_input_node_shared_ptr(1))->cast_vector()[0]; node.beta = ngraph::as_type_ptr(op->get_input_node_shared_ptr(2))->cast_vector()[0]; }}, - {ngraph::op::v0::Log::get_type_info_static(), [](const std::shared_ptr& op, Math& node) { - node.algorithm = Algorithm::MathLog; - }}, {ngraph::op::v0::Negative::get_type_info_static(), [](const std::shared_ptr& op, Math& node) { node.algorithm = Algorithm::MathNegative; }}, diff --git a/src/plugins/intel_cpu/src/nodes/matmul.cpp b/src/plugins/intel_cpu/src/nodes/matmul.cpp index 15ec4d6d206..1e2669579ae 100644 --- a/src/plugins/intel_cpu/src/nodes/matmul.cpp +++ b/src/plugins/intel_cpu/src/nodes/matmul.cpp @@ -679,6 +679,7 @@ const std::vector& MatMul::getPrimitivesPriority() { impl_desc_type::unknown, impl_desc_type::brgemm_avx512_amx, impl_desc_type::brgemm_avx512, + impl_desc_type::gemm_acl, impl_desc_type::gemm_blas, impl_desc_type::gemm_avx512, impl_desc_type::gemm_avx2, diff --git a/src/plugins/intel_cpu/src/nodes/mha.cpp b/src/plugins/intel_cpu/src/nodes/mha.cpp index 21841604df1..35c5a48b3ec 100644 --- a/src/plugins/intel_cpu/src/nodes/mha.cpp +++ b/src/plugins/intel_cpu/src/nodes/mha.cpp @@ -11,10 +11,10 @@ #include "common/cpu_memcpy.h" #include #include -#include "emitters/jit_dnnl_emitters.hpp" -#include "emitters/jit_load_store_emitters.hpp" +#include "emitters/x64/jit_dnnl_emitters.hpp" +#include "emitters/x64/jit_load_store_emitters.hpp" #include "common/cpu_convert.h" -#include "ngraph_transformations/op/mha.hpp" +#include "transformations/cpu_opset/x64/op/mha.hpp" #include "dnnl_extension_utils.h" #include diff --git a/src/plugins/intel_cpu/src/nodes/mvn.cpp b/src/plugins/intel_cpu/src/nodes/mvn.cpp index 1547059a6d9..6187d1f4ff3 100644 --- a/src/plugins/intel_cpu/src/nodes/mvn.cpp +++ b/src/plugins/intel_cpu/src/nodes/mvn.cpp @@ -13,8 +13,8 @@ #include #include "utils/bfloat16.hpp" #include "ie_parallel.hpp" -#include "emitters/jit_load_store_emitters.hpp" -#include "emitters/jit_bf16_emitters.hpp" +#include "emitters/x64/jit_load_store_emitters.hpp" +#include "emitters/x64/jit_bf16_emitters.hpp" #include #include @@ -41,7 +41,7 @@ namespace node { namespace { struct MVNKey { - MVN::MVNAttrs mvnAttrs; + MVNAttrs mvnAttrs; dnnl::primitive_attr attr; size_t hash() const; @@ -87,6 +87,8 @@ bool MVNKey::operator==(const MVNKey& rhs) const { } } // namespace +#if defined(OPENVINO_ARCH_X86_64) + // some utility functions static inline bool isFloatCompatible(Precision prc) { return Precision::FP32 == prc || Precision::BF16 == prc; @@ -1062,6 +1064,9 @@ private: } } }; + +#endif // OPENVINO_ARCH_X86_64 + ////////////////////////////////////////////////////////////////////////////////// bool MVN::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { @@ -1199,12 +1204,38 @@ void MVN::initSupportedPrimitiveDescriptors() { } auto& creatorsMap = BlockedDescCreator::getCommonCreators(); - auto pushDesc = [&](LayoutType format, impl_desc_type impl_type) { + auto pushDesc = [&](LayoutType format, impl_desc_type impl_type, bool useAclExecutor = false) { config.inConfs[0].setMemDesc(creatorsMap.at(format)->createSharedDesc(inputPrecision, getInputShapeAtPort(0))); config.outConfs[0].setMemDesc(creatorsMap.at(format)->createSharedDesc(outputPrecision, getOutputShapeAtPort(0))); - supportedPrimitiveDescriptors.push_back({config, impl_type}); + + if (useAclExecutor) { + std::vector srcMemoryDescs; + for (int i = 0; i < config.inConfs.size(); i++) { + srcMemoryDescs.push_back(config.inConfs[i].getMemDesc()); + } + std::vector dstMemoryDescs; + for (int i = 0; i < config.outConfs.size(); i++) { + dstMemoryDescs.push_back(config.outConfs[i].getMemDesc()); + } + + auto factory = std::make_shared(mvnAttrs, srcMemoryDescs, dstMemoryDescs, + std::make_shared(context, getPrimitivesPriority())); + if (!factory->isEmpty()) { + supportedPrimitiveDescriptors.push_back({config, impl_type, factory}); + } + } else { + supportedPrimitiveDescriptors.push_back({config, impl_type}); + } }; +#if defined(OV_CPU_WITH_ACL) + pushDesc(LayoutType::nspc, undef, true); + pushDesc(LayoutType::ncsp, undef, true); + canUseAclExecutor = !supportedPrimitiveDescriptors.empty(); + if (canUseAclExecutor) + return; +#endif // OV_CPU_WITH_ACL + impl_desc_type impl_type; if (mayiuse(cpu::x64::avx512_core)) { impl_type = impl_desc_type::jit_avx512; @@ -1239,14 +1270,14 @@ void MVN::initSupportedPrimitiveDescriptors() { pushDesc(LayoutType::ncsp, impl_type); } -MVN::MVNExecutor::MVNExecutor(const MVNAttrs& mvnAttrs) +MVN::MVNExecutorBase::MVNExecutorBase(const MVNAttrs& mvnAttrs) : mvnAttrs(mvnAttrs), src_data_size(mvnAttrs.src_prc.size()), dst_data_size(mvnAttrs.dst_prc.size()) {} MVN::MVNJitExecutor::MVNJitExecutor(const MVNAttrs& mvnAttrs, const dnnl::primitive_attr& attr): - MVNExecutor(mvnAttrs) { + MVNExecutorBase(mvnAttrs) { auto jcp = jit_mvn_config_params(); jcp.src_prc = mvnAttrs.src_prc; jcp.dst_prc = mvnAttrs.dst_prc; @@ -1257,6 +1288,7 @@ MVN::MVNJitExecutor::MVNJitExecutor(const MVNAttrs& mvnAttrs, jcp.across_channels = mvnAttrs.execAcrossChannels_; int N = 0; std::tie(N, jcp.C, jcp.D, jcp.H, jcp.W) = mvnAttrs.shape5D; +#if defined(OPENVINO_ARCH_X86_64) if (mayiuse(cpu::x64::avx512_core)) { mvn_kernel.reset(new jit_uni_mvn_kernel_f32(jcp, *attr.get())); jcp.normalize_variance = false; @@ -1284,7 +1316,7 @@ MVN::MVNJitExecutor::MVNJitExecutor(const MVNAttrs& mvnAttrs, } else { IE_THROW() << "Can't create jit MVN kernel"; } - +#endif // OPENVINO_ARCH_X86_64 if (mvn_kernel) mvn_kernel->create_ker(); if (mvn_mean_kernel) @@ -1306,7 +1338,7 @@ void MVN::MVNJitExecutor::exec(const uint8_t *src_data, uint8_t *dst_data, const } } -MVN::MVNRefExecutor::MVNRefExecutor(const MVNAttrs& mvnAttrs):MVNExecutor(mvnAttrs) {} +MVN::MVNRefExecutor::MVNRefExecutor(const MVNAttrs& mvnAttrs):MVNExecutorBase(mvnAttrs) {} void MVN::MVNRefExecutor::exec(const uint8_t *src_data, uint8_t *dst_data, const void *post_ops_data_) { mvn_ref(src_data, dst_data); @@ -1336,11 +1368,26 @@ void MVN::prepareParams() { mvnAttrs.layout = MVNLayoutType::mvn_block; } + if (canUseAclExecutor) { + std::vector srcMemoryDescs; + for (int i = 0; i < getParentEdges().size(); i++) { + srcMemoryDescs.push_back(getParentEdgeAt(i)->getMemoryPtr()->getDescPtr()); + } + std::vector dstMemoryDescs; + dstMemoryDescs.push_back(getChildEdgeAt(0)->getMemoryPtr()->getDescPtr()); + + auto selectedPD = getSelectedPrimitiveDescriptor(); + aclExecPtr = selectedPD->getExecutorFactoryAs()->makeExecutor(mvnAttrs, srcMemoryDescs, dstMemoryDescs, {}); + selectedPD->setImplementationType(aclExecPtr->getImplType()); + + return; + } + MVNKey key = {mvnAttrs, dnnl::primitive_attr()}; setPostOps(key.attr, true); - auto builder = [&](const MVNKey& key) -> std::shared_ptr { - std::shared_ptr executor; + auto builder = [&](const MVNKey& key) -> std::shared_ptr { + std::shared_ptr executor; if (mayiuse(cpu::x64::sse41)) { executor = std::make_shared(key.mvnAttrs, key.attr); } else { @@ -1411,15 +1458,18 @@ void MVN::executeDynamicImpl(dnnl::stream strm) { } void MVN::execute(dnnl::stream strm) { - if (!execPtr) { - IE_THROW() << "Can't execute MVN node. Primitive didn't created"; - } auto &dstMemPtr = getChildEdgeAt(0)->getMemoryPtr(); auto &srcMemPtr = getParentEdgeAt(0)->getMemoryPtr(); - uint8_t *dst_data = reinterpret_cast(dstMemPtr->GetPtr()); - uint8_t *src_data = reinterpret_cast(srcMemPtr->GetPtr()); - execPtr->exec(src_data, dst_data, postOpsDataPtrs.data()); + if (execPtr) { + uint8_t *dst_data = reinterpret_cast(dstMemPtr->GetPtr()); + uint8_t *src_data = reinterpret_cast(srcMemPtr->GetPtr()); + execPtr->exec(src_data, dst_data, postOpsDataPtrs.data()); + } else if (aclExecPtr) { + aclExecPtr->exec({srcMemPtr}, {dstMemPtr}, postOpsDataPtrs.data()); + } else { + IE_THROW() << "Can't execute Interpolate node. Primitive didn't created"; + } } void MVN::MVNJitExecutor::mvn_pln(const uint8_t* src_data, uint8_t* dst_data, const void *post_ops_data_) { @@ -2006,7 +2056,8 @@ bool MVN::canFuse(const NodePtr& node) const { // 1D only fused with unary int inputRank = getInputShapeAtPort(0).getRank(); bool unaryEltwise = one_of(node->getAlgorithm(), Algorithm::EltwiseRelu, - Algorithm::EltwiseGelu, + Algorithm::EltwiseGeluErf, + Algorithm::EltwiseGeluTanh, Algorithm::EltwiseElu, Algorithm::EltwiseSigmoid, Algorithm::EltwiseClamp, @@ -2034,4 +2085,4 @@ bool MVN::created() const { } // namespace node } // namespace intel_cpu -} // namespace ov +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/mvn.h b/src/plugins/intel_cpu/src/nodes/mvn.h index c394aa504f4..9d862820ceb 100644 --- a/src/plugins/intel_cpu/src/nodes/mvn.h +++ b/src/plugins/intel_cpu/src/nodes/mvn.h @@ -9,17 +9,12 @@ #include #include #include +#include "executors/mvn_list.hpp" namespace ov { namespace intel_cpu { namespace node { -enum MVNLayoutType { - mvn_planar, - mvn_block, - mvn_by_channel -}; - struct jit_mvn_config_params { MVNLayoutType layout; bool across_channels; @@ -103,24 +98,6 @@ public: bool canFuse(const NodePtr& node) const override; void prepareParams() override; - // Defines way to add epsilon: inside sqrt or outside. - enum MVNEpsMode { - INSIDE_SQRT, - OUTSIDE_SQRT - }; - - struct MVNAttrs { - MVNLayoutType layout; - std::tuple shape5D; - bool initAcrossChannels_; - bool execAcrossChannels_; - bool normalizeVariance_; - float epsValue_; - MVNEpsMode epsMode_; - InferenceEngine::Precision src_prc; - InferenceEngine::Precision dst_prc; - }; - private: void setPostOps(dnnl::primitive_attr &attr, bool initWeights = false); @@ -130,11 +107,11 @@ private: MVNAttrs mvnAttrs; - class MVNExecutor { + class MVNExecutorBase { public: - MVNExecutor(const MVNAttrs& mvnAttrs); + MVNExecutorBase(const MVNAttrs& mvnAttrs); virtual void exec(const uint8_t *in_ptr_, uint8_t *out_ptr_, const void *post_ops_data_) = 0; - virtual ~MVNExecutor() = default; + virtual ~MVNExecutorBase() = default; protected: MVNAttrs mvnAttrs; @@ -142,9 +119,11 @@ private: size_t dst_data_size = 0; }; - std::shared_ptr execPtr = nullptr; + std::shared_ptr execPtr = nullptr; + bool canUseAclExecutor = false; + std::shared_ptr aclExecPtr = nullptr; - class MVNJitExecutor : public MVNExecutor { + class MVNJitExecutor : public MVNExecutorBase { public: MVNJitExecutor(const MVNAttrs& mvnAttrs, const dnnl::primitive_attr &attr); @@ -161,7 +140,7 @@ private: std::shared_ptr mvn_kernel; }; - class MVNRefExecutor : public MVNExecutor { + class MVNRefExecutor : public MVNExecutorBase { public: MVNRefExecutor(const MVNAttrs& mvnAttrs); diff --git a/src/plugins/intel_cpu/src/nodes/ngram.cpp b/src/plugins/intel_cpu/src/nodes/ngram.cpp index 0c87d5c9266..c02a4cf50c3 100644 --- a/src/plugins/intel_cpu/src/nodes/ngram.cpp +++ b/src/plugins/intel_cpu/src/nodes/ngram.cpp @@ -8,7 +8,7 @@ #include "ngram.h" #include "ie_parallel.hpp" #include "common/cpu_memcpy.h" -#include "ngraph_transformations/op/ngram.hpp" +#include "transformations/cpu_opset/common/op/ngram.hpp" namespace ov { namespace intel_cpu { diff --git a/src/plugins/intel_cpu/src/nodes/non_max_suppression.cpp b/src/plugins/intel_cpu/src/nodes/non_max_suppression.cpp index 48fbea67c15..a5cfe2e9740 100644 --- a/src/plugins/intel_cpu/src/nodes/non_max_suppression.cpp +++ b/src/plugins/intel_cpu/src/nodes/non_max_suppression.cpp @@ -16,7 +16,7 @@ #include "utils/general_utils.h" #include "cpu/x64/jit_generator.hpp" -#include "emitters/jit_load_store_emitters.hpp" +#include "emitters/x64/jit_load_store_emitters.hpp" #include #include @@ -33,6 +33,7 @@ namespace ov { namespace intel_cpu { namespace node { +#if defined(OPENVINO_ARCH_X86_64) template struct jit_uni_nms_kernel_f32 : public jit_uni_nms_kernel, public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_nms_kernel_f32) @@ -551,6 +552,7 @@ private: dw(0x0001); } }; +#endif bool NonMaxSuppression::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { try { @@ -701,6 +703,7 @@ bool NonMaxSuppression::isExecutable() const { } void NonMaxSuppression::createJitKernel() { +#if defined(OPENVINO_ARCH_X86_64) auto jcp = jit_nms_config_params(); jcp.box_encode_type = boxEncodingType; jcp.is_soft_suppressed_by_iou = isSoftSuppressedByIOU; @@ -715,6 +718,7 @@ void NonMaxSuppression::createJitKernel() { if (nms_kernel) nms_kernel->create_ker(); +#endif } void NonMaxSuppression::executeDynamicImpl(dnnl::stream strm) { diff --git a/src/plugins/intel_cpu/src/nodes/non_max_suppression.h b/src/plugins/intel_cpu/src/nodes/non_max_suppression.h index a0666f66b63..2599fa3843f 100644 --- a/src/plugins/intel_cpu/src/nodes/non_max_suppression.h +++ b/src/plugins/intel_cpu/src/nodes/non_max_suppression.h @@ -148,7 +148,7 @@ private: void checkOutput(const Shape& shape, const std::vector& precList, const std::string& name, const size_t port); void createJitKernel(); - std::shared_ptr nms_kernel; + std::shared_ptr nms_kernel = nullptr; }; } // namespace node diff --git a/src/plugins/intel_cpu/src/nodes/normalize.cpp b/src/plugins/intel_cpu/src/nodes/normalize.cpp index a0b042b3db8..3f1d1157c0f 100644 --- a/src/plugins/intel_cpu/src/nodes/normalize.cpp +++ b/src/plugins/intel_cpu/src/nodes/normalize.cpp @@ -11,7 +11,7 @@ #include "utils/bfloat16.hpp" #include "utils/general_utils.h" #include -#include "emitters/jit_bf16_emitters.hpp" +#include "emitters/x64/jit_bf16_emitters.hpp" #include #include #include @@ -32,8 +32,9 @@ using namespace dnnl::impl::cpu::x64; using namespace dnnl::impl::utils; using namespace Xbyak; +#if defined(OPENVINO_ARCH_X86_64) #define GET_OFF(field) offsetof(jit_normalize_call_args, field) - +#endif #define THROW_ERROR IE_THROW() << "NormalizeL2 layer with name '" << getName() << "' " namespace ov { @@ -78,6 +79,8 @@ bool NormalizeKey::operator==(const NormalizeKey& rhs) const { } // namespace +#if defined(OPENVINO_ARCH_X86_64) + static inline bool isFloatCompatible(memory::data_type type) { return memory::data_type::f32 == type || memory::data_type::bf16 == type; } @@ -693,7 +696,7 @@ private: } } }; - +#endif bool NormalizeL2::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { try { auto norm = ov::as_type_ptr(op); @@ -965,7 +968,7 @@ private: // *=================* *======* *=================* // *=================* JIT case *=================* - +#if defined(OPENVINO_ARCH_X86_64) template class NormalizeL2::NormalizeL2JitExecutor : public NormalizeL2::NormalizeL2Executor { public: @@ -1295,7 +1298,7 @@ private: std::shared_ptr normalize_modulo_kernel; std::shared_ptr normalize_kernel; }; - +#endif // *=================* *======* *=================* // *=============* Reference case *===============* @@ -1491,8 +1494,10 @@ std::shared_ptr NormalizeL2::NormalizeL2Execut const NormalizeL2Attrs& attrs, const dnnl::primitive_attr& kernel_attrs, const VectorDims& dims) { if (attrs.cornerCase) return std::make_shared>(dims); +#if defined(OPENVINO_ARCH_X86_64) else if (mayiuse(cpu::x64::sse41)) return std::make_shared>(attrs, kernel_attrs, dims); +#endif else if (attrs.layout == LayoutType::ncsp) return std::make_shared>(attrs, kernel_attrs, dims); else diff --git a/src/plugins/intel_cpu/src/nodes/normalize.h b/src/plugins/intel_cpu/src/nodes/normalize.h index c5e5f0106e3..3b6c99bde42 100644 --- a/src/plugins/intel_cpu/src/nodes/normalize.h +++ b/src/plugins/intel_cpu/src/nodes/normalize.h @@ -17,7 +17,7 @@ namespace ov { namespace intel_cpu { namespace node { - +#if defined(OPENVINO_ARCH_X86_64) struct jit_normalize_config_params { bool is_nchw; bool is_nhwc; @@ -75,7 +75,7 @@ struct jit_uni_normalize_kernel { jit_normalize_config_params jcp_; const dnnl_primitive_attr &attr_; }; - +#endif class NormalizeL2 : public Node { public: NormalizeL2(const std::shared_ptr& op, const GraphContext::CPtr context); diff --git a/src/plugins/intel_cpu/src/nodes/pooling.cpp b/src/plugins/intel_cpu/src/nodes/pooling.cpp index b5d1775a677..622aa72f312 100644 --- a/src/plugins/intel_cpu/src/nodes/pooling.cpp +++ b/src/plugins/intel_cpu/src/nodes/pooling.cpp @@ -22,6 +22,11 @@ #include #include +#if defined(OV_CPU_WITH_ACL) +#include "executors/acl/acl_utils.hpp" +#include "utils/debug_capabilities.h" +#endif + using namespace dnnl; using namespace InferenceEngine; @@ -172,38 +177,45 @@ Pooling::Pooling(const std::shared_ptr& op, const GraphContext::CPtr c if (auto maxPoolOp_v8 = ov::as_type_ptr(op)) { isMaxPool8 = true; algorithm = Algorithm::PoolingMax; - exclude_pad = false; + poolingAttrs.exclude_pad = false; + poolingAttrs.rounding = maxPoolOp_v8->get_rounding_type(); + poolingAttrs.pad_type = maxPoolOp_v8->get_auto_pad(); - get_attributes(dilation, maxPoolOp_v8->get_dilations()); - get_attributes(stride, maxPoolOp_v8->get_strides()); - get_attributes(kernel, maxPoolOp_v8->get_kernel()); - get_attributes(data_pad_begin, maxPoolOp_v8->get_pads_begin()); - get_attributes(data_pad_end, maxPoolOp_v8->get_pads_end()); + get_attributes(poolingAttrs.dilation, maxPoolOp_v8->get_dilations()); + get_attributes(poolingAttrs.stride, maxPoolOp_v8->get_strides()); + get_attributes(poolingAttrs.kernel, maxPoolOp_v8->get_kernel()); + get_attributes(poolingAttrs.data_pad_begin, maxPoolOp_v8->get_pads_begin()); + get_attributes(poolingAttrs.data_pad_end, maxPoolOp_v8->get_pads_end()); - auto_pad = (maxPoolOp_v8->get_auto_pad() == ov::op::PadType::SAME_LOWER || maxPoolOp_v8->get_auto_pad() == ov::op::PadType::SAME_UPPER); + poolingAttrs.auto_pad = (maxPoolOp_v8->get_auto_pad() == ov::op::PadType::SAME_LOWER || maxPoolOp_v8->get_auto_pad() == ov::op::PadType::SAME_UPPER); } else if (auto maxPoolOp_v1 = ov::as_type_ptr(op)) { algorithm = Algorithm::PoolingMax; - exclude_pad = false; + poolingAttrs.exclude_pad = false; + poolingAttrs.pad_type = maxPoolOp_v1->get_auto_pad(); + poolingAttrs.rounding = maxPoolOp_v1->get_rounding_type(); - get_attributes(stride, maxPoolOp_v1->get_strides()); - get_attributes(kernel, maxPoolOp_v1->get_kernel()); - get_attributes(data_pad_begin, maxPoolOp_v1->get_pads_begin()); - get_attributes(data_pad_end, maxPoolOp_v1->get_pads_end()); - dilation.resize(kernel.size(), 1); + get_attributes(poolingAttrs.stride, maxPoolOp_v1->get_strides()); + get_attributes(poolingAttrs.kernel, maxPoolOp_v1->get_kernel()); + get_attributes(poolingAttrs.data_pad_begin, maxPoolOp_v1->get_pads_begin()); + get_attributes(poolingAttrs.data_pad_end, maxPoolOp_v1->get_pads_end()); + poolingAttrs.dilation.resize(poolingAttrs.kernel.size(), 1); - auto_pad = (maxPoolOp_v1->get_auto_pad() == ov::op::PadType::SAME_LOWER || maxPoolOp_v1->get_auto_pad() == ov::op::PadType::SAME_UPPER); + poolingAttrs.auto_pad = (maxPoolOp_v1->get_auto_pad() == ov::op::PadType::SAME_LOWER || maxPoolOp_v1->get_auto_pad() == ov::op::PadType::SAME_UPPER); } else if (auto avgPoolOp = ov::as_type_ptr(op)) { algorithm = Algorithm::PoolingAvg; - exclude_pad = avgPoolOp->get_exclude_pad(); + poolingAttrs.exclude_pad = avgPoolOp->get_exclude_pad(); + poolingAttrs.rounding = avgPoolOp->get_rounding_type(); - get_attributes(stride, avgPoolOp->get_strides()); - get_attributes(kernel, avgPoolOp->get_kernel()); - get_attributes(data_pad_begin, avgPoolOp->get_pads_begin()); - get_attributes(data_pad_end, avgPoolOp->get_pads_end()); - dilation.resize(kernel.size(), 1); + get_attributes(poolingAttrs.stride, avgPoolOp->get_strides()); + get_attributes(poolingAttrs.kernel, avgPoolOp->get_kernel()); + get_attributes(poolingAttrs.data_pad_begin, avgPoolOp->get_pads_begin()); + get_attributes(poolingAttrs.data_pad_end, avgPoolOp->get_pads_end()); + poolingAttrs.dilation.resize(poolingAttrs.kernel.size(), 1); - auto_pad = (avgPoolOp->get_auto_pad() == ov::op::PadType::SAME_LOWER || avgPoolOp->get_auto_pad() == ov::op::PadType::SAME_UPPER); + poolingAttrs.auto_pad = (avgPoolOp->get_auto_pad() == ov::op::PadType::SAME_LOWER || avgPoolOp->get_auto_pad() == ov::op::PadType::SAME_UPPER); } + + poolingAttrs.algorithm = algorithm; } std::vector Pooling::getAvailableFormatsForDims(const Shape &dims) const { @@ -223,22 +235,22 @@ std::vector Pooling::getAvailableFormatsForDims(const Shape } void Pooling::initEffectiveAttributes(const Shape &inShape, const Shape &outShape) { - effective_pad_begin = data_pad_begin; - effective_pad_end.resize(data_pad_end.size()); - effective_dilation.resize(dilation.size(), 0); + poolingAttrs.effective_pad_begin = poolingAttrs.data_pad_begin; + poolingAttrs.effective_pad_end.resize(poolingAttrs.data_pad_end.size()); + poolingAttrs.effective_dilation.resize(poolingAttrs.dilation.size(), 0); const auto &inDims = inShape.getStaticDims(); const auto &outDims = outShape.getStaticDims(); - for (int i = 0; i < effective_pad_end.size(); i++) { - int krn = kernel[i]; - int dil = dilation[i]; + for (int i = 0; i < poolingAttrs.effective_pad_end.size(); i++) { + int krn = poolingAttrs.kernel[i]; + int dil = poolingAttrs.dilation[i]; int src = inDims[2 + i]; int dst = outDims[2 + i]; - int calc_dst = (src - (1 + (krn - 1) * dil) + data_pad_begin[i]) / stride[i] + 1; - effective_pad_end[i] = (dst - calc_dst) * stride[i]; - effective_dilation[i] = dil - 1; + int calc_dst = (src - (1 + (krn - 1) * dil) + poolingAttrs.data_pad_begin[i]) / poolingAttrs.stride[i] + 1; + poolingAttrs.effective_pad_end[i] = (dst - calc_dst) * poolingAttrs.stride[i]; + poolingAttrs.effective_dilation[i] = dil - 1; } } @@ -254,6 +266,39 @@ void Pooling::getSupportedDescriptors() { InferenceEngine::Precision inputPrecision = getOriginalInputPrecisionAtPort(0); InferenceEngine::Precision outputPrecision = getOriginalOutputPrecisionAtPort(0); + const auto &parentShape = getInputShapeAtPort(0); + const auto &childShape = getOutputShapeAtPort(0); + const size_t inputRank = getInputShapeAtPort(0).getRank(); + +#if defined(OV_CPU_WITH_ACL) + // WA: we may specify any layout here (NCHW or NHWC) since both are supported by ACL + arm_compute::DataLayout dataLayout = (parentShape.getDims().size() == 5) ? arm_compute::DataLayout::NDHWC : arm_compute::DataLayout::NCHW; + arm_compute::TensorInfo srcTensorInfo = arm_compute::TensorInfo(shapeCast(parentShape.getDims()), + 1, + precisionToAclDataType(inputPrecision), + dataLayout); + arm_compute::TensorInfo dstTensorInfo = arm_compute::TensorInfo(shapeCast(childShape.getDims()), + 1, + precisionToAclDataType(outputPrecision), + dataLayout); + arm_compute::Pooling3dLayerInfo pool3d_info; + arm_compute::PoolingLayerInfo pool_info; + useACL = AclPoolingExecutor::isSupported(srcTensorInfo, + dstTensorInfo, + poolingAttrs, + parentShape.getDims().size(), + getOriginalOutputsNumber(), + dataLayout, + (getOriginalOutputsNumber() > 1) ? &getOutputShapeAtPort(1).getDims() : nullptr, + &pool_info, + &pool3d_info); + //FIXME: 5D tensors case is not assigned to ACL because there is no way to check layout here + //NEPooling3dLayer supports NDHWC only + if (parentShape.getDims().size() == 5) + useACL = false; +#endif + if (useACL) return; + // WA: LPT transformation has WA which allows average pooling has I8/U8 output precision instead of FP32, // so we explicitly set output precision as FP32 if (outputPrecision != Precision::I8 && inputPrecision != Precision::BF16) { @@ -275,10 +320,6 @@ void Pooling::getSupportedDescriptors() { auto inputDataType = DnnlExtensionUtils::IEPrecisionToDataType(inputPrecision); auto outputDataType = DnnlExtensionUtils::IEPrecisionToDataType(outputPrecision); - const auto &parentShape = getInputShapeAtPort(0); - const auto &childShape = getOutputShapeAtPort(0); - const size_t inputRank = getInputShapeAtPort(0).getRank(); - if ((inputRank < 3) || (inputRank > 5)) IE_THROW() << "Pooling layer. Unsupported mode. Only 3D, 4D and 5D blobs are supported as input."; @@ -290,7 +331,7 @@ void Pooling::getSupportedDescriptors() { auto inDims = inShape.getStaticDims(); for (size_t i = 0; i < inDims.size() - 2; i++) { if (origDims[i + 2] == Shape::UNDEFINED_DIM) { - inDims[i + 2] = std::min(origMaxDims[i + 2], std::max(inDims[i + 2], kernel[i])); + inDims[i + 2] = std::min(origMaxDims[i + 2], std::max(inDims[i + 2], poolingAttrs.kernel[i])); } } inShape = Shape(inDims); @@ -331,7 +372,7 @@ void Pooling::getSupportedDescriptors() { } void Pooling::prepareParams() { - const NodeDesc *selected_pd = getSelectedPrimitiveDescriptor(); + auto selected_pd = getSelectedPrimitiveDescriptor(); if (selected_pd == nullptr) IE_THROW() << "Pooling node with name '" << getName() << "' did not set preferable primitive descriptor"; @@ -345,84 +386,119 @@ void Pooling::prepareParams() { attr = initPrimitiveAttr(); } - auto inDesc = getParentEdgesAtPort(0)[0]->getMemory().GetDescWithType(); - auto outDesc = getChildEdgesAtPort(0)[0]->getMemory().GetDescWithType(); + if (useACL) { + auto& dstMemPtr = getChildEdgeAt(0)->getMemoryPtr(); + auto& srcMemPtr = getParentEdgeAt(0)->getMemoryPtr(); + if (!dstMemPtr || !dstMemPtr->isAllocated()) + IE_THROW() << "Destination memory didn't allocate."; + if (!srcMemPtr || !srcMemPtr->isAllocated()) + IE_THROW() << "Input memory didn't allocate."; - if (isDynamicNode()) { - if (auto_pad) { - data_pad_begin = shapeInference->get_pads_begin(); - data_pad_end = shapeInference->get_pads_end(); + std::vector srcMemoryDescs; + for (int i = 0; i < getOriginalInputsNumber(); i++) { + srcMemoryDescs.push_back(getParentEdgeAt(i)->getMemoryPtr()->getDescPtr()); + } + std::vector dstMemoryDescs; + for (int i = 0; i < getOriginalOutputsNumber(); i++) { + dstMemoryDescs.push_back(getChildEdgeAt(i)->getMemoryPtr()->getDescPtr()); } - initEffectiveAttributes(inDesc->getShape(), outDesc->getShape()); - } - dnnl::algorithm alg = getPoolingAlgorithm(); - PoolingKey key = {inDesc, - outDesc, - stride, - kernel, - effective_pad_begin, - effective_pad_end, - effective_dilation, - data_pad_end, - *attr, - alg, - selected_pd->getImplementationType()}; - auto engine = getEngine(); - auto builder = [&engine](const PoolingKey& key) -> executorPtr { - primitive_desc_iterator itpd = createDescriptorHelper(engine, - key.inp->getDnnlDesc(), - key.out->getDnnlDesc(), - key.alg, - key.stride, - key.kernel, - key.effective_pad_begin, - key.effective_pad_end, - key.effective_dilation, - key.data_pad_end, - key.attr); - dnnl::pooling_forward::primitive_desc prim_desc = itpd.get(); - while (static_cast(itpd)) { - impl_desc_type impl_type = parse_impl_name(itpd.impl_info_str()); + execPtr = selected_pd->getExecutorFactoryAs()->makeExecutor(poolingAttrs, + srcMemoryDescs, + dstMemoryDescs, + *attr); + selected_pd->setImplementationType(execPtr->getImplType()); + } else { + auto inDesc = getParentEdgesAtPort(0)[0]->getMemory().GetDescWithType(); + auto outDesc = getChildEdgesAtPort(0)[0]->getMemory().GetDescWithType(); - if (impl_type == key.implType) { - prim_desc = itpd.get(); - break; + if (isDynamicNode()) { + if (poolingAttrs.auto_pad) { + poolingAttrs.data_pad_begin = shapeInference->get_pads_begin(); + poolingAttrs.data_pad_end = shapeInference->get_pads_end(); } - if (!itpd.next_impl()) - break; + initEffectiveAttributes(inDesc->getShape(), outDesc->getShape()); } - return std::make_shared(prim_desc); - }; + dnnl::algorithm alg = getPoolingAlgorithm(); + PoolingKey key = {inDesc, + outDesc, + poolingAttrs.stride, + poolingAttrs.kernel, + poolingAttrs.effective_pad_begin, + poolingAttrs.effective_pad_end, + poolingAttrs.effective_dilation, + poolingAttrs.data_pad_end, + *attr, + alg, + selected_pd->getImplementationType()}; + auto engine = getEngine(); + auto builder = [&engine](const PoolingKey& key) -> executorPtr { + primitive_desc_iterator itpd = createDescriptorHelper(engine, + key.inp->getDnnlDesc(), + key.out->getDnnlDesc(), + key.alg, + key.stride, + key.kernel, + key.effective_pad_begin, + key.effective_pad_end, + key.effective_dilation, + key.data_pad_end, + key.attr); + dnnl::pooling_forward::primitive_desc prim_desc = itpd.get(); + while (static_cast(itpd)) { + impl_desc_type impl_type = parse_impl_name(itpd.impl_info_str()); - auto cache = context->getParamsCache(); - auto result = cache->getOrCreate(key, builder); + if (impl_type == key.implType) { + prim_desc = itpd.get(); + break; + } + if (!itpd.next_impl()) + break; + } - execPtr = result.first; + return std::make_shared(prim_desc); + }; - if (!execPtr) { - IE_THROW() << "Primitive descriptor was not found for node " << getName() << "."; - } + auto cache = context->getParamsCache(); + auto result = cache->getOrCreate(key, builder); - auto scratchpadMem = getScratchPadMem(execPtr->getScratchPadDesc()); - primArgs[DNNL_ARG_SCRATCHPAD] = scratchpadMem->GetPrimitive(); - primArgs[DNNL_ARG_SRC] = getParentEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive(); - primArgs[DNNL_ARG_DST] = getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive(); + dnnlExecPtr = result.first; - Node::appendPostOpArgs(*attr, primArgs, postOpsArgs); + if (!dnnlExecPtr) { + IE_THROW() << "Primitive descriptor was not found for node " << getName() << "."; + } + + auto scratchpadMem = getScratchPadMem(dnnlExecPtr->getScratchPadDesc()); + primArgs[DNNL_ARG_SCRATCHPAD] = scratchpadMem->GetPrimitive(); + primArgs[DNNL_ARG_SRC] = getParentEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive(); + primArgs[DNNL_ARG_DST] = getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive(); + + Node::appendPostOpArgs(*attr, primArgs, postOpsArgs); #ifdef CPU_DEBUG_CAPS - if (result.second == CacheEntryBase::LookUpStatus::Miss) { - auto pd = execPtr->getPrimitiveDesc(); - DEBUG_LOG("verbose##", getName(), "##", DnnlExtensionUtils::query_pd_info(pd), "\n"); - } + if (result.second == CacheEntryBase::LookUpStatus::Miss) { + auto pd = dnnlExecPtr->getPrimitiveDesc(); + DEBUG_LOG("verbose##", getName(), "##", DnnlExtensionUtils::query_pd_info(pd), "\n"); + } #endif + } } void Pooling::execute(dnnl::stream strm) { - if (execPtr) { - execPtr->exec(primArgs, strm); + if (dnnlExecPtr) { + dnnlExecPtr->exec(primArgs, strm); + } else if (execPtr) { + std::vector srcMemory; + for (int i = 0; i < getOriginalInputsNumber(); i++) { + srcMemory.push_back(getParentEdgeAt(i)->getMemoryPtr()); + } + std::vector dstMemory; + for (int i = 0; i < getOriginalOutputsNumber(); i++) { + dstMemory.push_back(getChildEdgeAt(i)->getMemoryPtr()); + } + + execPtr->exec(srcMemory, dstMemory, postOpsArgs); } else { IE_THROW() << "Pooling node with name '" << getName() << "' doesn't have an initialized executor"; } @@ -439,20 +515,20 @@ bool Pooling::created() const { dnnl::algorithm Pooling::getPoolingAlgorithm() const { if (algorithm == Algorithm::PoolingAvg) { bool not_zero_l = false; - for (auto lr : data_pad_begin) { + for (auto lr : poolingAttrs.data_pad_begin) { if (lr) { not_zero_l = true; break; } } bool not_zero_r = false; - for (auto pr : data_pad_end) { + for (auto pr : poolingAttrs.data_pad_end) { if (pr) { not_zero_r = true; break; } } - if (!exclude_pad && (not_zero_l || not_zero_r)) + if (!poolingAttrs.exclude_pad && (not_zero_l || not_zero_r)) return dnnl::algorithm::pooling_avg_include_padding; else return dnnl::algorithm::pooling_avg_exclude_padding; @@ -474,12 +550,12 @@ dnnl::pooling_forward::primitive_desc Pooling::createDescriptorInternal( in_candidate, out_candidate, alg, - stride, - kernel, - effective_pad_begin, - effective_pad_end, - effective_dilation, - data_pad_end, + poolingAttrs.stride, + poolingAttrs.kernel, + poolingAttrs.effective_pad_begin, + poolingAttrs.effective_pad_end, + poolingAttrs.effective_dilation, + poolingAttrs.data_pad_end, *attr); } @@ -493,9 +569,9 @@ void Pooling::createDescriptor(const std::vector &inputDesc, if (!outDesc->isDefined()) { auto outDims = shapeInferGeneric({Shape(inDesc->getShape().getStaticDims())}); outDesc = outDesc->cloneWithNewDims(outDims[0]); - if (auto_pad) { - data_pad_begin = shapeInference->get_pads_begin(); - data_pad_end = shapeInference->get_pads_end(); + if (poolingAttrs.auto_pad) { + poolingAttrs.data_pad_begin = shapeInference->get_pads_begin(); + poolingAttrs.data_pad_end = shapeInference->get_pads_end(); } initEffectiveAttributes(inDesc->getShape(), outDesc->getShape()); } @@ -513,51 +589,90 @@ void Pooling::initSupportedPrimitiveDescriptors() { dnnl::primitive_attr attr; setPostOps(attr); - for (auto& desc : descs) { - auto itpd = desc; - - while (static_cast(itpd)) { + if (useACL) { + auto& creatorsMap = BlockedDescCreator::getCommonCreators(); + auto pushDesc = [&](LayoutType format) { NodeConfig config; - config.dynBatchSupport = true; - for (size_t i = 0; i < descInputNumbers(); i++) { - PortConfig dataConfig; - dataConfig.inPlace(-1); - dataConfig.constant(false); - dataConfig.setMemDesc(getSrcMemDesc(itpd, i)); + config.dynBatchSupport = false; + config.inConfs.resize(getParentEdges().size()); + config.outConfs.resize(getOriginalOutputsNumber()); - config.inConfs.push_back(dataConfig); + config.inConfs[0].setMemDesc( + creatorsMap.at(format)->createSharedDesc(getOriginalInputPrecisionAtPort(0), getInputShapeAtPort(0))); + config.outConfs[0].setMemDesc( + creatorsMap.at(format)->createSharedDesc(getOriginalOutputPrecisionAtPort(0), getOutputShapeAtPort(0))); + + std::vector srcMemoryDescs; + for (int i = 0; i < config.inConfs.size(); i++) { + srcMemoryDescs.push_back(config.inConfs[i].getMemDesc()); + } + std::vector dstMemoryDescs; + for (int i = 0; i < config.outConfs.size(); i++) { + dstMemoryDescs.push_back(config.outConfs[i].getMemDesc()); } - for (size_t i = 0; i < descOutputNumbers(); i++) { - PortConfig dataConfig; - dataConfig.inPlace(canBeInPlace() ? 0 : -1); - dataConfig.constant(false); - dataConfig.setMemDesc(getDstMemDesc(itpd, i)); + auto factory = std::make_shared( + poolingAttrs, + srcMemoryDescs, + dstMemoryDescs, + std::make_shared(context, getPrimitivesPriority())); + supportedPrimitiveDescriptors.emplace_back(config, impl_desc_type::undef, factory); + }; + pushDesc(LayoutType::ncsp); + } else { + for (auto& desc : descs) { + auto itpd = desc; - config.outConfs.push_back(dataConfig); + while (static_cast(itpd)) { + NodeConfig config; + config.dynBatchSupport = true; + for (size_t i = 0; i < descInputNumbers(); i++) { + PortConfig dataConfig; + dataConfig.inPlace(-1); + dataConfig.constant(false); + dataConfig.setMemDesc(getSrcMemDesc(itpd, i)); + + config.inConfs.push_back(dataConfig); + } + + for (size_t i = 0; i < descOutputNumbers(); i++) { + PortConfig dataConfig; + dataConfig.inPlace(canBeInPlace() ? 0 : -1); + dataConfig.constant(false); + dataConfig.setMemDesc(getDstMemDesc(itpd, i)); + + config.outConfs.push_back(dataConfig); + } + + // CPU plugin doesn't support second output of MaxPool-8, but anyway we should have out config for second port as stub + if (isMaxPool8) { + auto& creatorsMap = BlockedDescCreator::getCommonCreators(); + PortConfig dataConfig; + dataConfig.inPlace(-1); + dataConfig.constant(false); + dataConfig.setMemDesc(creatorsMap.at(LayoutType::ncsp)->createSharedDesc(config.outConfs.front().getMemDesc()->getPrecision(), + getOutputShapeAtPort(1))); + + config.outConfs.push_back(dataConfig); + } + + impl_desc_type impl_type = parse_impl_name(itpd.impl_info_str()); + + supportedPrimitiveDescriptors.emplace_back(config, impl_type); + if (!itpd.next_impl()) + break; } - - // CPU plugin doesn't support second output of MaxPool-8, but anyway we should have out config for second port as stub - if (isMaxPool8) { - auto& creatorsMap = BlockedDescCreator::getCommonCreators(); - PortConfig dataConfig; - dataConfig.inPlace(-1); - dataConfig.constant(false); - dataConfig.setMemDesc(creatorsMap.at(LayoutType::ncsp)->createSharedDesc(config.outConfs.front().getMemDesc()->getPrecision(), - getOutputShapeAtPort(1))); - - config.outConfs.push_back(dataConfig); - } - - impl_desc_type impl_type = parse_impl_name(itpd.impl_info_str()); - - supportedPrimitiveDescriptors.emplace_back(config, impl_type); - if (!itpd.next_impl()) - break; } } } +void Pooling::initDescriptor(const NodeConfig& config) { + if (useACL) + return; + + Node::initDescriptor(config); +} + Node::AttrPtr Pooling::initPrimitiveAttr() { auto attr = std::make_shared(dnnl::primitive_attr()); diff --git a/src/plugins/intel_cpu/src/nodes/pooling.h b/src/plugins/intel_cpu/src/nodes/pooling.h index 6d76e3d4898..255b9fe2142 100644 --- a/src/plugins/intel_cpu/src/nodes/pooling.h +++ b/src/plugins/intel_cpu/src/nodes/pooling.h @@ -12,6 +12,8 @@ #include #include "common/dnnl_executor.h" +#include "executors/pooling_list.hpp" + namespace ov { namespace intel_cpu { namespace node { @@ -25,6 +27,7 @@ public: std::vector getAvailableFormatsForDims(const Shape &dims) const override; void getSupportedDescriptors() override; void initSupportedPrimitiveDescriptors() override; + void initDescriptor(const NodeConfig& config) override; bool created() const override; bool canBeInPlace() const override { return false; @@ -41,10 +44,14 @@ protected: private: using executorPtr = std::shared_ptr; - executorPtr execPtr = nullptr; + executorPtr dnnlExecPtr = nullptr; void setPostOps(dnnl::primitive_attr &attr); + PoolingAttrs poolingAttrs; + + std::shared_ptr execPtr = nullptr; + void initEffectiveAttributes(const Shape &inDims, const Shape &outDims); dnnl::algorithm getPoolingAlgorithm() const; dnnl::pooling_forward::primitive_desc createDescriptorInternal(const dnnl::memory::desc& in_candidate, @@ -56,28 +63,7 @@ private: Shape inShape; bool isMaxPool8 = false; - bool auto_pad = false; - bool exclude_pad = false; - std::vector dilation; - std::vector stride; - std::vector kernel; - - /// Effective padding. Used to define correct output shape by oneDNN - /// reshape formula: (iw - kernel + pad_l + pad_r) / strides[i - 2] + 1 - /// should be passed into pooling desc constructor. - std::vector effective_pad_begin; - std::vector effective_pad_end; - - /// Effective dilation. Used to define correct dilation for OneDNN. - /// For OneDNN default dilation is vector of zero - std::vector effective_dilation; - - /// Effective pad value. Describe how much zero element added to input - /// data tensor. May be less than "Effective padding" values. - /// If pooling window is out of this padding, the region of averaging - /// is decreased. - std::vector data_pad_begin; - std::vector data_pad_end; + bool useACL = false; }; } // namespace node diff --git a/src/plugins/intel_cpu/src/nodes/rdft.cpp b/src/plugins/intel_cpu/src/nodes/rdft.cpp index 9fbbad965f1..b1ed6f16dca 100644 --- a/src/plugins/intel_cpu/src/nodes/rdft.cpp +++ b/src/plugins/intel_cpu/src/nodes/rdft.cpp @@ -733,7 +733,7 @@ std::vector> RDFTExecutor::generateTwiddles(const std::vector } return twiddles; } - +#if defined(OPENVINO_ARCH_X86_64) struct RDFTJitExecutor : public RDFTExecutor { RDFTJitExecutor(bool inverse, NodeDesc* primDesc) : RDFTExecutor(inverse) { enum dft_type rdftType = isInverse ? complex_to_real : real_to_complex; @@ -841,7 +841,7 @@ struct RDFTJitExecutor : public RDFTExecutor { int vlen; }; - +#endif struct RDFTRefExecutor : public RDFTExecutor { RDFTRefExecutor(bool inverse) : RDFTExecutor(inverse) {} @@ -987,12 +987,14 @@ void RDFT::createPrimitive() { auto buildExecutor = [&] (const RDFTKey& key) -> std::shared_ptr { std::shared_ptr executor; NodeDesc* primDesc = getSelectedPrimitiveDescriptor(); +#if defined(OPENVINO_ARCH_X86_64) if (mayiuse(cpu::x64::sse41)) { executor = std::make_shared(key.isInverse, primDesc); - } else { - executor = std::make_shared(key.isInverse); - primDesc->setImplementationType(ref_any); + return executor; } +#endif + executor = std::make_shared(key.isInverse); + primDesc->setImplementationType(ref_any); return executor; }; diff --git a/src/plugins/intel_cpu/src/nodes/rdft.h b/src/plugins/intel_cpu/src/nodes/rdft.h index 9cf7e195713..acba5d03115 100644 --- a/src/plugins/intel_cpu/src/nodes/rdft.h +++ b/src/plugins/intel_cpu/src/nodes/rdft.h @@ -8,7 +8,7 @@ #include #include #include -#include "kernels/rdft_kernel.hpp" +#include "kernels/x64/rdft_kernel.hpp" namespace ov { namespace intel_cpu { diff --git a/src/plugins/intel_cpu/src/nodes/reduce.cpp b/src/plugins/intel_cpu/src/nodes/reduce.cpp index 0eefdf73146..67ba6a637a1 100644 --- a/src/plugins/intel_cpu/src/nodes/reduce.cpp +++ b/src/plugins/intel_cpu/src/nodes/reduce.cpp @@ -12,7 +12,7 @@ #include #include #include "utils/bfloat16.hpp" -#include "emitters/jit_bf16_emitters.hpp" +#include "emitters/x64/jit_bf16_emitters.hpp" #include "ie_parallel.hpp" #include @@ -102,6 +102,8 @@ bool ReduceKey::operator==(const ReduceKey &rhs) const { } } // namespace +#if defined(OPENVINO_ARCH_X86_64) + // some utility functions static inline bool isFloatCompatible(memory::data_type type) { return memory::data_type::f32 == type || memory::data_type::bf16 == type; @@ -1673,6 +1675,8 @@ private: } }; +#endif // OPENVINO_ARCH_X86_64 + const std::map&, Reduce&)>> Reduce::initializers = { {ngraph::opset4::ReduceL1::get_type_info_static(), [](const std::shared_ptr& op, Reduce& node) { node.algorithm = Algorithm::ReduceL1; @@ -1835,14 +1839,48 @@ void Reduce::initSupportedPrimitiveDescriptors() { auto& creatorsMap = BlockedDescCreator::getCommonCreators(); auto pushDesc = [&](LayoutType inFormat, LayoutType outFormat, InferenceEngine::Precision inPrecision, - InferenceEngine::Precision outPrecision, impl_desc_type impl_type) { + InferenceEngine::Precision outPrecision, impl_desc_type impl_type, bool useAclExecutor = false) { config.inConfs[REDUCE_DATA].setMemDesc(creatorsMap.at(inFormat)->createSharedDesc(inPrecision, getInputShapeAtPort(REDUCE_DATA))); config.inConfs[REDUCE_INDEXES].setMemDesc(creatorsMap.at(LayoutType::ncsp)->createSharedDesc(InferenceEngine::Precision::I32, getInputShapeAtPort(REDUCE_INDEXES))); config.outConfs[0].setMemDesc(creatorsMap.at(outFormat)->createSharedDesc(outPrecision, getOutputShapeAtPort(0))); - supportedPrimitiveDescriptors.push_back({config, impl_type}); + + if (useAclExecutor) { + std::vector srcMemoryDescs; + for (int i = 0; i < config.inConfs.size(); i++) { + srcMemoryDescs.push_back(config.inConfs[i].getMemDesc()); + } + std::vector dstMemoryDescs; + for (int i = 0; i < config.outConfs.size(); i++) { + dstMemoryDescs.push_back(config.outConfs[i].getMemDesc()); + } + + auto factory = std::make_shared(reduceAttrs, srcMemoryDescs, dstMemoryDescs, + std::make_shared(context, getPrimitivesPriority())); + if (!factory->isEmpty()) { + supportedPrimitiveDescriptors.push_back({config, impl_type, factory}); + } + } else { + supportedPrimitiveDescriptors.push_back({config, impl_type}); + } }; +#if defined (OV_CPU_WITH_ACL) + reduceAttrs.operation = algorithm; + reduceAttrs.keepDims = keep_dims; + reduceAttrs.axes = raw_axes; + for (auto &axis : reduceAttrs.axes) { + if (axis < 0) + axis += static_cast(getInputShapeAtPort(REDUCE_DATA).getRank()); + } + // TODO: Per-channel layout is disabled due to accuracy issue in ACL Reduce Executor + // pushDesc(LayoutType::nspc, LayoutType::nspc, input_prec, output_prec, undef, true); + pushDesc(LayoutType::ncsp, LayoutType::ncsp, input_prec, output_prec, impl_desc_type::undef, true); + canUseAclExecutor = !supportedPrimitiveDescriptors.empty(); + if (canUseAclExecutor) + return; +#endif + if (jit_mode) { impl_desc_type impl_type = impl_desc_type::jit_sse42; if (mayiuse(cpu::x64::avx512_core)) { @@ -1882,6 +1920,21 @@ bool Reduce::isExecutable() const { } void Reduce::prepareParams() { + if (canUseAclExecutor) { + std::vector srcMemoryDescs; + for (int i = 0; i < getParentEdges().size(); i++) { + srcMemoryDescs.push_back(getParentEdgeAt(i)->getMemoryPtr()->getDescPtr()); + } + std::vector dstMemoryDescs; + dstMemoryDescs.push_back(getChildEdgeAt(0)->getMemoryPtr()->getDescPtr()); + + auto selectedPD = getSelectedPrimitiveDescriptor(); + aclExecPtr = selectedPD->getExecutorFactoryAs()->makeExecutor(reduceAttrs, srcMemoryDescs, dstMemoryDescs, {}); + selectedPD->setImplementationType(aclExecPtr->getImplType()); + + return; + } + src_dims = getParentEdgesAtPort(REDUCE_DATA)[0]->getMemory().getDesc().getShape().getDims(); std::vector reduce_axes; if (jit_mode && jit_beyond_5D) { @@ -1900,7 +1953,7 @@ void Reduce::prepareParams() { auto builder = [&](const ReduceKey& key) -> std::shared_ptr { std::shared_ptr post_kernel; - +#if defined(OPENVINO_ARCH_X86_64) if (mayiuse(cpu::x64::avx512_core)) { post_kernel.reset(new jit_uni_reduce_post_kernel_f32(key.jcp, *attr.get())); } else if (mayiuse(cpu::x64::avx2)) { @@ -1908,6 +1961,7 @@ void Reduce::prepareParams() { } else if (mayiuse(cpu::x64::sse41)) { post_kernel.reset(new jit_uni_reduce_post_kernel_f32(key.jcp, *attr.get())); } +#endif // OPENVINO_ARCH_X86_64 if (post_kernel) post_kernel->create_ker(); @@ -1982,7 +2036,7 @@ void Reduce::createPrimitive() { prepareParams(); updateLastInputDims(); } - +#if defined(OPENVINO_ARCH_X86_64) if (mayiuse(cpu::x64::avx512_core)) { reduce_kernel.reset(new jit_uni_reduce_kernel_f32(jcp)); } else if (mayiuse(cpu::x64::avx2)) { @@ -1990,6 +2044,7 @@ void Reduce::createPrimitive() { } else if (mayiuse(cpu::x64::sse41)) { reduce_kernel.reset(new jit_uni_reduce_kernel_f32(jcp)); } +#endif // OPENVINO_ARCH_X86_64 if (reduce_kernel) reduce_kernel->create_ker(); jit_mode = jit_mode && reduce_kernel; @@ -2011,6 +2066,15 @@ void Reduce::execute(dnnl::stream strm) { dst_data = reinterpret_cast(prc_mem.get_data_handle()); } reduce_type(src_data, dst_data, dst_size); + } else if (aclExecPtr) { + std::vector srcMemory; + for (int i = 0; i < getParentEdges().size(); i++) { + srcMemory.push_back(getParentEdgeAt(i)->getMemoryPtr()); + } + std::vector dstMemory; + dstMemory.push_back(getChildEdgeAt(0)->getMemoryPtr()); + + aclExecPtr->exec(srcMemory, dstMemory, postOpsDataPtrs.data()); } else { if (layout == ReduceLayoutType::reduce_ncsp) { auto in_ptr = reinterpret_cast(src_data); @@ -3018,4 +3082,4 @@ bool Reduce::created() const { } // namespace node } // namespace intel_cpu -} // namespace ov +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/reduce.h b/src/plugins/intel_cpu/src/nodes/reduce.h index b55a66f5dc4..683e46cbede 100644 --- a/src/plugins/intel_cpu/src/nodes/reduce.h +++ b/src/plugins/intel_cpu/src/nodes/reduce.h @@ -9,6 +9,7 @@ #include #include #include +#include "executors/reduce_list.hpp" namespace ov { namespace intel_cpu { @@ -166,8 +167,12 @@ private: static const std::map& op, Reduce& node)>> initializers; std::string errorPrefix; + + ReduceAttrs reduceAttrs; + bool canUseAclExecutor = false; + std::shared_ptr aclExecPtr = nullptr; }; } // namespace node } // namespace intel_cpu -} // namespace ov +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/nodes/region_yolo.cpp b/src/plugins/intel_cpu/src/nodes/region_yolo.cpp index 4d7819236ab..3ebd0a5d093 100644 --- a/src/plugins/intel_cpu/src/nodes/region_yolo.cpp +++ b/src/plugins/intel_cpu/src/nodes/region_yolo.cpp @@ -12,7 +12,7 @@ #include #include "common/cpu_convert.h" #include -#include +#include "emitters/x64/jit_bf16_emitters.hpp" #include #include "utils/bfloat16.hpp" @@ -21,12 +21,14 @@ using namespace dnnl::impl::cpu; using namespace dnnl::impl::cpu::x64; using namespace dnnl::impl::utils; +#if defined(OPENVINO_ARCH_X86_64) #define GET_OFF(field) offsetof(jit_args_logistic, field) +#endif namespace ov { namespace intel_cpu { namespace node { - +#if defined(OPENVINO_ARCH_X86_64) template struct jit_uni_logistic_kernel_f32 : public jit_uni_logistic_kernel, public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_logistic_kernel_f32) @@ -226,6 +228,7 @@ private: } } }; +#endif bool RegionYolo::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { try { @@ -306,6 +309,7 @@ void RegionYolo::createPrimitive() { updateLastInputDims(); } +#if defined(OPENVINO_ARCH_X86_64) jit_logistic_config_params jcp; jcp.src_dt = jcp.dst_dt = output_prec; jcp.src_data_size = jcp.dst_data_size = output_prec.size(); @@ -322,10 +326,10 @@ void RegionYolo::createPrimitive() { block_size = 4; } - softmax_kernel = std::make_shared(input_prec, output_prec); - if (logistic_kernel) logistic_kernel->create_ker(); +#endif + softmax_kernel = std::make_shared(input_prec, output_prec); } inline float RegionYolo::logistic_scalar(float src) { diff --git a/src/plugins/intel_cpu/src/nodes/region_yolo.h b/src/plugins/intel_cpu/src/nodes/region_yolo.h index 660ec6d981b..da1a252edda 100644 --- a/src/plugins/intel_cpu/src/nodes/region_yolo.h +++ b/src/plugins/intel_cpu/src/nodes/region_yolo.h @@ -66,7 +66,7 @@ private: std::string errorPrefix; int block_size; - std::shared_ptr logistic_kernel; + std::shared_ptr logistic_kernel = nullptr; std::shared_ptr softmax_kernel; union U { diff --git a/src/plugins/intel_cpu/src/nodes/roi_align.cpp b/src/plugins/intel_cpu/src/nodes/roi_align.cpp index b86f6fa22ba..fea9f041e4e 100644 --- a/src/plugins/intel_cpu/src/nodes/roi_align.cpp +++ b/src/plugins/intel_cpu/src/nodes/roi_align.cpp @@ -15,7 +15,7 @@ #include #include -#include "emitters/jit_load_store_emitters.hpp" +#include "emitters/x64/jit_load_store_emitters.hpp" using namespace InferenceEngine; using namespace dnnl; @@ -31,7 +31,7 @@ namespace node { using ngPoolingMode = ngraph::opset9::ROIAlign::PoolingMode; using ngAlignedMode = ngraph::opset9::ROIAlign::AlignedMode; - +#if defined(OPENVINO_ARCH_X86_64) #define GET_OFF(field) offsetof(jit_roi_align_call_args, field) template @@ -648,7 +648,7 @@ private: } } }; - +#endif bool ROIAlign::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { try { auto roiAlign = ngraph::as_type_ptr(op); @@ -749,7 +749,7 @@ void ROIAlign::createJitKernel(const InferenceEngine::Precision& dataPrec, const jcp.layout = selectLayout; jcp.pooled_h = pooledH; jcp.pooled_w = pooledW; - +#if defined(OPENVINO_ARCH_X86_64) if (mayiuse(cpu::x64::avx512_core)) { roi_align_kernel.reset(new jit_uni_roi_align_kernel_f32(jcp)); } else if (mayiuse(cpu::x64::avx2)) { @@ -757,9 +757,9 @@ void ROIAlign::createJitKernel(const InferenceEngine::Precision& dataPrec, const } else if (mayiuse(cpu::x64::sse41)) { roi_align_kernel.reset(new jit_uni_roi_align_kernel_f32(jcp)); } - if (roi_align_kernel) roi_align_kernel->create_ker(); +#endif } void ROIAlign::initSupportedPrimitiveDescriptors() { @@ -792,7 +792,6 @@ void ROIAlign::initSupportedPrimitiveDescriptors() { } else { impl_type = impl_desc_type::ref; } - std::vector> supportedFormats { {LayoutType::ncsp, LayoutType::ncsp} }; diff --git a/src/plugins/intel_cpu/src/nodes/roi_pooling.cpp b/src/plugins/intel_cpu/src/nodes/roi_pooling.cpp index 496307ede5b..b082635f2a3 100644 --- a/src/plugins/intel_cpu/src/nodes/roi_pooling.cpp +++ b/src/plugins/intel_cpu/src/nodes/roi_pooling.cpp @@ -12,7 +12,7 @@ #include "ie_parallel.hpp" #include "utils/bfloat16.hpp" -#include "emitters/jit_load_store_emitters.hpp" +#include "emitters/x64/jit_load_store_emitters.hpp" #include #include @@ -36,6 +36,7 @@ namespace ov { namespace intel_cpu { namespace node { +#if defined(OPENVINO_ARCH_X86_64) template struct jit_uni_roi_pooling_kernel_f32 : public jit_uni_roi_pooling_kernel, public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_roi_pooling_kernel_f32); @@ -308,6 +309,7 @@ private: L(exit_label); } }; +#endif namespace { struct RoiPoolingKey { @@ -532,6 +534,7 @@ template class ROIPooling::ROIPoolingJitExecutor : public ROIPooling::ROIPoolingExecutor { public: ROIPoolingJitExecutor(const jit_roi_pooling_params &jpp) { +#if defined(OPENVINO_ARCH_X86_64) if (mayiuse(cpu::x64::avx512_core)) { roi_pooling_kernel.reset(new jit_uni_roi_pooling_kernel_f32(jpp)); } else if (mayiuse(cpu::x64::avx2)) { @@ -544,6 +547,7 @@ public: if (roi_pooling_kernel) roi_pooling_kernel->create_ker(); +#endif } void exec( @@ -891,10 +895,12 @@ std::pair ROIPooling::ROIPoolingExecutor::getXYForBilinearMode( template std::shared_ptr ROIPooling::ROIPoolingExecutor::makeExecutor( const jit_roi_pooling_params& jpp) { +#if defined(OPENVINO_ARCH_X86_64) if (mayiuse(cpu::x64::sse41)) return std::make_shared>(jpp); - else - return std::make_shared>(jpp); +#endif + + return std::make_shared>(jpp); } bool ROIPooling::created() const { diff --git a/src/plugins/intel_cpu/src/nodes/subgraph.cpp b/src/plugins/intel_cpu/src/nodes/subgraph.cpp index 387b695211f..39087f278b7 100644 --- a/src/plugins/intel_cpu/src/nodes/subgraph.cpp +++ b/src/plugins/intel_cpu/src/nodes/subgraph.cpp @@ -22,14 +22,14 @@ #include #include "snippets/pass/matmul_to_brgemm.hpp" -#include "emitters/cpu_generator.hpp" #include "utils/cpu_utils.hpp" -#include "snippets_transformations/enforce_precision.hpp" -#include "snippets_transformations/fuse_load_store_and_convert.hpp" -#include "snippets_transformations/mul_add_to_fma.hpp" -#include "snippets_transformations/brgemm_to_brgemm_cpu.hpp" -#include "snippets_transformations/remove_converts.hpp" -#include "ngraph_transformations/convert_to_swish_cpu.hpp" +#include "emitters/x64/cpu_generator.hpp" +#include "transformations/snippets/x64/pass/fuse_load_store_and_convert.hpp" +#include "transformations/snippets/x64/pass/mul_add_to_fma.hpp" +#include "transformations/snippets/x64/pass/brgemm_to_brgemm_cpu.hpp" +#include "transformations/snippets/x64/pass/remove_converts.hpp" +#include "transformations/snippets/x64/pass/enforce_precision.hpp" +#include "transformations/cpu_opset/common/pass/convert_to_swish_cpu.hpp" using namespace InferenceEngine; using namespace dnnl::impl::utils; diff --git a/src/plugins/intel_cpu/src/nodes/subgraph.h b/src/plugins/intel_cpu/src/nodes/subgraph.h index be9724981d2..4b1abef7b2a 100644 --- a/src/plugins/intel_cpu/src/nodes/subgraph.h +++ b/src/plugins/intel_cpu/src/nodes/subgraph.h @@ -7,7 +7,8 @@ #include #include -#include "emitters/jit_snippets_emitters.hpp" +#include +#include "emitters/x64/jit_snippets_emitters.hpp" #include #include "snippets/op/subgraph.hpp" diff --git a/src/plugins/intel_cpu/src/nodes/topk.cpp b/src/plugins/intel_cpu/src/nodes/topk.cpp index 12fa4717735..68c51e0006f 100644 --- a/src/plugins/intel_cpu/src/nodes/topk.cpp +++ b/src/plugins/intel_cpu/src/nodes/topk.cpp @@ -9,7 +9,7 @@ #include #include #include -#include "emitters/jit_load_store_emitters.hpp" +#include "emitters/x64/jit_load_store_emitters.hpp" #include "ie_parallel.hpp" #include #include @@ -32,6 +32,7 @@ namespace ov { namespace intel_cpu { namespace node { +#if defined(OPENVINO_ARCH_X86_64) #define GET_OFF(field) offsetof(jit_topk_call_args, field) #define vmm_mask Vmm(0) @@ -1787,6 +1788,7 @@ private: } } }; +#endif bool TopK::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { try { @@ -1898,7 +1900,11 @@ void TopK::initSupportedPrimitiveDescriptors() { impl_type = impl_desc_type::ref; } +#if defined(OPENVINO_ARCH_X86_64) jit_mode = mayiuse(cpu::x64::sse41); +#else + jit_mode = false; +#endif static const Precision supportedPrecision[] = { Precision::FP32, @@ -1923,9 +1929,11 @@ void TopK::initSupportedPrimitiveDescriptors() { std::vector> dataFomats{ {LayoutType::ncsp, LayoutType::ncsp}, +#if defined(OPENVINO_ARCH_X86_64) {LayoutType::nspc, LayoutType::nspc}, {LayoutType::nCsp16c, LayoutType::nCsp16c}, {LayoutType::nCsp8c, LayoutType::nCsp8c} +#endif }; for (const auto &df : dataFomats) { @@ -2110,7 +2118,7 @@ void TopK::createPrimitive() { calc_bitonic_idx(top_k, jcp.bitonic_k_idx_cnt, false); } } - +#if defined(OPENVINO_ARCH_X86_64) if (mayiuse(cpu::x64::avx512_core)) { topk_kernel.reset(new jit_uni_topk_kernel_f32(jcp)); } else if (mayiuse(cpu::x64::avx2)) { @@ -2121,6 +2129,7 @@ void TopK::createPrimitive() { if (topk_kernel) topk_kernel->create_ker(); +#endif } } diff --git a/src/plugins/intel_cpu/src/nodes/topk.h b/src/plugins/intel_cpu/src/nodes/topk.h index 29060026e70..cb2d202012a 100644 --- a/src/plugins/intel_cpu/src/nodes/topk.h +++ b/src/plugins/intel_cpu/src/nodes/topk.h @@ -144,7 +144,7 @@ private: std::vector vec_process_ptr; std::vector vec_process_idx_ptr; - std::shared_ptr topk_kernel; + std::shared_ptr topk_kernel = nullptr; std::string errorPrefix; }; diff --git a/src/plugins/intel_cpu/src/nodes_factory.cpp b/src/plugins/intel_cpu/src/nodes_factory.cpp index 85a216c0560..3afe8aaa32c 100644 --- a/src/plugins/intel_cpu/src/nodes_factory.cpp +++ b/src/plugins/intel_cpu/src/nodes_factory.cpp @@ -105,20 +105,19 @@ Node::NodesFactory::NodesFactory() INTEL_CPU_NODE(Generic, Type::Generic); INTEL_CPU_NODE(CumSum, Type::CumSum); INTEL_CPU_NODE(Convolution, Type::Convolution); + INTEL_CPU_NODE(BinaryConvolution, Type::BinaryConvolution); INTEL_CPU_NODE(SpaceToBatch, Type::SpaceToBatch); INTEL_CPU_NODE(Lrn, Type::Lrn); INTEL_CPU_NODE(BatchToSpace, Type::BatchToSpace); - INTEL_CPU_NODE(NormalizeL2, Type::NormalizeL2); + INTEL_CPU_NODE(DepthToSpace, Type::DepthToSpace); + INTEL_CPU_NODE(SpaceToDepth, Type::SpaceToDepth); INTEL_CPU_NODE(If, Type::If); - INTEL_CPU_NODE(Proposal, Type::Proposal); INTEL_CPU_NODE(Broadcast, Type::Broadcast); INTEL_CPU_NODE(ExperimentalDetectronTopKROIs, Type::ExperimentalDetectronTopKROIs); INTEL_CPU_NODE(Reorder, Type::Reorder); - INTEL_CPU_NODE(BinaryConvolution, Type::BinaryConvolution); INTEL_CPU_NODE(MatrixNms, Type::MatrixNms); INTEL_CPU_NODE(AdaptivePooling, Type::AdaptivePooling); INTEL_CPU_NODE(Pooling, Type::Pooling); - INTEL_CPU_NODE(Reduce, Type::Reduce); INTEL_CPU_NODE(Eltwise, Type::Eltwise); INTEL_CPU_NODE(SoftMax, Type::Softmax); INTEL_CPU_NODE(EmbeddingBagPackedSum, Type::EmbeddingBagPackedSum); @@ -127,22 +126,16 @@ Node::NodesFactory::NodesFactory() INTEL_CPU_NODE(MemoryInput, Type::MemoryInput); INTEL_CPU_NODE(MemoryOutput, Type::MemoryOutput); INTEL_CPU_NODE(Tile, Type::Tile); - INTEL_CPU_NODE(DFT, Type::DFT); - INTEL_CPU_NODE(RDFT, Type::RDFT); INTEL_CPU_NODE(GatherTree, Type::GatherTree); - INTEL_CPU_NODE(SpaceToDepth, Type::SpaceToDepth); INTEL_CPU_NODE(FullyConnected, Type::FullyConnected); INTEL_CPU_NODE(CTCGreedyDecoder, Type::CTCGreedyDecoder); INTEL_CPU_NODE(Transpose, Type::Transpose); - INTEL_CPU_NODE(DeformableConvolution, Type::DeformableConvolution); INTEL_CPU_NODE(ReorgYolo, Type::ReorgYolo); INTEL_CPU_NODE(EmbeddingSegmentsSum, Type::EmbeddingSegmentsSum); INTEL_CPU_NODE(ShapeOf, Type::ShapeOf); INTEL_CPU_NODE(ExperimentalDetectronGenerateProposalsSingleImage, Type::ExperimentalDetectronGenerateProposalsSingleImage); INTEL_CPU_NODE(GenerateProposals, Type::GenerateProposals); INTEL_CPU_NODE(ReverseSequence, Type::ReverseSequence); - INTEL_CPU_NODE(FakeQuantize, Type::FakeQuantize); - INTEL_CPU_NODE(NonMaxSuppression, Type::NonMaxSuppression); INTEL_CPU_NODE(ExperimentalDetectronPriorGridGenerator, Type::ExperimentalDetectronPriorGridGenerator); INTEL_CPU_NODE(GatherND, Type::GatherND); INTEL_CPU_NODE(LogSoftmax, Type::LogSoftmax); @@ -159,6 +152,7 @@ Node::NodesFactory::NodesFactory() INTEL_CPU_NODE(Math, Type::Math); INTEL_CPU_NODE(MultiClassNms, Type::MulticlassNms); INTEL_CPU_NODE(Convert, Type::Convert); + INTEL_CPU_NODE(ColorConvert, Type::ColorConvert); INTEL_CPU_NODE(EmbeddingBagOffsetSum, Type::EmbeddingBagOffsetsSum); INTEL_CPU_NODE(Roll, Type::Roll); INTEL_CPU_NODE(Pad, Type::Pad); @@ -168,34 +162,42 @@ Node::NodesFactory::NodesFactory() INTEL_CPU_NODE(ScatterUpdate, Type::ScatterUpdate); INTEL_CPU_NODE(ScatterUpdate, Type::ScatterElementsUpdate); INTEL_CPU_NODE(ScatterUpdate, Type::ScatterNDUpdate); - INTEL_CPU_NODE(Interpolate, Type::Interpolate); - INTEL_CPU_NODE(ROIPooling, Type::ROIPooling); + INTEL_CPU_NODE(ShuffleChannels, Type::ShuffleChannels); INTEL_CPU_NODE(TensorIterator, Type::TensorIterator); INTEL_CPU_NODE(Concat, Type::Concatenation); - INTEL_CPU_NODE(ExtractImagePatches, Type::ExtractImagePatches); INTEL_CPU_NODE(OneHot, Type::OneHot); INTEL_CPU_NODE(ExperimentalDetectronDetectionOutput, Type::ExperimentalDetectronDetectionOutput); - INTEL_CPU_NODE(ROIAlign, Type::ROIAlign); - INTEL_CPU_NODE(ShuffleChannels, Type::ShuffleChannels); - INTEL_CPU_NODE(DepthToSpace, Type::DepthToSpace); INTEL_CPU_NODE(Deconvolution, Type::Deconvolution); - INTEL_CPU_NODE(Gather, Type::Gather); - INTEL_CPU_NODE(GridSample, Type::GridSample); - INTEL_CPU_NODE(RegionYolo, Type::RegionYolo); + INTEL_CPU_NODE(DeformableConvolution, Type::DeformableConvolution); INTEL_CPU_NODE(Range, Type::Range); - INTEL_CPU_NODE(TopK, Type::TopK); INTEL_CPU_NODE(StridedSlice, Type::StridedSlice); INTEL_CPU_NODE(GRN, Type::GRN); INTEL_CPU_NODE(NonZero, Type::NonZero); - INTEL_CPU_NODE(Snippet, Type::Subgraph); - INTEL_CPU_NODE(ColorConvert, Type::ColorConvert); + INTEL_CPU_NODE(NormalizeL2, Type::NormalizeL2); INTEL_CPU_NODE(PriorBox, Type::PriorBox); INTEL_CPU_NODE(PriorBoxClustered, Type::PriorBoxClustered); INTEL_CPU_NODE(Eye, Type::Eye); - INTEL_CPU_NODE(Interaction, Type::Interaction); - INTEL_CPU_NODE(MHA, Type::MHA); INTEL_CPU_NODE(Unique, Type::Unique); INTEL_CPU_NODE(Ngram, Type::Ngram); + INTEL_CPU_NODE(Interpolate, Type::Interpolate); + INTEL_CPU_NODE(Reduce, Type::Reduce); + INTEL_CPU_NODE(Gather, Type::Gather); + INTEL_CPU_NODE(NonMaxSuppression, Type::NonMaxSuppression); + INTEL_CPU_NODE(ROIPooling, Type::ROIPooling); + INTEL_CPU_NODE(ROIAlign, Type::ROIAlign); + INTEL_CPU_NODE(TopK, Type::TopK); + INTEL_CPU_NODE(Proposal, Type::Proposal); + INTEL_CPU_NODE(RegionYolo, Type::RegionYolo); + INTEL_CPU_NODE(DFT, Type::DFT); + INTEL_CPU_NODE(RDFT, Type::RDFT); + INTEL_CPU_NODE(ExtractImagePatches, Type::ExtractImagePatches); +#if defined(OPENVINO_ARCH_X86_64) + INTEL_CPU_NODE(FakeQuantize, Type::FakeQuantize); + INTEL_CPU_NODE(GridSample, Type::GridSample); + INTEL_CPU_NODE(Interaction, Type::Interaction); + INTEL_CPU_NODE(MHA, Type::MHA); + INTEL_CPU_NODE(Snippet, Type::Subgraph); +#endif } #undef INTEL_CPU_NODE diff --git a/src/plugins/intel_cpu/src/onednn/iml_type_mapper.cpp b/src/plugins/intel_cpu/src/onednn/iml_type_mapper.cpp index 8d1cf648462..8f33b058b9a 100644 --- a/src/plugins/intel_cpu/src/onednn/iml_type_mapper.cpp +++ b/src/plugins/intel_cpu/src/onednn/iml_type_mapper.cpp @@ -36,8 +36,10 @@ impl_desc_type parse_impl_name(std::string impl_desc_name) { SEARCH_WORD(any); SEARCH_WORD(_1x1); SEARCH_WORD(_dw); + SEARCH_WORD_2(dw, _dw); SEARCH_WORD(reorder); SEARCH_WORD(sparse); + SEARCH_WORD(acl); if ((res & impl_desc_type::avx2) != impl_desc_type::avx2 && (res & impl_desc_type::avx512) != impl_desc_type::avx512) SEARCH_WORD(avx); @@ -63,6 +65,7 @@ const char* impl_type_to_string(impl_desc_type type) { } while (0) CASE(unknown); CASE(undef); + CASE(ref); CASE(ref_any); CASE(reorder); CASE(gemm_any); @@ -110,6 +113,10 @@ const char* impl_type_to_string(impl_desc_type type) { CASE(brgemm_uni); CASE(brgemm_avx512_amx); CASE(brgemm_sparse_avx512_amx); + CASE(acl); + CASE(dw_acl); + CASE(gemm_acl); + CASE(winograd_acl); #undef CASE return "unknown"; diff --git a/src/plugins/intel_cpu/src/onednn/iml_type_mapper.h b/src/plugins/intel_cpu/src/onednn/iml_type_mapper.h index 646cb0f868d..d91b6c6e139 100644 --- a/src/plugins/intel_cpu/src/onednn/iml_type_mapper.h +++ b/src/plugins/intel_cpu/src/onednn/iml_type_mapper.h @@ -28,15 +28,16 @@ enum impl_desc_type { blas = 1<<17, any = 1<<18, uni = 1<<19, + acl = 1<<20, // Other specificator - _1x1 = 1<<20, - _dw = 1<<21, + _1x1 = 1<<21, + _dw = 1<<22, // Other info - reorder = 1<<22, + reorder = 1<<23, // winograd - winograd = 1<<23, + winograd = 1<<24, // sparse - sparse = 1<<24, + sparse = 1<<25, // real types ref_any = ref | any, @@ -47,7 +48,6 @@ enum impl_desc_type { gemm_avx2 = gemm | avx2, gemm_avx = gemm | avx, gemm_sse42 = gemm | sse42, - jit_gemm = jit | gemm, jit_avx512_winograd = jit | avx512 | winograd, @@ -93,6 +93,10 @@ enum impl_desc_type { brgemm_uni = brgemm | uni, brgemm_avx512_amx = brgemm | avx512 | amx, brgemm_sparse_avx512_amx = brgemm | sparse | avx512 | amx, + + dw_acl = _dw | acl, + gemm_acl = gemm | acl, + winograd_acl = winograd | acl, }; const char * impl_type_to_string(impl_desc_type type); diff --git a/src/plugins/intel_cpu/src/plugin.cpp b/src/plugins/intel_cpu/src/plugin.cpp index 6db86eeaf7b..7c8ac9603f9 100644 --- a/src/plugins/intel_cpu/src/plugin.cpp +++ b/src/plugins/intel_cpu/src/plugin.cpp @@ -7,7 +7,7 @@ #include "openvino/runtime/properties.hpp" #include "plugin.h" -#include "transformation_pipeline.h" +#include "transformations/transformation_pipeline.h" #include "itt.h" #include "extension_mngr.h" #include "extension.h" diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_group_conv.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_group_conv.cpp new file mode 100644 index 00000000000..d9270a324e4 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_group_conv.cpp @@ -0,0 +1,65 @@ +// Copyright (C) 2020-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + + +#include "convert_group_conv.hpp" + +#include + +#include +#include +#include + +ov::intel_cpu::ConvertGroupConvolution::ConvertGroupConvolution() { + auto gconv = ngraph::pattern::wrap_type(); + + ngraph::matcher_pass_callback callback = [](ngraph::pattern::Matcher& m) { + enum Inputs {Data, Weights}; + auto gconv = std::dynamic_pointer_cast(m.get_match_root()); + if (!gconv) { + return false; + } + + auto data_shape = gconv->get_input_shape(Inputs::Data); + // Weights layout GOIYX + size_t groups = gconv->get_input_shape(Inputs::Weights)[0]; + if (groups == data_shape.at(1) && groups == gconv->get_output_shape(0)[1]) { // depthwise case + return false; + } + + ngraph::NodeVector replace_nodes; + auto split_weights = std::make_shared(gconv->input_value(Inputs::Weights), + ov::opset8::Constant::create(ngraph::element::i64, ngraph::Shape{}, {0}), + groups); + replace_nodes.push_back(split_weights); + + auto axis = ov::opset8::Constant::create(ngraph::element::i64, ngraph::Shape{}, {1}); + auto split = std::make_shared(gconv->input_value(Inputs::Data), axis, groups); + replace_nodes.push_back(split); + + ngraph::NodeVector concat_inputs; + for (size_t g = 0; g < groups; g++) { + auto out = split->output(g); + auto filter = std::make_shared(split_weights->output(g), + ov::opset8::Constant::create(ngraph::element::i64, ngraph::Shape{}, {0})); + auto conv = std::make_shared(out, + filter, + gconv->get_strides(), + gconv->get_pads_begin(), + gconv->get_pads_end(), + gconv->get_dilations(), + gconv->get_auto_pad()); + concat_inputs.push_back(conv); + replace_nodes.push_back(conv); + } + auto concat = std::make_shared(concat_inputs, 1); + replace_nodes.push_back(concat); + + concat->set_friendly_name(gconv->get_friendly_name()); + ngraph::copy_runtime_info(gconv, replace_nodes); + ngraph::replace_node(gconv, concat); + return true; + }; + auto m = std::make_shared(gconv, "ConvertGroupConvolution"); + register_matcher(m, callback); +} diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_group_conv.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_group_conv.hpp new file mode 100644 index 00000000000..c26b4651af2 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_group_conv.hpp @@ -0,0 +1,18 @@ +// Copyright (C) 2020-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +namespace ov { +namespace intel_cpu { + +class ConvertGroupConvolution: public ngraph::pass::MatcherPass { +public: + OPENVINO_RTTI("ConvertGroupConvolution", "0"); + ConvertGroupConvolution(); +}; +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_group_conv1d.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_group_conv1d.cpp new file mode 100644 index 00000000000..8b1632354d8 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_group_conv1d.cpp @@ -0,0 +1,73 @@ +// Copyright (C) 2020-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + + +#include "convert_group_conv1d.hpp" + +#include + +#include +#include +#include +#include + +template +ngraph::matcher_pass_callback ov::intel_cpu::ConvertConv1DBase::convert_conv1d_to_conv2d() { + return [&](ngraph::pattern::Matcher& m) { + auto conv = std::dynamic_pointer_cast(m.get_match_root()); + if (!conv) { + return false; + } + + auto input_shape = conv->get_input_shape(0); + // is Conv1D + if (input_shape.size() != 3) { + return false; + } + + auto input = conv->input_value(0); + auto weights = conv->input_value(1); + auto input2d_shape = input_shape; + input2d_shape.push_back(1); + auto in2d_shape = std::make_shared(ngraph::element::i64, ngraph::Shape{4}, input2d_shape); + + auto weights2d_shape = weights.get_shape(); + weights2d_shape.push_back(1); + auto w_shape = std::make_shared(ngraph::element::i64, ngraph::Shape{weights2d_shape.size()}, weights2d_shape); + + auto input2d = std::make_shared(input, in2d_shape, true); + auto weights2d = std::make_shared(weights, w_shape, true); + + auto conv2d = std::make_shared(input2d, + weights2d, + ngraph::Strides{conv->get_strides()[0], 1}, + ngraph::CoordinateDiff{conv->get_pads_begin()[0], 0}, + ngraph::CoordinateDiff{conv->get_pads_end()[0], 0}, + ngraph::Strides{conv->get_dilations()[0], 1}, + conv->get_auto_pad()); + + auto in_shape = std::make_shared(ngraph::element::i64, ngraph::Shape{3}, conv->get_output_shape(0)); + auto reshape = std::make_shared(conv2d, in_shape, true); + + reshape->set_friendly_name(conv->get_friendly_name()); + ngraph::copy_runtime_info(conv, {input2d, weights2d, conv2d, reshape}); + ngraph::replace_node(conv, reshape); + return true; + }; +} + +ov::intel_cpu::ConvertConv1D::ConvertConv1D() { + auto m = std::make_shared( + ngraph::pattern::wrap_type({ngraph::pattern::any_input(ngraph::pattern::has_static_shape()), + ngraph::pattern::any_input(ngraph::pattern::has_static_shape())}, + ngraph::pattern::has_static_shape()), "ConvertConvolutionToArm"); + register_matcher(m, convert_conv1d_to_conv2d()); +} + +ov::intel_cpu::ConvertGroupConv1D::ConvertGroupConv1D() { + auto m = std::make_shared( + ngraph::pattern::wrap_type({ngraph::pattern::any_input(ngraph::pattern::has_static_shape()), + ngraph::pattern::any_input(ngraph::pattern::has_static_shape())}, + ngraph::pattern::has_static_shape()), "ConvertGroupConvolutionToArm"); + register_matcher(m, convert_conv1d_to_conv2d()); +} \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_group_conv1d.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_group_conv1d.hpp new file mode 100644 index 00000000000..c5f197e64f6 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_group_conv1d.hpp @@ -0,0 +1,29 @@ +// Copyright (C) 2020-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +namespace ov { +namespace intel_cpu { +class ConvertConv1DBase: public ngraph::pass::MatcherPass { +protected: + OPENVINO_RTTI("ConvertConv1DBase", "0"); + template + ngraph::matcher_pass_callback convert_conv1d_to_conv2d(); +}; + +class ConvertConv1D: public ConvertConv1DBase { +public: + OPENVINO_RTTI("ConvertConv1D", "0"); + ConvertConv1D(); +}; + +class ConvertGroupConv1D: public ConvertConv1DBase { +public: + OPENVINO_RTTI("ConvertGroupConv1D", "0"); + ConvertGroupConv1D(); +}; +} // namespace intel_cpu +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_reduce_multi_axis.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_reduce_multi_axis.cpp new file mode 100644 index 00000000000..e5bdf01d472 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_reduce_multi_axis.cpp @@ -0,0 +1,77 @@ +// Copyright (C) 2020-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + + +#include "convert_reduce_multi_axis.hpp" + +#include + +#include +#include + +template +ngraph::matcher_pass_callback ov::intel_cpu::ConvertReduceMultiAxisBase::convert_reduce() { + return [&](ngraph::pattern::Matcher& m) { + auto reduce = m.get_match_root(); + if (!std::dynamic_pointer_cast(reduce)) { + return false; + } + if (ngraph::shape_size(reduce->input_value(1).get_shape()) <= 1) { + return false; + } + auto reduction_axes = std::dynamic_pointer_cast(reduce->input_value(1).get_node_shared_ptr()); + if (!reduction_axes) { + return false; + } + auto axes = reduction_axes->cast_vector(); + ngraph::NodeVector new_ops; + std::shared_ptr node = reduce->input_value(0).get_node_shared_ptr(); + for (auto axis : axes) { + auto reduction_axis = ov::opset8::Constant::create(ngraph::element::i64, ngraph::Shape{}, {axis}); + node = std::make_shared(node, reduction_axis, true); + new_ops.push_back(node); + } + + auto out_shape = reduce->get_output_shape(0); + auto dst_shape = std::make_shared(ngraph::element::i64, ngraph::Shape{out_shape.size()}, + std::vector(out_shape.begin(), out_shape.end())); + auto reshape = std::make_shared(node, dst_shape, true); + + reshape->set_friendly_name(reduce->get_friendly_name()); + ngraph::copy_runtime_info(reduce, new_ops); + ngraph::replace_node(reduce, reshape); + return true; + }; +} + +ov::intel_cpu::ConvertReduceProd::ConvertReduceProd() { + auto m = std::make_shared( + ngraph::pattern::wrap_type({ngraph::pattern::any_input(ngraph::pattern::has_static_shape()), + ngraph::pattern::wrap_type()}, + ngraph::pattern::has_static_shape()), "ConvertReduceProd"); + register_matcher(m, convert_reduce()); +} + +ov::intel_cpu::ConvertReduceMin::ConvertReduceMin() { + auto m = std::make_shared( + ngraph::pattern::wrap_type({ngraph::pattern::any_input(ngraph::pattern::has_static_shape()), + ngraph::pattern::wrap_type()}, + ngraph::pattern::has_static_shape()), "ConvertReduceMin"); + register_matcher(m, convert_reduce()); +} + +ov::intel_cpu::ConvertReduceMax::ConvertReduceMax() { + auto m = std::make_shared( + ngraph::pattern::wrap_type({ngraph::pattern::any_input(ngraph::pattern::has_static_shape()), + ngraph::pattern::wrap_type()}, + ngraph::pattern::has_static_shape()), "ConvertReduceMax"); + register_matcher(m, convert_reduce()); +} + +ov::intel_cpu::ConvertReduceSum::ConvertReduceSum() { + auto m = std::make_shared( + ngraph::pattern::wrap_type({ngraph::pattern::any_input(ngraph::pattern::has_static_shape()), + ngraph::pattern::wrap_type()}, + ngraph::pattern::has_static_shape()), "ConvertReduceSum"); + register_matcher(m, convert_reduce()); +} diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_reduce_multi_axis.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_reduce_multi_axis.hpp new file mode 100644 index 00000000000..1b2087e8594 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_reduce_multi_axis.hpp @@ -0,0 +1,55 @@ +// Copyright (C) 2020-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +namespace ov { +namespace intel_cpu { + +class ConvertReduceMultiAxisBase: public ngraph::pass::MatcherPass { +public: + OPENVINO_RTTI("ConvertReduceMultiAxisBase", "0"); + template + ngraph::matcher_pass_callback convert_reduce(); +}; + +class ConvertReduceProd: public ConvertReduceMultiAxisBase { +public: + OPENVINO_RTTI("ConvertReduceProd", "0"); + ConvertReduceProd(); +}; + +class ConvertReduceMin: public ConvertReduceMultiAxisBase { +public: + OPENVINO_RTTI("ConvertReduceMin", "0"); + ConvertReduceMin(); +}; + +class ConvertReduceMax: public ConvertReduceMultiAxisBase { +public: + OPENVINO_RTTI("ConvertReduceMax", "0"); + ConvertReduceMax(); +}; + +class ConvertReduceSum: public ConvertReduceMultiAxisBase { +public: + OPENVINO_RTTI("ConvertReduceSum", "0"); + ConvertReduceSum(); +}; + +class ConvertReduceMultiAxis: public ngraph::pass::GraphRewrite { +public: + OPENVINO_RTTI("ConvertReduceMultiAxis", "0"); + ConvertReduceMultiAxis() { + add_matcher(); + add_matcher(); + add_matcher(); + add_matcher(); + } +}; + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/mish_decomposition.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/mish_decomposition.cpp new file mode 100644 index 00000000000..ad7d17682fe --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/mish_decomposition.cpp @@ -0,0 +1,33 @@ +// Copyright (C) 2020-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + + +#include "mish_decomposition.hpp" + +#include +#include + +ov::intel_cpu::MishDecomposition::MishDecomposition() { + auto mish = ngraph::pattern::wrap_type(); + + ngraph::matcher_pass_callback callback = [](ngraph::pattern::Matcher& m) { + auto mish = std::dynamic_pointer_cast(m.get_match_root()); + if (!mish) { + return false; + } + + auto exp = std::make_shared(mish->input_value(0)); + auto add = std::make_shared(exp, opset4::Constant::create(mish->get_output_element_type(0), ngraph::Shape{}, {1.0f})); + auto log = std::make_shared(add); + auto tanh = std::make_shared(log); + auto mul = std::make_shared(mish->input_value(0), tanh); + + mul->set_friendly_name(mish->get_friendly_name()); + ngraph::copy_runtime_info(mish, {exp, add, log, tanh, mul}); + ngraph::replace_node(mish, mul); + return true; + }; + + auto m = std::make_shared(mish, "MishDecomposition"); + register_matcher(m, callback); +} \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/mish_decomposition.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/mish_decomposition.hpp new file mode 100644 index 00000000000..b56870679ed --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/mish_decomposition.hpp @@ -0,0 +1,19 @@ +// Copyright (C) 2020-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +namespace ov { +namespace intel_cpu { + +class MishDecomposition: public ngraph::pass::MatcherPass { +public: + OPENVINO_RTTI("MishDecomposition", "0"); + MishDecomposition(); +}; + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/fully_connected.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/fully_connected.cpp similarity index 99% rename from src/plugins/intel_cpu/src/ngraph_transformations/op/fully_connected.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/fully_connected.cpp index 3cfe828237c..23316961d10 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/op/fully_connected.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/fully_connected.cpp @@ -3,7 +3,7 @@ // #include "fully_connected.hpp" -#include "../itt.hpp" +#include "transformations/itt.hpp" ov::intel_cpu::FullyConnectedNode::FullyConnectedNode(const ngraph::Output& A, const ngraph::Output& B, diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/fully_connected.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/fully_connected.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/op/fully_connected.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/fully_connected.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/leaky_relu.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/leaky_relu.cpp similarity index 97% rename from src/plugins/intel_cpu/src/ngraph_transformations/op/leaky_relu.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/leaky_relu.cpp index 0ea80495d01..a4859da189b 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/op/leaky_relu.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/leaky_relu.cpp @@ -3,7 +3,7 @@ // #include "leaky_relu.hpp" -#include "../itt.hpp" +#include "transformations/itt.hpp" ov::intel_cpu::LeakyReluNode::LeakyReluNode(const ngraph::Output &data, const float &negative_slope, diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/leaky_relu.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/leaky_relu.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/op/leaky_relu.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/leaky_relu.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/ngram.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/ngram.cpp similarity index 98% rename from src/plugins/intel_cpu/src/ngraph_transformations/op/ngram.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/ngram.cpp index b099b02945a..942bb81e0ed 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/op/ngram.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/ngram.cpp @@ -3,7 +3,7 @@ // #include "ngram.hpp" -#include "../itt.hpp" +#include "transformations/itt.hpp" ov::intel_cpu::NgramNode::NgramNode(const ov::Output& embeddings, const ov::Output& batch_idces, const size_t k) : Op({embeddings, batch_idces}), m_k(k) { diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/ngram.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/ngram.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/op/ngram.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/ngram.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/power_static.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/power_static.cpp similarity index 97% rename from src/plugins/intel_cpu/src/ngraph_transformations/op/power_static.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/power_static.cpp index 664850c51d4..2a5cd2a8102 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/op/power_static.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/power_static.cpp @@ -3,7 +3,7 @@ // #include "power_static.hpp" -#include "../itt.hpp" +#include "transformations/itt.hpp" ov::intel_cpu::PowerStaticNode::PowerStaticNode(const ngraph::Output &data, const float &power, diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/power_static.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/power_static.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/op/power_static.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/power_static.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/swish_cpu.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/swish_cpu.cpp similarity index 96% rename from src/plugins/intel_cpu/src/ngraph_transformations/op/swish_cpu.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/swish_cpu.cpp index 3111c1bbb89..c398181b0f1 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/op/swish_cpu.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/swish_cpu.cpp @@ -3,7 +3,7 @@ // #include "swish_cpu.hpp" -#include "../itt.hpp" +#include "transformations/itt.hpp" ov::intel_cpu::SwishNode::SwishNode(const ngraph::Output & input, const float alpha) : Op({input}), m_alpha(alpha) { diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/swish_cpu.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/swish_cpu.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/op/swish_cpu.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/op/swish_cpu.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/align_matmul_input_ranks.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/align_matmul_input_ranks.cpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/align_matmul_input_ranks.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/align_matmul_input_ranks.cpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/align_matmul_input_ranks.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/align_matmul_input_ranks.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/align_matmul_input_ranks.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/align_matmul_input_ranks.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_broadcast_to_tiles.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_broadcast_to_tiles.cpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_broadcast_to_tiles.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_broadcast_to_tiles.cpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_broadcast_to_tiles.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_broadcast_to_tiles.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_broadcast_to_tiles.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_broadcast_to_tiles.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_fq_rnn_to_quantized_rnn.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_fq_rnn_to_quantized_rnn.cpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_fq_rnn_to_quantized_rnn.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_fq_rnn_to_quantized_rnn.cpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_fq_rnn_to_quantized_rnn.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_fq_rnn_to_quantized_rnn.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_fq_rnn_to_quantized_rnn.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_fq_rnn_to_quantized_rnn.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_matmul_to_fc.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_matmul_to_fc.cpp similarity index 99% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_matmul_to_fc.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_matmul_to_fc.cpp index 4bb8e32e7eb..a9c8b1147dc 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/convert_matmul_to_fc.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_matmul_to_fc.cpp @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 // +#include "transformations/cpu_opset/common/op/fully_connected.hpp" #include "convert_matmul_to_fc.hpp" -#include "op/fully_connected.hpp" #include #include #include diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_matmul_to_fc.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_matmul_to_fc.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_matmul_to_fc.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_matmul_to_fc.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_tile_to_seq_tiles.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_tile_to_seq_tiles.cpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_tile_to_seq_tiles.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_tile_to_seq_tiles.cpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_tile_to_seq_tiles.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_tile_to_seq_tiles.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_tile_to_seq_tiles.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_tile_to_seq_tiles.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_leaky_relu.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_leaky_relu.cpp similarity index 96% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_to_leaky_relu.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_leaky_relu.cpp index e79480ae0ca..113efee5b53 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_leaky_relu.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_leaky_relu.cpp @@ -7,7 +7,7 @@ #include #include #include -#include "op/leaky_relu.hpp" +#include "transformations/cpu_opset/common/op/leaky_relu.hpp" #include "itt.hpp" diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_leaky_relu.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_leaky_relu.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_to_leaky_relu.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_leaky_relu.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_power_static.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_power_static.cpp similarity index 98% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_to_power_static.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_power_static.cpp index 03ab33f9edc..1588aff87e9 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_power_static.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_power_static.cpp @@ -10,8 +10,8 @@ #include #include #include -#include "op/power_static.hpp" -#include "op/fully_connected.hpp" +#include "transformations/cpu_opset/common/op/power_static.hpp" +#include "transformations/cpu_opset/common/op/fully_connected.hpp" #include "utils/general_utils.h" #include "itt.hpp" diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_power_static.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_power_static.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_to_power_static.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_power_static.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_swish_cpu.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_swish_cpu.cpp similarity index 96% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_to_swish_cpu.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_swish_cpu.cpp index 2d56c565427..45c311a111c 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_swish_cpu.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_swish_cpu.cpp @@ -7,7 +7,7 @@ #include #include #include -#include "op/swish_cpu.hpp" +#include "transformations/cpu_opset/common/op/swish_cpu.hpp" #include "itt.hpp" diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_swish_cpu.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_swish_cpu.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_to_swish_cpu.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/convert_to_swish_cpu.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/fc_bias_fusion.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/fc_bias_fusion.cpp similarity index 98% rename from src/plugins/intel_cpu/src/ngraph_transformations/fc_bias_fusion.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/fc_bias_fusion.cpp index 0bba5f8ecbc..993922b9ccf 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/fc_bias_fusion.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/fc_bias_fusion.cpp @@ -3,7 +3,7 @@ // #include "fc_bias_fusion.hpp" -#include "op/fully_connected.hpp" +#include "transformations/cpu_opset/common/op/fully_connected.hpp" #include #include #include diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/fc_bias_fusion.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/fc_bias_fusion.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/fc_bias_fusion.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/fc_bias_fusion.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/move_eltwise_up_data_movement.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.cpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/move_eltwise_up_data_movement.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.cpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/move_eltwise_up_data_movement.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/move_eltwise_up_data_movement.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/ngram_fusion.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/ngram_fusion.cpp similarity index 99% rename from src/plugins/intel_cpu/src/ngraph_transformations/ngram_fusion.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/ngram_fusion.cpp index 3038c95a5c0..c9b761470a1 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/ngram_fusion.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/ngram_fusion.cpp @@ -3,7 +3,7 @@ // #include "ngram_fusion.hpp" -#include "op/ngram.hpp" +#include "transformations/cpu_opset/common/op/ngram.hpp" #include #include #include @@ -11,7 +11,7 @@ #include #include -#include "itt.hpp" +#include "transformations/itt.hpp" using namespace ov::pass::pattern; ov::intel_cpu::NgramFusion::NgramFusion() { diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/ngram_fusion.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/ngram_fusion.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/ngram_fusion.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/ngram_fusion.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/reshape_fc_fusion.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/reshape_fc_fusion.cpp similarity index 98% rename from src/plugins/intel_cpu/src/ngraph_transformations/reshape_fc_fusion.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/reshape_fc_fusion.cpp index d95b354b75b..61ec1dbebb9 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/reshape_fc_fusion.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/reshape_fc_fusion.cpp @@ -3,7 +3,7 @@ // #include "reshape_fc_fusion.hpp" -#include "op/fully_connected.hpp" +#include "transformations/cpu_opset/common/op/fully_connected.hpp" #include #include #include diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/reshape_fc_fusion.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/reshape_fc_fusion.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/reshape_fc_fusion.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/reshape_fc_fusion.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/rnn_sequences_optimization.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/rnn_sequences_optimization.cpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/rnn_sequences_optimization.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/rnn_sequences_optimization.cpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/rnn_sequences_optimization.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/rnn_sequences_optimization.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/rnn_sequences_optimization.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/rnn_sequences_optimization.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/swap_convert_transpose.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/swap_convert_transpose.cpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/swap_convert_transpose.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/swap_convert_transpose.cpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/swap_convert_transpose.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/swap_convert_transpose.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/swap_convert_transpose.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/swap_convert_transpose.hpp diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/convert_to_cpu_specific_opset.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/convert_to_cpu_specific_opset.hpp new file mode 100644 index 00000000000..f883f382696 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/convert_to_cpu_specific_opset.hpp @@ -0,0 +1,57 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include "common/pass/fc_bias_fusion.hpp" +#include "ngraph/op/fake_quantize.hpp" +#include "ngraph/pass/manager.hpp" +#include "common/pass/reshape_fc_fusion.hpp" +#include "common/pass/align_matmul_input_ranks.hpp" +#include "transformations/common_optimizations/reshape_prelu.hpp" +#include "common/pass/convert_broadcast_to_tiles.hpp" +#include "common/pass/convert_tile_to_seq_tiles.hpp" +#include "common/pass/convert_matmul_to_fc.hpp" +#include "common/pass/convert_to_power_static.hpp" +#include "common/pass/convert_to_leaky_relu.hpp" +#include "common/pass/convert_to_swish_cpu.hpp" +#include "transformations/convert_precision.hpp" +#include "transformations/utils/utils.hpp" +#include "common/pass/rnn_sequences_optimization.hpp" +#include "transformations/common_optimizations/reshape_sequence_fusion.hpp" +#include "common/pass/ngram_fusion.hpp" +#include "transformations/defs.hpp" + +#include "itt.hpp" + +namespace ov { +namespace intel_cpu { + +inline void ConvertToCPUSpecificOpset(std::shared_ptr &nGraphFunc) { + RUN_ON_FUNCTION_SCOPE(ConvertToCPUSpecificOpset); + + ngraph::pass::Manager manager; + manager.set_per_pass_validation(false); + CPU_REGISTER_PASS_COMMON(manager, ConvertMatMulToFC); + CPU_REGISTER_PASS_COMMON(manager, AlignMatMulInputRanks); + CPU_REGISTER_PASS_COMMON(manager, ConvertTileToSeqTiles); + CPU_REGISTER_PASS_COMMON(manager, FullyConnectedBiasFusion); + CPU_REGISTER_PASS_X64(manager, ConvertToPowerStatic); + CPU_REGISTER_PASS_COMMON(manager, ConvertToLeakyRelu); + CPU_REGISTER_PASS_COMMON(manager, ConvertToSwishCPU); + CPU_REGISTER_PASS_COMMON(manager, OptimizeSequenceTransposes); + if (!ov::op::util::has_op_with_type(nGraphFunc)) { + CPU_REGISTER_PASS_COMMON(manager, ReshapeFullyConnectedFusion); + } + // after transformation "MoveEltwiseUpThroughDataMov" there can be Reshape sequences that should be eliminated or fused + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ReshapeSequenceFusion); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConstantFolding); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConvertPrecision, precisions_map {{ ngraph::element::i64, ngraph::element::i32 }}); + CPU_REGISTER_PASS_COMMON(manager, NgramFusion); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::Validate); + + manager.run_passes(nGraphFunc); +} + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/interaction.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/x64/op/interaction.cpp similarity index 98% rename from src/plugins/intel_cpu/src/ngraph_transformations/op/interaction.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/x64/op/interaction.cpp index f9bd033984a..11b55be9670 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/op/interaction.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/x64/op/interaction.cpp @@ -3,7 +3,7 @@ // #include "interaction.hpp" -#include "../itt.hpp" +#include "transformations/itt.hpp" ov::intel_cpu::InteractionNode::InteractionNode(const OutputVector& args) : Op(args) { diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/interaction.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/x64/op/interaction.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/op/interaction.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/x64/op/interaction.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/mha.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/x64/op/mha.cpp similarity index 99% rename from src/plugins/intel_cpu/src/ngraph_transformations/op/mha.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/x64/op/mha.cpp index 694051efc76..375342e2a97 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/op/mha.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/x64/op/mha.cpp @@ -3,7 +3,7 @@ // #include "mha.hpp" -#include "../itt.hpp" +#include "transformations/itt.hpp" #include #include diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/mha.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/x64/op/mha.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/op/mha.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/x64/op/mha.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_interaction.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/x64/pass/convert_to_interaction.cpp similarity index 99% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_to_interaction.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/x64/pass/convert_to_interaction.cpp index 8df9ebcc43a..7f7873e54c2 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_interaction.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/x64/pass/convert_to_interaction.cpp @@ -13,7 +13,7 @@ #include "itt.hpp" #include "ov_ops/type_relaxed.hpp" -#include "op/interaction.hpp" +#include "transformations/cpu_opset/x64/op/interaction.hpp" #include "simplify_fakequantize.hpp" ov::intel_cpu::ConvertToInteraction::ConvertToInteraction() { diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_interaction.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/x64/pass/convert_to_interaction.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/convert_to_interaction.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/x64/pass/convert_to_interaction.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/mha_fusion.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/x64/pass/mha_fusion.cpp similarity index 99% rename from src/plugins/intel_cpu/src/ngraph_transformations/mha_fusion.cpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/x64/pass/mha_fusion.cpp index 53b189d6fca..27782991d98 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/mha_fusion.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/x64/pass/mha_fusion.cpp @@ -8,7 +8,7 @@ #include #include #include -#include "op/mha.hpp" +#include "transformations/cpu_opset/x64/op/mha.hpp" #include "simplify_fakequantize.hpp" #include "itt.hpp" diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/mha_fusion.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/x64/pass/mha_fusion.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/mha_fusion.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/x64/pass/mha_fusion.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/simplify_fakequantize.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/x64/pass/simplify_fakequantize.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/simplify_fakequantize.hpp rename to src/plugins/intel_cpu/src/transformations/cpu_opset/x64/pass/simplify_fakequantize.hpp diff --git a/src/plugins/intel_cpu/src/transformations/defs.hpp b/src/plugins/intel_cpu/src/transformations/defs.hpp new file mode 100644 index 00000000000..d0e3fe1f000 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/defs.hpp @@ -0,0 +1,55 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +namespace ov { +namespace intel_cpu { + +#define CPU_REGISTER_PASS_COMMON(MANAGER, PASS, ...) \ + MANAGER.register_pass(__VA_ARGS__); + +#define CPU_DISABLE_PASS_COMMON(MANAGER, PASS) \ + MANAGER.get_pass_config()->disable(); + +#define CPU_ENABLE_PASS_COMMON(MANAGER, PASS) \ + MANAGER.get_pass_config()->enable(); + +#define CPU_SET_CALLBACK_COMMON(MANAGER, CALLBACK, ...) \ + MANAGER.get_pass_config()->set_callback<__VA_ARGS__>(CALLBACK); + +#if defined(OPENVINO_ARCH_X86_64) + +#define CPU_REGISTER_PASS_X64(MANAGER, PASS, ...) CPU_REGISTER_PASS_COMMON(MANAGER, PASS, __VA_ARGS__) +#define CPU_DISABLE_PASS_X64(MANAGER, PASS) CPU_DISABLE_PASS_COMMON(MANAGER, PASS) +#define CPU_ENABLE_PASS_X64(MANAGER, PASS) CPU_ENABLE_PASS_COMMON(MANAGER, PASS) +#define CPU_SET_CALLBACK_X64(MANAGER, CALLBACK, ...) CPU_SET_CALLBACK_COMMON(MANAGER, CALLBACK, __VA_ARGS__) + +#else + +#define CPU_REGISTER_PASS_X64(MANAGER, PASS, ...) +#define CPU_DISABLE_PASS_X64(MANAGER, PASS) +#define CPU_ENABLE_PASS_X64(MANAGER, PASS) +#define CPU_SET_CALLBACK_X64(MANAGER, CALLBACK, ...) + +#endif + +#if defined(OPENVINO_ARCH_ARM) || defined(OPENVINO_ARCH_ARM64) + +#define CPU_REGISTER_PASS_ARM(MANAGER, PASS, ...) CPU_REGISTER_PASS_COMMON(MANAGER, PASS, __VA_ARGS__) +#define CPU_DISABLE_PASS_ARM(MANAGER, PASS) CPU_DISABLE_PASS_COMMON(MANAGER, PASS) +#define CPU_ENABLE_PASS_ARM(MANAGER, PASS) CPU_ENABLE_PASS_COMMON(MANAGER, PASS) +#define CPU_SET_CALLBACK_ARM(MANAGER, CALLBACK, ...) CPU_SET_CALLBACK_COMMON(MANAGER, CALLBACK, __VA_ARGS__) + +#else + +#define CPU_REGISTER_PASS_ARM(MANAGER, PASS, ...) +#define CPU_DISABLE_PASS_ARM(MANAGER, PASS) +#define CPU_ENABLE_PASS_ARM(MANAGER, PASS) +#define CPU_SET_CALLBACK_ARM(MANAGER, CALLBACK, ...) + +#endif + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/itt.hpp b/src/plugins/intel_cpu/src/transformations/itt.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/itt.hpp rename to src/plugins/intel_cpu/src/transformations/itt.hpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/op/brgemm_copy_b.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_copy_b.cpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/op/brgemm_copy_b.cpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_copy_b.cpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/op/brgemm_copy_b.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_copy_b.hpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/op/brgemm_copy_b.hpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_copy_b.hpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/op/brgemm_cpu.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_cpu.cpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/op/brgemm_cpu.cpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_cpu.cpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/op/brgemm_cpu.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_cpu.hpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/op/brgemm_cpu.hpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/op/brgemm_cpu.hpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/op/fused_mul_add.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/fused_mul_add.cpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/op/fused_mul_add.cpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/op/fused_mul_add.cpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/op/fused_mul_add.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/fused_mul_add.hpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/op/fused_mul_add.hpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/op/fused_mul_add.hpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/op/load_convert.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/load_convert.cpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/op/load_convert.cpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/op/load_convert.cpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/op/load_convert.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/load_convert.hpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/op/load_convert.hpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/op/load_convert.hpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/op/store_convert.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/store_convert.cpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/op/store_convert.cpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/op/store_convert.cpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/op/store_convert.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/op/store_convert.hpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/op/store_convert.hpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/op/store_convert.hpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/brgemm_to_brgemm_cpu.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/brgemm_to_brgemm_cpu.cpp similarity index 97% rename from src/plugins/intel_cpu/src/snippets_transformations/brgemm_to_brgemm_cpu.cpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/pass/brgemm_to_brgemm_cpu.cpp index 63779e5848b..c1b8e2a4841 100644 --- a/src/plugins/intel_cpu/src/snippets_transformations/brgemm_to_brgemm_cpu.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/brgemm_to_brgemm_cpu.cpp @@ -7,8 +7,8 @@ #include "brgemm_to_brgemm_cpu.hpp" #include "snippets/snippets_isa.hpp" #include "snippets/utils.hpp" -#include "op/brgemm_copy_b.hpp" -#include "op/brgemm_cpu.hpp" +#include "transformations/snippets/x64/op/brgemm_copy_b.hpp" +#include "transformations/snippets/x64/op/brgemm_cpu.hpp" #include "ngraph/rt_info.hpp" #include "ngraph/pattern/op/wrap_type.hpp" diff --git a/src/plugins/intel_cpu/src/snippets_transformations/brgemm_to_brgemm_cpu.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/brgemm_to_brgemm_cpu.hpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/brgemm_to_brgemm_cpu.hpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/pass/brgemm_to_brgemm_cpu.hpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/enforce_precision.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/enforce_precision.cpp similarity index 98% rename from src/plugins/intel_cpu/src/snippets_transformations/enforce_precision.cpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/pass/enforce_precision.cpp index 5f8aca1ac9a..d2d28defdc8 100644 --- a/src/plugins/intel_cpu/src/snippets_transformations/enforce_precision.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/enforce_precision.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "snippets_transformations/enforce_precision.hpp" +#include "enforce_precision.hpp" #include #include diff --git a/src/plugins/intel_cpu/src/snippets_transformations/enforce_precision.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/enforce_precision.hpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/enforce_precision.hpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/pass/enforce_precision.hpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/fuse_load_store_and_convert.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/fuse_load_store_and_convert.cpp similarity index 97% rename from src/plugins/intel_cpu/src/snippets_transformations/fuse_load_store_and_convert.cpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/pass/fuse_load_store_and_convert.cpp index 0c64a20b655..6571d719268 100644 --- a/src/plugins/intel_cpu/src/snippets_transformations/fuse_load_store_and_convert.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/fuse_load_store_and_convert.cpp @@ -7,8 +7,8 @@ #include "fuse_load_store_and_convert.hpp" #include "snippets/snippets_isa.hpp" -#include "snippets_transformations/op/load_convert.hpp" -#include "snippets_transformations/op/store_convert.hpp" +#include "transformations/snippets/x64/op/load_convert.hpp" +#include "transformations/snippets/x64/op/store_convert.hpp" #include "ngraph/rt_info.hpp" #include "ngraph/pattern/op/wrap_type.hpp" diff --git a/src/plugins/intel_cpu/src/snippets_transformations/fuse_load_store_and_convert.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/fuse_load_store_and_convert.hpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/fuse_load_store_and_convert.hpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/pass/fuse_load_store_and_convert.hpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/mul_add_to_fma.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/mul_add_to_fma.cpp similarity index 96% rename from src/plugins/intel_cpu/src/snippets_transformations/mul_add_to_fma.cpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/pass/mul_add_to_fma.cpp index e666f2ba9e9..f3be754294b 100644 --- a/src/plugins/intel_cpu/src/snippets_transformations/mul_add_to_fma.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/mul_add_to_fma.cpp @@ -6,7 +6,7 @@ #include "mul_add_to_fma.hpp" #include "snippets/snippets_isa.hpp" -#include "op/fused_mul_add.hpp" +#include "transformations/snippets/x64/op/fused_mul_add.hpp" #include #include diff --git a/src/plugins/intel_cpu/src/snippets_transformations/mul_add_to_fma.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/mul_add_to_fma.hpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/mul_add_to_fma.hpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/pass/mul_add_to_fma.hpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/remove_converts.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/remove_converts.cpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/remove_converts.cpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/pass/remove_converts.cpp diff --git a/src/plugins/intel_cpu/src/snippets_transformations/remove_converts.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/remove_converts.hpp similarity index 100% rename from src/plugins/intel_cpu/src/snippets_transformations/remove_converts.hpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/pass/remove_converts.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/snippets_mark_skipped.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/snippets_mark_skipped.cpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/snippets_mark_skipped.cpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/pass/snippets_mark_skipped.cpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/snippets_mark_skipped.hpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/snippets_mark_skipped.hpp similarity index 100% rename from src/plugins/intel_cpu/src/ngraph_transformations/snippets_mark_skipped.hpp rename to src/plugins/intel_cpu/src/transformations/snippets/x64/pass/snippets_mark_skipped.hpp diff --git a/src/plugins/intel_cpu/src/transformation_pipeline.cpp b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp similarity index 57% rename from src/plugins/intel_cpu/src/transformation_pipeline.cpp rename to src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp index de6fd09844d..3ee53279478 100644 --- a/src/plugins/intel_cpu/src/transformation_pipeline.cpp +++ b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp @@ -3,6 +3,7 @@ // #include "transformation_pipeline.h" +#include "defs.hpp" // Operations #include "openvino/opsets/opset1.hpp" @@ -88,13 +89,17 @@ #include "low_precision/group_convolution.hpp" // CPU specific transformations -#include "ngraph_transformations/convert_to_cpu_specific_opset.hpp" -#include "ngraph_transformations/snippets_mark_skipped.hpp" -#include "ngraph_transformations/mha_fusion.hpp" -#include "ngraph_transformations/convert_to_interaction.hpp" -#include "ngraph_transformations/convert_fq_rnn_to_quantized_rnn.hpp" -#include "ngraph_transformations/move_eltwise_up_data_movement.hpp" -#include "ngraph_transformations/swap_convert_transpose.hpp" +#include "transformations/cpu_opset/convert_to_cpu_specific_opset.hpp" +#include "transformations/snippets/x64/pass/snippets_mark_skipped.hpp" +#include "transformations/cpu_opset/x64/pass/mha_fusion.hpp" +#include "transformations/cpu_opset/x64/pass/convert_to_interaction.hpp" +#include "transformations/cpu_opset/arm/pass/convert_group_conv.hpp" +#include "transformations/cpu_opset/arm/pass/convert_group_conv1d.hpp" +#include "transformations/cpu_opset/arm/pass/convert_reduce_multi_axis.hpp" +#include "transformations/cpu_opset/arm/pass/mish_decomposition.hpp" +#include "transformations/cpu_opset/common/pass/convert_fq_rnn_to_quantized_rnn.hpp" +#include "transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.hpp" +#include "transformations/cpu_opset/common/pass/swap_convert_transpose.hpp" // Snippets #include "snippets/pass/tokenization.hpp" @@ -186,11 +191,11 @@ void Transformations::PreLpt(const std::vector& defaultPrecis ov::pass::Manager manager; manager.set_per_pass_validation(false); - manager.register_pass(); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::InitNodeInfo); const bool useLpt = !defaultPrecisions.empty(); if (useLpt) { - manager.register_pass(defaultPrecisions); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::MarkDequantizationSubgraph, defaultPrecisions); } auto get_convert_precisions = []() { @@ -215,56 +220,59 @@ void Transformations::PreLpt(const std::vector& defaultPrecis static const auto precisions = get_convert_precisions(); type_to_fuse_map type_to_fuse = {{ov::opset10::Convert::get_type_info_static(), fuse_type_to_convert}}; - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::AUGRUCellFusion); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::CommonOptimizations); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::WrapInterpolateIntoTransposes); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::TransposeSinking); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConvertSequenceToTensorIterator); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConvertOpSet3ToOpSet2); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConvertOpSet2ToOpSet1); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::LSTMCellDecomposition); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::GRUCellDecomposition); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::RNNCellDecomposition); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConvertNMS1ToNMS9); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConvertNMS3ToNMS9); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConvertNMS4ToNMS9); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConvertNMS5ToNMS9); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConvertNMS9ToNMSIEInternal); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::Validate); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConvertMulticlassNmsToMulticlassNmsIE); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::Validate); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConvertMatrixNmsToMatrixNmsIE); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::Validate); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::TransposeMatMul); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConstantFolding); if (useLpt) { CPU_LPT_SCOPE(LowPrecisionTransformations_Part2); - manager.register_pass(defaultPrecisions); + CPU_REGISTER_PASS_COMMON(manager, ngraph::pass::low_precision::ConvertSubtractConstant, defaultPrecisions); } - manager.register_pass(); - manager.register_pass(precisions, type_to_fuse); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); - - auto pass_config = manager.get_pass_config(); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::Validate); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConvertPrecision, precisions, type_to_fuse); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::EliminateConvert); + CPU_REGISTER_PASS_COMMON(manager, SwapConvertTranspose); + CPU_REGISTER_PASS_X64(manager, ConvertToInteraction); + CPU_REGISTER_PASS_X64(manager, ConvertInteractionInt8); + CPU_REGISTER_PASS_ARM(manager, ConvertReduceMultiAxis); + CPU_REGISTER_PASS_ARM(manager, MishDecomposition); + CPU_REGISTER_PASS_ARM(manager, ConvertConv1D); + CPU_REGISTER_PASS_ARM(manager, ConvertGroupConv1D); + CPU_REGISTER_PASS_ARM(manager, ConvertGroupConvolution); // SpaceToDepth/ DepthToSpace node implementation supports only equal input/output tensors with rank <= 5 - pass_config->set_callback( - [](const_node_ptr &node) -> bool { - return node->input_value(0).get_shape().size() <= 5lu && - node->input_value(0).get_shape().size() == node->get_output_shape(0).size(); - }); + CPU_SET_CALLBACK_COMMON(manager, + [](const_node_ptr &node) -> bool { + return node->input_value(0).get_shape().size() <= 5lu && + node->input_value(0).get_shape().size() == node->get_output_shape(0).size(); + }, + ov::pass::ConvertSpaceToDepth, ov::pass::ConvertDepthToSpace); - pass_config->set_callback( - [](const_node_ptr &node) -> bool { - const auto & rank = node->input(0).get_partial_shape().rank().get_length(); - return rank == 4lu || rank == 5lu; - }); + CPU_SET_CALLBACK_COMMON(manager, + [](const_node_ptr &node) -> bool { + const auto & rank = node->input(0).get_partial_shape().rank().get_length(); + return rank == 4lu || rank == 5lu; + }, + ov::pass::ConvertBatchToSpace, ov::pass::ConvertSpaceToBatch); auto isCellPrimitiveSupported = [](const_node_ptr &node) -> bool { if (const auto &rnn_cell = std::dynamic_pointer_cast(node)) { @@ -329,99 +337,109 @@ void Transformations::PreLpt(const std::vector& defaultPrecis return false; }; - pass_config->set_callback( - [isSequencePrimitiveSupported](const_node_ptr &node) -> bool { - return isSequencePrimitiveSupported(node); - }); + CPU_SET_CALLBACK_COMMON(manager, + [isSequencePrimitiveSupported](const_node_ptr &node) -> bool { + return isSequencePrimitiveSupported(node); + }, + ov::pass::ConvertRNNSequenceToTensorIterator, + ov::pass::ConvertGRUSequenceToTensorIterator, + ov::pass::ConvertLSTMSequenceToTensorIterator); - pass_config->set_callback( - [isCellPrimitiveSupported](const_node_ptr &node) -> bool { - return isCellPrimitiveSupported(node); - }); + CPU_SET_CALLBACK_COMMON(manager, + [isCellPrimitiveSupported](const_node_ptr &node) -> bool { + return isCellPrimitiveSupported(node); + }, + ov::pass::RNNCellDecomposition, + ov::pass::GRUCellDecomposition, + ov::pass::LSTMCellDecomposition); - pass_config->set_callback( + CPU_SET_CALLBACK_COMMON(manager, [](const_node_ptr &node) -> bool { std::string errorMessage; return node::MVN::isSupportedOperation(node, errorMessage); - }); + }, + ov::pass::MVN6Decomposition); - pass_config->set_callback( + CPU_SET_CALLBACK_COMMON(manager, [](const_node_ptr &node) -> bool { std::string errorMsg; return node::NormalizeL2::isSupportedOperation(node, errorMsg); - }); + }, + ov::pass::NormalizeL2Decomposition); - pass_config->enable(); - pass_config->set_callback( + CPU_ENABLE_PASS_COMMON(manager, ov::pass::SoftmaxDecomposition); + CPU_SET_CALLBACK_COMMON(manager, [](const_node_ptr &node) -> bool { return node->input_value(0).get_partial_shape().rank().get_length() <= 5; - }); + }, + ov::pass::SoftmaxDecomposition); // NMS-alike nodes are always transformed to NMSIEInternal node in case of legacy api, for compatibility. // And on the other hand in case of api 2.0, keep them internal dynamic for better performance and functionality. auto nmsCallback = [isLegacyApi](const_node_ptr &node) -> bool { return isLegacyApi ? false : true; }; - pass_config->set_callback(nmsCallback); - pass_config->set_callback(nmsCallback); - pass_config->set_callback(nmsCallback); + + CPU_SET_CALLBACK_COMMON(manager, nmsCallback, ov::pass::ConvertNMS9ToNMSIEInternal); + CPU_SET_CALLBACK_COMMON(manager, nmsCallback, ov::pass::ConvertMulticlassNmsToMulticlassNmsIE); + CPU_SET_CALLBACK_COMMON(manager, nmsCallback, ov::pass::ConvertMatrixNmsToMatrixNmsIE); // List of enabled/disabled transformations // Allow FP16 Converts to be folded and FP16 constants to be upgraded to FP32 data type - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::DisableDecompressionConvertConstantFolding); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertCompressedOnlyToLegacy); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::EyeDecomposition); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertGELU); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertShuffleChannels3); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::Gelu7Downgrade); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::SoftPlusDecomposition); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertMod); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertShuffleChannels3); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::WeightsDequantizeToFakeQuantize); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::SimplifyCTCGreedyDecoderSeqLen); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertGather7ToGather1); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertGather8ToGather7); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertMinimum); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertBroadcastToTiles); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertReduceMeanToPooling); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertReduceMaxToPooling); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertReduceSumToPooling); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::SliceToStridedSlice); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertDetectionOutput8ToDetectionOutput1); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertROIAlign9To3); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::SoftSignDecomposition); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::UniqueDecomposition); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertTopK3); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::ConvertTopK11ToTopK3); + CPU_DISABLE_PASS_COMMON(manager, ov::pass::HSwishDecomposition); + CPU_DISABLE_PASS_X64(manager, ov::pass::HSigmoidDecomposition); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); - pass_config->disable(); + CPU_DISABLE_PASS_X64(manager, ov::pass::ReduceL1Decomposition); + CPU_DISABLE_PASS_X64(manager, ov::pass::ReduceL2Decomposition); - pass_config->enable(); - pass_config->enable(); - pass_config->enable(); - pass_config->enable(); - pass_config->enable(); + CPU_ENABLE_PASS_COMMON(manager, ov::pass::NormalizeL2Decomposition); + CPU_ENABLE_PASS_COMMON(manager, ov::pass::ConvertInterpolate1ToInterpolate4); + CPU_ENABLE_PASS_COMMON(manager, ov::pass::ConvertGather1ToGather7); + CPU_ENABLE_PASS_COMMON(manager, ov::pass::ConvertDetectionOutput1ToDetectionOutput8); + CPU_ENABLE_PASS_COMMON(manager, ov::pass::ConvertROIAlign3To9); if (useLpt) { CPU_LPT_SCOPE(LowPrecisionTransformations_Part3); - pass_config->set_callback( - [](const_node_ptr &node) -> bool { - std::string errMsg; - return !node::FakeQuantize::isSupportedOperation(node, errMsg); - }); + CPU_SET_CALLBACK_COMMON(manager, + [](const_node_ptr &node) -> bool { + std::string errMsg; + return !node::FakeQuantize::isSupportedOperation(node, errMsg); + }, + ov::pass::AddFakeQuantizeFusion, + ov::pass::MulFakeQuantizeFusion, + ov::pass::FakeQuantizeMulFusion); - pass_config->set_callback([&defaultPrecisions](const_node_ptr &node) -> bool { - return ngraph::pass::low_precision::NetworkHelper::areQuantizeAndDequantizeSupportedForMultiply(node, defaultPrecisions); - }); + CPU_SET_CALLBACK_COMMON(manager, + [&defaultPrecisions](const_node_ptr &node) -> bool { + return ngraph::pass::low_precision::NetworkHelper::areQuantizeAndDequantizeSupportedForMultiply(node, defaultPrecisions); + }, + ov::pass::ConvertQuantizeDequantize); } manager.run_passes(model); @@ -482,23 +500,26 @@ void Transformations::Lpt(const bool hasINT16orINT32Levels, const std::vector( + CPU_REGISTER_PASS_COMMON(lptManager, ngraph::pass::low_precision::LowPrecision, supportedPrecisions, quantizationRestrictions, LayerTransformation::Params(updatePrecision, ov::element::f32, defaultPrecisions)); - lptManager.get_pass_config()->set_callback([](const_node_ptr& node) -> bool { - if (const auto mulitply = std::dynamic_pointer_cast(node)) { - return !MultiplyToGroupConvolutionTransformation::canBeTransformedToGroupConvolution(mulitply); - } - return false; - }); - lptManager.get_pass_config()->set_callback( + CPU_SET_CALLBACK_COMMON(lptManager, + [](const_node_ptr& node) -> bool { + if (const auto mulitply = std::dynamic_pointer_cast(node)) { + return !MultiplyToGroupConvolutionTransformation::canBeTransformedToGroupConvolution(mulitply); + } + return false; + }, + ngraph::pass::low_precision::MarkupPrecisions); + CPU_SET_CALLBACK_COMMON(lptManager, [&defaultPrecisions](const_node_ptr& node) -> bool { return LayerTransformation::isAsymmetricQuantization(node, defaultPrecisions) || WeightableLayerTransformation::isAsymmetricOnWeights(node, defaultPrecisions); - }); + }, + ngraph::pass::low_precision::ConvolutionBackpropDataTransformation); - lptManager.get_pass_config()->disable(); + CPU_DISABLE_PASS_COMMON(lptManager, ngraph::pass::low_precision::MultiplyToGroupConvolutionTransformation); lptManager.run_passes(model); } @@ -508,27 +529,31 @@ void Transformations::PostLpt() { ov::pass::Manager postLPTPassManager; postLPTPassManager.set_per_pass_validation(false); - postLPTPassManager.register_pass(); - postLPTPassManager.register_pass(); - postLPTPassManager.get_pass_config()->set_callback([](const_node_ptr &node) -> bool { - // UnrollTI transformation is disabled by default, is turned on by LowLatency transformation - return node->get_rt_info().count("UNROLL_TI") == 0; - }); - postLPTPassManager.register_pass(); - postLPTPassManager.get_pass_config()->set_callback([](const std::shared_ptr& node) -> bool { - if (node->get_input_size() >= 2) { - return node->get_input_element_type(1) == ov::element::i8 || node->get_input_element_type(1) == ov::element::u8; - } - return false; - }); + CPU_REGISTER_PASS_COMMON(postLPTPassManager, ov::pass::UnrollTensorIterator); + CPU_REGISTER_PASS_COMMON(postLPTPassManager, ov::pass::ReshapePRelu); + CPU_SET_CALLBACK_COMMON(postLPTPassManager, + [](const_node_ptr &node) -> bool { + // UnrollTI transformation is disabled by default, is turned on by LowLatency transformation + return node->get_rt_info().count("UNROLL_TI") == 0; + }, + ov::pass::UnrollTensorIterator); + CPU_REGISTER_PASS_COMMON(postLPTPassManager, MoveEltwiseUpThroughDataMov); + CPU_SET_CALLBACK_COMMON(postLPTPassManager, + [](const std::shared_ptr& node) -> bool { + if (node->get_input_size() >= 2) { + return node->get_input_element_type(1) == ov::element::i8 || node->get_input_element_type(1) == ov::element::u8; + } + return false; + }, + MoveEltwiseUpThroughDataMov); - postLPTPassManager.register_pass(); + CPU_REGISTER_PASS_COMMON(postLPTPassManager, ov::pass::ConstantFolding); // Snippets may brake MHA patterns so the fusion has to performed before - postLPTPassManager.register_pass(); - postLPTPassManager.register_pass(); - postLPTPassManager.get_pass_config()->set_callback + CPU_REGISTER_PASS_X64(postLPTPassManager, MHAFusion); + CPU_REGISTER_PASS_X64(postLPTPassManager, FuseFQtoInteraction); + + CPU_SET_CALLBACK_X64(postLPTPassManager, ([this](const std::shared_ptr& n) -> bool { std::string errorMessage; @@ -545,16 +570,17 @@ void Transformations::PostLpt() { } return false; - }); + }), + MHAFloatFusion, MHAFloatFusion2, MHAQuantFusion, MHAQuantFusion2); // Float MHA is supported by snippets now if (!enableBF16) { - postLPTPassManager.get_pass_config()->disable(); - postLPTPassManager.get_pass_config()->disable(); + CPU_DISABLE_PASS_X64(postLPTPassManager, MHAFloatFusion); + CPU_DISABLE_PASS_X64(postLPTPassManager, MHAFloatFusion2); } // Execute before snippets. Otherwise FQ will be converted to Subgraph - postLPTPassManager.register_pass(); + CPU_REGISTER_PASS_COMMON(postLPTPassManager, ConvertFqRnnToQuantizedRnn); postLPTPassManager.run_passes(model); } @@ -566,76 +592,78 @@ void Transformations::MainSnippets(void) { ngraph::pass::Manager snippetsManager; snippetsManager.set_per_pass_validation(false); if (snippetsMode != Config::SnippetsMode::IgnoreCallback) - snippetsManager.register_pass(enableBF16); - snippetsManager.register_pass(); + CPU_REGISTER_PASS_X64(snippetsManager, SnippetsMarkSkipped, enableBF16); + CPU_REGISTER_PASS_X64(snippetsManager, ngraph::snippets::pass::SnippetsTokenization); const bool isMHASupported = !enableBF16 && // TODO: Need to add BF16 support for MHA in Snippets dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core); // MHA has BRGEMM that is supported only on AVX512 platforms if (!isMHASupported) { - snippetsManager.get_pass_config()->disable(); + CPU_DISABLE_PASS_X64(snippetsManager, ngraph::snippets::pass::TokenizeMHASnippets); } if (snippetsMode != Config::SnippetsMode::IgnoreCallback) { - snippetsManager.get_pass_config()->set_callback( - [](const std::shared_ptr& n) -> bool { - const auto pshape = n->get_output_partial_shape(0); - const auto shape = pshape.get_shape(); - const auto parallel_work_amount = - std::accumulate(shape.rbegin() + 2, shape.rend(), 1, std::multiplies()); - const auto kernel_buffer_size = - std::accumulate(shape.rbegin(), shape.rbegin() + 2, 1, std::multiplies()) * - n->get_output_element_type(0).size(); - // Heuristic values: - // parallelism work amount - not enough work amount for parallelism - // kernel work amount - large shape for kernel execution, not cache-local - // TODO: The heuristics will be removed after - // - loop blocking support on code generation level - // - parallelism support on JIT level - const auto needed_num_of_threads = 12lu; - const auto l2_cache_size = dnnl::utils::get_cache_size(2, true); - const auto is_unsupported_parallel_work_amount = parallel_get_num_threads() / 2 > parallel_work_amount && - parallel_work_amount < needed_num_of_threads; - const auto is_unsupported_kernel_work_amount = kernel_buffer_size > l2_cache_size; - return is_unsupported_parallel_work_amount || is_unsupported_kernel_work_amount; - }); - snippetsManager.get_pass_config()->set_callback( - [](const std::shared_ptr& n) -> bool { - // CPU Plugin support Swish in Subgraph via conversion to SwichCPU which assumes second input to be constant - const bool is_unsupported_swish = - ov::is_type(n) && n->inputs().size() > 1 && - !ov::is_type(n->get_input_node_shared_ptr(1)); - // todo: general tokenization flow is not currently supported for these operations. - // they can be tokenized only as a part of complex patterns - const bool is_disabled_tokenization = (ov::is_type(n) || - ov::is_type(n) || - ov::is_type(n) || - ov::is_type(n) || - ov::is_type(n) || - ov::is_type(n)); - const auto& inputs = n->inputs(); - // todo: clarify whether we can evaluate snippets on const paths - const bool has_only_const_inputs = std::all_of(inputs.begin(), inputs.end(), - [](const ov::Input& in) { - return ov::is_type( - in.get_source_output().get_node_shared_ptr()); - }); - // todo: clarify whether we can evaluate snippets on inputs with larger ranks - auto rank_is_too_large = [](const ov::descriptor::Tensor& t) { - // callback is called has_supported_in_out(), so it's safe to assume that the shapes are static - return t.get_partial_shape().rank().get_length() > 6; - }; - const bool bad_input_rank = std::any_of(inputs.begin(), inputs.end(), - [&](const ov::Input& in) { - return rank_is_too_large(in.get_tensor()); + CPU_SET_CALLBACK_X64(snippetsManager, + [](const std::shared_ptr& n) -> bool { + const auto pshape = n->get_output_partial_shape(0); + const auto shape = pshape.get_shape(); + const auto parallel_work_amount = + std::accumulate(shape.rbegin() + 2, shape.rend(), 1, std::multiplies()); + const auto kernel_buffer_size = + std::accumulate(shape.rbegin(), shape.rbegin() + 2, 1, std::multiplies()) * + n->get_output_element_type(0).size(); + // Heuristic values: + // parallelism work amount - not enough work amount for parallelism + // kernel work amount - large shape for kernel execution, not cache-local + // TODO: The heuristics will be removed after + // - loop blocking support on code generation level + // - parallelism support on JIT level + const auto needed_num_of_threads = 12lu; + const auto l2_cache_size = dnnl::utils::get_cache_size(2, true); + const auto is_unsupported_parallel_work_amount = parallel_get_num_threads() / 2 > parallel_work_amount && + parallel_work_amount < needed_num_of_threads; + const auto is_unsupported_kernel_work_amount = kernel_buffer_size > l2_cache_size; + return is_unsupported_parallel_work_amount || is_unsupported_kernel_work_amount; + }, + ngraph::snippets::pass::TokenizeMHASnippets); + CPU_SET_CALLBACK_X64(snippetsManager, + [](const std::shared_ptr& n) -> bool { + // CPU Plugin support Swish in Subgraph via conversion to SwichCPU which assumes second input to be constant + const bool is_unsupported_swish = + ov::is_type(n) && n->inputs().size() > 1 && + !ov::is_type(n->get_input_node_shared_ptr(1)); + // todo: general tokenization flow is not currently supported for these operations. + // they can be tokenized only as a part of complex patterns + const bool is_disabled_tokenization = (ov::is_type(n) || + ov::is_type(n) || + ov::is_type(n) || + ov::is_type(n) || + ov::is_type(n) || + ov::is_type(n)); + const auto& inputs = n->inputs(); + // todo: clarify whether we can evaluate snippets on const paths + const bool has_only_const_inputs = std::all_of(inputs.begin(), inputs.end(), + [](const ov::Input& in) { + return ov::is_type( + in.get_source_output().get_node_shared_ptr()); }); - const auto& outputs = n->outputs(); - const bool bad_output_rank = std::any_of(outputs.begin(), outputs.end(), - [&](const ov::Output& out) { - return rank_is_too_large(out.get_tensor()); - }); - return has_only_const_inputs || bad_input_rank || bad_output_rank || is_unsupported_swish || - is_disabled_tokenization; - }); + // todo: clarify whether we can evaluate snippets on inputs with larger ranks + auto rank_is_too_large = [](const ov::descriptor::Tensor& t) { + // callback is called has_supported_in_out(), so it's safe to assume that the shapes are static + return t.get_partial_shape().rank().get_length() > 6; + }; + const bool bad_input_rank = std::any_of(inputs.begin(), inputs.end(), + [&](const ov::Input& in) { + return rank_is_too_large(in.get_tensor()); + }); + const auto& outputs = n->outputs(); + const bool bad_output_rank = std::any_of(outputs.begin(), outputs.end(), + [&](const ov::Output& out) { + return rank_is_too_large(out.get_tensor()); + }); + return has_only_const_inputs || bad_input_rank || bad_output_rank || is_unsupported_swish || + is_disabled_tokenization; + }, + ngraph::snippets::pass::TokenizeSnippets); } snippetsManager.run_passes(model); } @@ -643,12 +671,14 @@ void Transformations::MainSnippets(void) { void Transformations::PostSnippets(void) { ov::pass::Manager postSnippetsManager; postSnippetsManager.set_per_pass_validation(false); - postSnippetsManager.register_pass(); - postSnippetsManager.get_pass_config()->set_callback([](const_node_ptr& node) -> bool { - std::string errMsg; - return node::FakeQuantize::isSupportedOperation(node, errMsg); - }); - postSnippetsManager.register_pass(); + CPU_REGISTER_PASS_COMMON(postSnippetsManager, ov::pass::FakeQuantizeDecomposition); + CPU_SET_CALLBACK_COMMON(postSnippetsManager, + [](const_node_ptr& node) -> bool { + std::string errMsg; + return node::FakeQuantize::isSupportedOperation(node, errMsg); + }, + ov::pass::FakeQuantizeDecomposition); + CPU_REGISTER_PASS_COMMON(postSnippetsManager, ov::pass::ConstantFolding); postSnippetsManager.run_passes(model); } diff --git a/src/plugins/intel_cpu/src/transformation_pipeline.h b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.h similarity index 100% rename from src/plugins/intel_cpu/src/transformation_pipeline.h rename to src/plugins/intel_cpu/src/transformations/transformation_pipeline.h diff --git a/src/plugins/intel_cpu/src/utils/debug_capabilities.cpp b/src/plugins/intel_cpu/src/utils/debug_capabilities.cpp index b791f18cec1..b4bb869a0ca 100644 --- a/src/plugins/intel_cpu/src/utils/debug_capabilities.cpp +++ b/src/plugins/intel_cpu/src/utils/debug_capabilities.cpp @@ -93,6 +93,8 @@ void DebugLogEnabled::break_at(const std::string & log) { std::cout << "[ DEBUG ] " << " Debug log breakpoint hit" << std::endl; #if defined(_MSC_VER) __debugbreak(); +#elif defined(__APPLE__) || defined(___aarch64_) + __builtin_trap(); #else asm("int3"); #endif diff --git a/src/plugins/intel_cpu/src/utils/denormals.hpp b/src/plugins/intel_cpu/src/utils/denormals.hpp index 3efd6bbb4c1..3bfc2a4081f 100644 --- a/src/plugins/intel_cpu/src/utils/denormals.hpp +++ b/src/plugins/intel_cpu/src/utils/denormals.hpp @@ -4,7 +4,9 @@ #pragma once -#include +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) +# include +#endif #include "openvino/core/visibility.hpp" diff --git a/src/plugins/intel_cpu/tests/functional/single_layer_tests/activation.cpp b/src/plugins/intel_cpu/tests/functional/single_layer_tests/activation.cpp index c910695be13..0b95297adea 100644 --- a/src/plugins/intel_cpu/tests/functional/single_layer_tests/activation.cpp +++ b/src/plugins/intel_cpu/tests/functional/single_layer_tests/activation.cpp @@ -108,7 +108,7 @@ protected: inType = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(inPrecision); outType = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(outPrecision); - selectedType = getPrimitiveType() + "_" + netPrecision.name(); + selectedType = (activationType == ActivationTypes::Log ? "ref" : getPrimitiveType()) + "_" + netPrecision.name(); init_input_shapes(inputShapes); diff --git a/src/plugins/intel_cpu/tests/functional/single_layer_tests/eltwise.cpp b/src/plugins/intel_cpu/tests/functional/single_layer_tests/eltwise.cpp index 5dd0d742120..575440beec0 100644 --- a/src/plugins/intel_cpu/tests/functional/single_layer_tests/eltwise.cpp +++ b/src/plugins/intel_cpu/tests/functional/single_layer_tests/eltwise.cpp @@ -186,8 +186,10 @@ std::vector opTypes = { std::vector eltwiseOpTypesBinInp = { ngraph::helpers::EltwiseTypes::ADD, ngraph::helpers::EltwiseTypes::MULTIPLY, +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) // TODO: Fix CVS-105430 ngraph::helpers::EltwiseTypes::SUBTRACT, ngraph::helpers::EltwiseTypes::DIVIDE, +#endif ngraph::helpers::EltwiseTypes::FLOOR_MOD, ngraph::helpers::EltwiseTypes::SQUARED_DIFF, }; @@ -199,7 +201,11 @@ std::vector eltwiseOpTypesDiffInp = { // Differen ov::AnyMap additional_config; -std::vector netType = {ElementType::bf16, ElementType::f32}; +std::vector netType = { +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) + ElementType::bf16, +#endif + ElementType::f32}; std::vector cpuParams_4D = { CPUSpecificParams({nChw16c, nChw16c}, {nChw16c}, {}, {}), @@ -215,6 +221,7 @@ std::vector cpuParams_5D = { const std::vector fusingParamsSet{ emptyFusingSpec, +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) // eltwise fusingSigmoid, fusingPRelu1D, @@ -224,6 +231,7 @@ const std::vector fusingParamsSet{ fusingFakeQuantizePerTensorRelu, fusingFakeQuantizePerChannelRelu, fusingFQPerChannelSigmoidFQPerTensor +#endif }; std::vector> inShapes_4D = { @@ -329,14 +337,18 @@ INSTANTIATE_TEST_SUITE_P(smoke_CompareWithRefs_5D, EltwiseLayerCPUTest, params_5 std::vector eltwiseOpTypesI32 = { ngraph::helpers::EltwiseTypes::ADD, ngraph::helpers::EltwiseTypes::MULTIPLY, +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) // TODO: Fix CVS-105430 ngraph::helpers::EltwiseTypes::SUBTRACT, ngraph::helpers::EltwiseTypes::DIVIDE, +#endif ngraph::helpers::EltwiseTypes::SQUARED_DIFF, }; const std::vector fusingParamsSetI32{ emptyFusingSpec, +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) fusingMultiplyAddPerChannel, +#endif }; const auto params_5D_emptyCPUSpec_I32 = ::testing::Combine( @@ -353,8 +365,9 @@ const auto params_5D_emptyCPUSpec_I32 = ::testing::Combine( ::testing::Values(emptyCPUSpec), ::testing::ValuesIn(fusingParamsSetI32)); +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) INSTANTIATE_TEST_SUITE_P(smoke_CompareWithRefs_5D_I32, EltwiseLayerCPUTest, params_5D_emptyCPUSpec_I32, EltwiseLayerCPUTest::getTestCaseName); - +#endif std::vector> inShapes_4D_Blocked_Planar = { {{2, 17, 31, 3}, {2, 1, 31, 3}}, @@ -558,7 +571,9 @@ INSTANTIATE_TEST_SUITE_P(smoke_CompareWithRefs_5D_1D_Parameter, EltwiseLayerCPUT std::vector eltwiseOpTypesBinDyn = { ngraph::helpers::EltwiseTypes::ADD, ngraph::helpers::EltwiseTypes::MULTIPLY, +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) // TODO: Fix CVS-105430 ngraph::helpers::EltwiseTypes::SUBTRACT, +#endif ngraph::helpers::EltwiseTypes::SQUARED_DIFF, }; diff --git a/src/plugins/intel_cpu/tests/functional/single_layer_tests/fake_quantize.cpp b/src/plugins/intel_cpu/tests/functional/single_layer_tests/fake_quantize.cpp index 823f60bdbce..6d1ae4a81a6 100644 --- a/src/plugins/intel_cpu/tests/functional/single_layer_tests/fake_quantize.cpp +++ b/src/plugins/intel_cpu/tests/functional/single_layer_tests/fake_quantize.cpp @@ -199,7 +199,7 @@ std::vector rangesShapes4D_jit = { {{1, 16, 1, 1}, {1, 16, 1, 1}, {1, 16, 1, 1}, {1, 16, 1, 1}} }, }; - +#if defined(OPENVINO_ARCH_X86_64) const auto testParams4D_jit = ::testing::Combine(specificParams, ::testing::ValuesIn(rangesShapes4D_jit), ::testing::Values(Precision::FP32), @@ -207,7 +207,7 @@ const auto testParams4D_jit = ::testing::Combine(specificParams, ::testing::Values(false), ::testing::ValuesIn(filterCPUSpecificParams(memForm4D_jit))); INSTANTIATE_TEST_SUITE_P(smoke_FakeQuantizeLayerCPUTest_4D_jit, FakeQuantizeLayerCPUTest, testParams4D_jit, FakeQuantizeLayerCPUTest::getTestCaseName); - +#endif std::vector memForm4D_ref = { CPUSpecificParams({nchw}, {nchw}, {"ref_FP32"}, {"ref_FP32"}) @@ -232,7 +232,7 @@ const auto testParams4D_ref = ::testing::Combine(specificParams, ::testing::ValuesIn(memForm4D_ref)); INSTANTIATE_TEST_SUITE_P(smoke_FakeQuantizeLayerCPUTest_4D_ref, FakeQuantizeLayerCPUTest, testParams4D_ref, FakeQuantizeLayerCPUTest::getTestCaseName); - +#if defined(OPENVINO_ARCH_X86_64) std::vector memForm5D_jit = { CPUSpecificParams({ncdhw}, {ncdhw}, {}, {}), CPUSpecificParams({ndhwc}, {ndhwc}, {}, {}), @@ -266,7 +266,7 @@ const auto testParams5D_jit = ::testing::Combine(specificParams, ::testing::ValuesIn(filterCPUSpecificParams(memForm5D_jit))); INSTANTIATE_TEST_SUITE_P(smoke_FakeQuantizeLayerCPUTest_5D_jit, FakeQuantizeLayerCPUTest, testParams5D_jit, FakeQuantizeLayerCPUTest::getTestCaseName); - +#endif std::vector memForm5D_ref = { CPUSpecificParams({ncdhw}, {ncdhw}, {"ref_FP32"}, {"ref_FP32"}) diff --git a/src/plugins/intel_cpu/tests/functional/single_layer_tests/interpolate.cpp b/src/plugins/intel_cpu/tests/functional/single_layer_tests/interpolate.cpp index 21a96de9592..b728e310ebd 100644 --- a/src/plugins/intel_cpu/tests/functional/single_layer_tests/interpolate.cpp +++ b/src/plugins/intel_cpu/tests/functional/single_layer_tests/interpolate.cpp @@ -328,8 +328,10 @@ const std::vector cubeCoefs = { const std::vector interpolateFusingParamsSet{ emptyFusingSpec, +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) fusingSwish, fusingFakeQuantizePerTensorRelu, +#endif }; std::vector> filterAdditionalConfig() { @@ -458,6 +460,7 @@ const std::vector shapeParams4D_fixed_C = { } }; +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) INSTANTIATE_TEST_SUITE_P(smoke_InterpolateNN_Layout_PerChannelFuse_Test, InterpolateLayerCPUTest, ::testing::Combine( interpolateCasesNN_Smoke, @@ -477,6 +480,7 @@ INSTANTIATE_TEST_SUITE_P(InterpolateNN_Layout_PerChannelFuse_Test, InterpolateLa ::testing::Values(fusingFakeQuantizePerChannelRelu), ::testing::ValuesIn(filterAdditionalConfig())), InterpolateLayerCPUTest::getTestCaseName); +#endif const auto interpolateCasesLinearOnnx_Smoke = ::testing::Combine( ::testing::Values(ngraph::op::v4::Interpolate::InterpolateMode::LINEAR_ONNX), diff --git a/src/plugins/intel_cpu/tests/functional/single_layer_tests/mvn.cpp b/src/plugins/intel_cpu/tests/functional/single_layer_tests/mvn.cpp index 4c59dbd6b5f..ff957df357a 100644 --- a/src/plugins/intel_cpu/tests/functional/single_layer_tests/mvn.cpp +++ b/src/plugins/intel_cpu/tests/functional/single_layer_tests/mvn.cpp @@ -277,40 +277,62 @@ const std::vector epsilon = { const std::vector emptyReductionAxes = {{}}; -std::vector inpPrc = {ElementType::i8, ElementType::bf16, ElementType::f32}; -std::vector outPrc = {ElementType::bf16, ElementType::f32}; +std::vector inpPrc = { + ElementType::i8, + ElementType::f32, + #if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) + ElementType::bf16 + #endif +}; +std::vector outPrc = { + ElementType::f32, + #if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) + ElementType::bf16 + #endif +}; std::vector cpuParams_4D = { - CPUSpecificParams({nhwc}, {nhwc}, {}, {}), - CPUSpecificParams({nChw16c}, {nChw16c}, {}, {}), - CPUSpecificParams({nchw}, {nchw}, {}, {}) + CPUSpecificParams({nchw}, {nchw}, {}, {}), + #if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) + // TODO: enable nspc test cases for ARM + CPUSpecificParams({nhwc}, {nhwc}, {}, {}), + CPUSpecificParams({nChw16c}, {nChw16c}, {}, {}) + #endif }; std::vector cpuParams_5D = { - CPUSpecificParams({ndhwc}, {ndhwc}, {}, {}), - CPUSpecificParams({nCdhw16c}, {nCdhw16c}, {}, {}), - CPUSpecificParams({ncdhw}, {ncdhw}, {}, {}) + CPUSpecificParams({ncdhw}, {ncdhw}, {}, {}), + #if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) + // TODO: enable nspc test cases for ARM + CPUSpecificParams({ndhwc}, {ndhwc}, {}, {}), + CPUSpecificParams({nCdhw16c}, {nCdhw16c}, {}, {}) + #endif }; std::vector fusingParamsSet { - emptyFusingSpec, - /* activations */ - fusingRelu, - fusingElu, - fusingTanh, - fusingSwish, - /* FQ */ - fusingFakeQuantizePerTensorRelu, - /* another patterns */ - fusingAddPerTensor + emptyFusingSpec, + #if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) + /* activations */ + fusingRelu, + fusingElu, + fusingTanh, + fusingSwish, + /* FQ */ + fusingFakeQuantizePerTensorRelu, + /* another patterns */ + fusingAddPerTensor + #endif }; std::vector fusingParamsSetStaticShape { + emptyFusingSpec, + #if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) /* FQ */ fusingFakeQuantizePerChannel, fusingFakeQuantizePerChannelRelu, /* another patterns */ fusingScaleShift, + #endif }; const auto Mvn3D = ::testing::Combine( @@ -360,11 +382,14 @@ INSTANTIATE_TEST_SUITE_P(smoke_CompareWithRefs_Mvn5D, MvnLayerCPUTest, Mvn5D, Mv // 1D 2D case std::vector fusingUnaryEltwiseParamsSet { + emptyFusingSpec, + #if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) /* activations */ fusingRelu, fusingElu, fusingTanh, fusingSwish, + #endif }; const auto Mvn1D = ::testing::Combine( diff --git a/src/plugins/intel_cpu/tests/functional/single_layer_tests/normalize.cpp b/src/plugins/intel_cpu/tests/functional/single_layer_tests/normalize.cpp index 7b50b243100..2307d98ec63 100755 --- a/src/plugins/intel_cpu/tests/functional/single_layer_tests/normalize.cpp +++ b/src/plugins/intel_cpu/tests/functional/single_layer_tests/normalize.cpp @@ -107,21 +107,27 @@ namespace { /* ============= Common params ============= */ std::vector fusingParamsSet { emptyFusingSpec, +#if defined(OPENVINO_ARCH_X86_64) fusingMultiplyPerTensor, fusingRelu, fusingPReluPerChannel +#endif }; std::vector fusingParamsSetDynamic { emptyFusingSpec, +#if defined(OPENVINO_ARCH_X86_64) fusingMultiplyPerTensor, fusingRelu, fusingFakeQuantizePerTensor +#endif }; std::vector fusingParamsSetPerChannel { +#if defined(OPENVINO_ARCH_X86_64) fusingPReluPerChannel, fusingFakeQuantizePerChannel +#endif }; const float epsilon = 1e-4f; diff --git a/src/plugins/intel_cpu/tests/functional/single_layer_tests/reduce_ops.cpp b/src/plugins/intel_cpu/tests/functional/single_layer_tests/reduce_ops.cpp index f41e74dd422..7678d8dd430 100644 --- a/src/plugins/intel_cpu/tests/functional/single_layer_tests/reduce_ops.cpp +++ b/src/plugins/intel_cpu/tests/functional/single_layer_tests/reduce_ops.cpp @@ -304,15 +304,19 @@ std::vector> inputShapes_SmallChannel = { }; std::vector cpuParams_4D = { +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) CPUSpecificParams({nChw16c}, {nChw16c}, {}, {}), + CPUSpecificParams({nhwc}, {nhwc}, {}, {}), +#endif CPUSpecificParams({nchw}, {nchw}, {}, {}), - CPUSpecificParams({nhwc}, {nhwc}, {}, {}) }; std::vector cpuParams_5D = { +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) CPUSpecificParams({nCdhw16c}, {nCdhw16c}, {}, {}), + CPUSpecificParams({ndhwc}, {ndhwc}, {}, {}), +#endif CPUSpecificParams({ncdhw}, {ncdhw}, {}, {}), - CPUSpecificParams({ndhwc}, {ndhwc}, {}, {}) }; std::vector cpuParams_HybridLayout_4D = { @@ -392,6 +396,7 @@ const auto params_MultiAxis_5D = testing::Combine( testing::ValuesIn(filterCPUSpecificParams(cpuParams_5D)), testing::Values(emptyFusingSpec)); +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) const auto params_MultiAxis_4D_Hybrid = testing::Combine( testing::Combine( testing::ValuesIn(axesND), @@ -417,6 +422,7 @@ const auto params_MultiAxis_5D_Hybrid = testing::Combine( testing::ValuesIn(inputShapes_5D)), testing::ValuesIn(filterCPUSpecificParams(cpuParams_HybridLayout_5D)), testing::Values(emptyFusingSpec)); +#endif const auto params_MultiAxis_6D = testing::Combine( testing::Combine( @@ -478,6 +484,7 @@ INSTANTIATE_TEST_SUITE_P( ReduceCPULayerTest::getTestCaseName ); +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) INSTANTIATE_TEST_SUITE_P( smoke_Reduce_MultiAxis_4D_Hybrid_CPU, ReduceCPULayerTest, @@ -491,6 +498,7 @@ INSTANTIATE_TEST_SUITE_P( params_MultiAxis_5D_Hybrid, ReduceCPULayerTest::getTestCaseName ); +#endif INSTANTIATE_TEST_SUITE_P( smoke_Reduce_MultiAxis_6D_CPU, @@ -553,6 +561,7 @@ const auto params_MultiAxis_5D_Logical = testing::Combine( testing::ValuesIn(filterCPUSpecificParams(cpuParams_5D)), testing::Values(emptyFusingSpec)); +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) const auto params_MultiAxis_4D_Hybrid_Logical = testing::Combine( testing::Combine( testing::ValuesIn(axesND), @@ -578,6 +587,7 @@ const auto params_MultiAxis_5D_Hybrid_Logical = testing::Combine( testing::ValuesIn(inputShapes_5D)), testing::ValuesIn(filterCPUSpecificParams(cpuParams_HybridLayout_5D)), testing::Values(emptyFusingSpec)); +#endif const auto params_MultiAxis_6D_Logical = testing::Combine( testing::Combine( @@ -613,6 +623,7 @@ INSTANTIATE_TEST_SUITE_P( ReduceCPULayerTest::getTestCaseName ); +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) INSTANTIATE_TEST_SUITE_P( smoke_Reduce_MultiAxis_4D_Hybrid_Logical_CPU, ReduceCPULayerTest, @@ -626,6 +637,7 @@ INSTANTIATE_TEST_SUITE_P( params_MultiAxis_5D_Hybrid_Logical, ReduceCPULayerTest::getTestCaseName ); +#endif INSTANTIATE_TEST_SUITE_P( smoke_Reduce_MultiAxis_6D_Logical_CPU, @@ -635,6 +647,7 @@ INSTANTIATE_TEST_SUITE_P( ); /* ================================ 2.1 Fusion - KeepDims ================================ */ +#if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) const auto params_OneAxis_fusing = testing::Combine( testing::Combine( testing::ValuesIn(axes), @@ -755,6 +768,8 @@ INSTANTIATE_TEST_SUITE_P( params_MultiAxis_5D_Hybrid_fusing_KeepNoDims, ReduceCPULayerTest::getTestCaseName ); +#endif + } // namespace } // namespace CPULayerTestsDefinitions diff --git a/src/plugins/intel_cpu/tests/unit/CMakeLists.txt b/src/plugins/intel_cpu/tests/unit/CMakeLists.txt index 0f21ef99343..eb1a8fddd51 100644 --- a/src/plugins/intel_cpu/tests/unit/CMakeLists.txt +++ b/src/plugins/intel_cpu/tests/unit/CMakeLists.txt @@ -12,6 +12,16 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") ie_add_compiler_flags(/wd5051) endif() +if (NOT OPENVINO_ARCH_X86_64) + set(EXCLUDED_SOURCE_PATHS_FOR_NON_X64 ${CMAKE_CURRENT_SOURCE_DIR}/jit_kernel_test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/registers_pool.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ngraph_transformations/convert_to_interaction.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ngraph_transformations/snipptes_mark_skipped.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ngraph_transformations/mul_add_to_fma.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/snippets_transformations + ${CMAKE_CURRENT_SOURCE_DIR}/nodes/eltwise_node_test.cpp) +endif() + addIeTargetTest( NAME ${TARGET_NAME} ROOT ${CMAKE_CURRENT_SOURCE_DIR} @@ -24,6 +34,8 @@ addIeTargetTest( $ PRIVATE $/include + EXCLUDED_SOURCE_PATHS + ${EXCLUDED_SOURCE_PATHS_FOR_NON_X64} OBJECT_FILES ${OBJ_LIB} LINK_LIBRARIES diff --git a/src/plugins/intel_cpu/tests/unit/jit_kernel_test.cpp b/src/plugins/intel_cpu/tests/unit/jit_kernel_test.cpp index a9fc864a47a..53576c629bc 100644 --- a/src/plugins/intel_cpu/tests/unit/jit_kernel_test.cpp +++ b/src/plugins/intel_cpu/tests/unit/jit_kernel_test.cpp @@ -3,7 +3,7 @@ // #include -#include +#include #include using namespace ov::intel_cpu; diff --git a/src/plugins/intel_cpu/tests/unit/ngraph_transformations/convert_matmul_test.cpp b/src/plugins/intel_cpu/tests/unit/ngraph_transformations/convert_matmul_test.cpp index c08104dec58..e63711dc76d 100644 --- a/src/plugins/intel_cpu/tests/unit/ngraph_transformations/convert_matmul_test.cpp +++ b/src/plugins/intel_cpu/tests/unit/ngraph_transformations/convert_matmul_test.cpp @@ -11,9 +11,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include #include diff --git a/src/plugins/intel_cpu/tests/unit/ngraph_transformations/convert_to_interaction.cpp b/src/plugins/intel_cpu/tests/unit/ngraph_transformations/convert_to_interaction.cpp index 216a6a288a8..b380da6dee7 100644 --- a/src/plugins/intel_cpu/tests/unit/ngraph_transformations/convert_to_interaction.cpp +++ b/src/plugins/intel_cpu/tests/unit/ngraph_transformations/convert_to_interaction.cpp @@ -9,8 +9,8 @@ #include #include -#include -#include +#include +#include #include #include #include diff --git a/src/plugins/intel_cpu/tests/unit/ngraph_transformations/convert_to_leaky_relu_test.cpp b/src/plugins/intel_cpu/tests/unit/ngraph_transformations/convert_to_leaky_relu_test.cpp index 6e74e89018d..3c152a24cbd 100644 --- a/src/plugins/intel_cpu/tests/unit/ngraph_transformations/convert_to_leaky_relu_test.cpp +++ b/src/plugins/intel_cpu/tests/unit/ngraph_transformations/convert_to_leaky_relu_test.cpp @@ -9,8 +9,8 @@ #include #include -#include -#include +#include +#include #include #include #include diff --git a/src/plugins/intel_cpu/tests/unit/ngraph_transformations/move_eltwise_up_data_movement_test.cpp b/src/plugins/intel_cpu/tests/unit/ngraph_transformations/move_eltwise_up_data_movement_test.cpp index 7ac2fadbc64..2a826146267 100644 --- a/src/plugins/intel_cpu/tests/unit/ngraph_transformations/move_eltwise_up_data_movement_test.cpp +++ b/src/plugins/intel_cpu/tests/unit/ngraph_transformations/move_eltwise_up_data_movement_test.cpp @@ -11,7 +11,7 @@ #include #include "common_test_utils/ngraph_test_utils.hpp" -#include +#include using namespace testing; diff --git a/src/plugins/intel_cpu/tests/unit/ngraph_transformations/optimize_sequence_transposes_test.cpp b/src/plugins/intel_cpu/tests/unit/ngraph_transformations/optimize_sequence_transposes_test.cpp index 55342f19fbb..09066433da1 100644 --- a/src/plugins/intel_cpu/tests/unit/ngraph_transformations/optimize_sequence_transposes_test.cpp +++ b/src/plugins/intel_cpu/tests/unit/ngraph_transformations/optimize_sequence_transposes_test.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/plugins/intel_cpu/tests/unit/ngraph_transformations/snipptes_mark_skipped.cpp b/src/plugins/intel_cpu/tests/unit/ngraph_transformations/snipptes_mark_skipped.cpp index f3eaa9a38d6..5d038fd9b6d 100644 --- a/src/plugins/intel_cpu/tests/unit/ngraph_transformations/snipptes_mark_skipped.cpp +++ b/src/plugins/intel_cpu/tests/unit/ngraph_transformations/snipptes_mark_skipped.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include "snippets/pass/tokenization.hpp" namespace ov { diff --git a/src/plugins/intel_cpu/tests/unit/registers_pool.cpp b/src/plugins/intel_cpu/tests/unit/registers_pool.cpp index be81870e19f..d69144a9e6f 100644 --- a/src/plugins/intel_cpu/tests/unit/registers_pool.cpp +++ b/src/plugins/intel_cpu/tests/unit/registers_pool.cpp @@ -3,7 +3,7 @@ // #include -#include "nodes/kernels/registers_pool.hpp" +#include "nodes/kernels/x64/registers_pool.hpp" #include "common/nstl.hpp" using namespace ov::intel_cpu; diff --git a/src/plugins/intel_cpu/tests/unit/shape_inference_test/avg_pool_shape_inference_test.cpp b/src/plugins/intel_cpu/tests/unit/shape_inference_test/avg_pool_shape_inference_test.cpp index abb8dd8f6f3..c23e3efc4c8 100644 --- a/src/plugins/intel_cpu/tests/unit/shape_inference_test/avg_pool_shape_inference_test.cpp +++ b/src/plugins/intel_cpu/tests/unit/shape_inference_test/avg_pool_shape_inference_test.cpp @@ -42,9 +42,9 @@ TEST_F(AvgPoolV1StaticShapeInferenceTest, no_auto_pad_round_floor) { const auto data = std::make_shared(element::f64, PartialShape{-1, -1, -1, -1}); const Strides strides{1, 1}; - const Shape pads_begin{2, 2}; - const Shape pads_end{2, 1}; - const Shape kernel_shape{3, 2}; + const ov::Shape pads_begin{2, 2}; + const ov::Shape pads_end{2, 1}; + const ov::Shape kernel_shape{3, 2}; const auto rounding_mode = op::RoundingType::FLOOR; const auto pad_type = op::PadType::EXPLICIT; @@ -64,9 +64,9 @@ TEST_F(AvgPoolV1StaticShapeInferenceTest, auto_padding_same_lower_round_ceil) { const auto data = std::make_shared(element::f64, PartialShape::dynamic()); const Strides strides{1, 3, 2}; - const Shape pads_begin{2, 2, 1}; - const Shape pads_end{2, 1, 10}; - const Shape kernel_shape{5, 5, 5}; + const ov::Shape pads_begin{2, 2, 1}; + const ov::Shape pads_end{2, 1, 10}; + const ov::Shape kernel_shape{5, 5, 5}; const auto rounding_mode = op::RoundingType::CEIL; const auto pad_type = op::PadType::SAME_LOWER; @@ -86,9 +86,9 @@ TEST_F(AvgPoolV1StaticShapeInferenceTest, auto_padding_same_upper_round_floor_ex const auto data = std::make_shared(element::f64, PartialShape::dynamic()); const Strides strides{1, 3, 2}; - const Shape pads_begin{2, 2, 1}; - const Shape pads_end{2, 1, 10}; - const Shape kernel_shape{5, 5, 5}; + const ov::Shape pads_begin{2, 2, 1}; + const ov::Shape pads_end{2, 1, 10}; + const ov::Shape kernel_shape{5, 5, 5}; const auto rounding_mode = op::RoundingType::FLOOR; const auto pad_type = op::PadType::SAME_UPPER; @@ -108,9 +108,9 @@ TEST_F(AvgPoolV1StaticShapeInferenceTest, 5d_auto_padding_same_upper_round_floor const auto data = std::make_shared(element::f64, PartialShape::dynamic()); const Strides strides{1, 1, 1}; - const Shape pads_begin{0, 0, 0}; - const Shape pads_end{0, 0, 0}; - const Shape kernel_shape{2, 2, 2}; + const ov::Shape pads_begin{0, 0, 0}; + const ov::Shape pads_end{0, 0, 0}; + const ov::Shape kernel_shape{2, 2, 2}; const auto rounding_mode = op::RoundingType::FLOOR; const auto pad_type = op::PadType::SAME_UPPER; diff --git a/src/plugins/intel_cpu/tests/unit/shape_inference_test/max_pool_shape_inference_test.cpp b/src/plugins/intel_cpu/tests/unit/shape_inference_test/max_pool_shape_inference_test.cpp index bd3e6c8e7c9..b5957226850 100644 --- a/src/plugins/intel_cpu/tests/unit/shape_inference_test/max_pool_shape_inference_test.cpp +++ b/src/plugins/intel_cpu/tests/unit/shape_inference_test/max_pool_shape_inference_test.cpp @@ -42,9 +42,9 @@ TEST_F(MaxPoolV1StaticShapeInferenceTest, no_auto_pad_round_floor) { const auto data = std::make_shared(element::f64, PartialShape{-1, -1, -1, -1}); const Strides strides{1, 1}; - const Shape pads_begin{2, 2}; - const Shape pads_end{2, 1}; - const Shape kernel_shape{3, 2}; + const ov::Shape pads_begin{2, 2}; + const ov::Shape pads_end{2, 1}; + const ov::Shape kernel_shape{3, 2}; const auto rounding_mode = op::RoundingType::FLOOR; const auto pad_type = op::PadType::EXPLICIT; @@ -64,9 +64,9 @@ TEST_F(MaxPoolV1StaticShapeInferenceTest, auto_padding_same_lower_round_ceil) { const auto data = std::make_shared(element::f64, PartialShape::dynamic()); const Strides strides{1, 3, 2}; - const Shape pads_begin{2, 2, 1}; - const Shape pads_end{2, 1, 10}; - const Shape kernel_shape{5, 5, 5}; + const ov::Shape pads_begin{2, 2, 1}; + const ov::Shape pads_end{2, 1, 10}; + const ov::Shape kernel_shape{5, 5, 5}; const auto rounding_mode = op::RoundingType::CEIL; const auto pad_type = op::PadType::SAME_LOWER; @@ -114,9 +114,9 @@ TEST_F(MaxPoolV8StaticShapeInferenceTest, no_dilation) { const Strides strides{1, 1}; const Strides dilations{1, 1}; - const Shape pads_begin{1, 1}; - const Shape pads_end{0, 0}; - const Shape kernel_shape{2, 2}; + const ov::Shape pads_begin{1, 1}; + const ov::Shape pads_end{0, 0}; + const ov::Shape kernel_shape{2, 2}; op = make_op(data, strides, dilations, pads_begin, pads_end, kernel_shape); @@ -135,9 +135,9 @@ TEST_F(MaxPoolV8StaticShapeInferenceTest, with_dilations) { const Strides strides{1, 1}; const Strides dilations{2, 3}; - const Shape pads_begin{0, 0}; - const Shape pads_end{1, 1}; - const Shape kernel_shape{2, 2}; + const ov::Shape pads_begin{0, 0}; + const ov::Shape pads_end{1, 1}; + const ov::Shape kernel_shape{2, 2}; op = make_op(data, strides, dilations, pads_begin, pads_end, kernel_shape); diff --git a/src/plugins/intel_cpu/tests/unit/snippets_transformations/enforce_precision.cpp b/src/plugins/intel_cpu/tests/unit/snippets_transformations/enforce_precision.cpp index 83717a71350..fb429d527cd 100644 --- a/src/plugins/intel_cpu/tests/unit/snippets_transformations/enforce_precision.cpp +++ b/src/plugins/intel_cpu/tests/unit/snippets_transformations/enforce_precision.cpp @@ -10,7 +10,7 @@ #include #include "openvino/core/type/element_type.hpp" -#include "snippets_transformations/enforce_precision.hpp" +#include "transformations/snippets/x64/pass/enforce_precision.hpp" #include "common_test_utils/common_utils.hpp" #include "two_binary_ops_function.hpp" diff --git a/src/plugins/intel_cpu/tests/unit/snippets_transformations/fake_quantize_tokenization_test.cpp b/src/plugins/intel_cpu/tests/unit/snippets_transformations/fake_quantize_tokenization_test.cpp index 518d2dfb1cc..5705486b4f4 100644 --- a/src/plugins/intel_cpu/tests/unit/snippets_transformations/fake_quantize_tokenization_test.cpp +++ b/src/plugins/intel_cpu/tests/unit/snippets_transformations/fake_quantize_tokenization_test.cpp @@ -9,7 +9,7 @@ #include "snippets/pass/tokenization.hpp" #include "fake_quantize_function.hpp" #include "snippets/op/subgraph.hpp" -#include "ngraph_transformations/snippets_mark_skipped.hpp" +#include "transformations/snippets/x64/pass/snippets_mark_skipped.hpp" #include "function_helper.hpp" namespace ov { diff --git a/src/plugins/intel_cpu/tests/unit/snippets_transformations/mul_add_to_fma.cpp b/src/plugins/intel_cpu/tests/unit/snippets_transformations/mul_add_to_fma.cpp index 5431cbb2626..e46bbd57a0c 100644 --- a/src/plugins/intel_cpu/tests/unit/snippets_transformations/mul_add_to_fma.cpp +++ b/src/plugins/intel_cpu/tests/unit/snippets_transformations/mul_add_to_fma.cpp @@ -4,8 +4,8 @@ #include #include -#include -#include +#include +#include #include "snippets/pass/loop_helpers.hpp" #include "lowering_utils.hpp" diff --git a/src/plugins/intel_cpu/thirdparty/CMakeLists.txt b/src/plugins/intel_cpu/thirdparty/CMakeLists.txt index f468bf8667c..0f513dbfcd3 100644 --- a/src/plugins/intel_cpu/thirdparty/CMakeLists.txt +++ b/src/plugins/intel_cpu/thirdparty/CMakeLists.txt @@ -2,6 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 # +project(intel_cpu_thirdparty) + if((CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") AND (MSVC_VERSION VERSION_GREATER_EQUAL "1910")) # 1910 version of Visual Studio 2017 # This flagis needed for enabling SIMD vectorization with command '#pragma omp simd'. @@ -45,6 +47,51 @@ function(ie_add_onednn) set(OpenMP_cmake_included ON) ## to skip "omp simd" inside a code. Lead to some crashes inside NDK LLVM.. endif() + if(SUGGEST_OVERRIDE_SUPPORTED) + # xbyak compilation fails + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-suggest-override") + endif() + if(X86_64) + set(DNNL_TARGET_ARCH "X64" CACHE STRING "" FORCE) + elseif(X86) + set(DNNL_TARGET_ARCH "X86" CACHE STRING "" FORCE) + elseif(RISCV64) + set(DNNL_TARGET_ARCH "RV64" CACHE STRING "" FORCE) + elseif(AARCH64) + set(DNNL_TARGET_ARCH "AARCH64" CACHE STRING "" FORCE) + set(DNNL_AARCH64_USE_ACL ON CACHE BOOL "" FORCE) + # TODO: fix warning + if(CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG) + ie_add_compiler_flags(-Wno-macro-redefined) + endif() + + # move to separate ACL cmake + if(APPLE) + # Apple M1 / M2 is assumed + set(ARM_COMPUTE_TARGET_ARCH_DEFAULT armv8.2-a) + else() + set(ARM_COMPUTE_TARGET_ARCH_DEFAULT arm64-v8a) + endif() + set(ARM_COMPUTE_TARGET_ARCHS arm64-v8a arm64-v8.2-a arm64-v8.2-a-sve arm64-v8.2-a-sve2 + armv8a armv8.2-a armv8.2-a-sve armv8.6-a armv8.6-a-sve armv8.6-a-sve2 + armv8r64) + + set(ARM_COMPUTE_TARGET_ARCH "${ARM_COMPUTE_TARGET_ARCH_DEFAULT}" CACHE STRING "Architecture for ARM ComputeLibrary") + set_property(CACHE ARM_COMPUTE_TARGET_ARCH PROPERTY STRINGS ${ARM_COMPUTE_TARGET_ARCHS}) + else() + message(FATAL_ERROR "Unsupported system processor ${CMAKE_SYSTEM_PROCESSOR}") + endif() + if(CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG) + ie_add_compiler_flags(-Wno-undef) + ie_add_compiler_flags(-Wno-missing-declarations) + if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12 AND CMAKE_COMPILER_IS_GNUCXX) + ie_add_compiler_flags(-Wno-error=array-bounds) + ie_add_compiler_flags(-Wno-error=stringop-overflow=) + endif() + elseif(UNIX AND CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -diag-disable=10121") + endif() + # WA for old TBBConfig.cmake like tbb2019_20180718oss # they don't check that imported target is already created if(TBB_FOUND) @@ -85,11 +132,15 @@ function(ie_add_onednn) # C4334 '<<': result of 32-bit shift implicitly converted to 64 bits ie_add_compiler_flags(/wd4334) endif() + if(SUGGEST_OVERRIDE_SUPPORTED) # xbyak compilation fails set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-suggest-override") endif() + # to find our FindACL.cmake + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") + add_subdirectory(onednn EXCLUDE_FROM_ALL) ov_install_static_lib(dnnl cpu) endfunction() diff --git a/src/plugins/intel_cpu/thirdparty/ComputeLibrary b/src/plugins/intel_cpu/thirdparty/ComputeLibrary new file mode 160000 index 00000000000..1b3192e8a23 --- /dev/null +++ b/src/plugins/intel_cpu/thirdparty/ComputeLibrary @@ -0,0 +1 @@ +Subproject commit 1b3192e8a23513031163dc14d248f47671986121 diff --git a/src/plugins/intel_cpu/thirdparty/FindACL.cmake b/src/plugins/intel_cpu/thirdparty/FindACL.cmake new file mode 100644 index 00000000000..870be7fac8b --- /dev/null +++ b/src/plugins/intel_cpu/thirdparty/FindACL.cmake @@ -0,0 +1,260 @@ +# Copyright (C) 2018-2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +if(ARM_COMPUTE_INCLUDE_DIR OR ARM_COMPUTE_LIB_DIR) + if (NOT ARM_COMPUTE_INCLUDE_DIR) + message(FATAL_ERROR "Undefined ARM_COMPUTE_INCLUDE_DIR input variable should be set manually") + else() + message(STATUS "Using ${ARM_COMPUTE_INCLUDE_DIR} to include arm compute library headers") + endif() + + if (NOT ARM_COMPUTE_LIB_DIR) + message(FATAL_ERROR "Undefined ARM_COMPUTE_LIB_DIR input variable should be set manually") + else() + find_library( + ARM_COMPUTE_LIB + arm_compute-static + PATHS ${ARM_COMPUTE_LIB_DIR} + ) + message(STATUS "Found arm_compute-static: ${ARM_COMPUTE_LIB}") + add_library(arm_compute STATIC IMPORTED GLOBAL) + set_target_properties(arm_compute PROPERTIES + IMPORTED_LOCATION ${ARM_COMPUTE_LIB} + INTERFACE_INCLUDE_DIRECTORIES ${ARM_COMPUTE_INCLUDE_DIR}) + find_library( + ARM_COMPUTE_CORE_LIB + arm_compute_core-static + PATHS ${ARM_COMPUTE_LIB_DIR} + ) + message(STATUS "Found arm_compute_core-static: ${ARM_COMPUTE_CORE_LIB}") + add_library(arm_compute_core STATIC IMPORTED GLOBAL) + set_target_properties(arm_compute_core PROPERTIES + IMPORTED_LOCATION ${ARM_COMPUTE_CORE_LIB} + INTERFACE_INCLUDE_DIRECTORIES ${ARM_COMPUTE_INCLUDE_DIR}) + endif() + + add_library(half INTERFACE IMPORTED GLOBAL) + set_target_properties(half PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${ARM_COMPUTE_INCLUDE_DIR}) +else() + find_host_program(SCONS scons) + + if(NOT SCONS) + message(FATAL_ERROR "Scons tool is not found!") + endif() + + set(ARM_COMPUTE_SOURCE_DIR ${intel_cpu_thirdparty_SOURCE_DIR}/ComputeLibrary) + set(ARM_COMPUTE_BINARY_DIR ${intel_cpu_thirdparty_BINARY_DIR}/ComputeLibrary) + + file(GLOB_RECURSE SOURCES + ${ARM_COMPUTE_SOURCE_DIR}/*.cpp + ${ARM_COMPUTE_SOURCE_DIR}/*.hpp + ${ARM_COMPUTE_SOURCE_DIR}/*.h + ) + + set(extra_cxx_flags "-fPIC ${CMAKE_CXX_FLAGS} -Wno-undef") + + set(ARM_COMPUTE_OPTIONS + neon=1 + opencl=0 + openmp=0 + cppthreads=1 + examples=0 + Werror=0 + gemm_tuner=0 + reference_openmp=0 + validation_tests=0 + benchmark_tests=0 + # TODO: check this for Apple Silicon + # multi_isa=1 + # TODO: use CC for ARM compute library to minimize binary size + # build_config= + # TODO: use data_type_support to disable useless kernels + data_layout_support=all + build_dir=${ARM_COMPUTE_BINARY_DIR} + install_dir=${ARM_COMPUTE_BINARY_DIR}/install + arch=${ARM_COMPUTE_TARGET_ARCH} + ) + + if(ARM_COMPUTE_SCONS_JOBS) + list(APPEND ARM_COMPUTE_OPTIONS --jobs=${ARM_COMPUTE_SCONS_JOBS}) + endif() + + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + list(APPEND ARM_COMPUTE_OPTIONS + debug=1 + asserts=1 + logging=1) + endif() + + if(EMSCRIPTEN OR LINUX) + list(APPEND ARM_COMPUTE_OPTIONS os=linux) + elseif(ANDROID) + list(APPEND ARM_COMPUTE_OPTIONS os=android) + elseif(APPLE) + list(APPEND ARM_COMPUTE_OPTIONS os=macos) + endif() + + if(CMAKE_CROSSCOMPILING) + list(APPEND ARM_COMPUTE_OPTIONS build=cross_compile) + else() + list(APPEND ARM_COMPUTE_OPTIONS build=native) + endif() + + if (CMAKE_CXX_COMPILER_LAUNCHER) + list(APPEND ARM_COMPUTE_OPTIONS compiler_cache=${CMAKE_CXX_COMPILER_LAUNCHER}) + endif() + + # used to build for yocto + if(ARM_COMPUTE_TOOLCHAIN_PREFIX) + list(APPEND ARM_COMPUTE_OPTIONS toolchain_prefix=${ARM_COMPUTE_TOOLCHAIN_PREFIX}) + endif() + + if(ANDROID) + if(ANDROID_PLATFORM_LEVEL LESS 18) + message(FATAL_ERROR "ARM compute library requires Android API 18 level and higher" + "Please, speficy -DANDROID_PLATFORM=android-18 at least") + endif() + + if(ANDROID_NDK_REVISION LESS "23.0") + list(APPEND ARM_COMPUTE_OPTIONS toolchain_prefix="${ANDROID_TOOLCHAIN_PREFIX}") + else() + string(REGEX REPLACE "/bin/[^/]+-" "/bin/llvm-" ANDROID_TOOLCHAIN_PREFIX_FIXED "${ANDROID_TOOLCHAIN_PREFIX}") + message(STATUS "SCONS: using ANDROID_TOOLCHAIN_PREFIX=${ANDROID_TOOLCHAIN_PREFIX_FIXED}") + list(APPEND ARM_COMPUTE_OPTIONS toolchain_prefix="${ANDROID_TOOLCHAIN_PREFIX_FIXED}") + endif() + + list(APPEND ARM_COMPUTE_OPTIONS + compiler_prefix="${ANDROID_TOOLCHAIN_ROOT}/bin/") + + set(extra_flags "${extra_flags} --target=${ANDROID_LLVM_TRIPLE}") + set(extra_flags "${extra_flags} --gcc-toolchain=${ANDROID_TOOLCHAIN_ROOT}") + set(extra_flags "${extra_flags} --sysroot=${CMAKE_SYSROOT}") + + set(extra_link_flags "${extra_link_flags} ${extra_flags}") + set(extra_cxx_flags "${extra_cxx_flags} ${extra_flags}") + elseif(CMAKE_CROSSCOMPILING AND LINUX) + get_filename_component(cxx_compiler "${CMAKE_CXX_COMPILER}" NAME) + get_filename_component(c_compiler "${CMAKE_C_COMPILER}" NAME) + get_filename_component(compiler_prefix "${CMAKE_CXX_COMPILER}" DIRECTORY) + set(cmake_build_env + CC=${c_compiler} + CXX=${cxx_compiler}) + list(APPEND ARM_COMPUTE_OPTIONS compiler_prefix="${compiler_prefix}/") + elseif(EMSCRIPTEN) + set(cmake_build_env + CC=emcc + CXX=em++ + RANLIB=emranlib + AR=emar) + # EMSDK: Passing any of -msse, -msse2, -msse3, -mssse3, -msse4.1, -msse4.2, + # -msse4, -mavx, -mfpu=neon flags also requires passing -msimd128 (or -mrelaxed-simd)! + set(extra_cxx_flags "${extra_cxx_flags} -msimd128") + # clang-16: error: argument unused during compilation: '-mthumb' [-Werror,-Wunused-command-line-argument] + # clang-16: error: argument unused during compilation: '-mfloat-abi=hard' [-Werror,-Wunused-command-line-argument] + set(extra_cxx_flags "${extra_cxx_flags} \ + -Wno-unused-command-line-argument \ + -Wno-unknown-warning-option \ + -Wno-unused-function \ + -Wno-unused-but-set-variable") + + get_filename_component(toolchain_prefix "${CMAKE_CXX_COMPILER}" DIRECTORY) + list(APPEND ARM_COMPUTE_OPTIONS toolchain_prefix="${toolchain_prefix}/") + elseif(APPLE) + if(CMAKE_OSX_DEPLOYMENT_TARGET) + set(extra_cxx_flags "${extra_cxx_flags} -mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") + set(minos_added ON) + endif() + + if(HOST_X86_64) + if(NOT minos_added) + message(FATAL_ERROR "Please, specify either env MACOSX_DEPLOYMENT_TARGET or cmake CMAKE_OSX_DEPLOYMENT_TARGET variables") + endif() + set(extra_cxx_flags "${extra_cxx_flags} --sysroot ${CMAKE_OSX_SYSROOT}") + endif() + + set(extra_cxx_flags "${extra_cxx_flags} -Wno-error=return-stack-address") + get_filename_component(compiler_prefix "${CMAKE_CXX_COMPILER}" DIRECTORY) + list(APPEND ARM_COMPUTE_OPTIONS compiler_prefix="${compiler_prefix}/") + + if(CMAKE_OSX_ARCHITECTURES) + foreach(arch IN LISTS CMAKE_OSX_ARCHITECTURES) + set(extra_cxx_flags "${extra_cxx_flags} -arch ${arch}") + endforeach() + endif() + endif() + + if(ENABLE_LTO) + if((CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG) AND (NOT CMAKE_CROSSCOMPILING)) + set(extra_cxx_flags "${extra_cxx_flags} -flto=thin") + set(extra_link_flags "${extra_link_flags} -flto=thin") + endif() + endif() + + if(extra_link_flags) + list(APPEND ARM_COMPUTE_OPTIONS extra_link_flags=${extra_link_flags}) + endif() + if(extra_cxx_flags) + list(APPEND ARM_COMPUTE_OPTIONS extra_cxx_flags=${extra_cxx_flags}) + endif() + + if(NOT CMAKE_VERBOSE_MAKEFILE) + list(APPEND ARM_COMPUTE_OPTIONS --silent) + endif() + + set(arm_compute ${ARM_COMPUTE_BINARY_DIR}/libarm_compute-static.a) + set(arm_compute_core ${ARM_COMPUTE_BINARY_DIR}/libarm_compute_core-static.a) + + add_custom_command( + OUTPUT + ${arm_compute} + ${arm_compute_core} + COMMAND ${CMAKE_COMMAND} -E env ${cmake_build_env} + ${SCONS} ${ARM_COMPUTE_OPTIONS} + ${arm_compute} + ${arm_compute_core} + WORKING_DIRECTORY ${ARM_COMPUTE_SOURCE_DIR} + COMMENT "Build Arm Compute Library" + DEPENDS ${SOURCES} + ${CMAKE_CURRENT_LIST_FILE} + ${ARM_COMPUTE_SOURCE_DIR}/SConscript + ${ARM_COMPUTE_SOURCE_DIR}/SConstruct) + + # Import targets + + add_custom_target(arm_compute_static_libs + DEPENDS + ${arm_compute} + ${arm_compute_core} + ) + + add_library(arm_compute::arm_compute STATIC IMPORTED GLOBAL) + set_target_properties(arm_compute::arm_compute PROPERTIES + IMPORTED_LOCATION ${arm_compute} + INTERFACE_INCLUDE_DIRECTORIES ${ARM_COMPUTE_SOURCE_DIR}) + add_dependencies(arm_compute::arm_compute arm_compute_static_libs) + + add_library(arm_compute::arm_compute_core STATIC IMPORTED GLOBAL) + set_target_properties(arm_compute::arm_compute_core PROPERTIES + IMPORTED_LOCATION ${arm_compute_core} + INTERFACE_INCLUDE_DIRECTORIES ${ARM_COMPUTE_SOURCE_DIR}) + add_dependencies(arm_compute::arm_compute_core arm_compute_static_libs) + + add_library(arm_compute::half INTERFACE IMPORTED GLOBAL) + set_target_properties(arm_compute::half PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${ARM_COMPUTE_SOURCE_DIR}/include) + + set(ACL_FOUND ON) + set(ACL_LIBRARIES arm_compute::arm_compute + arm_compute::arm_compute_core + arm_compute::half) + + foreach(acl_library IN LISTS ACL_LIBRARIES) + list(APPEND ACL_INCLUDE_DIRS + $) + endforeach() + + # required by oneDNN to attempt to parse ACL version + set(ENV{ACL_ROOT_DIR} "${ARM_COMPUTE_SOURCE_DIR}") +endif() diff --git a/src/plugins/intel_cpu/thirdparty/onednn b/src/plugins/intel_cpu/thirdparty/onednn index 4b2424253b1..f9127156d14 160000 --- a/src/plugins/intel_cpu/thirdparty/onednn +++ b/src/plugins/intel_cpu/thirdparty/onednn @@ -1 +1 @@ -Subproject commit 4b2424253b1dcde315fdd4df796927fa5fb0484b +Subproject commit f9127156d148393502d1d2254d9a48f564dc9adb