From 835fda6a0bdf9bdb6c7ecb503594fdee59df8f20 Mon Sep 17 00:00:00 2001 From: Egor Duplenskii Date: Tue, 16 Apr 2024 10:11:14 +0200 Subject: [PATCH] [CPU] Introduce empty undefined memory descriptor (#23437) ### Details: - The example use case is to generalize processing of the operation with and without bias. When the operation has no bias, it is possible to create memory using empty memory descriptor, and use the exact same code as if bias actually exists. - Many headers are affected to break the chain of implicit intersecting includes ### Tickets: - 131590 ### Todo: - [ ] Add unit tests --- src/plugins/intel_cpu/src/cpu_memory.cpp | 1 + src/plugins/intel_cpu/src/cpu_memory.h | 1 - src/plugins/intel_cpu/src/cpu_shape.cpp | 3 +- src/plugins/intel_cpu/src/cpu_types.cpp | 22 ++++- src/plugins/intel_cpu/src/cpu_types.h | 3 + src/plugins/intel_cpu/src/infer_request.cpp | 2 +- .../memory_desc/cpu_blocked_memory_desc.cpp | 2 +- .../src/memory_desc/cpu_blocked_memory_desc.h | 4 +- .../src/memory_desc/cpu_memory_desc.h | 11 ++- .../src/memory_desc/cpu_memory_desc_utils.cpp | 39 ++++---- .../src/memory_desc/cpu_memory_desc_utils.h | 14 ++- .../memory_desc/dnnl_blocked_memory_desc.cpp | 2 +- .../memory_desc/dnnl_blocked_memory_desc.h | 5 +- .../src/memory_desc/empty_memory_desc.h | 94 +++++++++++++++++++ src/plugins/intel_cpu/src/memory_state.cpp | 1 + .../common/arbitrary_order_desc_creator.cpp | 3 +- .../dnnl/dnnl_convolution_primitive.cpp | 5 +- .../executors/dnnl/dnnl_fullyconnected.hpp | 3 +- .../dnnl/dnnl_fullyconnected_primitive.cpp | 15 +-- .../fullyconnected_implementations.cpp | 16 +++- .../intel_cpu/src/nodes/fullyconnected.cpp | 19 +--- .../intel_cpu/src/nodes/fullyconnected.h | 1 - src/plugins/intel_cpu/src/nodes/memory.cpp | 3 +- .../src/nodes/non_max_suppression.cpp | 7 +- src/plugins/intel_cpu/src/nodes/rnn.cpp | 2 +- .../unit/memory_desc/empty_memory_desc.cpp | 45 +++++++++ 26 files changed, 240 insertions(+), 83 deletions(-) create mode 100644 src/plugins/intel_cpu/src/memory_desc/empty_memory_desc.h create mode 100644 src/plugins/intel_cpu/tests/unit/memory_desc/empty_memory_desc.cpp diff --git a/src/plugins/intel_cpu/src/cpu_memory.cpp b/src/plugins/intel_cpu/src/cpu_memory.cpp index eac89b17bef..bc28450d072 100644 --- a/src/plugins/intel_cpu/src/cpu_memory.cpp +++ b/src/plugins/intel_cpu/src/cpu_memory.cpp @@ -3,6 +3,7 @@ // #include "cpu_memory.h" +#include "memory_desc/cpu_memory_desc_utils.h" #include #include "nodes/reorder.h" #if defined(__linux__) diff --git a/src/plugins/intel_cpu/src/cpu_memory.h b/src/plugins/intel_cpu/src/cpu_memory.h index 27a502db9a6..69b70cb6e58 100644 --- a/src/plugins/intel_cpu/src/cpu_memory.h +++ b/src/plugins/intel_cpu/src/cpu_memory.h @@ -6,7 +6,6 @@ #include "memory_desc/cpu_memory_desc.h" #include "dnnl_extension_utils.h" -#include "memory_desc/cpu_memory_desc_utils.h" #include #include diff --git a/src/plugins/intel_cpu/src/cpu_shape.cpp b/src/plugins/intel_cpu/src/cpu_shape.cpp index 5da5c62c6c4..4c6b5793d9f 100644 --- a/src/plugins/intel_cpu/src/cpu_shape.cpp +++ b/src/plugins/intel_cpu/src/cpu_shape.cpp @@ -4,7 +4,6 @@ #include "cpu_shape.h" #include "utils/general_utils.h" -#include "memory_desc/cpu_memory_desc_utils.h" namespace ov { namespace intel_cpu { @@ -39,7 +38,7 @@ std::string Shape::toString() const { size_t i = 0; do { if (dims[i] == Shape::UNDEFINED_DIM) { - output << MemoryDescUtils::dim2str(minDims[i]) << " - " << MemoryDescUtils::dim2str(maxDims[i]); + output << dim2str(minDims[i]) << " - " << dim2str(maxDims[i]); } else { output << dims[i]; } diff --git a/src/plugins/intel_cpu/src/cpu_types.cpp b/src/plugins/intel_cpu/src/cpu_types.cpp index 534466b1be8..73ad7b36b4a 100644 --- a/src/plugins/intel_cpu/src/cpu_types.cpp +++ b/src/plugins/intel_cpu/src/cpu_types.cpp @@ -2,13 +2,33 @@ // SPDX-License-Identifier: Apache-2.0 // #include "cpu_types.h" +#include "cpu_shape.h" #include -#include +#include namespace ov { namespace intel_cpu { +std::string dim2str(Dim dim) { + return dim == Shape::UNDEFINED_DIM ? "?" : std::to_string(dim); +} + +std::string dims2str(const VectorDims& dims) { + std::stringstream output; + output << "{"; + + if (!dims.empty()) { + auto itr = dims.begin(); + do { + output << dim2str(*itr); + } while (++itr != dims.end() && output << ", "); + } + + output << "}"; + return output.str(); +} + using TypeToNameMap = ov::intel_cpu::caseless_unordered_map; static const TypeToNameMap& get_type_to_name_tbl() { diff --git a/src/plugins/intel_cpu/src/cpu_types.h b/src/plugins/intel_cpu/src/cpu_types.h index 49defda252e..45c3617f9b8 100644 --- a/src/plugins/intel_cpu/src/cpu_types.h +++ b/src/plugins/intel_cpu/src/cpu_types.h @@ -15,6 +15,9 @@ namespace intel_cpu { using Dim = std::size_t; using VectorDims = std::vector; +std::string dim2str(Dim dim); +std::string dims2str(const VectorDims& dims); + enum class Type { Unknown, If, diff --git a/src/plugins/intel_cpu/src/infer_request.cpp b/src/plugins/intel_cpu/src/infer_request.cpp index 93bfb0117f7..db20ed1e7ac 100644 --- a/src/plugins/intel_cpu/src/infer_request.cpp +++ b/src/plugins/intel_cpu/src/infer_request.cpp @@ -10,7 +10,7 @@ #include "itt.h" #include "memory_state.h" #include "nodes/common/cpu_convert.h" -#include "nodes/common/cpu_memcpy.h" +#include "memory_desc/cpu_memory_desc_utils.h" #include "nodes/memory.hpp" #include "openvino/core/shape.hpp" #include "openvino/runtime/make_tensor.hpp" diff --git a/src/plugins/intel_cpu/src/memory_desc/cpu_blocked_memory_desc.cpp b/src/plugins/intel_cpu/src/memory_desc/cpu_blocked_memory_desc.cpp index 0eea64f69e8..cb458b09c06 100644 --- a/src/plugins/intel_cpu/src/memory_desc/cpu_blocked_memory_desc.cpp +++ b/src/plugins/intel_cpu/src/memory_desc/cpu_blocked_memory_desc.cpp @@ -3,8 +3,8 @@ // #include "cpu_blocked_memory_desc.h" -#include "cpu_memory.h" #include "dnnl_blocked_memory_desc.h" +#include "utils/general_utils.h" namespace ov { namespace intel_cpu { diff --git a/src/plugins/intel_cpu/src/memory_desc/cpu_blocked_memory_desc.h b/src/plugins/intel_cpu/src/memory_desc/cpu_blocked_memory_desc.h index 5214390e13b..28badb4dac1 100644 --- a/src/plugins/intel_cpu/src/memory_desc/cpu_blocked_memory_desc.h +++ b/src/plugins/intel_cpu/src/memory_desc/cpu_blocked_memory_desc.h @@ -5,11 +5,13 @@ #pragma once #include "blocked_memory_desc.h" -#include "utils/general_utils.h" +#include "dnnl_extension_utils.h" namespace ov { namespace intel_cpu { +class DnnlBlockedMemoryDesc; + class CpuBlockedMemoryDesc : public BlockedMemoryDesc { public: CpuBlockedMemoryDesc(ov::element::Type prc, const Shape& shape); diff --git a/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc.h b/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc.h index 9c37ba2c369..c3936528abe 100644 --- a/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc.h +++ b/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc.h @@ -6,7 +6,6 @@ #include "cpu_shape.h" #include "cpu_types.h" -#include "memory_desc/cpu_memory_desc_utils.h" #include "openvino/core/type/element_type.hpp" /** @@ -35,8 +34,8 @@ enum MemoryDescType { Undef = 0, Blocked = 1, Dnnl = 1 << 1, - - DnnlBlocked = Blocked | Dnnl + DnnlBlocked = Blocked | Dnnl, + Empty = 1 << 2, }; enum class LayoutType : unsigned { @@ -90,7 +89,7 @@ public: OPENVINO_THROW("ParameterMismatch: Can not clone with new dims. Descriptor's shape: ", getShape().toString(), " is incompatible with provided dimensions: ", - MemoryDescUtils::dims2str(dims), + dims2str(dims), "."); } @@ -132,6 +131,10 @@ public: return getMaxMemSize() != MemoryDesc::UNDEFINED_SIZE; } + bool empty() const { + return type == Empty; + } + template ::value && !std::is_reference::value, int>::type = 0, typename std::enable_if::value, int>::type = 0> diff --git a/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc_utils.cpp b/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc_utils.cpp index 3145d407a84..0ae17d6c003 100644 --- a/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc_utils.cpp +++ b/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc_utils.cpp @@ -2,10 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 // +#include "memory_desc/cpu_memory_desc_utils.h" + #include "memory_desc/cpu_blocked_memory_desc.h" #include "memory_desc/dnnl_blocked_memory_desc.h" +#include "graph_context.h" #include "cpu_memory_desc.h" -#include "memory_desc/cpu_memory_desc_utils.h" +#include "memory_desc/empty_memory_desc.h" #include #include #include @@ -24,6 +27,8 @@ DnnlMemoryDescPtr MemoryDescUtils::convertToDnnlMemoryDesc(const MemoryDescPtr & return std::shared_ptr(new DnnlBlockedMemoryDesc(cpuDesc->getPrecision(), cpuDesc->getShape(), cpuDesc->getBlockDims(), cpuDesc->getOrder(), cpuDesc->getOffsetPadding(), cpuDesc->getOffsetPaddingToData(), cpuDesc->getStrides())); + } else if (MemoryDescType::Empty == desc->getType()) { + return DnnlExtensionUtils::makeDescriptor(dnnl::memory::desc()); } else if (MemoryDescType::Dnnl & desc->getType()) { return std::dynamic_pointer_cast(desc); } else { @@ -39,7 +44,7 @@ DnnlBlockedMemoryDesc MemoryDescUtils::convertToDnnlBlockedMemoryDesc(const Memo return DnnlBlockedMemoryDesc(cpuDesc->getPrecision(), cpuDesc->getShape(), cpuDesc->getBlockDims(), cpuDesc->getOrder(), cpuDesc->getOffsetPadding(), cpuDesc->getOffsetPaddingToData(), cpuDesc->getStrides()); } else { - OPENVINO_THROW("Cannot convert MemoryDesc to DnnlMemoryDesc"); + OPENVINO_THROW("Cannot convert MemoryDesc to DnnlBlockedMemoryDesc"); } } @@ -47,7 +52,7 @@ BlockedMemoryDescPtr MemoryDescUtils::convertToBlockedMemoryDesc(const MemoryDes if (desc->getType() & MemoryDescType::Blocked) { return std::dynamic_pointer_cast(desc); } else { - OPENVINO_THROW("Can not convert unsupported memory descriptor"); + OPENVINO_THROW("Cannot convert MemoryDesc to BlockedMemoryDesc"); } } @@ -92,30 +97,20 @@ CpuBlockedMemoryDescPtr MemoryDescUtils::generateCpuBlockedMemoryDesc(const ov:: blk_strides); } -std::string MemoryDescUtils::dim2str(Dim dim) { - return dim == Shape::UNDEFINED_DIM ? "?" : std::to_string(dim); -} - -std::string MemoryDescUtils::dims2str(const VectorDims& dims) { - std::stringstream output; - output << "{"; - - if (!dims.empty()) { - auto itr = dims.begin(); - do { - output << dim2str(*itr); - } while (++itr != dims.end() && output << ", "); - } - - output << "}"; - return output.str(); -} - std::shared_ptr MemoryDescUtils::makeDummyDesc(const MemoryDesc &desc, Dim dummyVal) { auto dummyShape = makeDummyShape(desc.getShape(), dummyVal); return desc.cloneWithNewDims(dummyShape.getStaticDims()); } +std::shared_ptr MemoryDescUtils::makeEmptyDesc() { + static auto emptyDesc = std::make_shared(); + return emptyDesc; +} + +std::shared_ptr MemoryDescUtils::makeEmptyMemory(const GraphContext::CPtr context) { + return std::make_shared(context->getEngine(), makeEmptyDesc(), nullptr); +} + Shape MemoryDescUtils::makeDummyShape(const Shape &shape, Dim dummyVal) { const auto& minDims = shape.getMinDims(); const auto& maxDims = shape.getMaxDims(); diff --git a/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc_utils.h b/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc_utils.h index 2df36a60c32..a4acd3eb2aa 100644 --- a/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc_utils.h +++ b/src/plugins/intel_cpu/src/memory_desc/cpu_memory_desc_utils.h @@ -4,12 +4,12 @@ #pragma once +#include #include "cpu_shape.h" #include "cpu_types.h" #include "openvino/runtime/itensor.hpp" #include "openvino/runtime/so_ptr.hpp" - -#include "onednn/dnnl.h" +#include "graph_context.h" namespace ov { namespace intel_cpu { @@ -19,7 +19,9 @@ class DnnlMemoryDesc; class BlockedMemoryDesc; class DnnlBlockedMemoryDesc; class CpuBlockedMemoryDesc; +class EmptyMemoryDesc; class IMemory; +class Memory; class MemoryDescUtils { public: @@ -63,6 +65,14 @@ public: */ static std::shared_ptr makeDummyDesc(const MemoryDesc& desc, Dim dummyVal = DEFAULT_DUMMY_VAL); + /** + * @brief Make an empty memory descriptor + * @note Shape{0}, undefined + * @return empty memory descriptor + */ + static std::shared_ptr makeEmptyDesc(); + static std::shared_ptr makeEmptyMemory(const GraphContext::CPtr context); + /** * @brief Makes a static dummy shape where all undefined values are replaced with the smallest value between the parameter and the upper bound dim * @param shape a Shape object from which the new static shape is generated diff --git a/src/plugins/intel_cpu/src/memory_desc/dnnl_blocked_memory_desc.cpp b/src/plugins/intel_cpu/src/memory_desc/dnnl_blocked_memory_desc.cpp index 95d14e9dd65..a24b55831c2 100644 --- a/src/plugins/intel_cpu/src/memory_desc/dnnl_blocked_memory_desc.cpp +++ b/src/plugins/intel_cpu/src/memory_desc/dnnl_blocked_memory_desc.cpp @@ -444,7 +444,7 @@ static dnnl::memory::desc cloneDescWithNewDims(const dnnl::memory::desc& desc, dnnl::impl::memory_desc_t& newCdesc = *newMklDesc.get(); auto retCode = dnnl::impl::fill_blocked(newCdesc, perm, innerBlks, innerIdxs); if (retCode != dnnl::impl::status::success) { - OPENVINO_THROW("Can not clone DnnlBlockedMemoryDesc with dims: ", MemoryDescUtils::dims2str(dims)); + OPENVINO_THROW("Can not clone DnnlBlockedMemoryDesc with dims: ", dims2str(dims)); } // dnnl::impl::fill_blocked always set offset0 to 0 // so we need to restore actual value diff --git a/src/plugins/intel_cpu/src/memory_desc/dnnl_blocked_memory_desc.h b/src/plugins/intel_cpu/src/memory_desc/dnnl_blocked_memory_desc.h index acb6bd774ac..a6c6a3297ba 100644 --- a/src/plugins/intel_cpu/src/memory_desc/dnnl_blocked_memory_desc.h +++ b/src/plugins/intel_cpu/src/memory_desc/dnnl_blocked_memory_desc.h @@ -5,15 +5,16 @@ #pragma once #include "dnnl_memory_desc.h" -#include "blocked_memory_desc.h" +#include "memory_desc/blocked_memory_desc.h" #include "openvino/util/util.hpp" -#include "cpu_memory.h" #include "dnnl_extension_utils.h" #include namespace ov { namespace intel_cpu { +class CpuBlockedMemoryDesc; + OPENVINO_DISABLE_WARNING_MSVC_BEGIN(4250) // Visual Studio warns us about inheritance via dominance but it's done intentionally // so turn it off class DnnlBlockedMemoryDesc : public BlockedMemoryDesc, public DnnlMemoryDesc { diff --git a/src/plugins/intel_cpu/src/memory_desc/empty_memory_desc.h b/src/plugins/intel_cpu/src/memory_desc/empty_memory_desc.h new file mode 100644 index 00000000000..4b641669262 --- /dev/null +++ b/src/plugins/intel_cpu/src/memory_desc/empty_memory_desc.h @@ -0,0 +1,94 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "cpu_memory_desc.h" + +#include "cpu_shape.h" +#include "openvino/core/except.hpp" +#include "openvino/core/type/element_type.hpp" +#include "utils/general_utils.h" + +namespace ov { +namespace intel_cpu { + +/** + * @brief Represents an empty memory descriptor. + * + * The main purpose is to create an empty Memory. + * Empty Memory is used to generalize passing an optional memory (such as bias) + * when both pointer to the memory data and nullptr are valid + */ +class EmptyMemoryDesc : public MemoryDesc { +public: + EmptyMemoryDesc(): + MemoryDesc(Shape{0}, Empty) { + /* status never changes for an empty memory desc + * so "define" beforehand to ensure isDefined() is thread safe */ + status = MemoryDesc::descStatus::Defined; + } + + MemoryDescPtr clone() const override { + return std::make_shared(*this); + } + + bool isCompatible(const MemoryDesc& rhs) const override { + return everyone_is(this->getType(), rhs.getType(), Empty); + }; + + ov::element::Type getPrecision() const override { + return ov::element::undefined; + } + + size_t getOffsetPadding() const override { + return 0; + } + + bool hasLayoutType(LayoutType layoutType) const override { + return false; + } + + std::string serializeFormat() const override { + return "empty"; + } + + size_t getMaxMemSize() const override { + return 0; + } + + MemoryDescPtr cloneWithNewPrecision(const ov::element::Type prec) const override { + OPENVINO_THROW("Clone an empty memory desc with any precision (", prec, ") is prohibited"); + } + +private: + size_t getElementOffset(size_t elemNumber) const override { + return 0; + } + bool canComputeMemSizeZeroDims() const override { + return false; + } + size_t getCurrentMemSizeImp() const override { + return 0; + } + size_t getOffset(const VectorDims& v) const { + return 0; + } + bool isDefinedImp() const override { + return true; + } + MemoryDescPtr cloneWithNewDimsImp(const VectorDims& dims) const override { + OPENVINO_THROW("Clone an empty memory desc with any new dimensions is prohibited"); + } + + void setPrecision(ov::element::Type prc) override { + OPENVINO_THROW("Setting any precision (", prc, ") for an empty memory desc is prohibited"); + } +}; + +using EmptyMemoryDescPtr = std::shared_ptr; +using EmptyMemoryDescCPtr = std::shared_ptr; + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/memory_state.cpp b/src/plugins/intel_cpu/src/memory_state.cpp index 29f249a6fe2..c4cf3ee2f57 100644 --- a/src/plugins/intel_cpu/src/memory_state.cpp +++ b/src/plugins/intel_cpu/src/memory_state.cpp @@ -7,6 +7,7 @@ #include #include "cpu_memory.h" #include "memory_desc/cpu_blocked_memory_desc.h" +#include "memory_desc/cpu_memory_desc_utils.h" #include "dnnl_extension_utils.h" #include "cpu_tensor.h" #include "utils/plain_tensor.hpp" diff --git a/src/plugins/intel_cpu/src/nodes/common/arbitrary_order_desc_creator.cpp b/src/plugins/intel_cpu/src/nodes/common/arbitrary_order_desc_creator.cpp index de0cd2d6f27..a7d3adc50d6 100644 --- a/src/plugins/intel_cpu/src/nodes/common/arbitrary_order_desc_creator.cpp +++ b/src/plugins/intel_cpu/src/nodes/common/arbitrary_order_desc_creator.cpp @@ -3,6 +3,7 @@ // #include "arbitrary_order_desc_creator.h" +#include "utils/general_utils.h" namespace ov { namespace intel_cpu { @@ -36,4 +37,4 @@ size_t ArbitraryOrderDescCreator::getMinimalRank() const { } } // namespace intel_cpu -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp index c8426b4b28c..dac4fc044c7 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp @@ -193,10 +193,7 @@ std::shared_ptr DnnlConvolutionPrimitive::create( const DnnlShapeAgnosticDataPtr& shapeAgnosticData) { const auto& srcDesc = MemoryDescUtils::convertToDnnlMemoryDesc(memory.at(ARG_SRC)->getDescPtr()); const auto& weiDesc = MemoryDescUtils::convertToDnnlMemoryDesc(memory.at(ARG_WEI)->getDescPtr()); - // @todo remove after empty memory desc is introduced - const DnnlMemoryDescPtr biaDesc = memory.at(ARG_BIAS)->getDescPtr()->getCurrentMemSize() != 0 - ? MemoryDescUtils::convertToDnnlMemoryDesc(memory.at(ARG_BIAS)->getDescPtr()) - : DnnlExtensionUtils::makeDescriptor(dnnl::memory::desc{}); + const auto& biaDesc = MemoryDescUtils::convertToDnnlMemoryDesc(memory.at(ARG_BIAS)->getDescPtr()); const auto& dstDesc = MemoryDescUtils::convertToDnnlMemoryDesc(memory.at(ARG_DST)->getDescPtr()); const Key dnnlConvKey{srcDesc, weiDesc, biaDesc, dstDesc, shapeAgnosticData->primAttrs.attr}; diff --git a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected.hpp b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected.hpp index 17f239dc32e..ae5ecc1e2e3 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected.hpp +++ b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected.hpp @@ -8,12 +8,11 @@ #include #include "cpu_memory.h" -#include "memory_desc/cpu_memory_desc.h" #include "nodes/executors/dnnl/dnnl_fullyconnected_primitive.hpp" #include "nodes/executors/dnnl/dnnl_convolution_primitive.hpp" #include "nodes/executors/dnnl/dnnl_aliases.hpp" #include "nodes/executors/executor.hpp" -#include "nodes/executors/executor_config.hpp" +#include "memory_desc/cpu_memory_desc_utils.h" #include "nodes/executors/memory_arguments.hpp" namespace ov { diff --git a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp index 45968e22f37..2abf1e7b28d 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp @@ -83,9 +83,7 @@ std::shared_ptr DnnlFCPrimitive::create(const MemoryArgs& memor const DnnlShapeAgnosticDataPtr& shapeAgnosticData) { const auto& srcDesc = MemoryDescUtils::convertToDnnlMemoryDesc(memory.at(ARG_SRC)->getDescPtr()); const auto& weiDesc = MemoryDescUtils::convertToDnnlMemoryDesc(memory.at(ARG_WEI)->getDescPtr()); - const DnnlMemoryDescPtr biaDesc = memory.at(ARG_BIAS)->getDescPtr()->getCurrentMemSize() != 0 - ? MemoryDescUtils::convertToDnnlMemoryDesc(memory.at(ARG_BIAS)->getDescPtr()) - : DnnlExtensionUtils::makeDescriptor(dnnl::memory::desc{}); + const auto& biaDesc = MemoryDescUtils::convertToDnnlMemoryDesc(memory.at(ARG_BIAS)->getDescPtr()); const auto& dstDesc = MemoryDescUtils::convertToDnnlMemoryDesc(memory.at(ARG_DST)->getDescPtr()); Key dnnlFCKey{ @@ -188,7 +186,7 @@ static DnnlPrimitiveAttrs createPrimitiveAttrs(const FCAttrs& attrs, isINT8, 1 << 0, attrs.dequantizationScales, - attrs.withBias, + !memory.at(ARG_BIAS)->getDesc().empty(), outputDataType); if (attrs.decompressionMultiplyPtr) @@ -314,6 +312,7 @@ DnnlShapeAgnosticDataPtr DnnlFCPrimitive::createShapeAgnosticData(const FCAttrs& DEBUG_LOG("Creating shape agnostic data"); auto srcDesc = memory.at(ARG_SRC)->getDescPtr(); const auto& weiDesc = memory.at(ARG_WEI)->getDescPtr(); + const auto& biasDesc = memory.at(ARG_BIAS)->getDescPtr(); auto dstDesc = memory.at(ARG_DST)->getDescPtr(); const auto useWeightsDecompression = useWeightsDecompressionImpl(srcDesc->getPrecision(), weiDesc->getPrecision()); @@ -339,13 +338,9 @@ DnnlShapeAgnosticDataPtr DnnlFCPrimitive::createShapeAgnosticData(const FCAttrs& const dnnl::memory::desc srcDnnlDesc = MemoryDescUtils::convertToDnnlMemoryDesc(srcDesc)->getDnnlDesc(); const dnnl::memory::desc weiDnnlDesc = MemoryDescUtils::convertToDnnlMemoryDesc(weiDesc)->getDnnlDesc(); const dnnl::memory::desc dstDnnlDesc = MemoryDescUtils::convertToDnnlMemoryDesc(dstDesc)->getDnnlDesc(); + const dnnl::memory::desc biaDnnlDesc = MemoryDescUtils::convertToDnnlMemoryDesc(biasDesc)->getDnnlDesc(); + const auto useSparseWeights = attrs.sparseWeights; - - const dnnl::memory::desc biaDnnlDesc = - memory.at(ARG_BIAS)->getDescPtr()->getCurrentMemSize() != 0 - ? MemoryDescUtils::convertToDnnlMemoryDesc(memory.at(ARG_BIAS)->getDescPtr())->getDnnlDesc() - : dnnl::memory::desc{}; - const auto primDesc = createPrimitiveDesc(srcDnnlDesc, weiDnnlDesc, biaDnnlDesc, diff --git a/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp b/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp index d90909ff6fd..95600b7677d 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp @@ -98,8 +98,15 @@ static bool fullyMatchConfiguration(const MemoryDescArgs& currentDescriptors, for (size_t i = 0; i < typeConfig.size(); i++) { const auto& type = typeConfig[i]; const auto& desc = currentDescriptors.at(notation[i]); - if ((!one_of(desc->getPrecision(), type, ov::element::undefined)) || !desc->hasLayoutType(layoutConfig[i])) - return false; + + if (desc->empty()) + continue; + + if (desc->getPrecision() != type) + return false; // type mismatch + + if (!desc->hasLayoutType(layoutConfig[i])) + return false; // layout mismatch } return true; @@ -118,7 +125,10 @@ static MemoryDescArgs createOptimalDescriptors(const MemoryDescArgs& currentDesc const auto& type = typeConfig[i]; const auto& layout = layoutConfig[i]; - if (one_of(descType, ov::element::undefined, type)) { + if (desc->empty()) + continue; + + if (descType == type && desc->hasLayoutType(layout)) { continue; } diff --git a/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp b/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp index ab79dbde80f..d8df1acaefa 100644 --- a/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp +++ b/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp @@ -16,7 +16,6 @@ #include "memory_desc/blocked_memory_desc.h" #include "memory_desc/cpu_memory_desc.h" #include "memory_desc/cpu_memory_desc_utils.h" -#include "memory_desc/dnnl_blocked_memory_desc.h" #include "nodes/executors/executor.hpp" #include "nodes/executors/fullyconnected_config.hpp" #include "openvino/core/type/element_type.hpp" @@ -63,16 +62,6 @@ FullyConnected::FullyConnected(const std::shared_ptr& op, const GraphC std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); - - auto createEmptyMemoryDesc = [](const ov::element::Type type) { - return std::make_shared(type, Shape{0}); - }; - - auto createEmptyMemory = [&createEmptyMemoryDesc](const GraphContext::CPtr context, const ov::element::Type type) { - return std::make_shared(context->getEngine(), createEmptyMemoryDesc(type), nullptr); - }; - - emptyMemory = createEmptyMemory(context, ov::element::undefined); } bool FullyConnected::canBeExecutedInInt8() const { @@ -236,14 +225,10 @@ void FullyConnected::initSupportedPrimitiveDescriptors() { dstDescs.push_back(dstDesc); } - auto createEmptyMemoryDesc = [](const ov::element::Type type) { - return std::make_shared(type, Shape{0}); - }; - MemoryDescArgs descs{ {ARG_SRC, srcDescs[0]}, {ARG_WEI, srcDescs[1]}, - {ARG_BIAS, attrs.withBias ? srcDescs[2] : createEmptyMemoryDesc(ov::element::undefined)}, + {ARG_BIAS, attrs.withBias ? srcDescs[2] : MemoryDescUtils::makeEmptyDesc()}, {ARG_DST, dstDescs[0]}, }; @@ -265,7 +250,7 @@ void FullyConnected::initSupportedPrimitiveDescriptors() { void FullyConnected::createPrimitive() { memory[ARG_SRC] = getSrcMemoryAtPort(DATA_ID); memory[ARG_WEI] = getSrcMemoryAtPort(WEIGHTS_ID); - memory[ARG_BIAS] = attrs.withBias ? getSrcMemoryAtPort(BIAS_ID) : emptyMemory; + memory[ARG_BIAS] = attrs.withBias ? getSrcMemoryAtPort(BIAS_ID) : MemoryDescUtils::makeEmptyMemory(context); memory[ARG_DST] = getDstMemoryAtPort(0); // @todo should we preconfigure only for dynamic shapes? // Since for static shapes primitive is created in scope of compile_model() anyway diff --git a/src/plugins/intel_cpu/src/nodes/fullyconnected.h b/src/plugins/intel_cpu/src/nodes/fullyconnected.h index 7fef9c6ecb9..06b83d6a784 100644 --- a/src/plugins/intel_cpu/src/nodes/fullyconnected.h +++ b/src/plugins/intel_cpu/src/nodes/fullyconnected.h @@ -76,7 +76,6 @@ private: FCAttrs attrs; PostOps postOps; MemoryArgs memory; - MemoryPtr emptyMemory; ExecutorFactoryPtr factory; ExecutorPtr executor = nullptr; std::string errorPrefix; diff --git a/src/plugins/intel_cpu/src/nodes/memory.cpp b/src/plugins/intel_cpu/src/nodes/memory.cpp index ccd777f8997..5b3379ba78a 100644 --- a/src/plugins/intel_cpu/src/nodes/memory.cpp +++ b/src/plugins/intel_cpu/src/nodes/memory.cpp @@ -8,10 +8,9 @@ #include "memory.hpp" #include "scaled_attn.h" #include "common/cpu_convert.h" -#include "common/cpu_memcpy.h" #include "utils/general_utils.h" #include "memory_desc/dnnl_blocked_memory_desc.h" -#include "utils/ngraph_utils.hpp" +#include "memory_desc/cpu_memory_desc_utils.h" #include "shape_inference/shape_inference_pass_through.hpp" #include "common/arbitrary_order_desc_creator.h" diff --git a/src/plugins/intel_cpu/src/nodes/non_max_suppression.cpp b/src/plugins/intel_cpu/src/nodes/non_max_suppression.cpp index 9eb47e038c9..944318a3c24 100644 --- a/src/plugins/intel_cpu/src/nodes/non_max_suppression.cpp +++ b/src/plugins/intel_cpu/src/nodes/non_max_suppression.cpp @@ -9,6 +9,7 @@ #include "non_max_suppression.h" +#include "cpu_types.h" #include "openvino/core/parallel.hpp" #include "utils/general_utils.h" #include "shape_inference/shape_inference_internal_dyn.hpp" @@ -18,8 +19,6 @@ #include - - namespace ov { namespace intel_cpu { namespace node { @@ -891,14 +890,14 @@ void NonMaxSuppression::check1DInput(const Shape& shape, const std::string& name THROW_CPU_NODE_ERR("has unsupported '", name, "' input rank: ", shape.getRank()); if (shape.getRank() == 1) if (shape.getDims()[0] != 1) - THROW_CPU_NODE_ERR("has unsupported '", name, "' input 1st dimension size: ", MemoryDescUtils::dim2str(shape.getDims()[0])); + THROW_CPU_NODE_ERR("has unsupported '", name, "' input 1st dimension size: ", dim2str(shape.getDims()[0])); } void NonMaxSuppression::checkOutput(const Shape& shape, const std::string& name, const size_t port) { if (shape.getRank() != 2) THROW_CPU_NODE_ERR("has unsupported '", name, "' output rank: ", shape.getRank()); if (shape.getDims()[1] != 3) - THROW_CPU_NODE_ERR("has unsupported '", name, "' output 2nd dimension size: ", MemoryDescUtils::dim2str(shape.getDims()[1])); + THROW_CPU_NODE_ERR("has unsupported '", name, "' output 2nd dimension size: ", dim2str(shape.getDims()[1])); } bool NonMaxSuppression::isExecutable() const { diff --git a/src/plugins/intel_cpu/src/nodes/rnn.cpp b/src/plugins/intel_cpu/src/nodes/rnn.cpp index eaeb4b4aba3..564a2de9b07 100644 --- a/src/plugins/intel_cpu/src/nodes/rnn.cpp +++ b/src/plugins/intel_cpu/src/nodes/rnn.cpp @@ -6,10 +6,10 @@ #include "utils/general_utils.h" #include "nodes/common/cpu_memcpy.h" #include "nodes/common/cpu_convert.h" -#include "utils/bfloat16.hpp" #include "input.h" #include "dnnl_extension_utils.h" #include "memory_desc/dnnl_blocked_memory_desc.h" +#include "memory_desc/cpu_memory_desc_utils.h" #include "common/primitive_hashing_utils.hpp" #include #include "shape_inference/shape_inference_ngraph.hpp" diff --git a/src/plugins/intel_cpu/tests/unit/memory_desc/empty_memory_desc.cpp b/src/plugins/intel_cpu/tests/unit/memory_desc/empty_memory_desc.cpp new file mode 100644 index 00000000000..e9dc2b60e15 --- /dev/null +++ b/src/plugins/intel_cpu/tests/unit/memory_desc/empty_memory_desc.cpp @@ -0,0 +1,45 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include "dnnl_extension_utils.h" +#include "memory_desc/cpu_blocked_memory_desc.h" +#include "memory_desc/dnnl_blocked_memory_desc.h" +#include "memory_desc/cpu_memory_desc.h" +#include "memory_desc/cpu_memory_desc_utils.h" +#include "openvino/core/type/element_type.hpp" + +using namespace ov::intel_cpu; + +TEST(MemoryTest, EmptyMemoryDescVerifyPublicInterface) { + const auto emptyDesc = MemoryDescUtils::makeEmptyDesc(); + ASSERT_EQ(emptyDesc->getType(), MemoryDescType::Empty); + + ASSERT_EQ(emptyDesc->getShape(), Shape{0}); + + ASSERT_TRUE(emptyDesc->empty()); + + ASSERT_TRUE(emptyDesc->clone()->empty()); + + ASSERT_EQ(emptyDesc->getPrecision(), ov::element::undefined); + + ASSERT_EQ(emptyDesc->getOffsetPadding(), 0); + + for (const auto& layout : {LayoutType::ncsp, LayoutType::nspc, LayoutType::nCsp8c, LayoutType::nCsp16c}) { + ASSERT_FALSE(emptyDesc->hasLayoutType(layout)); + } + + ASSERT_EQ(emptyDesc->serializeFormat(), "empty"); + + ASSERT_EQ(emptyDesc->getMaxMemSize(), 0); + + ASSERT_THROW(emptyDesc->cloneWithNewPrecision(ov::element::f32), ov::Exception); + + // compatible with empty memory desc + ASSERT_TRUE(emptyDesc->isCompatible(*emptyDesc->clone())); + // not compatible with any other memory desc + ASSERT_FALSE(emptyDesc->isCompatible(CpuBlockedMemoryDesc{ov::element::f32, Shape{1, 2, 3}})); + ASSERT_FALSE(emptyDesc->isCompatible(DnnlBlockedMemoryDesc{ov::element::u8, Shape{1}})); + ASSERT_FALSE(emptyDesc->isCompatible(CpuBlockedMemoryDesc{ov::element::undefined, Shape{0}})); +}