natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CPU] Use Dnnl executor to avoid extra dnnl primitve desc query (#16372)

This commit is contained in:
Maksim Kutakov 2023-03-23 13:25:39 +01:00 committed by GitHub
parent 3b8d9c568c
commit 8a246a8bf2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 205 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/plugins/intel_cpu/src/memory_desc/dnnl_memory_desc.cpp
View File

@ -36,13 +36,18 @@ MemoryDescPtr DnnlMemoryDesc::cloneWithNewPrecision(const InferenceEngine::Preci
}
bool DnnlMemoryDesc::isCompatible(const MemoryDesc &rhs) const {
if (MemoryDescType::Dnnl == rhs.getType()) {
return this->desc == rhs.as<DnnlMemoryDesc>()->desc;
if (MemoryDescType::Dnnl & rhs.getType()) {
auto* dnnMemDesc = rhs.as<DnnlMemoryDesc>();
return isCompatible(*dnnMemDesc);
} else {
return false;
}
}
bool DnnlMemoryDesc::isCompatible(const DnnlMemoryDesc& rhs) const {
return this->desc == rhs.desc;
}
std::string DnnlMemoryDesc::serializeFormat() const {
dnnl::impl::memory_desc_wrapper wrapped(desc.get());
if (wrapped.is_wino_desc()) {

1
src/plugins/intel_cpu/src/memory_desc/dnnl_memory_desc.h
View File

@ -26,6 +26,7 @@ public:
MemoryDescPtr cloneWithNewPrecision(const InferenceEngine::Precision prec) const override;
bool isCompatible(const MemoryDesc& rhs) const override;
bool isCompatible(const DnnlMemoryDesc& rhs) const;
bool hasLayoutType(LayoutType layoutType) const override { return false; }

6
src/plugins/intel_cpu/src/node.cpp
View File

@ -550,12 +550,6 @@ std::vector<memory::format_tag> Node::getAvailableFormatsForDims(const Shape &di
return {memory::format_tag::any};
}
void Node::execute(dnnl::stream strm) {
if (prim) {
prim.execute(strm, primArgs);
}
}
void Node::updateShapes() {
IE_ASSERT(isDynamicNode()) << "Node::updateShapes() is called to a static shape node of type: " << getTypeStr() << " with name: " << getName();
if (needShapeInfer()) {

10
src/plugins/intel_cpu/src/node.h
View File

@ -334,7 +334,7 @@ public:
void resolveInPlaceEdges();
virtual void execute(dnnl::stream strm);
virtual void execute(dnnl::stream strm) = 0;
void updateShapes();
void updateDynamicParams();
void executeDynamic(dnnl::stream strm);
@ -578,7 +578,6 @@ protected:
std::vector<NodeDesc> supportedPrimitiveDescriptors;
std::unordered_map<int, dnnl::memory> primArgs;
std::unordered_map<int, MemoryPtr> postOpsArgs;
dnnl::primitive prim;
std::vector<dnnl::primitive_desc> descs;
const GraphContext::CPtr context;
@ -649,9 +648,10 @@ protected:
IE_THROW(NotImplemented) << "[DS] prapareParams not implemented for node with type " << NameFromType(getType());
}
MemoryPtr getScratchPadMem(const const_dnnl_primitive_desc_t& pd) {
auto scratchpadMemoryDesc = DnnlExtensionUtils::query_md(pd, dnnl::query::scratchpad_md);
scratchpadMem = context->getScratchPad()->createScratchPadMem(scratchpadMemoryDesc);
MemoryPtr getScratchPadMem(const DnnlMemoryDescPtr& desc) {
if (!scratchpadMem || !scratchpadMem->getDesc().isCompatible(*desc)) {
scratchpadMem = context->getScratchPad()->createScratchPadMem(desc);
}
return scratchpadMem;
}

39
src/plugins/intel_cpu/src/nodes/common/dnnl_executor.cpp
View File

@ -9,6 +9,14 @@ using namespace dnnl;
namespace ov {
namespace intel_cpu {
DnnlExecutor::DnnlExecutor(const dnnl::primitive_desc& pd) {
execPrim = dnnl::primitive(pd);
src_md = DnnlExtensionUtils::makeDescriptor(pd.src_desc());
dst_md = DnnlExtensionUtils::makeDescriptor(pd.dst_desc());
wghts_md = DnnlExtensionUtils::makeDescriptor(pd.weights_desc());
scrch_md = DnnlExtensionUtils::makeDescriptor(pd.scratchpad_desc());
}
DnnlExecutor::IntermReorder::IntermReorder(const dnnl::memory::desc& descSrc,
const dnnl::memory::desc& descDst,
const dnnl::engine& engine) : m_descSrc(descSrc), m_descDst(descDst) {
@ -20,7 +28,15 @@ void DnnlExecutor::IntermReorder::exec(dnnl::memory& memSrc, dnnl::memory& memDs
m_reorder.execute(strm, memSrc, memDst);
}
void DnnlExecutor::exec(std::unordered_map<int, dnnl::memory> primArgs, dnnl::stream strm) {
void DnnlExecutor::exec(const std::unordered_map<int, dnnl::memory>& primArgs, dnnl::stream strm) {
if (inputReorders.empty() && outputReorders.empty()) {
execPrim.execute(strm, primArgs);
} else {
reorder_exec(primArgs, strm);
}
}
void DnnlExecutor::reorder_exec(std::unordered_map<int, dnnl::memory> primArgs, dnnl::stream strm) {
for (auto &inReorder : inputReorders) {
if (primArgs.count(inReorder.first)) {
dnnl::memory memDst(inReorder.second.getDstDesc(), strm.get_engine());
@ -58,27 +74,6 @@ const_dnnl_primitive_desc_t DnnlExecutor::getPrimitiveDesc() const {
return execPrim.get_primitive_desc();
}
dnnl::memory::desc DnnlExecutor::getSrcDesc() const {
auto pd = getPrimitiveDesc();
auto md = DnnlExtensionUtils::query_md(pd, dnnl::query::src_md);
return md->getDnnlDesc();
}
dnnl::memory::desc DnnlExecutor::getWeightDesc() const {
auto pd = getPrimitiveDesc();
auto md = DnnlExtensionUtils::query_md(pd, dnnl::query::weights_md);
return md->getDnnlDesc();
}
dnnl::memory::desc DnnlExecutor::getDstDesc() const {
auto pd = getPrimitiveDesc();
auto md = DnnlExtensionUtils::query_md(pd, dnnl::query::dst_md);
return md->getDnnlDesc();
}
impl_desc_type DnnlExecutor::getImplementationType() const {
auto pd = getPrimitiveDesc();
return parse_impl_name(DnnlExtensionUtils::query_impl_info_str(pd));

40
src/plugins/intel_cpu/src/nodes/common/dnnl_executor.h
View File

@ -26,22 +26,52 @@ class DnnlExecutor {
};
public:
void exec(std::unordered_map<int, dnnl::memory> primArgs, dnnl::stream strm);
explicit DnnlExecutor(const dnnl::primitive_desc& pd);
void exec(const std::unordered_map<int, dnnl::memory>& primArgs, dnnl::stream strm);
bool needReordering() const;
virtual ~DnnlExecutor() = default;
dnnl::primitive getExecPrim() const;
const_dnnl_primitive_desc_t getPrimitiveDesc() const;
dnnl::memory::desc getSrcDesc() const;
dnnl::memory::desc getWeightDesc() const;
dnnl::memory::desc getDstDesc() const;
impl_desc_type getImplementationType() const;
DnnlMemoryDescPtr getSrcDesc() const {
return src_md;
}
DnnlMemoryDescPtr getWeightDesc() const {
return wghts_md;
}
DnnlMemoryDescPtr getDstDesc() const {
return dst_md;
}
DnnlMemoryDescPtr getScratchPadDesc() const {
return scrch_md;
}
const dnnl::memory::desc& getDnnlSrcDesc() const {
return src_md->getDnnlDesc();
}
const dnnl::memory::desc& getDnnlWeightDesc() const {
return wghts_md->getDnnlDesc();
}
const dnnl::memory::desc& getDnnlDstDesc() const {
return dst_md->getDnnlDesc();
}
const dnnl::memory::desc& getDnnlScratchPadDesc() const {
return scrch_md->getDnnlDesc();
}
protected:
void reorder_exec(std::unordered_map<int, dnnl::memory> primArgs, dnnl::stream strm);
protected:
DnnlExecutor() = default;
dnnl::primitive execPrim;
// key is the port number for the primitive that needs memory reordering
std::unordered_map<int, IntermReorder> inputReorders;
std::unordered_map<int, IntermReorder> outputReorders;
DnnlMemoryDescPtr src_md;
DnnlMemoryDescPtr wghts_md;
DnnlMemoryDescPtr dst_md;
DnnlMemoryDescPtr scrch_md;
};
} // namespace intel_cpu

1
src/plugins/intel_cpu/src/nodes/concat.h
View File

@ -52,6 +52,7 @@ private:
InferenceEngine::Precision outputPrecision = InferenceEngine::Precision::FP32;
bool canExecRef = false;
static constexpr size_t MAX_RANK_REF = 6;
dnnl::primitive prim;
};
} // namespace node

19
src/plugins/intel_cpu/src/nodes/conv.cpp
View File

@ -1490,8 +1490,7 @@ void Convolution::prepareParams() {
Node::appendPostOpArgs(*pAttrLocal, primArgs, convPostOpsArgs[preferLegacyPostOps]);
auto pd = execPtr->getPrimitiveDesc();
auto scratchpadMem = getScratchPadMem(pd);
auto scratchpadMem = getScratchPadMem(execPtr->getScratchPadDesc());
primArgs[DNNL_ARG_SCRATCHPAD] = scratchpadMem->GetPrimitive();
#ifdef CPU_DEBUG_CAPS
@ -1508,19 +1507,17 @@ Convolution::ConvolutionExecutor::ConvolutionExecutor(const dnnl::convolution_fo
const dnnl::memory::desc& inMemDesc,
const dnnl::memory::desc& weightMemDesc,
const dnnl::memory::desc& outMemDesc,
const dnnl::engine& engine) {
execPrim = dnnl::convolution_forward(pd);
if (inMemDesc != pd.src_desc()) {
inputReorders.insert({DNNL_ARG_SRC, IntermReorder(inMemDesc, pd.src_desc(), engine)});
const dnnl::engine& engine) : DnnlExecutor(pd) {
if (inMemDesc != getDnnlSrcDesc()) {
inputReorders.insert({DNNL_ARG_SRC, IntermReorder(inMemDesc, getDnnlSrcDesc(), engine)});
}
if (weightMemDesc != pd.weights_desc()) {
inputReorders.insert({DNNL_ARG_WEIGHTS, IntermReorder(weightMemDesc, pd.weights_desc(), engine)});
if (weightMemDesc != getDnnlWeightDesc()) {
inputReorders.insert({DNNL_ARG_WEIGHTS, IntermReorder(weightMemDesc, getDnnlWeightDesc(), engine)});
}
if (outMemDesc != pd.dst_desc()) {
outputReorders.insert({DNNL_ARG_DST, IntermReorder(pd.dst_desc(), outMemDesc, engine)});
if (outMemDesc != getDnnlDstDesc()) {
outputReorders.insert({DNNL_ARG_DST, IntermReorder(getDnnlDstDesc(), outMemDesc, engine)});
}
}

23
src/plugins/intel_cpu/src/nodes/deconv.cpp
View File

@ -991,8 +991,7 @@ void Deconvolution::prepareParams() {
}
Node::appendPostOpArgs(*pAttrLocal, primArgs, postOpsArgs);
auto pd = execPtr->getPrimitiveDesc();
auto scratchpadMem = getScratchPadMem(pd);
auto scratchpadMem = getScratchPadMem(execPtr->getScratchPadDesc());
primArgs[DNNL_ARG_SCRATCHPAD] = scratchpadMem->GetPrimitive();
#ifdef CPU_DEBUG_CAPS
if (result.second == CacheEntryBase::LookUpStatus::Miss) {
@ -1094,9 +1093,7 @@ Deconvolution::DeconvExecutorDefault::DeconvExecutorDefault(const dnnl::convolut
const dnnl::memory::desc& inMemDesc,
const dnnl::memory::desc& weightMemDesc,
const dnnl::memory::desc& outMemDesc,
const dnnl::engine& engine) {
execPrim = dnnl::convolution_backward_data(pd);
const dnnl::engine& engine) : DnnlExecutor(pd) {
if (inMemDesc != pd.diff_dst_desc()) {
inputReorders.insert({DNNL_ARG_DIFF_DST, IntermReorder(inMemDesc, pd.diff_dst_desc(), engine)});
}
@ -1114,19 +1111,17 @@ Deconvolution::DeconvExecutorInt8::DeconvExecutorInt8(const dnnl::deconvolution_
const dnnl::memory::desc& inMemDesc,
const dnnl::memory::desc& weightMemDesc,
const dnnl::memory::desc& outMemDesc,
const dnnl::engine& engine) {
execPrim = dnnl::deconvolution_forward(pd);
if (inMemDesc != pd.src_desc()) {
inputReorders.insert({DNNL_ARG_SRC, IntermReorder(inMemDesc, pd.src_desc(), engine)});
const dnnl::engine& engine) : DnnlExecutor(pd) {
if (inMemDesc != getDnnlSrcDesc()) {
inputReorders.insert({DNNL_ARG_SRC, IntermReorder(inMemDesc, getDnnlSrcDesc(), engine)});
}
if (weightMemDesc != pd.weights_desc()) {
inputReorders.insert({DNNL_ARG_WEIGHTS, IntermReorder(weightMemDesc, pd.weights_desc(), engine)});
if (weightMemDesc != getDnnlWeightDesc()) {
inputReorders.insert({DNNL_ARG_WEIGHTS, IntermReorder(weightMemDesc, getDnnlWeightDesc(), engine)});
}
if (outMemDesc != pd.dst_desc()) {
outputReorders.insert({DNNL_ARG_DST, IntermReorder(pd.dst_desc(), outMemDesc, engine)});
if (outMemDesc != getDnnlDstDesc()) {
outputReorders.insert({DNNL_ARG_DST, IntermReorder(getDnnlDstDesc(), outMemDesc, engine)});
}
}

49
src/plugins/intel_cpu/src/nodes/fullyconnected.cpp
View File

@ -311,7 +311,7 @@ void FullyConnected::prepareParams() {
implementationTypeIP,
useConv1x1};
auto engine = getEngine();
auto& engine = getEngine();
auto builder = [&engine](const FCKey& key) -> executorPtr {
executorPtr execPtr = nullptr;
@ -333,7 +333,7 @@ void FullyConnected::prepareParams() {
}
if (prim_desc) {
execPtr = std::make_shared<ExecutorConv1x1>(prim_desc);
execPtr = std::make_shared<DnnlExecutor>(prim_desc);
}
}
// fallback
@ -388,7 +388,7 @@ void FullyConnected::prepareParams() {
}
}
execPtr = std::make_shared<ExecutorInnerProduct>(prim_desc);
execPtr = std::make_shared<DnnlExecutor>(prim_desc);
}
return execPtr;
};
@ -404,26 +404,20 @@ void FullyConnected::prepareParams() {
execPtr = result.first;
if (execPtr) {
// no executor yet or shapes changed
if (!prevExecPtr || prevExecPtr->getSrcDesc() != execPtr->getSrcDesc()) {
auto oldMem = srcMemPtr->GetPrimitive();
// fast path: wanted is same with parent node output, typical is static shape with inner product
if (execPtr->getSrcDesc() == inDesc->getDnnlDesc()) {
primArgs[DNNL_ARG_SRC] = std::move(oldMem);
} else {
primArgs[DNNL_ARG_SRC] = dnnl::memory(execPtr->getSrcDesc(), oldMem.get_engine(), oldMem.get_data_handle());
}
if (execPtr->getSrcDesc()->isCompatible(*inDesc)) {
primArgs[DNNL_ARG_SRC] = srcMemPtr->GetPrimitive();
} else {
primArgs[DNNL_ARG_SRC] = dnnl::memory(execPtr->getDnnlSrcDesc(), engine, srcMemPtr->GetData());
}
if (!prevExecPtr || prevExecPtr->getDstDesc() != execPtr->getDstDesc()) {
auto oldMem = dstMemPtr->GetPrimitive();
if (execPtr->getDstDesc() == outDesc->getDnnlDesc()) {
primArgs[DNNL_ARG_DST] = std::move(oldMem);
} else {
primArgs[DNNL_ARG_DST] = dnnl::memory(execPtr->getDstDesc(), oldMem.get_engine(), oldMem.get_data_handle());
}
if (execPtr->getDstDesc()->isCompatible(*outDesc)) {
primArgs[DNNL_ARG_DST] = dstMemPtr->GetPrimitive();
} else {
primArgs[DNNL_ARG_DST] = dnnl::memory(execPtr->getDnnlDstDesc(), engine, dstMemPtr->GetData());
}
if (!prevExecPtr || prevExecPtr->getWeightDesc() != execPtr->getWeightDesc()) {
primArgs[DNNL_ARG_WEIGHTS] = prepareWeightMemory(DnnlExtensionUtils::makeDescriptor(execPtr->getWeightDesc()))->GetPrimitive();
if (!prevExecPtr || !execPtr->getWeightDesc()->isCompatible(*(prevExecPtr->getWeightDesc()))) {
primArgs[DNNL_ARG_WEIGHTS] = prepareWeightMemory(execPtr->getWeightDesc())->GetPrimitive();
}
// changed shapes may also cause the kernel type changed
selected_pd->setImplementationType(execPtr->getImplementationType());
@ -438,9 +432,8 @@ void FullyConnected::prepareParams() {
primArgs[DNNL_ARG_BIAS] = biasMemPtr->GetPrimitive();
}
auto pd = execPtr->getPrimitiveDesc();
auto scratchpadMem = getScratchPadMem(pd);
primArgs[DNNL_ARG_SCRATCHPAD] = scratchpadMem->GetPrimitive();
auto schratchpadMem = getScratchPadMem(execPtr->getScratchPadDesc());
primArgs[DNNL_ARG_SCRATCHPAD] = schratchpadMem->GetPrimitive();
#ifdef CPU_DEBUG_CAPS
if (result.second == CacheEntryBase::LookUpStatus::Miss) {
DEBUG_LOG("verbose##", getName(), "##", pd->info(), "\n");
@ -919,14 +912,6 @@ bool FullyConnected::canBeExecutedInConv1x1() const {
return retVal;
}
FullyConnected::ExecutorInnerProduct::ExecutorInnerProduct(const dnnl::inner_product_forward::primitive_desc& pd) {
execPrim = dnnl::inner_product_forward(pd);
}
FullyConnected::ExecutorConv1x1::ExecutorConv1x1(const dnnl::convolution_forward::primitive_desc& pd) {
execPrim = dnnl::convolution_forward(pd);
}
MemoryPtr FullyConnected::prepareWeightMemory(DnnlMemoryDescPtr weightDesc) {
if (!getParentEdgeAt(1)->getParent()->isConstant())
IE_THROW() << "Weight input is not const for node " << getName() << ".";

10
src/plugins/intel_cpu/src/nodes/fullyconnected.h
View File

@ -90,16 +90,6 @@ private:
std::unordered_map<std::string, MemoryPtr> privateWeightCache;
dnnl::primitive_attr attr;
class ExecutorInnerProduct : public DnnlExecutor {
public:
ExecutorInnerProduct(const dnnl::inner_product_forward::primitive_desc& pd);
};
class ExecutorConv1x1 : public DnnlExecutor {
public:
ExecutorConv1x1(const dnnl::convolution_forward::primitive_desc& pd);
};
static dnnl::convolution_forward::primitive_desc
createDescriptorInternalForConv(DnnlMemoryDescCPtr inputDescPtr,
DnnlMemoryDescCPtr weightDescPtr,

1
src/plugins/intel_cpu/src/nodes/input.h
View File

@ -31,6 +31,7 @@ public:
void withMeanImage();
MemoryCPtr getMemoryPtr() const;
void execute(dnnl::stream strm) override {}
void executeDynamicImpl(dnnl::stream strm) override {}
bool isExecutable() const override {
return false;

1
src/plugins/intel_cpu/src/nodes/interaction.h
View File

@ -60,6 +60,7 @@ public:
private:
void execRef(dnnl::stream strm);
dnnl::primitive prim;
size_t batchSize = 0;
size_t featureSize = 0;
size_t inputSizes = 0;

27
src/plugins/intel_cpu/src/nodes/lrn.cpp
View File

@ -182,7 +182,7 @@ void Lrn::prepareParams() {
LrnKey key = {inpDesc, selected_pd->getImplementationType(), alg, size, k, alpha, beta, attr};
auto engine = getEngine();
auto builder = [&engine](const LrnKey& key) -> dnnl::primitive {
auto builder = [&engine](const LrnKey& key) -> executorPtr {
auto desc = std::make_shared<dnnl::lrn_forward::primitive_desc>(
engine,
dnnl::prop_kind::forward_inference,
@ -205,25 +205,24 @@ void Lrn::prepareParams() {
break;
}
if (!itpd.next_impl())
return dnnl::lrn_forward();
return nullptr;
}
return dnnl::lrn_forward(prim_desc);
return std::make_shared<DnnlExecutor>(prim_desc);
};
auto cache = context->getParamsCache();
auto result = cache->getOrCreate(key, builder);
if (!result.first) {
execPtr = result.first;
if (!execPtr) {
IE_THROW() << "Primitive descriptor was not found for node " << getName() << ".";
}
prim = result.first;
auto pd = prim.get_primitive_desc();
auto scratchpadMem = getScratchPadMem(pd);
auto scratchpadMem = getScratchPadMem(execPtr->getScratchPadDesc());
auto src = srcMemPtr->GetPrimitive();
auto dst = dstMemPtr->GetPrimitive();
primArgs = { {DNNL_ARG_SRC, src}, {DNNL_ARG_DST, dst}, {DNNL_ARG_SCRATCHPAD, scratchpadMem->GetPrimitive()} };
primArgs[DNNL_ARG_SCRATCHPAD] = scratchpadMem->GetPrimitive();
primArgs[DNNL_ARG_SRC] = srcMemPtr->GetPrimitive();
primArgs[DNNL_ARG_DST] = dstMemPtr->GetPrimitive();
}
bool Lrn::created() const {
@ -250,6 +249,14 @@ void Lrn::createDescriptor(const std::vector<MemoryDescPtr> &inputDesc,
descs.push_back(desc);
}
void Lrn::execute(dnnl::stream strm) {
if (execPtr) {
execPtr->exec(primArgs, strm);
} else {
IE_THROW() << errorPrefix << " doesn't have an initialized executor";
}
}
void Lrn::executeDynamicImpl(dnnl::stream strm) {
execute(strm);
}

4
src/plugins/intel_cpu/src/nodes/lrn.h
View File

@ -9,6 +9,7 @@
#include <string>
#include <memory>
#include <vector>
#include "common/dnnl_executor.h"
namespace ov {
namespace intel_cpu {
@ -31,11 +32,14 @@ public:
}
void prepareParams() override;
void execute(dnnl::stream strm) override;
void executeDynamicImpl(dnnl::stream strm) override;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
using executorPtr = std::shared_ptr<DnnlExecutor>;
executorPtr execPtr = nullptr;
dnnl::algorithm alg;
size_t size = 1;
int k = 1;

22
src/plugins/intel_cpu/src/nodes/matmul.cpp
View File

@ -593,7 +593,7 @@ void MatMul::prepareParams() {
auto engine = getEngine();
auto builder = [&engine](const MatMulKey& key) -> dnnl::primitive {
auto builder = [&engine](const MatMulKey& key) -> executorPtr {
dnnl::matmul::primitive_desc matmul_desc;
if (key.bias) {
@ -633,22 +633,20 @@ void MatMul::prepareParams() {
break;
}
}
return matmul(prim_desc);
return std::make_shared<DnnlExecutor>(prim_desc);
};
auto cache = context->getParamsCache();
auto result = cache->getOrCreate(key, builder);
if (!result.first) {
execPtr = result.first;
if (!execPtr) {
IE_THROW() << "Primitive descriptor was not found for node " << getName() << ".";
}
prim = result.first;
auto schratchpadMem = getScratchPadMem(execPtr->getScratchPadDesc());
auto pd = prim.get_primitive_desc();
auto scratchpadMem = getScratchPadMem(pd);
primArgs[DNNL_ARG_SCRATCHPAD] = scratchpadMem->GetPrimitive();
primArgs[DNNL_ARG_SCRATCHPAD] = schratchpadMem->GetPrimitive();
primArgs[DNNL_ARG_SRC_0] = src0MemPtr->GetPrimitive();
primArgs[DNNL_ARG_WEIGHTS_0] = src1MemPtr->GetPrimitive();
primArgs[DNNL_ARG_DST] = dstMemPtr->GetPrimitive();
@ -658,6 +656,14 @@ void MatMul::prepareParams() {
appendPostOpArgs(*attr, primArgs, postOpsArgs);
}
void MatMul::execute(dnnl::stream strm) {
if (execPtr) {
execPtr->exec(primArgs, strm);
} else {
IE_THROW() << errorPrefix << " doesn't have an initialized executor";
}
}
void MatMul::executeDynamicImpl(dnnl::stream strm) {
execute(strm);
}

4
src/plugins/intel_cpu/src/nodes/matmul.h
View File

@ -10,6 +10,7 @@
#include <vector>
#include <array>
#include "memory_desc/dnnl_blocked_memory_desc.h"
#include "common/dnnl_executor.h"
namespace ov {
namespace intel_cpu {
@ -38,6 +39,7 @@ public:
}
void prepareParams() override;
void execute(dnnl::stream strm) override;
void executeDynamicImpl(dnnl::stream strm) override;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
@ -48,6 +50,8 @@ protected:
AttrPtr initPrimitiveAttr(const VectorDims& dims);
private:
using executorPtr = std::shared_ptr<DnnlExecutor>;
executorPtr execPtr = nullptr;
dnnl::memory::desc getBiasDescFrom(const DnnlMemoryDescCPtr outMemDesc);
std::pair<Shape, Shape> makeDummyInputShapes(const Shape& in0, const Shape& in1) const;

27
src/plugins/intel_cpu/src/nodes/pooling.cpp
View File

@ -369,7 +369,7 @@ void Pooling::prepareParams() {
alg,
selected_pd->getImplementationType()};
auto engine = getEngine();
auto builder = [&engine](const PoolingKey& key) -> dnnl::primitive {
auto builder = [&engine](const PoolingKey& key) -> executorPtr {
primitive_desc_iterator itpd = createDescriptorHelper(engine,
key.inp->getDnnlDesc(),
key.out->getDnnlDesc(),
@ -393,27 +393,34 @@ void Pooling::prepareParams() {
break;
}
return pooling_forward(prim_desc);
return std::make_shared<DnnlExecutor>(prim_desc);
};
auto cache = context->getParamsCache();
auto result = cache->getOrCreate(key, builder);
if (!result.first) {
execPtr = result.first;
if (!execPtr) {
IE_THROW() << "Primitive descriptor was not found for node " << getName() << ".";
}
prim = result.first;
auto pd = prim.get_primitive_desc();
auto scratchpadMem = getScratchPadMem(pd);
auto src = getParentEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive();
auto dst = getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive();
primArgs = {{DNNL_ARG_SRC, src}, {DNNL_ARG_DST, dst}, {DNNL_ARG_SCRATCHPAD, scratchpadMem->GetPrimitive()}};
auto scratchpadMem = getScratchPadMem(execPtr->getScratchPadDesc());
primArgs[DNNL_ARG_SCRATCHPAD] = scratchpadMem->GetPrimitive();
primArgs[DNNL_ARG_SRC] = getParentEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive();
primArgs[DNNL_ARG_DST] = getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive();
Node::appendPostOpArgs(*attr, primArgs, postOpsArgs);
}
void Pooling::execute(dnnl::stream strm) {
if (execPtr) {
execPtr->exec(primArgs, strm);
} else {
IE_THROW() << "Pooling node with name '" << getName() << "' doesn't have an initialized executor";
}
}
void Pooling::executeDynamicImpl(dnnl::stream strm) {
execute(strm);
}

5
src/plugins/intel_cpu/src/nodes/pooling.h
View File

@ -10,6 +10,7 @@
#include <string>
#include <memory>
#include <vector>
#include "common/dnnl_executor.h"
namespace ov {
namespace intel_cpu {
@ -30,6 +31,7 @@ public:
}
void prepareParams() override;
void execute(dnnl::stream strm) override;
void executeDynamicImpl(dnnl::stream strm) override;
static bool isSupportedOperation(const std::shared_ptr<const ov::Node>& op, std::string& errorMessage) noexcept;
@ -38,6 +40,9 @@ protected:
AttrPtr initPrimitiveAttr() override;
private:
using executorPtr = std::shared_ptr<DnnlExecutor>;
executorPtr execPtr = nullptr;
void setPostOps(dnnl::primitive_attr &attr);
void initEffectiveAttributes(const Shape &inDims, const Shape &outDims);

6
src/plugins/intel_cpu/src/nodes/reorder.cpp
View File

@ -336,7 +336,11 @@ void Reorder::execute(dnnl::stream strm) {
src_blocked->setDataHandle(getParentEdgeAt(0)->getMemory().GetData());
dst_blocked->setDataHandle(getChildEdgeAt(0)->getMemory().GetData());
Node::execute(strm);
if (prim) {
prim.execute(strm, primArgs);
} else {
IE_THROW() << "Reorder node with name " << getName() << " doesn't have an initialized primitive";
}
}
}

1
src/plugins/intel_cpu/src/nodes/reorder.h
View File

@ -66,6 +66,7 @@ public:
static void reorderData(const Memory &input, const Memory &output, MultiCachePtr cache = nullptr);
private:
dnnl::reorder::primitive prim;
std::shared_ptr<MemoryDesc> input;
std::shared_ptr<MemoryDesc> output;

19
src/plugins/intel_cpu/src/nodes/rnn.cpp
View File

@ -1062,7 +1062,7 @@ void RNN::prepareParams() {
RNNKey key = { inDataDescs, outDataDescs, wDescs, cell_type, cell_act, direction, *attr };
auto engine = getEngine();
auto builder = [&engine](const RNNKey& key) -> dnnl::primitive {
auto builder = [&engine](const RNNKey& key) -> executorPtr {
const auto descPtr = createPrimitiveDescriptor(engine,
key.cellType,
key.cellAct,
@ -1072,23 +1072,22 @@ void RNN::prepareParams() {
key.wDescs,
key.attr);
return dnnl::primitive(descPtr);
return std::make_shared<DnnlExecutor>(descPtr);
};
auto cache = context->getParamsCache();
auto result = cache->getOrCreate(key, builder);
if (!result.first) {
execPtr = result.first;
if (!execPtr) {
IE_THROW() << "Primitive descriptor was not found for node " << getName() << ".";
}
prim = result.first;
auto pd = prim.get_primitive_desc();
scratchpadMem = getScratchPadMem(pd);
scratchpadMem = getScratchPadMem(execPtr->getScratchPadDesc());
if (!wasMemoryPrepared || wFormatWasChanged) {
auto pd = prim.get_primitive_desc();
auto pd = execPtr->getPrimitiveDesc();
auto query_weights_md = [&](int idx = 0) -> dnnl::memory::desc {
auto what = dnnl::convert_to_c(dnnl::query::weights_md);
const_dnnl_memory_desc_t cdesc = dnnl_primitive_desc_query_md(pd, what, idx);
@ -1118,7 +1117,7 @@ std::shared_ptr<MemoryDesc> RNN::getDstMemDesc(dnnl::primitive_desc_iterator& pr
}
void RNN::execute(dnnl::stream strm) {
if (!prim)
if (!execPtr)
THROW_ERROR << "does not have initialized primitive to execute.";
const auto src_data_mem = getParentEdgeAt(0)->getMemoryPtr();
@ -1160,7 +1159,7 @@ void RNN::execute(dnnl::stream strm) {
}
}
prim.execute(strm, args);
execPtr->exec(args, strm);
}
void RNN::executeDynamicImpl(dnnl::stream strm) {

5
src/plugins/intel_cpu/src/nodes/rnn.h
View File

@ -11,6 +11,8 @@
#include <memory>
#include <vector>
#include "common/dnnl_executor.h"
namespace ov {
namespace intel_cpu {
namespace node {
@ -66,6 +68,9 @@ private:
void copyWeightsData();
using executorPtr = std::shared_ptr<DnnlExecutor>;
executorPtr execPtr = nullptr;
/** Specify mode Cell or Seq. true - Cell, false - Seq */
bool is_cell = false;

26
src/plugins/intel_cpu/src/nodes/softmax.cpp
View File

@ -170,7 +170,7 @@ void SoftMax::prepareParams() {
SoftmaxKey key = {inpDesc, selected_pd->getImplementationType(), axis, *attr};
auto engine = getEngine();
auto builder = [&engine](const SoftmaxKey& key) -> dnnl::primitive {
auto builder = [&engine](const SoftmaxKey& key) -> executorPtr {
softmax_forward::primitive_desc prim_desc;
auto desc = std::make_shared<softmax_forward::primitive_desc>(
engine,
@ -196,26 +196,32 @@ void SoftMax::prepareParams() {
break;
}
if (!itpd.next_impl())
return softmax_forward();
return nullptr;
}
return softmax_forward(prim_desc);
return std::make_shared<DnnlExecutor>(prim_desc);
};
auto cache = context->getParamsCache();
auto result = cache->getOrCreate(key, builder);
if (!result.first) {
execPtr = result.first;
if (!execPtr) {
IE_THROW() << "Primitive descriptor was not found for node " << getName() << ".";
}
prim = result.first;
auto scratchpadMem = getScratchPadMem(execPtr->getScratchPadDesc());
auto pd = prim.get_primitive_desc();
auto scratchpadMem = getScratchPadMem(pd);
primArgs[DNNL_ARG_SCRATCHPAD] = scratchpadMem->GetPrimitive();
primArgs[DNNL_ARG_SRC] = getParentEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive();
primArgs[DNNL_ARG_DST] = getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive();
}
auto src = getParentEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive();
auto dst = getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive();
primArgs = {{DNNL_ARG_SRC, src}, {DNNL_ARG_DST, dst}, {DNNL_ARG_SCRATCHPAD, scratchpadMem->GetPrimitive()}};
void SoftMax::execute(dnnl::stream strm) {
if (execPtr) {
execPtr->exec(primArgs, strm);
} else {
IE_THROW() << "Softmax node with name '" << getName() << "' doesn't have an initialized executor";
}
}
void SoftMax::executeDynamicImpl(dnnl::stream strm) {

5
src/plugins/intel_cpu/src/nodes/softmax.h
View File

@ -11,6 +11,8 @@
#include <memory>
#include <vector>
#include "common/dnnl_executor.h"
namespace ov {
namespace intel_cpu {
namespace node {
@ -26,11 +28,14 @@ public:
bool created() const override;
AttrPtr initPrimitiveAttr() override;
void prepareParams() override;
void execute(dnnl::stream strm) override;
void executeDynamicImpl(dnnl::stream strm) override;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
using executorPtr = std::shared_ptr<DnnlExecutor>;
executorPtr execPtr = nullptr;
size_t axis = 0;
};

1
src/plugins/intel_cpu/src/nodes/transpose.h
View File

@ -48,6 +48,7 @@ private:
};
using executorPtr = std::shared_ptr<TransposeExecutor>;
executorPtr execPtr = nullptr;
dnnl::primitive prim;
struct TransposeJitExecutor : public TransposeExecutor {
TransposeJitExecutor(const PermuteParams& params);