diff --git a/src/core/src/op/random_uniform.cpp b/src/core/src/op/random_uniform.cpp index 296b115979c..b07044960f4 100644 --- a/src/core/src/op/random_uniform.cpp +++ b/src/core/src/op/random_uniform.cpp @@ -92,6 +92,8 @@ bool RandomUniform::evaluate(TensorVector& outputs, const TensorVector& inputs) const auto& t_out = get_out_type(); OPENVINO_ASSERT(validate::out_et(t_out), "Unsupported type of RandomUniform: " + t_out.get_type_name()); + outputs[0].set_shape(out_shape); + auto state = ov::reference::random_uniform(out_dims.data(), static_cast(inputs[1].data()), static_cast(inputs[2].data()), diff --git a/src/core/tests/copy.cpp b/src/core/tests/copy.cpp index d7b2b4256f4..f902d7485a1 100644 --- a/src/core/tests/copy.cpp +++ b/src/core/tests/copy.cpp @@ -447,12 +447,12 @@ TEST(copy, random_uniform) { const auto min_val_param = make_shared(element::f32, Shape{1}); const auto max_val_param = make_shared(element::f32, Shape{1}); - auto out_shape = make_shared(element::i64, Shape{3}, std::vector{1, 2, 3}); + auto out_shape = make_shared(element::i64, Shape{3}, shape); auto ru = std::make_shared(out_shape, min_val_param, max_val_param, element::f32, 150, 10); // Call `evaluate` to update m_state - auto outputs = ov::TensorVector{{element::i64, out_shape->get_shape(), shape.data()}}; + auto outputs = ov::TensorVector{{element::i64, {1lu, 2lu, 3lu}}}; ru->evaluate(outputs, ov::TensorVector{{element::i64, out_shape->get_shape(), shape.data()}, {element::f32, min_val_param->get_shape(), &min}, diff --git a/src/frontends/onnx/frontend/src/op/random_uniform.cpp b/src/frontends/onnx/frontend/src/op/random_uniform.cpp index 6215dcc491c..a26ed672a0c 100644 --- a/src/frontends/onnx/frontend/src/op/random_uniform.cpp +++ b/src/frontends/onnx/frontend/src/op/random_uniform.cpp @@ -29,6 +29,7 @@ OutputVector random_uniform(const Node& node) { const auto target_type = common::get_ngraph_element_type(dtype); const uint64_t global_seed = 0; + // TODO: This multiplication leads to a mismatch in accuracy. Issue: 123003 const auto seed_uint64 = static_cast(seed * 1000); return {std::make_shared(target_shape_const, diff --git a/src/plugins/intel_cpu/src/cpu_types.cpp b/src/plugins/intel_cpu/src/cpu_types.cpp index 03fbe1a9923..6f5a84701b1 100644 --- a/src/plugins/intel_cpu/src/cpu_types.cpp +++ b/src/plugins/intel_cpu/src/cpu_types.cpp @@ -123,6 +123,7 @@ static const TypeToNameMap& get_type_to_name_tbl() { { "ScatterElementsUpdate", Type::ScatterElementsUpdate}, { "ScatterNDUpdate", Type::ScatterNDUpdate}, { "Interpolate", Type::Interpolate}, + { "RandomUniform", Type::RandomUniform}, { "ReduceL1", Type::Reduce}, { "ReduceL2", Type::Reduce}, { "ReduceLogicalAnd", Type::Reduce}, @@ -317,6 +318,7 @@ std::string NameFromType(const Type type) { CASE(PriorBox); CASE(PriorBoxClustered) CASE(MHA); + CASE(RandomUniform); CASE(Unique); CASE(Ngram); CASE(Unknown); diff --git a/src/plugins/intel_cpu/src/cpu_types.h b/src/plugins/intel_cpu/src/cpu_types.h index 403ed62d482..9afbe2d7485 100644 --- a/src/plugins/intel_cpu/src/cpu_types.h +++ b/src/plugins/intel_cpu/src/cpu_types.h @@ -110,6 +110,7 @@ enum class Type { PriorBoxClustered, Interaction, MHA, + RandomUniform, Unique, Ngram }; diff --git a/src/plugins/intel_cpu/src/node.cpp b/src/plugins/intel_cpu/src/node.cpp index e8fe6b89a00..ab02ae44dd6 100644 --- a/src/plugins/intel_cpu/src/node.cpp +++ b/src/plugins/intel_cpu/src/node.cpp @@ -95,8 +95,6 @@ Node::Node(const std::shared_ptr& op, typeStr(op->get_type_name()), type(TypeFromName(op->get_type_name())), profiling(op->get_friendly_name()) { - const std::string errorPrefix = "Ngraph operation " + std::string(op->get_type_name()) + " with name " + op->get_friendly_name(); - for (size_t i = 0; i < op->get_input_size(); i++) { const auto &shape = op->get_input_partial_shape(i); if (shape.rank().is_dynamic()) { @@ -480,6 +478,8 @@ std::string Node::getPrimitiveDescriptorType() const { SEARCH_TYPE(_dw); SEARCH_TYPE(_1x1); +#undef SEARCH_TYPE + if (type == impl_desc_type::unknown) str_type = "unknown"; else if (str_type.empty()) diff --git a/src/plugins/intel_cpu/src/node.h b/src/plugins/intel_cpu/src/node.h index 5becbfa9863..864c08a95b0 100644 --- a/src/plugins/intel_cpu/src/node.h +++ b/src/plugins/intel_cpu/src/node.h @@ -41,6 +41,8 @@ #include "nodes/executors/mvn_list.hpp" #include "nodes/executors/executor.hpp" +#define THROW_CPU_NODE_ERR(...) OPENVINO_THROW(getTypeStr(), " node with name '", getName(), "' ", __VA_ARGS__) + namespace ov { namespace intel_cpu { @@ -353,7 +355,7 @@ public: inplace = InPlaceType::Unknown; } - std::string getPrimitiveDescriptorType() const; + virtual std::string getPrimitiveDescriptorType() const; PerfCount &PerfCounter() { return perfCounter; } diff --git a/src/plugins/intel_cpu/src/nodes/grid_sample.cpp b/src/plugins/intel_cpu/src/nodes/grid_sample.cpp index 798b0407835..6868e907fa7 100644 --- a/src/plugins/intel_cpu/src/nodes/grid_sample.cpp +++ b/src/plugins/intel_cpu/src/nodes/grid_sample.cpp @@ -10,9 +10,11 @@ #include using namespace InferenceEngine; -using namespace dnnl::impl::cpu; using namespace ov::intel_cpu; using namespace ov::intel_cpu::node; +#if defined(OPENVINO_ARCH_X86_64) +using namespace dnnl::impl::cpu; +#endif // OPENVINO_ARCH_X86_64 #define THROW_ERROR IE_THROW() << getTypeStr() << " node with name '" << getName() << "' " @@ -23,10 +25,14 @@ bool GridSample::isSupportedOperation(const std::shared_ptr& op, errorMessage = "Not supported GridSample operation version. CPU plug-in supports only 9th version."; return false; } +#if defined(OPENVINO_ARCH_X86_64) if (!x64::mayiuse(x64::sse41)) { errorMessage = "Not supported CPU instructions set."; return false; } +#else + return false; +#endif // OPENVINO_ARCH_X86_64 } catch (...) { return false; } @@ -34,6 +40,8 @@ bool GridSample::isSupportedOperation(const std::shared_ptr& op, return true; } +#if defined(OPENVINO_ARCH_X86_64) + GridSample::GridSample(const std::shared_ptr& op, const GraphContext::CPtr context) : Node(op, context, NgraphShapeInferFactory(op, PortMask(1))) { std::string errorMessage; @@ -110,7 +118,7 @@ void GridSample::initSupportedPrimitiveDescriptors() { } void GridSample::createPrimitive() { - GridSampleKernelConfParams jcp; + kernel::GridSampleKernelConfParams jcp; jcp.inDataPrc = dataPrecision; jcp.gridPrc = gridPrecision; @@ -133,15 +141,13 @@ void GridSample::createPrimitive() { jcp.cannelNum = jcp.dynamicChannel ? 1lu : srcDataDims[1]; } -#if defined(OPENVINO_ARCH_X86_64) if (x64::mayiuse(x64::avx512_core)) { - jitKernel.reset(new GridSampleKernel(jcp)); + jitKernel.reset(new kernel::GridSampleKernel(jcp)); } else if (x64::mayiuse(x64::avx2)) { - jitKernel.reset(new GridSampleKernel(jcp)); + jitKernel.reset(new kernel::GridSampleKernel(jcp)); } else if (x64::mayiuse(x64::sse41)) { - jitKernel.reset(new GridSampleKernel(jcp)); + jitKernel.reset(new kernel::GridSampleKernel(jcp)); } -#endif // OPENVINO_ARCH_X86_64 if (!jitKernel) { THROW_ERROR << " could not create JIT kernel."; } @@ -268,7 +274,7 @@ void GridSample::execute(dnnl::stream strm) { auto threadBody = [&](const int ithr, const int nthr) { const auto& p = execParamsPerThread[ithr]; - auto arg = GridSamplesKernelExecArgs(); + auto arg = kernel::GridSamplesKernelExecArgs(); if (p.workAmount == 0lu) { return; } @@ -311,3 +317,5 @@ void GridSample::executeDynamicImpl(dnnl::stream strm) { bool GridSample::created() const { return getType() == Type::GridSample; } + +#endif // OPENVINO_ARCH_X86_64 diff --git a/src/plugins/intel_cpu/src/nodes/grid_sample.hpp b/src/plugins/intel_cpu/src/nodes/grid_sample.hpp index 89a1a409764..78b5f9d6671 100644 --- a/src/plugins/intel_cpu/src/nodes/grid_sample.hpp +++ b/src/plugins/intel_cpu/src/nodes/grid_sample.hpp @@ -72,7 +72,7 @@ private: static constexpr size_t IN_DATA = 0; static constexpr size_t IN_GRID = 1; - std::shared_ptr jitKernel; + std::shared_ptr jitKernel; }; } // namespace node diff --git a/src/plugins/intel_cpu/src/nodes/kernels/x64/grid_sample.cpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/grid_sample.cpp index 7501dd60642..89e658a7d6a 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/x64/grid_sample.cpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/x64/grid_sample.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2022 Intel Corporation +// Copyright (C) 2018-2023 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -8,12 +8,13 @@ using namespace dnnl::impl::cpu; namespace ov { namespace intel_cpu { +namespace kernel { #define GET_OFF(field) offsetof(GridSamplesKernelExecArgs, field) template GridSampleKernel::GridSampleKernel(const GridSampleKernelConfParams& jcp) : - GridSampleKernelBase(jit_name(), jcp) { + GridSampleKernelBase(jit_name(), jcp, isa) { vlen = x64::cpu_isa_traits::vlen; dataTypeSize = jcp.inDataPrc.size(); gridTypeSize = jcp.gridPrc.size(); @@ -2085,5 +2086,6 @@ template class GridSampleKernel; template class GridSampleKernel; template class GridSampleKernel; +} // namespace kernel } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/kernels/x64/grid_sample.hpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/grid_sample.hpp index c24100259cd..295c715fb81 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/x64/grid_sample.hpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/x64/grid_sample.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2022 Intel Corporation +// Copyright (C) 2018-2023 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -14,6 +14,12 @@ namespace intel_cpu { enum class GridSampleInterpolationMode { BILINEAR, BICUBIC, NEAREST }; enum class GridSamplePaddingMode { ZEROS, BORDER, REFLECTION }; +namespace kernel { + +class GridSampleKernelBase; + +#if defined(OPENVINO_ARCH_X86_64) + struct GridSampleKernelConfParams { bool dynamicShapes = false; bool dynamicBatch = false; @@ -66,7 +72,8 @@ public: assert(ker_); ker_(args); } - explicit GridSampleKernelBase(const char* name, const GridSampleKernelConfParams& jcp) : JitKernelBase(name), ker_(nullptr), jcp(jcp) {} + explicit GridSampleKernelBase(const char* name, const GridSampleKernelConfParams& jcp, dnnl::impl::cpu::x64::cpu_isa_t isa) + : JitKernelBase(name, isa), ker_(nullptr), jcp(jcp) {} virtual void create_ker() = 0; uint64_t getVecLen() { @@ -173,5 +180,8 @@ private: void hwShiftPs2dq(const Vmm& vDst, const Vmm& vHCoord, const Vmm& vWCoord, const Vmm& vWidth); }; +#endif // OPENVINO_ARCH_X86_64 + +} // namespace kernel } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel_base.cpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel_base.cpp index 6afbecf143f..bc0daaf6e33 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel_base.cpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel_base.cpp @@ -1,172 +1,243 @@ -// Copyright (C) 2022 Intel Corporation +// Copyright (C) 2018-2023 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // #include "jit_kernel_base.hpp" -using namespace ov; -using namespace intel_cpu; using namespace dnnl::impl::cpu; +namespace ov { +namespace intel_cpu { +namespace kernel { -void JitKernelBase::uni_vfmsub132ps(const Xbyak::Xmm& vDst, - const Xbyak::Xmm& vSrc, +JitKernelBase::JitKernelBase(const char* name, x64::cpu_isa_t isa) + : x64::jit_generator(name, nullptr, x64::MAX_CODE_SIZE, true, isa), m_isa(isa) { + vlen = x64::isa_max_vlen(isa); +} + +void JitKernelBase::uni_vfmsub132ps(const Xbyak::Xmm& v_dst, + const Xbyak::Xmm& v_src, const Xbyak::Operand& op) { if (isValidIsa(x64::avx2)) { - vfmsub132ps(vDst, vSrc, op); + vfmsub132ps(v_dst, v_src, op); } else if (isValidIsa(x64::avx)) { - assert(vDst.getIdx() != vSrc.getIdx()); - vmulps(vDst, vDst, op); - vsubps(vDst, vDst, vSrc); + assert(v_dst.getIdx() != v_src.getIdx()); + vmulps(v_dst, v_dst, op); + vsubps(v_dst, v_dst, v_src); } else { - assert(vDst.getIdx() != vSrc.getIdx()); - mulps(vDst, op); - subps(vDst, vSrc); + assert(v_dst.getIdx() != v_src.getIdx()); + mulps(v_dst, op); + subps(v_dst, v_src); } } -void JitKernelBase::uni_vfnmadd132ps(const Xbyak::Xmm& vDst, - const Xbyak::Xmm& vSrc, +void JitKernelBase::uni_vfnmadd132ps(const Xbyak::Xmm& v_dst, + const Xbyak::Xmm& v_src, const Xbyak::Operand& op) { if (isValidIsa(x64::avx2)) { - vfnmadd132ps(vDst, vSrc, op); + vfnmadd132ps(v_dst, v_src, op); } else if (isValidIsa(x64::avx)) { - assert(vDst.getIdx() != vSrc.getIdx()); - vmulps(vDst, vDst, op); - vsubps(vDst, vSrc, vDst); + assert(v_dst.getIdx() != v_src.getIdx()); + vmulps(v_dst, v_dst, op); + vsubps(v_dst, v_src, v_dst); } else { - assert(vDst.getIdx() != vSrc.getIdx()); - mulps(vDst, op); - subps(vSrc, vDst); - movups(vDst, vSrc); + assert(v_dst.getIdx() != v_src.getIdx()); + mulps(v_dst, op); + subps(v_src, v_dst); + movups(v_dst, v_src); } } -void JitKernelBase::uni_vfmsub231ps(const Xbyak::Xmm& vDst, - const Xbyak::Xmm& vSrc, +void JitKernelBase::uni_vfmsub231ps(const Xbyak::Xmm& v_dst, + const Xbyak::Xmm& v_src, const Xbyak::Operand& op) { if (isValidIsa(x64::avx2)) { - vfmsub231ps(vDst, vSrc, op); + vfmsub231ps(v_dst, v_src, op); } else if (isValidIsa(x64::avx)) { - assert(!vDst.isEqualIfNotInherited(op)); - vmulps(vSrc, vSrc, op); - vsubps(vDst, vSrc, vDst); + assert(!v_dst.isEqualIfNotInherited(op)); + vmulps(v_src, v_src, op); + vsubps(v_dst, v_src, v_dst); } else { - assert(!vDst.isEqualIfNotInherited(op)); - mulps(vSrc, op); - subps(vSrc, vDst); - movups(vDst, vSrc); + assert(!v_dst.isEqualIfNotInherited(op)); + mulps(v_src, op); + subps(v_src, v_dst); + movups(v_dst, v_src); } } -void JitKernelBase::uni_vpaddd(const Xbyak::Ymm& vDst, - const Xbyak::Ymm& vSrc, +void JitKernelBase::uni_vpaddd(const Xbyak::Ymm& v_dst, + const Xbyak::Ymm& v_src, const Xbyak::Operand& op) { if (isValidIsa(x64::avx2)) { - vpaddd(vDst, vSrc, op); + vpaddd(v_dst, v_src, op); } else if (isValidIsa(x64::avx)) { - Xbyak::Xmm xmmDst(vDst.getIdx()); - vmovups(vDst, vSrc); + Xbyak::Xmm xmmDst(v_dst.getIdx()); + vmovups(v_dst, v_src); if (op.isYMM()) { Xbyak::Ymm ymmOp(op.getIdx()); Xbyak::Xmm xmmOp(op.getIdx()); paddd(xmmDst, xmmOp); - vperm2f128(vDst, vDst, vDst, 0x1); + vperm2f128(v_dst, v_dst, v_dst, 0x1); vperm2f128(ymmOp, ymmOp, ymmOp, 0x1); paddd(xmmDst, xmmOp); - vperm2f128(vDst, vDst, vDst, 0x1); + vperm2f128(v_dst, v_dst, v_dst, 0x1); vperm2f128(ymmOp, ymmOp, ymmOp, 0x1); } else if (op.isMEM()) { const int vlen = x64::cpu_isa_traits::vlen; paddd(xmmDst, op.getAddress()); - vperm2f128(vDst, vDst, vDst, 0x1); + vperm2f128(v_dst, v_dst, v_dst, 0x1); paddd(xmmDst, ptr[op.getAddress().getRegExp() + vlen]); - vperm2f128(vDst, vDst, vDst, 0x1); + vperm2f128(v_dst, v_dst, v_dst, 0x1); } else { IE_THROW() << "Not supported operand type."; } } else if (isValidIsa(x64::sse41)) { - assert(vDst.getIdx() != vSrc.getIdx()); - paddd(vDst, op); + assert(v_dst.getIdx() != v_src.getIdx()); + paddd(v_dst, op); } else { IE_THROW() << "Not defined behavior for instruction 'vpaddd' in current instructions set."; } } -void JitKernelBase::uni_vpsubd(const Xbyak::Ymm& vDst, - const Xbyak::Ymm& vSrc, +void JitKernelBase::uni_vpaddq(const Xbyak::Xmm& v_dst, + const Xbyak::Xmm& v_src, const Xbyak::Operand& op) { if (isValidIsa(x64::avx2)) { - vpsubd(vDst, vSrc, op); + vpaddq(v_dst, v_src, op); + } else { + if (v_dst.getIdx() != v_src.getIdx()) { + movups(v_dst, v_src); + } + paddq(v_dst, op); + } +} + +void JitKernelBase::uni_vpsubd(const Xbyak::Ymm& v_dst, + const Xbyak::Ymm& v_src, + const Xbyak::Operand& op) { + if (isValidIsa(x64::avx2)) { + vpsubd(v_dst, v_src, op); } else if (isValidIsa(x64::avx)) { - Xbyak::Xmm xmmDst(vDst.getIdx()); - vmovups(vDst, vSrc); + Xbyak::Xmm xmmDst(v_dst.getIdx()); + vmovups(v_dst, v_src); if (op.isYMM()) { Xbyak::Ymm ymmOp(op.getIdx()); Xbyak::Xmm xmmOp(op.getIdx()); psubd(xmmDst, xmmOp); - vperm2f128(vDst, vDst, vDst, 0x1); + vperm2f128(v_dst, v_dst, v_dst, 0x1); vperm2f128(ymmOp, ymmOp, ymmOp, 0x1); psubd(xmmDst, xmmOp); - vperm2f128(vDst, vDst, vDst, 0x1); + vperm2f128(v_dst, v_dst, v_dst, 0x1); vperm2f128(ymmOp, ymmOp, ymmOp, 0x1); } else if (op.isMEM()) { const int vlen = x64::cpu_isa_traits::vlen; psubd(xmmDst, op.getAddress()); - vperm2f128(vDst, vDst, vDst, 0x1); + vperm2f128(v_dst, v_dst, v_dst, 0x1); psubd(xmmDst, ptr[op.getAddress().getRegExp() + vlen]); - vperm2f128(vDst, vDst, vDst, 0x1); + vperm2f128(v_dst, v_dst, v_dst, 0x1); } else { IE_THROW() << "Not supported operand type."; } } else if (isValidIsa(x64::sse41)) { - assert(vDst.getIdx() != vSrc.getIdx()); - psubd(vDst, op); + assert(v_dst.getIdx() != v_src.getIdx()); + psubd(v_dst, op); } else { IE_THROW() << "Not defined behavior for instruction 'vpsubd' in current instructions set."; } } -void JitKernelBase::uni_vdivps(const Xbyak::Xmm& vDst, +void JitKernelBase::uni_vsubpd(const Xbyak::Xmm& v_dst, + const Xbyak::Xmm& v_src, + const Xbyak::Operand& op) { + if (isValidIsa(x64::avx)) { + vsubpd(v_dst, v_src, op); + } else { + if (v_dst.getIdx() != v_src.getIdx()) { + movups(v_dst, v_src); + } + subpd(v_dst, op); + } +} + +void JitKernelBase::uni_vmulpd(const Xbyak::Xmm& v_dst, + const Xbyak::Xmm& v_src, + const Xbyak::Operand& op) { + if (isValidIsa(x64::avx)) { + vmulpd(v_dst, v_src, op); + } else { + if (v_dst.getIdx() != v_src.getIdx()) { + movups(v_dst, v_src); + } + mulpd(v_dst, op); + } +} + +void JitKernelBase::uni_vpmuludq(const Xbyak::Xmm& v_dst, + const Xbyak::Xmm& v_src, + const Xbyak::Operand& op) { + if (isValidIsa(x64::avx2)) { + vpmuludq(v_dst, v_src, op); + } else { + if (v_dst.getIdx() != v_src.getIdx()) { + movups(v_dst, v_src); + } + pmuludq(v_dst, op); + } +} + +void JitKernelBase::uni_vdivps(const Xbyak::Xmm& v_dst, const Xbyak::Operand& op1, const Xbyak::Operand& op2) { if (isValidIsa(x64::avx)) { - vdivps(vDst, op1, op2); + vdivps(v_dst, op1, op2); } else { - if (!vDst.isEqualIfNotInherited(op1)) { - movups(vDst, op1); + if (!v_dst.isEqualIfNotInherited(op1)) { + movups(v_dst, op1); } - divps(vDst, op2); + divps(v_dst, op2); } } -void JitKernelBase::uni_vandps(const Xbyak::Xmm& vDst, +void JitKernelBase::uni_vdivpd(const Xbyak::Xmm& v_dst, + const Xbyak::Xmm& v_src, + const Xbyak::Operand& op) { + if (isValidIsa(x64::avx)) { + vdivpd(v_dst, v_src, op); + } else { + if (v_dst.getIdx() != v_src.getIdx()) { + movups(v_dst, v_src); + } + divpd(v_dst, op); + } +} + +void JitKernelBase::uni_vandps(const Xbyak::Xmm& v_dst, const Xbyak::Xmm& vSrs, const Xbyak::Operand &op) { if (isValidIsa(x64::avx)) { - vandps(vDst, vSrs, op); + vandps(v_dst, vSrs, op); } else { - if (!vDst.isEqualIfNotInherited(vSrs)) { - movups(vDst, vSrs); + if (!v_dst.isEqualIfNotInherited(vSrs)) { + movups(v_dst, vSrs); } - andps(vDst, op); + andps(v_dst, op); } } -void JitKernelBase::uni_vandnps(const Xbyak::Xmm& vDst, +void JitKernelBase::uni_vandnps(const Xbyak::Xmm& v_dst, const Xbyak::Xmm& vSrs, const Xbyak::Operand &op) { if (isValidIsa(x64::avx)) { - vandnps(vDst, vSrs, op); + vandnps(v_dst, vSrs, op); } else { - if (!vDst.isEqualIfNotInherited(vSrs)) { - movups(vDst, vSrs); + if (!v_dst.isEqualIfNotInherited(vSrs)) { + movups(v_dst, vSrs); } - andnps(vDst, op); + andnps(v_dst, op); } } -void JitKernelBase::gatherdd(const Xbyak::Xmm& vDst, +void JitKernelBase::gatherdd(const Xbyak::Xmm& v_dst, const Xbyak::Reg64& rSrcPtr, const Xbyak::Xmm& vSrcShift, const Xbyak::Opmask& kReadMask, @@ -178,28 +249,28 @@ void JitKernelBase::gatherdd(const Xbyak::Xmm& vDst, if (!useMask) kxnord(kReadMask, kReadMask, kReadMask); if (zeroFill) - uni_vpxor(vDst, vDst, vDst); + uni_vpxor(v_dst, v_dst, v_dst); - vpgatherdd(vDst | kReadMask, ptr[rSrcPtr + vSrcShift]); + vpgatherdd(v_dst | kReadMask, ptr[rSrcPtr + vSrcShift]); } -void JitKernelBase::gatherdd(const Xbyak::Xmm& vDst, +void JitKernelBase::gatherdd(const Xbyak::Xmm& v_dst, const Xbyak::Reg64& rSrcPtr, const Xbyak::Xmm& vSrcShift, const Xbyak::Xmm& vReadMask, const bool useMask, const bool zeroFill) { - if (vDst.getIdx() == vSrcShift.getIdx() || vDst.getIdx() == vReadMask.getIdx() || vSrcShift.getIdx() == vReadMask.getIdx()) { + if (v_dst.getIdx() == vSrcShift.getIdx() || v_dst.getIdx() == vReadMask.getIdx() || vSrcShift.getIdx() == vReadMask.getIdx()) { IE_THROW() << "Any pair of the index, mask, or destination registers cannot be the same."; } if (zeroFill) - pxor(vDst, vDst); // Don't use vpxor. It zeros the rest of the YMM register. + pxor(v_dst, v_dst); // Don't use vpxor. It zeros the rest of the YMM register. if (isValidIsa(x64::avx2)) { if (!useMask) uni_vpcmpeqd(vReadMask, vReadMask, vReadMask); - vpgatherdd(vDst, ptr[rSrcPtr + vSrcShift], vReadMask); + vpgatherdd(v_dst, ptr[rSrcPtr + vSrcShift], vReadMask); } else { auto rAux = getReg64(); Xbyak::Reg32 r32Aux = Xbyak::Reg32(rAux.getIdx()); @@ -213,7 +284,7 @@ void JitKernelBase::gatherdd(const Xbyak::Xmm& vDst, je(lLoopNext, T_NEAR); } uni_vpextrd(r32Aux, vSrcShift, i); - pinsrd(vDst, ptr[rSrcPtr + rAux], i); + pinsrd(v_dst, ptr[rSrcPtr + rAux], i); if (useMask) L(lLoopNext); @@ -221,30 +292,30 @@ void JitKernelBase::gatherdd(const Xbyak::Xmm& vDst, } } -void JitKernelBase::gatherdd(const Xbyak::Ymm& vDst, +void JitKernelBase::gatherdd(const Xbyak::Ymm& v_dst, const Xbyak::Reg64& rSrcPtr, const Xbyak::Ymm& vSrcShift, const Xbyak::Ymm& vReadMask, const bool useMask, const bool zeroFill) { - if (vDst.getIdx() == vSrcShift.getIdx() || vDst.getIdx() == vReadMask.getIdx() || vSrcShift.getIdx() == vReadMask.getIdx()) { + if (v_dst.getIdx() == vSrcShift.getIdx() || v_dst.getIdx() == vReadMask.getIdx() || vSrcShift.getIdx() == vReadMask.getIdx()) { IE_THROW() << "Any pair of the index, mask, or destination registers cannot be the same."; } if (isValidIsa(x64::avx2)) { if (!useMask) uni_vpcmpeqd(vReadMask, vReadMask, vReadMask); if (zeroFill) - uni_vpxor(vDst, vDst, vDst); + uni_vpxor(v_dst, v_dst, v_dst); - vpgatherdd(vDst, ptr[rSrcPtr + vSrcShift], vReadMask); + vpgatherdd(v_dst, ptr[rSrcPtr + vSrcShift], vReadMask); } else { - Xbyak::Xmm xmmDst = Xbyak::Xmm(vDst.getIdx()), + Xbyak::Xmm xmmDst = Xbyak::Xmm(v_dst.getIdx()), xmmSrcShft = Xbyak::Xmm(vSrcShift.getIdx()), xmmReadMask = Xbyak::Xmm(vReadMask.getIdx()); for (uint8_t i = 0; i < 2; i++) { gatherdd(xmmDst, rSrcPtr, xmmSrcShft, xmmReadMask, useMask, zeroFill); - vperm2f128(vDst, vDst, vDst, 0x1); + vperm2f128(v_dst, v_dst, v_dst, 0x1); vperm2f128(vSrcShift, vSrcShift, vSrcShift, 0x1); if (useMask) vperm2f128(vReadMask, vReadMask, vReadMask, 0x1); @@ -252,6 +323,15 @@ void JitKernelBase::gatherdd(const Xbyak::Ymm& vDst, } } +void JitKernelBase::uni_vpbroadcastq(const Xbyak::Xmm &x, const Xbyak::Operand &op) { + if (isValidIsa(x64::avx2)) { + vpbroadcastq(x, op); + } else { + movsd(x, op); + shufpd(x, x, 0x0); + } +} + void JitKernelBase::uni_vpbroadcastd(const Xbyak::Xmm &x, const Xbyak::Operand &op) { if (isValidIsa(x64::avx2)) { vpbroadcastd(x, op); @@ -285,6 +365,57 @@ void JitKernelBase::uni_vpbroadcastd(const Xbyak::Ymm &x, const Xbyak::Operand & } } +void JitKernelBase::uni_vroundpd(const Xbyak::Xmm& v_dst, const Xbyak::Operand& op, const uint8_t imm) { + if (isValidIsa(x64::avx512_core)) { + vrndscalepd(v_dst, op, imm & 0x3); + } else if (isValidIsa(x64::avx)) { + vroundpd(v_dst, op, imm); + } else { + roundpd(v_dst, op, imm); + } +} + +void JitKernelBase::uni_vcvtdq2pd(const Xbyak::Xmm& v_dst, + const Xbyak::Operand& op) { + if (isValidIsa(x64::avx)) { + vcvtdq2pd(v_dst, op); + } else { + cvtdq2pd(v_dst, op); + } +} + +void JitKernelBase::uni_vcvtpd2dq(const Xbyak::Xmm& v_dst, + const Xbyak::Operand& op) { + if (isValidIsa(x64::avx)) { + vcvtpd2dq(v_dst, op); + } else { + cvtpd2dq(v_dst, op); + } +} + +void JitKernelBase::uni_vpmovzxdq(const Xbyak::Xmm& v_dst, + const Xbyak::Operand& op) { + if (isValidIsa(x64::avx2)) { + vpmovzxdq(v_dst, op); + } else { + pmovzxdq(v_dst, op); + } +} + +void JitKernelBase::uni_vshufpd(const Xbyak::Xmm& v_dst, + const Xbyak::Xmm& v_src, + const Xbyak::Operand& op, + uint8_t imm) { + if (isValidIsa(x64::avx)) { + vshufpd(v_dst, v_src, op, imm); + } else { + if (v_dst.getIdx() != v_src.getIdx()) { + movups(v_dst, v_src); + } + shufpd(v_dst, op, imm); + } +} + void JitKernelBase::fillRestWorkMask(const Xbyak::Opmask& dstMask, const Xbyak::Reg64& rWorkRest) { auto rOnes = getReg64(); @@ -362,7 +493,7 @@ void JitKernelBase::fillRestWorkMask(const Xbyak::Ymm& ymmDstMask, L(lEnd); } -void JitKernelBase::load(const Xbyak::Xmm& vDst, +void JitKernelBase::load(const Xbyak::Xmm& v_dst, const Xbyak::Address& srcAddr, const Xbyak::Reg64& rLoadNum, const size_t typeSize, @@ -373,7 +504,7 @@ void JitKernelBase::load(const Xbyak::Xmm& vDst, const uint8_t elPerVec = x64::cpu_isa_traits::vlen / typeSize; Xbyak::Label lEnd; if (zeroFilling) - pxor(vDst, vDst); + pxor(v_dst, v_dst); for (uint8_t i = 0; i < elPerVec; i++) { cmp(rLoadNum, i); @@ -381,18 +512,18 @@ void JitKernelBase::load(const Xbyak::Xmm& vDst, const size_t offset = i * typeSize; if (typeSize == 1) - pinsrb(vDst, ptr[srcAddr.getRegExp() + offset], i); + pinsrb(v_dst, ptr[srcAddr.getRegExp() + offset], i); else if (typeSize == 2) - pinsrw(vDst, ptr[srcAddr.getRegExp() + offset], i); + pinsrw(v_dst, ptr[srcAddr.getRegExp() + offset], i); else if (typeSize == 4) - pinsrd(vDst, ptr[srcAddr.getRegExp() + offset], i); + pinsrd(v_dst, ptr[srcAddr.getRegExp() + offset], i); else if (typeSize == 8) - pinsrq(vDst, ptr[srcAddr.getRegExp() + offset], i); + pinsrq(v_dst, ptr[srcAddr.getRegExp() + offset], i); } L(lEnd); } -void JitKernelBase::load(const Xbyak::Ymm& vDst, +void JitKernelBase::load(const Xbyak::Ymm& v_dst, const Xbyak::Address& srcAddr, const Xbyak::Reg64& rLoadNum, const size_t typeSize, @@ -403,8 +534,8 @@ void JitKernelBase::load(const Xbyak::Ymm& vDst, const size_t elPerXmm = x64::cpu_isa_traits::vlen / typeSize; Xbyak::Label lEnd; if (zeroFilling) - uni_vpxor(vDst, vDst, vDst); - Xbyak::Xmm xmmDst(vDst.getIdx()); + uni_vpxor(v_dst, v_dst, v_dst); + Xbyak::Xmm xmmDst(v_dst.getIdx()); for (size_t i = 0lu; i < 2lu; i++) { Xbyak::Label lPerm; @@ -427,13 +558,13 @@ void JitKernelBase::load(const Xbyak::Ymm& vDst, } L(lPerm); - vperm2f128(vDst, vDst, vDst, 0x1); + vperm2f128(v_dst, v_dst, v_dst, 0x1); } L(lEnd); } void JitKernelBase::store(const Xbyak::Address& dstAddr, - const Xbyak::Xmm& vSrc, + const Xbyak::Xmm& v_src, const Xbyak::Reg64& rToStoreNum, const size_t typeSize) { if (!one_of(typeSize, 1u, 2u, 4u, 8u)) { @@ -448,27 +579,27 @@ void JitKernelBase::store(const Xbyak::Address& dstAddr, const size_t offset = i * typeSize; if (typeSize == 1) { - uni_vpextrb(ptr[dstAddr.getRegExp() + offset], vSrc, i); + uni_vpextrb(ptr[dstAddr.getRegExp() + offset], v_src, i); } else if (typeSize == 2) { - uni_vpextrw(ptr[dstAddr.getRegExp() + offset], vSrc, i); + uni_vpextrw(ptr[dstAddr.getRegExp() + offset], v_src, i); } else if (typeSize == 4) { - uni_vpextrd(ptr[dstAddr.getRegExp() + offset], vSrc, i); + uni_vpextrd(ptr[dstAddr.getRegExp() + offset], v_src, i); } else if (typeSize == 8) { - uni_vpextrq(ptr[dstAddr.getRegExp() + offset], vSrc, i); + uni_vpextrq(ptr[dstAddr.getRegExp() + offset], v_src, i); } } L(lEnd); } void JitKernelBase::store(const Xbyak::Address& dstAddr, - const Xbyak::Ymm& vSrc, + const Xbyak::Ymm& v_src, const Xbyak::Reg64& rToStoreNum, const size_t typeSize) { if (!one_of(typeSize, 1u, 2u, 4u, 8u)) { IE_THROW() << "Could not store data with type size " << typeSize; } Xbyak::Label lEnd; - Xbyak::Xmm xmmSrc(vSrc.getIdx()); + Xbyak::Xmm xmmSrc(v_src.getIdx()); const size_t elPerXmm = x64::cpu_isa_traits::vlen / typeSize; for (int i = 0; i < 2; i++) { @@ -493,7 +624,7 @@ void JitKernelBase::store(const Xbyak::Address& dstAddr, } L(lPerm); - vperm2f128(vSrc, vSrc, vSrc, 0x1); + vperm2f128(v_src, v_src, v_src, 0x1); } L(lEnd); } @@ -575,3 +706,7 @@ void JitKernelBase::memMovDD(const Xbyak::Reg64& rDst, } L(lEnd); } + +} // namespace kernel +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel_base.hpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel_base.hpp index e39efde753b..f17eb9a02d8 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel_base.hpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel_base.hpp @@ -1,14 +1,23 @@ -// Copyright (C) 2022 Intel Corporation +// Copyright (C) 2018-2023 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // #pragma once +#include "openvino/core/visibility.hpp" + +#if defined(OPENVINO_ARCH_X86_64) #include "cpu/x64/jit_generator.hpp" #include "registers_pool.hpp" +#endif // OPENVINO_ARCH_X86_64 namespace ov { namespace intel_cpu { +namespace kernel { + +class JitKernelBase; + +#if defined(OPENVINO_ARCH_X86_64) #define getReg64() RegistersPool::Reg(registersPool) #define getReg32() RegistersPool::Reg(registersPool) @@ -17,7 +26,11 @@ namespace intel_cpu { class JitKernelBase: public dnnl::impl::cpu::x64::jit_generator { public: - JitKernelBase(const char* name) : dnnl::impl::cpu::x64::jit_generator(name) {} + JitKernelBase(const char* name, dnnl::impl::cpu::x64::cpu_isa_t max_cpu_isa); + + dnnl::impl::cpu::x64::cpu_isa_t getIsa() { return m_isa; } + + size_t getVectorLen() { return vlen; } void uni_vfmsub132ps(const Xbyak::Xmm& vDst, const Xbyak::Xmm& vSrc, const Xbyak::Operand& op); @@ -31,14 +44,24 @@ public: void uni_vpaddd(const Xbyak::Ymm& vDst, const Xbyak::Ymm& vSrc, const Xbyak::Operand& op); + void uni_vpaddq(const Xbyak::Xmm& vDst, const Xbyak::Xmm& vSrc, const Xbyak::Operand& op); + void uni_vpsubd(const Xbyak::Xmm& vDst, const Xbyak::Xmm& vSrc, const Xbyak::Operand& op) { jit_generator::uni_vpsubd(vDst, vSrc, op); } void uni_vpsubd(const Xbyak::Ymm& vDst, const Xbyak::Ymm& vSrc, const Xbyak::Operand& op); + void uni_vsubpd(const Xbyak::Xmm& v_dst, const Xbyak::Xmm& v_src, const Xbyak::Operand& op); + + void uni_vmulpd(const Xbyak::Xmm& v_dst, const Xbyak::Xmm& v_src, const Xbyak::Operand& op); + + void uni_vpmuludq(const Xbyak::Xmm& v_dst, const Xbyak::Xmm& op_1, const Xbyak::Operand& op_2); + void uni_vdivps(const Xbyak::Xmm& vDst, const Xbyak::Operand& op1, const Xbyak::Operand& op2); + void uni_vdivpd(const Xbyak::Xmm& v_dst, const Xbyak::Xmm& v_src, const Xbyak::Operand& op2); + void uni_vandps(const Xbyak::Xmm& vDst, const Xbyak::Xmm& vSrs, const Xbyak::Operand &op); void uni_vandnps(const Xbyak::Xmm& vDst, const Xbyak::Xmm& vSrs, const Xbyak::Operand &op); @@ -63,6 +86,18 @@ public: void uni_vpbroadcastd(const Xbyak::Ymm &x, const Xbyak::Operand &op); + void uni_vpbroadcastq(const Xbyak::Xmm &x, const Xbyak::Operand &op); + + void uni_vroundpd(const Xbyak::Xmm& v_dst, const Xbyak::Operand& op, const uint8_t imm); + + void uni_vcvtdq2pd(const Xbyak::Xmm& v_dst, const Xbyak::Operand& op); + + void uni_vcvtpd2dq(const Xbyak::Xmm& v_dst, const Xbyak::Operand& op); + + void uni_vpmovzxdq(const Xbyak::Xmm& v_dst, const Xbyak::Operand& op); + + void uni_vshufpd(const Xbyak::Xmm& v_dst, const Xbyak::Xmm& v_srs, const Xbyak::Operand& op, uint8_t imm); + void gatherdd(const Xbyak::Xmm& vDst, const Xbyak::Reg64& rSrcPtr, const Xbyak::Xmm& vSrcShift, @@ -140,7 +175,9 @@ protected: return dnnl::impl::cpu::x64::mayiuse(isa); } + const dnnl::impl::cpu::x64::cpu_isa_t m_isa; RegistersPool::Ptr registersPool; + size_t vlen; enum { // Comparison predicate operand (immediate byte) for single-precision floating-point values. @@ -155,5 +192,70 @@ protected: }; }; +template +class JitKernel : public JitKernelBase { +public: + using KernelFunc = void (*)(const CallArgs *); + + explicit JitKernel(const char* name, const CompileParams& jcp, dnnl::impl::cpu::x64::cpu_isa_t max_cpu_isa) + : JitKernelBase{name, max_cpu_isa}, m_jcp{jcp}, m_func{nullptr} {} + + ~JitKernel() override = default; + + dnnl::impl::status_t create_kernel() override { + const dnnl::impl::status_t code = jit_generator::create_kernel(); + if (code != dnnl::impl::status::success) { + OPENVINO_THROW("Could not create kernel. Error code: ", std::to_string(code), ". ", + "Xbyak error code: ", Xbyak::ConvertErrorToString(Xbyak::GetError())); + } + m_func = (decltype(m_func))jit_ker(); + return code; + } + + void operator()(const CallArgs* args) const { + assert(m_func); + m_func(args); + } + + void operator()(const CallArgs& args) const { + this->operator()(&args); + } + + template class KernelT> + static std::shared_ptr> createInstance(const CompileParams& jcp) { + std::shared_ptr> res; + + try { +#define IF_ISA_CASE(ISA) \ + if (dnnl::impl::cpu::x64::mayiuse(ISA)) \ + res.reset(new KernelT(jcp)); \ + else + + IF_ISA_CASE(dnnl::impl::cpu::x64::avx512_core) + IF_ISA_CASE(dnnl::impl::cpu::x64::avx2) + IF_ISA_CASE(dnnl::impl::cpu::x64::sse41); + +#undef IF_ISA_CASE + + if (res) { + res->create_kernel(); + } + } catch (...) { + return nullptr; + } + + return res; + } + +protected: + CompileParams m_jcp; + +private: + KernelFunc m_func; +}; + +#endif // OPENVINO_ARCH_X86_64 + +} // namespace kernel } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/kernels/x64/random_uniform.cpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/random_uniform.cpp new file mode 100644 index 00000000000..301c2f7e08f --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/kernels/x64/random_uniform.cpp @@ -0,0 +1,635 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "random_uniform.hpp" + +using namespace dnnl::impl::cpu; + +namespace ov { +namespace intel_cpu { +namespace kernel { + +#define GET_OFF(field) offsetof(RandomUniformCallArgs, field) + +template +RandomUniform::RandomUniform(const RandomUniformCompileParams& jcp) : + JitKernel(jit_name(), jcp, isa) { +} + +template +void RandomUniform::generate() { + this->preamble(); + registersPool = RegistersPool::create(isa, {rax, rcx, rsp, rdi, k0}); + + r64_dst = getReg64(); + r64_work_amount = getReg64(); + + mov(r64_work_amount, ptr[r64_params + GET_OFF(work_amount)]); + mov(r64_dst, ptr[r64_params + GET_OFF(dst_ptr)]); + + initVectors(); + process(); + + registersPool.reset(); + this->postamble(); +} + +template <> +void RandomUniform::initVectors() { + const auto r64_aux = getReg64(); + const auto r32_aux = Xbyak::Reg32(r64_aux.getIdx()); + const auto r16_aux = Xbyak::Reg16(r64_aux.getIdx()); + + v_max_mul_n_64 = getVmm(); + v_max_mul_c_64 = getVmm(); + v_add_low_k = getVmm(); + v_add_up_k = getVmm(); + v_n_inc = getVmm(); + v_range = getVmm(); + v_min = getVmm(); + v_key_64 = getVmm(); + v_counter_64 = getVmm(); + v_n_64 = getVmm(); + v_res_perm = getVmm(); + + if (m_jcp.out_data_type.is_real()) { + v_convert_0 = getVmm(); + v_convert_1 = getVmm(); + } + + // Initialize constants. +#define BROADCAST_R(F, V, R, C) \ + mov(R, C); \ + F(V, R); +#define BROADCAST_P(F, V, R, C) \ + mov(R, ptr[r64_params + GET_OFF(C)]); \ + F(V, ptr[R]); + + BROADCAST_R(vpbroadcastq, v_max_mul_n_64, r64_aux, STATISTIC_MAXIMIZING_MULTIPLIER_N) + BROADCAST_R(vpbroadcastq, v_max_mul_c_64, r64_aux, STATISTIC_MAXIMIZING_MULTIPLIER_COUNTER) + BROADCAST_R(vpbroadcastd, v_add_low_k, r32_aux, CRUSH_RESISTANCE_CONST_LOWER_VALUE) + BROADCAST_R(vpbroadcastd, v_add_up_k, r32_aux, CRUSH_RESISTANCE_CONST_UPPER_VALUE) + BROADCAST_R(vpbroadcastq, v_n_inc, r64_aux, 0x00000008) + + if (m_jcp.out_data_type == element::f32) { + BROADCAST_R(vpbroadcastd, v_convert_0, r32_aux, 0x3f800000) + BROADCAST_R(vpbroadcastd, v_convert_1, r32_aux, 0x007fffff) + BROADCAST_P(vpbroadcastd, v_range, r64_aux, range_ptr) + BROADCAST_P(vpbroadcastd, v_min, r64_aux, min_ptr) + } else if (m_jcp.out_data_type == element::f16 && x64::mayiuse(x64::avx512_core_fp16)) { + BROADCAST_R(vpbroadcastw, v_convert_0, r16_aux, 0x3c00) + BROADCAST_R(vpbroadcastw, v_convert_1, r16_aux, 0x03ff) + BROADCAST_P(vpbroadcastw, v_range, r64_aux, range_ptr) + BROADCAST_P(vpbroadcastw, v_min, r64_aux, min_ptr) + } else if (m_jcp.out_data_type == element::bf16 && x64::mayiuse(x64::avx512_core_bf16)) { + v_convert_2 = getVmm(); + const auto ymm_min = Xbyak::Ymm(v_min.getIdx()); + const auto ymm_range = Xbyak::Ymm(v_range.getIdx()); + + BROADCAST_R(vpbroadcastw, v_convert_0, r16_aux, 0x3f80) + BROADCAST_R(vpbroadcastw, v_convert_1, r16_aux, 0x007f) + BROADCAST_R(vpbroadcastd, v_convert_2, r32_aux, 0x3f800000) + + BROADCAST_P(vpbroadcastw, v_range, r64_aux, range_ptr) + vpmovzxwd(v_range, ymm_range); + uni_vpslld(v_range, v_range, 16); + + BROADCAST_P(vpbroadcastw, v_min, r64_aux, min_ptr) + vpmovzxwd(v_min, ymm_min); + uni_vpslld(v_min, v_min, 16); + } else if (m_jcp.out_data_type == element::i32) { + const auto ymm_range = Xbyak::Ymm(v_range.getIdx()); + + BROADCAST_P(vpbroadcastd, v_range, r64_aux, range_ptr) + BROADCAST_P(vpbroadcastd, v_min, r64_aux, min_ptr) + + uni_vcvtdq2pd(v_range, ymm_range); + } else { + OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info()); + } + + // Initialize inputs. + BROADCAST_P(vpbroadcastq, v_key_64, r64_aux, key_ptr) + BROADCAST_P(vpbroadcastq, v_counter_64, r64_aux, counter_ptr) + BROADCAST_P(vpbroadcastq, v_n_64, r64_aux, n_ptr) + + if (m_jcp.out_data_type.size() <= 4) { + static const uint64_t n_inc_arr[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + mov(r64_aux, reinterpret_cast(n_inc_arr)); + } else { + static const uint64_t n_inc_arr[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; // TODO: i64 + mov(r64_aux, reinterpret_cast(n_inc_arr)); + } + uni_vpaddq(v_n_64, v_n_64, ptr[r64_aux]); + + // Initialize auxiliary vectors. + static const uint32_t res_perm_mask[16] = { 0b00000000, 0b00010000, 0b00001000, 0b00011000, 0b00000010, 0b00010010, 0b00001010, 0b00011010, + 0b00000100, 0b00010100, 0b00001100, 0b00011100, 0b00000110, 0b00010110, 0b00001110, 0b00011110 }; + mov(r64_aux, reinterpret_cast(res_perm_mask)); + uni_vmovups(v_res_perm, ptr[r64_aux]); + + if (m_jcp.out_data_type == element::f16 && x64::mayiuse(x64::avx512_core_fp16)) { + v_perm_16 = getVmm(); + static const uint16_t perm_16[32] = { 0b00000000, 0b00000010, 0b00000100, 0b00000110, 0b00001000, 0b00001010, 0b00001100, 0b00001110, + 0b00010000, 0b00010010, 0b00010100, 0b00010110, 0b00011000, 0b00011010, 0b00011100, 0b00011110, + 0b00100000, 0b00100010, 0b00100100, 0b00100110, 0b00101000, 0b00101010, 0b00101100, 0b00101110, + 0b00110000, 0b00110010, 0b00110100, 0b00110110, 0b00111000, 0b00111010, 0b00111100, 0b00111110 }; + mov(r64_aux, reinterpret_cast(perm_16)); + uni_vmovups(v_perm_16, ptr[r64_aux]); + } + +#undef BROADCAST_R +#undef BROADCAST_P +} + +template // Works for AVX2, SSE41 +void RandomUniform::initVectors() { + const auto r64_aux = getReg64(); + + v_max_mul_n_64 = getVmm(); + v_max_mul_c_64 = getVmm(); + v_add_low_k = getVmm(); + v_add_up_k = getVmm(); + v_range = getVmm(); + v_key_64 = getVmm(); + v_counter_64 = getVmm(); + v_n_64 = getVmm(); + + r64_n_inc = getReg64(); + r64_min = getReg64(); + +#define INIT_ARR(A, V, R, T) \ + static const T A[8] = { V, V, V, V, V, V, V, V }; \ + if (isa == x64::avx2) { \ + mov(R, reinterpret_cast(A)); \ + } else { \ + static const T* A##_aligned = A + (reinterpret_cast(A) % 16) / sizeof(T); \ + mov(R, reinterpret_cast(A##_aligned)); \ + } + + // Initialize constants. + INIT_ARR(max_mul_n_64, STATISTIC_MAXIMIZING_MULTIPLIER_N, r64_aux, uint64_t); + uni_vmovups(v_max_mul_n_64, ptr[r64_aux]); + + INIT_ARR(max_mul_c_64, STATISTIC_MAXIMIZING_MULTIPLIER_COUNTER, r64_aux, uint64_t); + uni_vmovups(v_max_mul_c_64, ptr[r64_aux]); + + INIT_ARR(add_low_k, CRUSH_RESISTANCE_CONST_LOWER_VALUE, r64_aux, uint32_t); + uni_vmovups(v_add_low_k, ptr[r64_aux]); + + INIT_ARR(add_up_k, CRUSH_RESISTANCE_CONST_UPPER_VALUE, r64_aux, uint32_t); + uni_vmovups(v_add_up_k, ptr[r64_aux]); + + INIT_ARR(n_inc_step, isa == x64::avx2 ? 4 : 2, r64_n_inc, uint64_t); + + if (m_jcp.out_data_type == element::f32) { + r64_convert_0 = getReg64(); + r64_convert_1 = getReg64(); + + INIT_ARR(convert_0, 0x3f800000, r64_convert_0, uint32_t); + INIT_ARR(convert_1, 0x007fffff, r64_convert_1, uint32_t); + + mov(r64_aux, ptr[r64_params + GET_OFF(range_ptr)]); + uni_vpbroadcastd(v_range, ptr[r64_aux]); + + auto v_aux = getVmm(); + mov(r64_aux, ptr[r64_params + GET_OFF(min_ptr)]); + uni_vpbroadcastd(v_aux, ptr[r64_aux]); + static uint32_t min_arr[8]; + mov(r64_min, reinterpret_cast(min_arr)); + uni_vmovups(ptr[r64_min], v_aux); + } else if (m_jcp.out_data_type == element::i32) { + r64_f64_pow_52 = getReg64(); + const auto v_aux = getVmm(); + const auto xmm_range = Xbyak::Xmm(v_range.getIdx()); + + INIT_ARR(f64_pow_52, 0x4330000000000000, r64_f64_pow_52, uint64_t); + + mov(r64_aux, ptr[r64_params + GET_OFF(range_ptr)]); + uni_vpbroadcastd(v_range, ptr[r64_aux]); + + mov(r64_aux, ptr[r64_params + GET_OFF(min_ptr)]); + uni_vpbroadcastd(v_aux, ptr[r64_aux]); + static uint32_t min_arr[8]; + mov(r64_min, reinterpret_cast(min_arr)); + uni_vmovups(ptr[r64_min], v_aux); + + uni_vcvtdq2pd(v_range, xmm_range); + } else { + OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info()); + } + + // Initialize inputs. + mov(r64_aux, ptr[r64_params + GET_OFF(key_ptr)]); + uni_vpbroadcastq(v_key_64, ptr[r64_aux]); + + mov(r64_aux, ptr[r64_params + GET_OFF(counter_ptr)]); + uni_vpbroadcastq(v_counter_64, ptr[r64_aux]); + + mov(r64_aux, ptr[r64_params + GET_OFF(n_ptr)]); + uni_vpbroadcastq(v_n_64, ptr[r64_aux]); + + if (m_jcp.out_data_type.size() <= 4) { + if (isa == x64::avx2) { + static const uint64_t n_inc_arr[4] = { 0, 1, 2, 3 }; + mov(r64_aux, reinterpret_cast(n_inc_arr)); + } else { + static uint64_t n_inc_arr[4]; + static uint64_t* n_inc_arr_aligned = n_inc_arr + (reinterpret_cast(n_inc_arr) % 16) / sizeof(uint64_t); + n_inc_arr_aligned[0] = 0; + n_inc_arr_aligned[1] = 1; + mov(r64_aux, reinterpret_cast(n_inc_arr_aligned)); + } + } else { + static const uint64_t n_inc_arr[4] = { 0, 1, 2, 3 }; // TODO: i64 + mov(r64_aux, reinterpret_cast(n_inc_arr)); + } + + uni_vpaddq(v_n_64, v_n_64, ptr[r64_aux]); + +#undef INIT_ARR +} + +template +void RandomUniform::process() { + auto v_dst_0 = getVmm(); + auto v_dst_1 = getVmm(); + std::vector v_res{ v_dst_0, v_dst_1 }; + + auto step = vlen; + if (one_of(m_jcp.out_data_type.size(), 2lu, 4lu)) { + step = vlen * 2 / sizeof(uint32_t); + } else if (m_jcp.out_data_type.size() == 8) { + step = vlen / sizeof(uint32_t); + } + + Xbyak::Label l_loop, l_tail; + L(l_loop); { + cmp(r64_work_amount, step); + jl(l_tail, T_NEAR); + + runPhilox(v_res, v_key_64, v_counter_64, v_n_64); + convert(v_res, v_res); + + uni_vmovups(ptr[r64_dst], v_dst_0); + add(r64_dst, vlen); + if (one_of(m_jcp.out_data_type.size(), 4lu, 8lu)) { + uni_vmovups(ptr[r64_dst], v_dst_1); + add(r64_dst, vlen); + } + + if (isa == x64::avx512_core) { + uni_vpaddd(v_n_64, v_n_64, v_n_inc); + } else { + uni_vpaddd(v_n_64, v_n_64, ptr[r64_n_inc]); + } + + sub(r64_work_amount, step); + jmp(l_loop, T_NEAR); + } + + L(l_tail); + tail(v_res); +} + +template +void RandomUniform::calculateRound(const Vmm& vmm_k_0, const Vmm& vmm_k_1, const Vmm& vmm_c_0, const Vmm& vmm_c_1, + const Vmm& vmm_n_0, const Vmm& vmm_n_1, const Vmm& vmm_aux_0, const Vmm& vmm_aux_1) { + uni_vpmuludq(vmm_aux_0, vmm_n_0, v_max_mul_n_64); // {p0,p1,p0,p1} = {n0,_,n0,_} * {m0,_,m0,_} + uni_vpmuludq(vmm_aux_1, vmm_c_0, v_max_mul_c_64); // {r0,r1,r0,r1} = {c0,_,c0,_} * {m0,_,m0,_} + + uni_vpshufd(vmm_c_0, vmm_aux_0, 0b10110001); // {p1,p0,p1,p0} = shuf {p0,p1,p0,p1} + uni_vxorps(vmm_c_0, vmm_c_0, vmm_c_1); // {c0,_,c0,_} = {p1,_,p1,_} ^ {c1,_,c1,_} + uni_vxorps(vmm_c_0, vmm_c_0, vmm_k_1); // {c0,_,c0,_} = {c0,_,c0,_} ^ {k1,_,k1,_} + + uni_vpshufd(vmm_n_0, vmm_aux_1, 0b10110001); // {r1,r0,r1,r0} = shuf {r0,r1,r0,r1} + uni_vxorps(vmm_n_0, vmm_n_0, vmm_n_1); // {n0,_,n0,_} = {r1,_,r1,_} ^ {n1,_,n1,_} + uni_vxorps(vmm_n_0, vmm_n_0, vmm_k_0); // {n0,_,n0,_} = {n0,_,n0,_} ^ {k0,_,k0,_} +} + +template +void RandomUniform::runPhilox(const std::vector& vmm_dst, const Vmm& vmm_key, const Vmm& vmm_counter, const Vmm& vmm_n) { + auto vmm_k_0 = getVmm(); + auto vmm_k_1 = getVmm(); + auto vmm_n_0 = getVmm(); + auto vmm_n_1 = vmm_dst[0]; + auto vmm_c_0 = getVmm(); + auto vmm_c_1 = getVmm(); + auto vmm_aux_0 = getVmm(); + auto vmm_aux_1 = vmm_dst[1]; + + uni_vmovups(vmm_k_0, vmm_key); // {k0,k1,k0,k1} -> {k0,_,k0,_} + uni_vpshufd(vmm_k_1, vmm_key, 0b10110001); // {k0,k1,k0,k1} -> {k1,_,k1,_} + + uni_vpmuludq(vmm_aux_0, vmm_n, v_max_mul_n_64); // {p0,p1,p0,p1} = {n0,_,n0,_} * {m0,_,m0,_} + uni_vpmuludq(vmm_aux_1, vmm_counter, v_max_mul_c_64); // {r0,r1,r0,r1} = {c0,_,c0,_} * {m0,_,m0,_} + + uni_vxorps(vmm_c_0, vmm_aux_0, vmm_counter); // {_,c0,_,c0} = {_,p1,_,p1} ^ {_,c1,_,c1} + uni_vxorps(vmm_c_0, vmm_c_0, vmm_key); // {_,c0,_,c0} = {_,c0,_,c0} ^ {_,k1,_,k1} + uni_vpshufd(vmm_c_0, vmm_c_0, 0b10110001); // {_,c0,_,c0} -> {c0,_,c0,_} + + uni_vxorps(vmm_n_0, vmm_aux_1, vmm_n); // {_,n0,_,n0} = {_,r1,_,r1} ^ {_,n1,_,n1} + uni_vpshufd(vmm_n_0, vmm_n_0, 0b10110001); // {_,n0,_,n0} -> {n0,_,n0,_} + uni_vxorps(vmm_n_0, vmm_n_0, vmm_key); // {n0,_,n0,_} = {n0,_,n0,_} ^ {k0,_,k0,_} + + for (size_t i = 0lu; i < ROUNDS_NUMBER - 1; i++) { + raiseKey(vmm_k_0, vmm_k_1); + + std::swap(vmm_c_1, vmm_aux_0); + std::swap(vmm_n_1, vmm_aux_1); + calculateRound(vmm_k_0, vmm_k_1, vmm_c_0, vmm_c_1, vmm_n_0, vmm_n_1, vmm_aux_0, vmm_aux_1); + } + std::swap(vmm_c_1, vmm_aux_0); + std::swap(vmm_n_1, vmm_aux_1); + + if (isa == x64::avx512_core) { + vpermt2d(vmm_n_0, v_res_perm, vmm_n_1); // {n0,n1,n0,n1} = perm {n0,_,n0,_} {n1,_,n1,_} + vpermt2d(vmm_c_0, v_res_perm, vmm_c_1); // {c0,c1,c0,c1} = perm {c0,_,c0,_} {c1,_,c1,_} + vshufpd(vmm_dst[0], vmm_n_0, vmm_c_0, 0b00000000); // {n0,n1,c0,c1} = shuf {n0,n1,n0,n1} {c0,c1,c0,c1} + vshufpd(vmm_dst[1], vmm_n_0, vmm_c_0, 0b11111111); // {n0,n1,c0,c1} = shuf {n0,n1,n0,n1} {c0,c1,c0,c1} + } else if (isa == x64::avx2) { + auto ymm_dst_0 = Xbyak::Ymm(vmm_dst[0].getIdx()); + auto ymm_dst_1 = Xbyak::Ymm(vmm_dst[1].getIdx()); + auto ymm_c_0 = Xbyak::Ymm(vmm_c_0.getIdx()); + + uni_vshufps(vmm_n_0, vmm_n_0, vmm_n_1, 0b10001000); // {n0,n0,n1,n1} = shuf {n0,_,n0,_} {n1,_,n1,_} + uni_vshufps(vmm_c_0, vmm_c_0, vmm_c_1, 0b10001000); // {c0,c0,c1,c1} = shuf {c0,_,c0,_} {c1,_,c1,_} + uni_vshufps(ymm_dst_1, vmm_n_0, vmm_c_0, 0b10001000); // {n0,n1,c0,c1} = shuf {n0,n0,n1,n1} {c0,c0,c1,c1} + uni_vshufps(vmm_c_0, vmm_n_0, vmm_c_0, 0b11011101); // {n0,n1,c0,c1} = shuf {n0,n0,n1,n1} {c0,c0,c1,c1} + vperm2f128(ymm_dst_0, ymm_dst_1, ymm_c_0, 0b00100000); + vperm2f128(ymm_dst_1, ymm_dst_1, ymm_c_0, 0b00110001); + } else { + uni_vshufps(vmm_n_0, vmm_n_0, vmm_n_1, 0b10001000); + uni_vshufps(vmm_c_0, vmm_c_0, vmm_c_1, 0b10001000); + uni_vshufps(vmm_dst[0], vmm_n_0, vmm_c_0, 0b10001000); + uni_vshufps(vmm_dst[1], vmm_n_0, vmm_c_0, 0b11011101); + } +} + +template +void RandomUniform::raiseKey(const Vmm& vmm_k_0, const Vmm& vmm_k_1) { + uni_vpaddd(vmm_k_0, vmm_k_0, v_add_low_k); // {k0,_,k0,_} + {l0,_,l0,_} + uni_vpaddd(vmm_k_1, vmm_k_1, v_add_up_k); // {k1,_,k1,_} + {u0,_,u0,_} +} + +template <> +void RandomUniform::convert(const std::vector& v_dst, const std::vector& v_src) { + if (m_jcp.out_data_type.size() == 4) { + for (size_t i = 0lu; i < v_src.size(); i++) { + const auto& vmm_src = v_src[i]; + const auto& vmm_dst = v_dst[i]; + + if (m_jcp.out_data_type == element::f32) { + uni_vandps(vmm_dst, vmm_src, v_convert_1); + uni_vorps(vmm_dst, vmm_dst, v_convert_0); + uni_vsubps(vmm_dst, vmm_dst, v_convert_0); + vfmadd132ps(vmm_dst, v_min, v_range); + } else if (m_jcp.out_data_type == element::i32) { + // x % (max - min) + min + const auto v_aux_0 = getVmm(); + const auto v_aux_1 = getVmm(); + const auto ymm_src = Xbyak::Ymm(vmm_src.getIdx()); + const auto ymm_dst = Xbyak::Ymm(vmm_dst.getIdx()); + const auto ymm_aux_1 = Xbyak::Ymm(v_aux_1.getIdx()); + + // Divide in the f64 due to the f32 loses accuracy here. + vcvtudq2pd(v_aux_0, ymm_src); + uni_vdivpd(v_aux_1, v_aux_0, v_range); + uni_vroundpd(v_aux_1, v_aux_1, 3); + vfnmadd132pd(v_aux_1, v_aux_0, v_range); + + vextractf64x4(ymm_dst, vmm_src, 1); + vcvtudq2pd(v_aux_0, ymm_dst); + uni_vcvtpd2dq(ymm_dst, v_aux_1); + uni_vdivpd(v_aux_1, v_aux_0, v_range); + uni_vroundpd(v_aux_1, v_aux_1, 3); + vfnmadd132pd(v_aux_1, v_aux_0, v_range); + uni_vcvtpd2dq(ymm_aux_1, v_aux_1); + vshuff64x2(vmm_dst, vmm_dst, v_aux_1, 0b01000100); + + uni_vpaddd(vmm_dst, vmm_dst, v_min); + } else { + OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info()); + } + } + } else if (m_jcp.out_data_type.size() == 2) { + if (m_jcp.out_data_type == element::f16 && x64::mayiuse(x64::avx512_core_fp16)) { + const auto& vmm_dst = v_dst[0]; + + if (v_src[0].getIdx() != vmm_dst.getIdx()) { + uni_vmovups(vmm_dst, v_src[0]); + } + vpermt2w(vmm_dst, v_perm_16, v_src[1]); + + uni_vandps(vmm_dst, vmm_dst, v_convert_1); + uni_vorps(vmm_dst, vmm_dst, v_convert_0); + vsubph(vmm_dst, vmm_dst, v_convert_0); + vfmadd132ph(vmm_dst, v_min, v_range); + } else if (m_jcp.out_data_type == element::bf16 && x64::mayiuse(x64::avx512_core_bf16)) { + for (size_t i = 0lu; i < v_src.size(); i++) { + const auto& vmm_dst = v_dst[i]; + + uni_vandps(vmm_dst, v_src[i], v_convert_1); + uni_vorps(vmm_dst, vmm_dst, v_convert_0); + uni_vpslld(vmm_dst, vmm_dst, 16); + + uni_vsubps(vmm_dst, vmm_dst, v_convert_2); + vfmadd132ps(vmm_dst, v_min, v_range); + } + + vcvtne2ps2bf16(v_dst[0], v_dst[1], v_dst[0]); + } else { + OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info()); + } + } else if (m_jcp.out_data_type.size() == 8) { + if (m_jcp.out_data_type == element::i64) { + // TODO: in scope of i64 enabling. + } + OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info()); + } else { + OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info()); + } +} + +template // Works for AVX2, SSE41 +void RandomUniform::convert(const std::vector& v_dst, const std::vector& v_src) { + if (m_jcp.out_data_type.size() == 4) { + for (size_t i = 0lu; i < v_src.size(); i++) { + auto vmm_src = v_src[i]; + auto vmm_dst = v_dst[i]; + + if (m_jcp.out_data_type == element::f32) { + uni_vandps(vmm_dst, vmm_src, ptr[r64_convert_1]); + uni_vorps(vmm_dst, vmm_dst, ptr[r64_convert_0]); + uni_vsubps(vmm_dst, vmm_dst, ptr[r64_convert_0]); + if (isa == x64::avx2) { + vfmadd213ps(vmm_dst, v_range, ptr[r64_min]); + } else { + uni_vmulps(vmm_dst, vmm_dst, v_range); + uni_vaddps(vmm_dst, vmm_dst, ptr[r64_min]); + } + } else if (m_jcp.out_data_type == element::i32) { + // x % (max - min) + min + const auto v_aux_0 = getVmm(); + const auto v_aux_1 = getVmm(); + const auto xmm_dst = Xbyak::Xmm(vmm_dst.getIdx()); + const auto ymm_dst = Xbyak::Ymm(vmm_dst.getIdx()); + const auto xmm_aux_1 = Xbyak::Xmm(v_aux_1.getIdx()); + + // Convert u32->f64. TODO: move to convert emitter after i64 enabling. + uni_vpmovzxdq(v_aux_0, xmm_dst); + uni_vorps(v_aux_0, v_aux_0, ptr[r64_f64_pow_52]); + uni_vsubpd(v_aux_0, v_aux_0, ptr[r64_f64_pow_52]); + + // Divide in the f64 due to the f32 loses accuracy here. + uni_vdivpd(v_aux_1, v_aux_0, v_range); + uni_vroundpd(v_aux_1, v_aux_1, 3); + if (isa == x64::avx2) { + vfnmadd132pd(v_aux_1, v_aux_0, v_range); + } else { + uni_vmulpd(v_aux_1, v_aux_1, v_range); + uni_vsubpd(v_aux_0, v_aux_0, v_aux_1); + uni_vmovups(v_aux_1, v_aux_0); + } + + if (isa == x64::avx2) { + vperm2f128(ymm_dst, ymm_dst, ymm_dst, 0b00000001); + } else { + uni_vshufpd(vmm_dst, vmm_dst, vmm_dst, 0b00000001); + } + // Convert u32->f64. TODO: move to convert emitter after i64 enabling. + uni_vpmovzxdq(v_aux_0, xmm_dst); + uni_vorps(v_aux_0, v_aux_0, ptr[r64_f64_pow_52]); + uni_vsubpd(v_aux_0, v_aux_0, ptr[r64_f64_pow_52]); + + uni_vcvtpd2dq(xmm_dst, v_aux_1); + uni_vdivpd(v_aux_1, v_aux_0, v_range); + uni_vroundpd(v_aux_1, v_aux_1, 3); + if (isa == x64::avx2) { + vfnmadd132pd(v_aux_1, v_aux_0, v_range); + } else { + uni_vmulpd(v_aux_1, v_aux_1, v_range); + uni_vsubpd(v_aux_0, v_aux_0, v_aux_1); + uni_vmovups(v_aux_1, v_aux_0); + } + uni_vcvtpd2dq(xmm_aux_1, v_aux_1); + if (isa == x64::avx2) { + vperm2f128(ymm_dst, ymm_dst, v_aux_1, 0b00100000); + } else { + uni_vshufpd(vmm_dst, vmm_dst, v_aux_1, 0b00000000); + } + + uni_vpaddd(vmm_dst, vmm_dst, ptr[r64_min]); + } else { + OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info()); + } + } + } else if (m_jcp.out_data_type.size() == 8) { + if (m_jcp.out_data_type == element::i64) { + // TODO: in scope of i64 enabling. + } + OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info()); + } else { + OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info()); + } +} + +template <> +void RandomUniform::tail(const std::vector& vmm_dst) { + Xbyak::Label l_end; + const auto k_rest_mask = getMask(); + + cmp(r64_work_amount, 0); + jle(l_end, T_NEAR); + + runPhilox(vmm_dst, v_key_64, v_counter_64, v_n_64); + convert(vmm_dst, vmm_dst); + + if (m_jcp.out_data_type.size() == 4) { + Xbyak::Label l_0; + const auto step = vlen / sizeof(uint32_t); + + cmp(r64_work_amount, step); + jl(l_0, T_NEAR); + + uni_vmovups(ptr[r64_dst], vmm_dst[0]); + add(r64_dst, vlen); + sub(r64_work_amount, step); + fillRestWorkMask(k_rest_mask, r64_work_amount); + uni_vmovups(ptr[r64_dst] | k_rest_mask, vmm_dst[1]); + jmp(l_end, T_NEAR); + + L(l_0); + fillRestWorkMask(k_rest_mask, r64_work_amount); + uni_vmovups(ptr[r64_dst] | k_rest_mask, vmm_dst[0]); + } else if (m_jcp.out_data_type.size() == 2) { + fillRestWorkMask(k_rest_mask, r64_work_amount); + vmovdqu16(ptr[r64_dst] | k_rest_mask, vmm_dst[0]); + } + + L(l_end); +} + +template <> +void RandomUniform::tail(const std::vector& vmm_dst) { + Xbyak::Label l_0, l_end; + const auto step = vlen / sizeof(uint32_t); + + cmp(r64_work_amount, 0); + jle(l_end, T_NEAR); + + runPhilox(vmm_dst, v_key_64, v_counter_64, v_n_64); + convert(vmm_dst, vmm_dst); + const auto v_rest_mask = getVmm(); + + cmp(r64_work_amount, step); + jl(l_0, T_NEAR); + + uni_vmovups(ptr[r64_dst], vmm_dst[0]); + add(r64_dst, vlen); + sub(r64_work_amount, step); + fillRestWorkMask(v_rest_mask, r64_work_amount, m_jcp.out_data_type.size()); + vmaskmovps(ptr[r64_dst], v_rest_mask, vmm_dst[1]); + jmp(l_end, T_NEAR); + + L(l_0); + fillRestWorkMask(v_rest_mask, r64_work_amount, m_jcp.out_data_type.size()); + vmaskmovps(ptr[r64_dst], v_rest_mask, vmm_dst[0]); + + L(l_end); +} + +template +void RandomUniform::tail(const std::vector& vmm_dst) { + Xbyak::Label l_0, l_end; + const auto step = vlen / sizeof(uint32_t); + + cmp(r64_work_amount, 0); + jle(l_end, T_NEAR); + + runPhilox(vmm_dst, v_key_64, v_counter_64, v_n_64); + convert(vmm_dst, vmm_dst); + + cmp(r64_work_amount, step); + jl(l_0, T_NEAR); + + uni_vmovups(ptr[r64_dst], vmm_dst[0]); + add(r64_dst, vlen); + sub(r64_work_amount, step); + store(ptr[r64_dst], vmm_dst[1], r64_work_amount, m_jcp.out_data_type.size()); + jmp(l_end, T_NEAR); + + L(l_0); + store(ptr[r64_dst], vmm_dst[0], r64_work_amount, m_jcp.out_data_type.size()); + + L(l_end); +} + +template class RandomUniform; +template class RandomUniform; +template class RandomUniform; + +} // namespace kernel +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/kernels/x64/random_uniform.hpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/random_uniform.hpp new file mode 100644 index 00000000000..366be4c3a13 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/kernels/x64/random_uniform.hpp @@ -0,0 +1,99 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "jit_kernel_base.hpp" + +#if defined(OPENVINO_ARCH_X86_64) + +namespace ov { +namespace intel_cpu { +namespace kernel { + +struct RandomUniformCompileParams { + element::Type out_data_type = element::f32; +}; + +struct RandomUniformCallArgs { + void* dst_ptr; + const void* key_ptr; + const void* counter_ptr; + const void* n_ptr; + const void* min_ptr; + const void* range_ptr; + uint64_t work_amount = 0lu; +}; + +template +class RandomUniform : public JitKernel { +public: + DECLARE_CPU_JIT_AUX_FUNCTIONS(RandomUniform) + + explicit RandomUniform(const RandomUniformCompileParams& jcp); + + void generate() override; + +private: + using Vmm = typename dnnl::impl::utils::conditional3::type; + using Vmask = typename dnnl::impl::utils::conditional3::type; + + RegistersPool::Reg r64_dst; + RegistersPool::Reg r64_work_amount; + RegistersPool::Reg r64_n_inc; + RegistersPool::Reg r64_convert_0; + RegistersPool::Reg r64_convert_1; + RegistersPool::Reg r64_min; + RegistersPool::Reg r64_f64_pow_52; + + const Xbyak::Reg64 r64_params = Xbyak::Reg64(dnnl::impl::cpu::x64::abi_param_regs[0]); + + // Vector registers. + RegistersPool::Reg v_max_mul_n_64; + RegistersPool::Reg v_max_mul_c_64; + RegistersPool::Reg v_add_low_k; + RegistersPool::Reg v_add_up_k; + RegistersPool::Reg v_convert_0; + RegistersPool::Reg v_convert_1; + RegistersPool::Reg v_convert_2; + RegistersPool::Reg v_n_inc; + RegistersPool::Reg v_key_64; + RegistersPool::Reg v_counter_64; + RegistersPool::Reg v_n_64; + RegistersPool::Reg v_min; + RegistersPool::Reg v_range; + RegistersPool::Reg v_res_perm; + RegistersPool::Reg v_perm_16; + + void initVectors(); + + void process(); + + void runPhilox(const std::vector& vmm_res, const Vmm& vmm_key, const Vmm& vmm_counter, const Vmm& vmm_n); + + void calculateRound(const Vmm& vmm_k_0, const Vmm& vmm_k_1, const Vmm& vmm_c_0, const Vmm& vmm_c_1, + const Vmm& vmm_n_0, const Vmm& vmm_n_1, const Vmm& vmm_aux_0, const Vmm& vmm_aux_1); + + void raiseKey(const Vmm& vmm_k_0, const Vmm& vmm_k_1); + + void convert(const std::vector& vmm_dst, const std::vector& vmm_src); + + void tail(const std::vector& vmm_dst); + + static constexpr uint64_t ROUNDS_NUMBER = 10lu; + static constexpr uint32_t CRUSH_RESISTANCE_CONST_LOWER_VALUE = 0x9E3779B9; + static constexpr uint32_t CRUSH_RESISTANCE_CONST_UPPER_VALUE = 0xBB67AE85; + static constexpr uint64_t STATISTIC_MAXIMIZING_MULTIPLIER_N = 0xD2511F53; + static constexpr uint64_t STATISTIC_MAXIMIZING_MULTIPLIER_COUNTER = 0xCD9E8D57; +}; + +} // namespace kernel +} // namespace intel_cpu +} // namespace ov + +#endif // OPENVINO_ARCH_X86_64 diff --git a/src/plugins/intel_cpu/src/nodes/random_uniform.cpp b/src/plugins/intel_cpu/src/nodes/random_uniform.cpp new file mode 100644 index 00000000000..77d823710c9 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/random_uniform.cpp @@ -0,0 +1,532 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "random_uniform.hpp" + +#include "ie_parallel.hpp" +#include "ie_ngraph_utils.hpp" +#include +#include +#include "shape_inference/custom/random_uniform.hpp" + +namespace ov { +namespace intel_cpu { +namespace node { + +bool RandomUniform::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { + try { + if (op->get_type_info() != op::v8::RandomUniform::get_type_info_static()) { + errorMessage = "Only RandomUniform operation from the opset8 is supported by the CPU plugin."; + return false; + } + } catch (...) { + return false; + } + return true; +} + +RandomUniform::RandomUniform(const std::shared_ptr& op, const GraphContext::CPtr& context) + : Node(op, context, RandomUniformShapeInferFactory(op)) { + std::string errorMessage; + if (!isSupportedOperation(op, errorMessage)) { + THROW_CPU_NODE_ERR(errorMessage); + } + + // RandomUniform should generate new sequence each run even if all inputs are constants. So that method Node::IsConstant() + // doesn't return 'True' for RandomUniform with all constant inputs and the node generates new values for each inference, + // we set 'NoConst' value for 'ConstantType' in ctor. + constant = ConstantType::NoConst; + + auto rnd_op = as_type_ptr(op); + m_global_seed = rnd_op->get_global_seed(); + m_op_seed = rnd_op->get_op_seed(); + + m_output_prc = op->get_output_element_type(0); + + for (size_t i = 0lu; i < op->get_input_size(); i++) { + if (is_type(op->get_input_node_ptr(i))) { + m_const_inputs[i] = true; + } + } + + if (m_algo == STL) { + m_generator = std::default_random_engine{static_cast(m_op_seed)}; + } +} + +void RandomUniform::getSupportedDescriptors() { + if (getParentEdges().size() != 3) { + THROW_CPU_NODE_ERR("has incorrect number of input edges."); + } + if (getChildEdges().empty()) { + THROW_CPU_NODE_ERR("has incorrect number of output edges."); + } +} + +void RandomUniform::initSupportedPrimitiveDescriptors() { + auto shape_prc = getOriginalInputPrecisionAtPort(SHAPE); + if (!one_of(shape_prc, InferenceEngine::Precision::I32, InferenceEngine::Precision::I64)) { + shape_prc = InferenceEngine::Precision::I32; + } + + auto out_prc = getOriginalOutputPrecisionAtPort(0); + if (out_prc.is_float() && ((m_algo == PHILOX && + !one_of(out_prc, InferenceEngine::Precision::FP32, InferenceEngine::Precision::FP16, InferenceEngine::Precision::BF16)) || + (m_algo == STL && !one_of(out_prc, InferenceEngine::Precision::FP32)))) { + out_prc = InferenceEngine::Precision::FP32; + } + if (!out_prc.is_float() && !one_of(out_prc, InferenceEngine::Precision::I32, InferenceEngine::Precision::I64)) { + out_prc = InferenceEngine::Precision::I32; + } + m_output_prc = InferenceEngine::details::convertPrecision(out_prc); + + addSupportedPrimDesc({{LayoutType::ncsp, shape_prc, m_const_inputs[SHAPE]}, + {LayoutType::ncsp, out_prc, m_const_inputs[MIN_VAL]}, + {LayoutType::ncsp, out_prc, m_const_inputs[MAX_VAL]}}, + {{LayoutType::ncsp, out_prc}}, + ref_any); +} + +void RandomUniform::createPrimitive() { + if (m_const_inputs[MIN_VAL]) { + initEdgeValues(m_min_val, getParentEdgeAt(MIN_VAL)->getMemoryPtr()->getData(), m_output_prc); + } + if (m_const_inputs[MAX_VAL]) { + initEdgeValues(m_max_val, getParentEdgeAt(MAX_VAL)->getMemoryPtr()->getData(), m_output_prc); + evalRange(); + } + + if (m_algo == PHILOX) { +#if defined(OPENVINO_ARCH_X86_64) + kernel::RandomUniformCompileParams jcp; + + jcp.out_data_type = m_output_prc; + + m_jit_kernel = kernel::JitKernel::createInstance(jcp); + + if (m_jit_kernel) { + if (auto selected_pd = getSelectedPrimitiveDescriptor()) { + using namespace dnnl::impl::cpu; + if (m_jit_kernel->getIsa() == x64::avx512_core) { + selected_pd->setImplementationType(jit_avx512); + } else if (m_jit_kernel->getIsa() == x64::avx2) { + selected_pd->setImplementationType(jit_avx2); + } else if (m_jit_kernel->getIsa() == x64::sse41) { + selected_pd->setImplementationType(jit_sse42); + } + } + } +#endif // OPENVINO_ARCH_X86_64 + } + + if (m_const_inputs[SHAPE]) { + Node::createPrimitive(); + } +} + +bool RandomUniform::needPrepareParams() const { + if (m_out_shape != getChildEdgeAt(0)->getMemoryPtr()->getShape().getStaticDims()) { + return true; + } + return false; +} + +void RandomUniform::prepareParams() { + m_out_shape = getChildEdgeAt(0)->getMemoryPtr()->getShape().getStaticDims(); + m_out_el_num = std::accumulate(m_out_shape.begin(), m_out_shape.end(), 1lu, std::multiplies()); + + if (m_algo == PHILOX) { + m_skip_count = m_out_el_num * SKIP_CONST; + + if (m_out_el_num < PHILOX_PARALLEL_EXECUTION_THRESHOLD) { + m_threads_num = 1; + } else { + m_threads_num = parallel_get_max_threads(); + } + m_thread_params.resize(m_threads_num); + + parallel_nt(m_threads_num, [&](const int ithr, const int nthr) { + auto& p = m_thread_params[ithr]; + uint64_t start = 0lu, end = 0lu; + + if (m_jit_kernel) { +#if defined(OPENVINO_ARCH_X86_64) + const auto block_size = (m_jit_kernel->getVectorLen() / m_output_prc.size()) * 2; + const auto blocks_num = (m_out_el_num + block_size - 1) / block_size; + const auto blocks_per_thr = (blocks_num + nthr - 1) / nthr; + + start = ithr * blocks_per_thr * block_size; + end = (ithr + 1) * blocks_per_thr * block_size; +#endif // OPENVINO_ARCH_X86_64 + } else { + const auto groups_num = (m_out_el_num + PHILOX_GROUP_SIZE - 1) / PHILOX_GROUP_SIZE; + const auto groups_per_thr = (groups_num + nthr - 1) / nthr; + + start = ithr * groups_per_thr * PHILOX_GROUP_SIZE; + end = (ithr + 1) * groups_per_thr * PHILOX_GROUP_SIZE; + + p.step = m_output_prc.size() > 4 ? 2 : 4; + } + + if (end > m_out_el_num) { + end = m_out_el_num; + } + if (start > end) { + start = end; + } + p.work_amount = end - start; + p.n_shift = start / PHILOX_GROUP_SIZE; + p.dst_shift = start * m_output_prc.size(); + }); + } +} + +void RandomUniform::execute(dnnl::stream strm) { + if (!m_const_inputs[MIN_VAL]) { + initEdgeValues(m_min_val, getParentEdgeAt(MIN_VAL)->getMemoryPtr()->getData(), m_output_prc); + if (m_const_inputs[MAX_VAL]) { + evalRange(); + } + } + if (!m_const_inputs[MAX_VAL]) { + initEdgeValues(m_max_val, getParentEdgeAt(MAX_VAL)->getMemoryPtr()->getData(), m_output_prc); + evalRange(); + } + + auto data = getChildEdgeAt(0)->getMemoryPtr()->getData(); + + if (m_algo == PHILOX) { + m_state = computePhilox(data, m_out_el_num, m_state); + } else if (m_algo == STL) { + computeStl(data, m_out_el_num); + } else { + THROW_CPU_NODE_ERR("unsupported algorithm."); + } +} + +void RandomUniform::executeDynamicImpl(dnnl::stream strm) { + execute(strm); +} + +////////////// PHILOX algo /////////////// + +namespace { +// Following const values are taken from the original paper: +// https://www.thesalmons.org/john/random123/papers/random123sc11.pdf +constexpr uint32_t CRUSH_RESISTANCE_CONST_LOWER_VALUE = 0x9E3779B9; +constexpr uint32_t CRUSH_RESISTANCE_CONST_UPPER_VALUE = 0xBB67AE85; +constexpr uint64_t STATISTIC_MAXIMIZING_MULTIPLIER_N = 0xD2511F53; +constexpr uint64_t STATISTIC_MAXIMIZING_MULTIPLIER_COUNTER = 0xCD9E8D57; +constexpr uint64_t ROUNDS_NUMBER = 10llu; + +inline void calculateRound(const uint32_t* key, uint32_t* counter, uint32_t* n) { + uint64_t prod_0 = STATISTIC_MAXIMIZING_MULTIPLIER_N * n[0]; + uint64_t prod_1 = STATISTIC_MAXIMIZING_MULTIPLIER_COUNTER * counter[0]; + n[0] = static_cast(prod_1 >> 32) ^ n[1] ^ key[0]; + n[1] = static_cast(prod_1); + counter[0] = static_cast(prod_0 >> 32) ^ counter[1] ^ key[1]; + counter[1] = static_cast(prod_0); +} + +inline void raiseKey(uint32_t* key) { + key[0] += CRUSH_RESISTANCE_CONST_LOWER_VALUE; + key[1] += CRUSH_RESISTANCE_CONST_UPPER_VALUE; +} + +inline void runPhilox(uint64_t key, uint64_t counter, uint64_t n, uint32_t* res) { + uint32_t* key_32 = reinterpret_cast(&key); + uint32_t* counter_32 = reinterpret_cast(&counter); + uint32_t* n_32 = reinterpret_cast(&n); + + for (size_t i = 0lu; i < ROUNDS_NUMBER; i++) { + calculateRound(key_32, counter_32, n_32); + if (i < ROUNDS_NUMBER - 1) + raiseKey(key_32); + } + + res[0] = n_32[0]; + res[1] = n_32[1]; + res[2] = counter_32[0]; + res[3] = counter_32[1]; +} + +inline void convertToOutputType(const uint32_t* in, + float min, + float range, + float* out, + size_t el_to_copy) { + RandomUniform::OutputType out_val; + + for (size_t i = 0lu; i < el_to_copy; i++) { + out_val.u32 = 0x3f800000 | (in[i] & 0x7fffffu); + out[i] = (out_val.f32 - 1.f) * range + min; + } +} + +inline void convertToOutputType(const uint32_t* in, + float16 min, + float16 range, + float16* out, + size_t el_to_copy) { + RandomUniform::OutputType out_val; + + for (size_t i = 0lu; i < el_to_copy; i++) { + uint16_t x_uint16 = static_cast(in[i]); + out_val.u16 = 0x3c00 | (x_uint16 & 0x03ffu); + out[i] = (out_val.f16 - static_cast(1)) * range + min; + } +} + +inline void convertToOutputType(const uint32_t* in, + bfloat16 min, + bfloat16 range, + bfloat16* out, + size_t el_to_copy) { + RandomUniform::OutputType out_val; + + for (size_t i = 0lu; i < el_to_copy; i++) { + uint16_t x_uint16 = static_cast(in[i]); + out_val.u16 = 0x3f80 | (x_uint16 & 0x7fu); + out[i] = (out_val.bf16 - static_cast(1)) * range + min; + } +} + +inline void convertToOutputType(const uint32_t* in, + int32_t min, + int32_t range, + int32_t* out, + size_t el_to_copy) { + for (size_t i = 0lu; i < el_to_copy; i++) { + out[i] = static_cast(in[i] % range + min); + } +} + +inline void convertToOutputType(const uint32_t* in, + int64_t min, + int64_t range, + int64_t* out, + size_t el_to_copy) { + for (size_t i = 0lu; i < el_to_copy; i++) { + out[i] = static_cast(((static_cast(in[i * 2]) << 32) + in[i * 2 + 1]) % range + min); + } +} + +} // namespace + +std::pair RandomUniform::computePhilox(void* out, size_t out_el_num, const std::pair& prev_state) { + // When both seed values are equal to zero RandomUniform should generate non-deterministic sequence. + if (m_global_seed == 0lu && m_op_seed == 0lu) { + std::srand(static_cast(std::time(nullptr))); + m_global_seed = std::rand(); + } + + uint64_t n_state = prev_state.first; + uint64_t counter_state = prev_state.second; + + uint64_t counter = counter_state > 0 ? counter_state : m_op_seed; + + auto out_u8 = reinterpret_cast(out); + + if (m_jit_kernel) { +#if defined(OPENVINO_ARCH_X86_64) + parallel_nt(m_threads_num, [&](const int ithr, const int nthr) { + auto& p = m_thread_params[ithr]; + if (p.work_amount == 0lu) { + return; + } + auto n = n_state + p.n_shift; + + kernel::RandomUniformCallArgs args; + + args.dst_ptr = (out_u8 + p.dst_shift); + args.key_ptr = &m_global_seed; + args.counter_ptr = &counter; + args.n_ptr = &n; + args.min_ptr = &m_min_val; + args.range_ptr = &m_range_val; + args.work_amount = p.work_amount; + + (*m_jit_kernel)(&args); + }); +#endif // OPENVINO_ARCH_X86_64 + } else { + auto threadBody = [&](const int ithr, const int nthr) { + auto& p = m_thread_params[ithr]; + if (p.work_amount == 0lu) { + return; + } + auto n = n_state + p.n_shift; + auto out_cur = out_u8 + p.dst_shift; + auto work_rest = static_cast(p.work_amount); + uint32_t res[4]; + +#define EXEC_CASE(P) \ + case element::P: { \ + auto out_t = reinterpret_cast::value_type *>(out_cur); \ + for (; work_rest > 0l; work_rest -= p.step, out_t += p.step) { \ + runPhilox(m_global_seed, counter, n, res); \ + auto el_to_copy = std::min(p.step, static_cast(work_rest)); \ + convertToOutputType(res, m_min_val.P, m_range_val.P, out_t, el_to_copy); \ + if (++n == 0) { \ + counter++; \ + } \ + } \ + } break; + + switch (m_output_prc) { + EXEC_CASE(f32) + EXEC_CASE(f16) + EXEC_CASE(bf16) + EXEC_CASE(i32) + EXEC_CASE(i64) + default: THROW_CPU_NODE_ERR("Unsupported type of RandomUniform: ", m_output_prc.to_string()); + } + +#undef EXEC_CASE + }; + + parallel_nt(m_threads_num, threadBody); + } + + // Calculate counter values for next RandomUniform run. + n_state += m_skip_count; + if (n_state < m_skip_count) { + counter_state++; + } + + return { n_state, counter_state }; +} + +////////////// STL algo /////////////// +void RandomUniform::computeStl(void* out, size_t work_amount) { + switch (m_output_prc) { + case element::f32: { + generateData>( + std::uniform_real_distribution{m_min_val.f32, m_max_val.f32}, out, work_amount); + } break; + case element::i32: { + generateData>( + std::uniform_int_distribution{m_min_val.i32, m_max_val.i32}, out, work_amount); + } break; + case element::i64: { + generateData>( + std::uniform_int_distribution{m_min_val.i64, m_max_val.i64}, out, work_amount); + } break; + default: + THROW_CPU_NODE_ERR("has unsupported output type: ", m_output_prc); + } +} + +template +void RandomUniform::generateData(DISTR_TYPE distribution, void* out, size_t work_amount) { + auto dst = reinterpret_cast(out); + for (size_t i = 0; i < work_amount; i++) { + *dst = distribution(m_generator); + dst++; + } +} +////////////////////////////////// + +void RandomUniform::initEdgeValues(OutputType& dst, const void* src, const element::Type& output_type) { +#define EL_CASE(E) \ + case element::E: \ + dst.E = *reinterpret_cast::value_type *>(src); \ + break; + + switch (output_type) { + EL_CASE(f32) + EL_CASE(f16) + EL_CASE(bf16) + EL_CASE(i32) + EL_CASE(i64) + EL_CASE(f64) + default: + THROW_CPU_NODE_ERR("has unsupported output precision: ", output_type); + } + +#undef EL_CASE +} + +void RandomUniform::evalRange() { +#define EL_CASE(E) \ + case element::E: \ + m_range_val.E = m_max_val.E - m_min_val.E; \ + break; + + switch (m_output_prc) { + EL_CASE(f32) + EL_CASE(f16) + EL_CASE(bf16) + EL_CASE(i32) + EL_CASE(i64) + EL_CASE(f64) + default: + THROW_CPU_NODE_ERR("has unsupported output precision: ", m_output_prc); + } + +#undef EL_CASE +} + +std::string RandomUniform::getPrimitiveDescriptorType() const { + auto selectedPrimitiveDesc = getSelectedPrimitiveDescriptor(); + + impl_desc_type type = impl_desc_type::undef; + if (selectedPrimitiveDesc) { + type = selectedPrimitiveDesc->getImplementationType(); + } + + std::string str_type; + + auto add_type = [&](std::string t) { + if (!str_type.empty() && t.c_str()[0] != '_') + str_type += "_"; + str_type += t; + }; + +#define SEARCH_TYPE(_type) \ + if ((type & impl_desc_type::_type) == impl_desc_type::_type) \ + add_type(#_type) + + SEARCH_TYPE(undef); + SEARCH_TYPE(jit); + SEARCH_TYPE(ref); + + SEARCH_TYPE(avx512); + SEARCH_TYPE(avx2); + SEARCH_TYPE(sse42); + SEARCH_TYPE(any); + +#undef SEARCH_TYPE + + if (type == impl_desc_type::unknown) + str_type = "unknown"; + else if (str_type.empty()) + str_type = "undef"; + + if (selectedPrimitiveDesc) { + if (selectedPrimitiveDesc->getConfig().outConfs[0].getMemDesc()->getPrecision() != InferenceEngine::Precision::U8) { + str_type += "_" + std::string(selectedPrimitiveDesc->getConfig().outConfs[0].getMemDesc()->getPrecision().name()); + } else { + str_type += "_I8"; + } + } + + return str_type; +} + +bool RandomUniform::needShapeInfer() const { + return !m_const_inputs[SHAPE]; +} + +bool RandomUniform::isExecutable() const { + return !isInputTensorAtPortEmpty(SHAPE); +} + +bool RandomUniform::created() const { + return getType() == Type::RandomUniform; +} + +} // namespace node +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/random_uniform.hpp b/src/plugins/intel_cpu/src/nodes/random_uniform.hpp new file mode 100644 index 00000000000..ecbfebdf5d7 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/random_uniform.hpp @@ -0,0 +1,120 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include "kernels/x64/random_uniform.hpp" + +namespace ov { +namespace intel_cpu { +namespace node { + +class RandomUniform : public Node { +public: + union OutputType { + float f32; + float16 f16; + bfloat16 bf16; + double f64; + int32_t i32; + uint32_t u32; + uint16_t u16; + int64_t i64; + }; + + RandomUniform(const std::shared_ptr& op, const GraphContext::CPtr& context); + + void getSupportedDescriptors() override; + + void initSupportedPrimitiveDescriptors() override; + + bool needPrepareParams() const override; + + void prepareParams() override; + + void execute(dnnl::stream strm) override; + + void executeDynamicImpl(dnnl::stream strm) override; + + bool isExecutable() const override; + + void createPrimitive() override; + + bool created() const override; + + bool canBeInPlace() const override { return false; } + + static bool isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept; + + std::string getPrimitiveDescriptorType() const override; + +protected: + bool needShapeInfer() const override; + +private: + void computeStl(void* out, size_t work_amount); + + std::pair computePhilox(void* out, size_t work_amount, const std::pair& prev_state); + + template + void generateData(DISTR_TYPE distribution, void* out, size_t work_amount); + + void initOutShape(VectorDims& dst, const void* src, const element::Type& shape_type, size_t len); + + void initEdgeValues(OutputType& dst, const void* src, const element::Type& output_type); + + void evalRange(); + + enum { SHAPE = 0, MIN_VAL, MAX_VAL }; + enum AlgoType { STL, PHILOX }; + + bool m_const_inputs[3] = {false, false, false}; + + ov::element::Type m_output_prc; + uint64_t m_global_seed = 0lu; + uint64_t m_op_seed = 0lu; + std::pair m_state {0lu, 0lu}; + + VectorDims m_out_shape = {}; + uint64_t m_out_el_num = 1lu; + OutputType m_min_val; + OutputType m_max_val; + OutputType m_range_val; + AlgoType m_algo = PHILOX; + + std::default_random_engine m_generator; + + struct ThreadParams { + uint64_t work_amount = 0lu; + uint64_t dst_shift = 0lu; + uint64_t n_shift = 0lu; + uint64_t step = 0lu; + }; + + uint64_t m_threads_num = 0lu; + std::vector m_thread_params; + + ///// PHILOX constants ///// + + // Determines how many sequence elements of RNG sequence are skipped between runs. + // Can be any positive value, 256 is chosen for parity with Tensorflow. + static constexpr uint64_t SKIP_CONST = 256lu; + + // Philox algorithm returns 4 elements of RNG sequence per each invocation + static constexpr uint64_t PHILOX_GROUP_SIZE = 4lu; + + // Output elements number threshold to execute on one thread. + static constexpr uint64_t PHILOX_PARALLEL_EXECUTION_THRESHOLD = 1000lu; + + uint64_t m_skip_count = 0lu; + ///////////////////////////////////////////////////////////////////////////////// + + std::shared_ptr m_jit_kernel; +}; + +} // namespace node +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/reference.cpp b/src/plugins/intel_cpu/src/nodes/reference.cpp index b42dc99b390..091e3181312 100644 --- a/src/plugins/intel_cpu/src/nodes/reference.cpp +++ b/src/plugins/intel_cpu/src/nodes/reference.cpp @@ -2,18 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 // -#include - -#include -#include - -#include "common/blocked_desc_creator.h" -#include "common/cpu_memcpy.h" -#include "openvino/core/shape_util.hpp" -#include "openvino/runtime/tensor.hpp" #include "reference.h" -using namespace dnnl; +#include "common/cpu_memcpy.h" +#include +#include "openvino/core/shape_util.hpp" + using namespace InferenceEngine; using namespace InferenceEngine::details; @@ -21,21 +15,15 @@ namespace ov { namespace intel_cpu { namespace node { -Reference::Reference(const std::shared_ptr& op, const GraphContext::CPtr context, +Reference::Reference(const std::shared_ptr& op, const GraphContext::CPtr& context, const std::string& errorMessage) : - Node(op, context, NgraphShapeInferFactory(op, FULL_PORT_MASK)), ngraphOp(op), additionalErrorMessage(errorMessage) { + Node(op, context, NgraphShapeInferFactory(op, FULL_PORT_MASK)), ovCoreNode(op), additionalErrorMessage(errorMessage) { if (!op->has_evaluate()) { IE_THROW(NotImplemented) << "Cannot fallback on ngraph reference implementation (Ngraph::Node::evaluate() is not implemented)"; } + setType(Type::Reference); setTypeStr("Reference"); - - // RandomUniform should generate new sequence each run even if all inputs are constants. So that method Node::IsConstant() - // doesn't return 'True' for RandomUniform with all constant inputs and the node generates new values for each inference, - // we set 'NoConst' value for 'ConstantType' in ctor - if (ov::is_type(ngraphOp)) { - constant = ConstantType::NoConst; - } } void Reference::getSupportedDescriptors() {} @@ -47,13 +35,13 @@ void Reference::initSupportedPrimitiveDescriptors() { std::vector inputConfigurators; inputConfigurators.reserve(inputShapes.size()); for (size_t i = 0; i < inputShapes.size(); i++) { - inputConfigurators.emplace_back(LayoutType::ncsp, convertPrecision(ngraphOp->get_input_element_type(i)), inputShapes[i]); + inputConfigurators.emplace_back(LayoutType::ncsp, convertPrecision(ovCoreNode->get_input_element_type(i)), inputShapes[i]); } std::vector outputConfigurators; outputConfigurators.reserve(inputShapes.size()); for (size_t i = 0; i < outputShapes.size(); i++) { - outputConfigurators.emplace_back(LayoutType::ncsp, convertPrecision(ngraphOp->get_output_element_type(i)), outputShapes[i]); + outputConfigurators.emplace_back(LayoutType::ncsp, convertPrecision(ovCoreNode->get_output_element_type(i)), outputShapes[i]); } addSupportedPrimDesc(inputConfigurators, outputConfigurators, impl_desc_type::ref); @@ -64,8 +52,8 @@ void Reference::createPrimitive() {} void Reference::execute(dnnl::stream strm) { auto inputs = prepareInputs(); auto outputs = prepareOutputs(); - if (!ngraphOp->evaluate(outputs, inputs)) { - IE_THROW() << "Evaluation failed on node of type: " << std::string(ngraphOp->get_type_name()) << " name: " << getName(); + if (!ovCoreNode->evaluate(outputs, inputs)) { + THROW_CPU_NODE_ERR("evaluation failed for core operation: ", std::string(ovCoreNode->get_type_name())); } } @@ -81,18 +69,16 @@ void Reference::executeDynamicImpl(dnnl::stream strm) { for (size_t i = 0; i < outputShapes.size(); ++i) { auto mem_desc = getBaseMemDescAtOutputPort(i); if (mem_desc->isDefined()) { - outputs.emplace_back(ngraphOp->get_output_element_type(i), mem_desc->getShape().getStaticDims()); + outputs.emplace_back(ovCoreNode->get_output_element_type(i), mem_desc->getShape().getStaticDims()); } else { - outputs.emplace_back(ngraphOp->get_output_element_type(i), ov::util::make_dynamic_shape()); + outputs.emplace_back(ovCoreNode->get_output_element_type(i), ov::util::make_dynamic_shape()); } } } else { - IE_THROW(Unexpected) << - "Unexpected shape infer result status during the inference of a node with type " << - getTypeStr() << " and name " << getName(); + THROW_CPU_NODE_ERR("got unexpected shape infer result status during the inference."); } - if (!ngraphOp->evaluate(outputs, inputs)) { - IE_THROW() << "Evaluation failed on node of type: " << std::string(ngraphOp->get_type_name()) << " name: " << getName(); + if (!ovCoreNode->evaluate(outputs, inputs)) { + THROW_CPU_NODE_ERR("evaluation failed for core operation: ", std::string(ovCoreNode->get_type_name())); } if (ShapeInferStatus::skip == result.status) { std::vector newOutputDims; @@ -105,8 +91,7 @@ void Reference::executeDynamicImpl(dnnl::stream strm) { auto memory = getChildEdgesAtPort(i)[0]->getMemoryPtr(); auto& tensor = outputs[i]; if (memory->getSize() != tensor.get_byte_size()) { - IE_THROW(Unexpected) << "Output tensor data size mismatch occurred during the inference of a node with type " << - getTypeStr() << " and name " << getName() << " on output port number " << i; + THROW_CPU_NODE_ERR("output tensor data size mismatch occurred during the inference on output port number ", i); } cpu_memcpy(memory->getData(), tensor.data(), tensor.get_byte_size()); } @@ -125,9 +110,9 @@ ov::TensorVector Reference::prepareInputs() const { ov::TensorVector inputs; for (size_t i = 0; i < inputShapes.size(); i++) { void *srcDataPtr = getParentEdgesAtPort(i)[0]->getMemory().getData(); - ov::Shape shape = ngraphOp->get_input_partial_shape(i).rank().get_length() == 0 ? + ov::Shape shape = ovCoreNode->get_input_partial_shape(i).rank().get_length() == 0 ? ov::Shape{} : getParentEdgesAtPort(i)[0]->getMemory().getStaticDims(); - inputs.push_back(ov::Tensor(ngraphOp->get_input_element_type(i), shape, srcDataPtr)); + inputs.push_back(ov::Tensor(ovCoreNode->get_input_element_type(i), shape, srcDataPtr)); } return inputs; } @@ -136,9 +121,9 @@ ov::TensorVector Reference::prepareOutputs() const { ov::TensorVector outputs; for (size_t i = 0; i < outputShapes.size(); i++) { void *dstDataPtr = getChildEdgesAtPort(i)[0]->getMemory().getData(); - ov::Shape shape = ngraphOp->get_output_partial_shape(i).rank().get_length() == 0 ? + ov::Shape shape = ovCoreNode->get_output_partial_shape(i).rank().get_length() == 0 ? ov::Shape{} : getChildEdgesAtPort(i)[0]->getMemory().getStaticDims(); - outputs.push_back(ov::Tensor(ngraphOp->get_output_element_type(i), shape, dstDataPtr)); + outputs.push_back(ov::Tensor(ovCoreNode->get_output_element_type(i), shape, dstDataPtr)); } return outputs; } diff --git a/src/plugins/intel_cpu/src/nodes/reference.h b/src/plugins/intel_cpu/src/nodes/reference.h index 4c2a8a13108..c2453835229 100644 --- a/src/plugins/intel_cpu/src/nodes/reference.h +++ b/src/plugins/intel_cpu/src/nodes/reference.h @@ -12,7 +12,7 @@ namespace node { class Reference : public Node { public: - Reference(const std::shared_ptr& op, const GraphContext::CPtr context, const std::string& errorMessage); + Reference(const std::shared_ptr& op, const GraphContext::CPtr& context, const std::string& errorMessage); void getSupportedDescriptors() override; void initSupportedPrimitiveDescriptors() override; @@ -29,7 +29,7 @@ private: ov::TensorVector prepareOutputs() const; private: - const std::shared_ptr ngraphOp; + const std::shared_ptr ovCoreNode; const std::string additionalErrorMessage; }; diff --git a/src/plugins/intel_cpu/src/nodes_factory.cpp b/src/plugins/intel_cpu/src/nodes_factory.cpp index 3afe8aaa32c..7add05741f0 100644 --- a/src/plugins/intel_cpu/src/nodes_factory.cpp +++ b/src/plugins/intel_cpu/src/nodes_factory.cpp @@ -79,6 +79,7 @@ #include "nodes/experimental_detectron_generate_proposals_single_image.h" #include "nodes/generate_proposals.h" #include "nodes/embedding_bag_packed_sum.h" +#include "nodes/random_uniform.hpp" #include "nodes/reduce.h" #include "nodes/if.h" #include "nodes/ctc_greedy_decoder.h" @@ -180,6 +181,7 @@ Node::NodesFactory::NodesFactory() INTEL_CPU_NODE(Unique, Type::Unique); INTEL_CPU_NODE(Ngram, Type::Ngram); INTEL_CPU_NODE(Interpolate, Type::Interpolate); + INTEL_CPU_NODE(RandomUniform, Type::RandomUniform); INTEL_CPU_NODE(Reduce, Type::Reduce); INTEL_CPU_NODE(Gather, Type::Gather); INTEL_CPU_NODE(NonMaxSuppression, Type::NonMaxSuppression); diff --git a/src/plugins/intel_cpu/src/shape_inference/custom/random_uniform.cpp b/src/plugins/intel_cpu/src/shape_inference/custom/random_uniform.cpp new file mode 100644 index 00000000000..cca3c74cce8 --- /dev/null +++ b/src/plugins/intel_cpu/src/shape_inference/custom/random_uniform.cpp @@ -0,0 +1,47 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "random_uniform.hpp" +#include + +namespace ov { +namespace intel_cpu { +namespace node { + +// TODO: remove after fixing the issue 123011 +IShapeInfer::Result RandomUniformShapeInfer::infer( + const std::vector>& input_shapes, + const std::unordered_map& data_dependency) { + VectorDims dims; + const auto& mem = data_dependency.at(0); + const auto rank = mem->getShape().getElementsCount(); + auto shape_prc = mem->getDesc().getPrecision(); + switch (shape_prc) { + case InferenceEngine::Precision::I32: { + auto data = reinterpret_cast(mem->getData()); + dims.assign(data, data + rank); + } break; + case InferenceEngine::Precision::I64: { + auto data = reinterpret_cast(mem->getData()); + dims.assign(data, data + rank); + } break; + default: + OPENVINO_THROW("Unexpected Shape input precision: ", shape_prc); + } + + return {{dims}, ShapeInferStatus::success}; +} + +RandomUniformShapeInferFactory::RandomUniformShapeInferFactory(const std::shared_ptr& op) : m_op(op) { + OPENVINO_ASSERT(ov::is_type(m_op), + "Unexpected op type in RandomUniform shape inference factory: ", m_op->get_type_name()); +} + +ShapeInferPtr RandomUniformShapeInferFactory::makeShapeInfer() const { + return std::make_shared(); +} + +} // namespace node +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/shape_inference/custom/random_uniform.hpp b/src/plugins/intel_cpu/src/shape_inference/custom/random_uniform.hpp new file mode 100644 index 00000000000..ce87a966a9c --- /dev/null +++ b/src/plugins/intel_cpu/src/shape_inference/custom/random_uniform.hpp @@ -0,0 +1,37 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "shape_inference/shape_inference_cpu.hpp" +#include + +#pragma once + +namespace ov { +namespace intel_cpu { +namespace node { + +class RandomUniformShapeInfer : public ShapeInferEmptyPads { +public: + explicit RandomUniformShapeInfer() {} + IShapeInfer::Result infer( + const std::vector>& input_shapes, + const std::unordered_map& data_dependency) override; + + port_mask_t get_port_mask() const override { + return PortMask(0); + } +}; + +class RandomUniformShapeInferFactory : public ShapeInferFactory { +public: + explicit RandomUniformShapeInferFactory(const std::shared_ptr& op); + ShapeInferPtr makeShapeInfer() const override; + +private: + std::shared_ptr m_op; +}; + +} // namespace node +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp index 9faf421c26a..4eb40365fa9 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp @@ -189,6 +189,8 @@ std::vector disabledTestPatterns() { R"(smoke_LSTMSequenceCommon.*LSTMSequenceTest.Inference.*CONVERT_TO_TI.*)", // Issue: 122094 R"(smoke_Interpolate_Basic_Down_Sample_Tail/InterpolateLayerTest.Inference.*(asymmetric|align_corners).*f16.*)", + // Need to generate sequence exactly in the i64 data type. Enable in scope of i64 enabling. + R"(.*RandomUniformLayerTestCPU.*OutPrc=i64.*)", }; #if defined(OPENVINO_ARCH_X86) diff --git a/src/plugins/intel_cpu/tests/functional/single_layer_tests/classes/random_uniform.cpp b/src/plugins/intel_cpu/tests/functional/single_layer_tests/classes/random_uniform.cpp new file mode 100644 index 00000000000..2f9706e7d25 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/single_layer_tests/classes/random_uniform.cpp @@ -0,0 +1,265 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "random_uniform.hpp" +#include "ov_models/builders.hpp" + +using namespace CPUTestUtils; +using namespace ov::test; + +namespace CPULayerTestsDefinitions { + +std::string RandomUniformLayerTestCPU::getTestCaseName(const testing::TestParamInfo& obj) { + const auto& out_shape = std::get<0>(obj.param); + const auto& min_max = std::get<1>(obj.param); + + std::ostringstream result; + + result << "IS={" << out_shape.size(); + result << "}_OS=" << out_shape; + result << "_Min=" << std::get<0>(min_max); + result << "_Max=" << std::get<1>(min_max); + result << "_ShapePrc=" << std::get<2>(obj.param); + result << "_OutPrc=" << std::get<3>(obj.param); + result << "_GlobalSeed=" << std::get<4>(obj.param); + result << "_OperationalSeed=" << std::get<5>(obj.param); + result << "_ConstIn={" << utils::bool2str(std::get<6>(obj.param)) << "," + << utils::bool2str(std::get<7>(obj.param)) << "," + << utils::bool2str(std::get<8>(obj.param)) << "}"; + + result << CPUTestsBase::getTestCaseName(std::get<9>(obj.param)); + + const auto& config = std::get<10>(obj.param); + if (!config.empty()) { + result << "_PluginConf={"; + for (const auto& conf_item : config) { + result << "_" << conf_item.first << "="; + conf_item.second.print(result); + } + result << "}"; + } + + return result.str(); +} + +void RandomUniformLayerTestCPU::SetUp() { + targetDevice = utils::DEVICE_CPU; + + const auto& params = this->GetParam(); + m_output_shape = std::get<0>(params); + const auto& min_max = std::get<1>(params); + const auto& shape_prc = std::get<2>(params); + const auto& output_prc = std::get<3>(params); + m_global_seed = std::get<4>(params); + m_operational_seed = std::get<5>(params); + const auto& const_in_1 = std::get<6>(params); + const auto& const_in_2 = std::get<7>(params); + const auto& const_in_3 = std::get<8>(params); + const auto& cpu_params = std::get<9>(params); + configuration = std::get<10>(params); + + m_min_val = std::get<0>(min_max); + m_max_val = std::get<1>(min_max); + std::tie(inFmts, outFmts, priority, selectedType) = cpu_params; + +#if defined(OV_CPU_WITH_ACL) + updateSelectedType("ref_any", output_prc, configuration); +#else + if (output_prc == ElementType::i64) { + updateSelectedType(getPrimitiveType(), ElementType::i32, configuration); + } else if (output_prc == ElementType::f64) { + updateSelectedType(getPrimitiveType(), ElementType::f32, configuration); + } else if (output_prc == ElementType::f16) { + if (InferenceEngine::with_cpu_x86_avx512_core_fp16()) { + updateSelectedType(getPrimitiveType(), ElementType::f16, configuration); + } else { + updateSelectedType(getPrimitiveType(), ElementType::f32, configuration); + } + } else if (output_prc == ElementType::bf16) { + if (InferenceEngine::with_cpu_x86_bfloat16()) { + updateSelectedType(getPrimitiveType(), ElementType::bf16, configuration); + } else { + updateSelectedType("ref_any", ElementType::bf16, configuration); + } + } else { + updateSelectedType(getPrimitiveType(), output_prc, configuration); + } +#endif + + std::vector in_shapes; + ov::ParameterVector in_params; + std::vector> inputs; + + if (!const_in_1) { + in_shapes.push_back({{}, {{m_output_shape.size()}}}); + in_params.push_back(std::make_shared(shape_prc, ov::PartialShape{static_cast(m_output_shape.size())})); + in_params.back()->set_friendly_name("shape"); + inputs.push_back(in_params.back()); + } else { + inputs.push_back(ngraph::builder::makeConstant(shape_prc, {m_output_shape.size()}, m_output_shape)); + } + if (!const_in_2) { + in_shapes.push_back({{}, {{1}}}); + in_params.push_back(std::make_shared(output_prc, ov::PartialShape{1})); + in_params.back()->set_friendly_name("minval"); + inputs.push_back(in_params.back()); + } else { + inputs.push_back(ngraph::builder::makeConstant(output_prc, {1}, std::vector{m_min_val})); + } + if (!const_in_3) { + in_shapes.push_back({{}, {{1}}}); + in_params.push_back(std::make_shared(output_prc, ov::PartialShape{1})); + in_params.back()->set_friendly_name("maxval"); + inputs.push_back(in_params.back()); + } else { + inputs.push_back(ngraph::builder::makeConstant(output_prc, {1}, std::vector{m_max_val})); + } + + init_input_shapes(in_shapes); + + const auto rnd_op = std::make_shared(inputs[0], inputs[1], inputs[2], output_prc, m_global_seed, m_operational_seed); + const ov::ResultVector results{std::make_shared(rnd_op)}; + + function = std::make_shared(results, in_params, "RandomUniformLayerTestCPU"); +} + +template +void fill_data(TD* dst, const TS* src, size_t len) { + for (size_t i = 0llu; i < len; i++) { + dst[i] = static_cast(src[i]); + } +} + +void RandomUniformLayerTestCPU::generate_inputs(const std::vector& targetInputStaticShapes) { + inputs.clear(); + const auto& func_inputs = function->inputs(); + + for (size_t i = 0llu; i < func_inputs.size(); ++i) { + const auto& func_input = func_inputs[i]; + const auto& name = func_input.get_node()->get_friendly_name(); + const auto& in_prc = func_input.get_element_type(); + auto tensor = ov::Tensor(in_prc, targetInputStaticShapes[i]); + +#define CASE(P, S, L) \ +case P : \ +fill_data(tensor.data::value_type>(), S, L); break; + + if (name == "shape") { + switch (in_prc) { + CASE(ElementType::i32, m_output_shape.data(), m_output_shape.size()) + CASE(ElementType::i64, m_output_shape.data(), m_output_shape.size()) + default: + OPENVINO_THROW("RandomUniform does not support precision ", in_prc, " for the Shape input."); + } + } else if (name == "minval") { + switch (in_prc) { + CASE(ElementType::f32, &m_min_val, 1) + CASE(ElementType::f16, &m_min_val, 1) + CASE(ElementType::bf16, &m_min_val, 1) + CASE(ElementType::i32, &m_min_val, 1) + CASE(ElementType::i64, &m_min_val, 1) + CASE(ElementType::f64, &m_min_val, 1) + default: + OPENVINO_THROW("RandomUniform does not support precision ", in_prc, " for the Minval input."); + } + } else if (name == "maxval") { + switch (in_prc) { + CASE(ElementType::f32, &m_max_val, 1) + CASE(ElementType::f16, &m_max_val, 1) + CASE(ElementType::bf16, &m_max_val, 1) + CASE(ElementType::i32, &m_max_val, 1) + CASE(ElementType::i64, &m_max_val, 1) + CASE(ElementType::f64, &m_max_val, 1) + default: + OPENVINO_THROW("RandomUniform does not support precision ", in_prc, " for the Maxval input."); + } + } + +#undef CASE + + inputs.insert({func_input.get_node_shared_ptr(), tensor}); + } +} + +void RandomUniformLayerTestCPU::compare(const std::vector& expected, const std::vector& actual) { + if (m_global_seed != 0lu || m_operational_seed != 0lu) { + SubgraphBaseTest::compare(expected, actual); + return; + } + + // When both seed values are equal to zero, RandomUniform should generate non-deterministic sequence. + // In this case will use Mean and Variance metrics. + +#define CASE(X) case X : rndUCompare::value_type>(expected[0], actual[0]); break; + + switch (expected[0].get_element_type()) { + CASE(ElementType::f32) + CASE(ElementType::i32) + CASE(ElementType::f16) + CASE(ElementType::bf16) + CASE(ElementType::i64) + CASE(ElementType::f64) + default: OPENVINO_THROW("Unsupported element type: ", expected[0].get_element_type()); + } + +#undef CASE +} + +precisions_map RandomUniformLayerTestCPU::get_ref_precisions_convert_map() { + precisions_map precisions; + + if (!InferenceEngine::with_cpu_x86_avx512_core()) { + precisions.insert({ ov::element::bf16, ov::element::f32 }); + } + if (!InferenceEngine::with_cpu_x86_avx512_core_fp16()) { + precisions.insert({ ov::element::f16, ov::element::f32 }); + } + + return precisions; +} + +inline double less_or_equal(double a, double b) { + return (b - a) >= (std::fmax(std::fabs(a), std::fabs(b)) * std::numeric_limits::epsilon()); +} + +template +void RandomUniformLayerTestCPU::rndUCompare(const ov::Tensor& expected, const ov::Tensor& actual) { + auto actual_data = actual.data(); + size_t shape_size_cnt = ov::shape_size(expected.get_shape()); + double act_mean = 0.0; + double act_variance = 0.0; + const double exp_mean = (m_max_val + m_min_val) / 2.0; + const double exp_variance = std::pow(m_max_val - m_min_val, 2) / 12.0; + + for (size_t i = 0; i < shape_size_cnt; ++i) { + auto actual_value = static_cast(actual_data[i]); + if (std::isnan(actual_value)) { + std::ostringstream out_stream; + out_stream << "Actual value is NAN on coordinate: " << i; + throw std::runtime_error(out_stream.str()); + } + act_mean += actual_value; + act_variance += std::pow(actual_value - exp_mean, 2); + } + act_mean /= shape_size_cnt; + act_variance /= shape_size_cnt; + + auto rel_mean = (exp_mean - act_mean) / (m_max_val - m_min_val); + auto rel_variance = (exp_variance - act_variance) / std::pow(m_max_val - m_min_val, 2); + + if (!(less_or_equal(rel_mean, m_mean_threshold) && less_or_equal(rel_variance, m_variance_threshold))) { + std::ostringstream out_stream; + out_stream << "rel_mean < m_mean_threshold && rel_variance < m_variance_threshold" << + "\n\t rel_mean: " << rel_mean << + "\n\t rel_variance: " << rel_variance; + throw std::runtime_error(out_stream.str()); + } +} + +TEST_P(RandomUniformLayerTestCPU, CompareWithRefs) { + run(); + CheckPluginRelatedResults(compiledModel, "RandomUniform"); +} + +} // namespace CPULayerTestsDefinitions diff --git a/src/plugins/intel_cpu/tests/functional/single_layer_tests/classes/random_uniform.hpp b/src/plugins/intel_cpu/tests/functional/single_layer_tests/classes/random_uniform.hpp new file mode 100644 index 00000000000..1cb9f5fccc4 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/single_layer_tests/classes/random_uniform.hpp @@ -0,0 +1,53 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "shared_test_classes/base/ov_subgraph.hpp" +#include "test_utils/cpu_test_utils.hpp" + +namespace CPULayerTestsDefinitions { + +typedef std::tuple< + ov::Shape, // Output shapes + std::tuple, // Min and Max values + ov::test::ElementType, // Shape precision + ov::test::ElementType, // Output precision + uint64_t, // Global seed + uint64_t, // Operational seed + bool, // Is 1st input constant + bool, // Is 2nd input constant + bool, // Is 3rd input constant + CPUTestUtils::CPUSpecificParams, // CPU specific params + ov::AnyMap // Additional plugin configuration +> RandomUniformLayerTestCPUParamSet; + +class RandomUniformLayerTestCPU : public testing::WithParamInterface, + public ov::test::SubgraphBaseTest, public CPUTestUtils::CPUTestsBase { +public: + static std::string getTestCaseName(const testing::TestParamInfo& obj); + +protected: + void SetUp() override; + + void generate_inputs(const std::vector& target_shapes) override; + + void compare(const std::vector& expected, const std::vector& actual) override; + + precisions_map get_ref_precisions_convert_map() override; + + template + void rndUCompare(const ov::Tensor& expected, const ov::Tensor& actual); + +private: + ov::Shape m_output_shape; + uint64_t m_global_seed; + uint64_t m_operational_seed; + double m_min_val; + double m_max_val; + static constexpr double m_mean_threshold = 0.05; + static constexpr double m_variance_threshold = 0.1; +}; + +} // namespace CPULayerTestsDefinitions diff --git a/src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/common/random_uniform.cpp b/src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/common/random_uniform.cpp new file mode 100644 index 00000000000..f319fb6ada2 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/common/random_uniform.cpp @@ -0,0 +1,68 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "single_layer_tests/classes/random_uniform.hpp" + +using namespace CPUTestUtils; +using namespace ov::test; + +namespace CPULayerTestsDefinitions { +namespace RandomUniform { + +static const std::vector shape_prc = { + ElementType::i32, + ElementType::i64 +}; + +static const std::vector output_shapes = { + {500}, + {4, 3, 210} +}; + +static const std::vector global_seed = { + 0, 8 +}; + +static const std::vector operational_seed = { + 0, 3, 5 +}; + +static const std::vector> min_max = { + {0, 50}, + {-50, 50}, + {-50, 0} +}; + +INSTANTIATE_TEST_SUITE_P(smoke_Param, RandomUniformLayerTestCPU, + ::testing::Combine( + ::testing::ValuesIn(output_shapes), + ::testing::ValuesIn(min_max), + ::testing::ValuesIn(shape_prc), + ::testing::Values(ElementType::f32, ElementType::i32), + ::testing::ValuesIn(global_seed), + ::testing::ValuesIn(operational_seed), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(false), + ::testing::Values(emptyCPUSpec), + ::testing::Values(empty_plugin_config)), + RandomUniformLayerTestCPU::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_ParamConst, RandomUniformLayerTestCPU, + ::testing::Combine( + ::testing::Values(output_shapes[0]), + ::testing::Values(min_max[0]), + ::testing::Values(ElementType::i32), + ::testing::Values(ElementType::f32), + ::testing::Values(1), + ::testing::Values(0), + ::testing::Values(true, false), + ::testing::Values(true, false), + ::testing::Values(true, false), + ::testing::Values(emptyCPUSpec), + ::testing::Values(empty_plugin_config)), + RandomUniformLayerTestCPU::getTestCaseName); + +} // namespace RandomUniform +} // namespace CPULayerTestsDefinitions diff --git a/src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/x64/random_uniform.cpp b/src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/x64/random_uniform.cpp new file mode 100644 index 00000000000..8fec42f3824 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/x64/random_uniform.cpp @@ -0,0 +1,46 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "single_layer_tests/classes/random_uniform.hpp" + +using namespace CPUTestUtils; +using namespace ov::test; + +namespace CPULayerTestsDefinitions { +namespace RandomUniform { + +static const std::vector output_prc_nightly = { + ElementType::f32, + ElementType::f16, + ElementType::bf16, + ElementType::i32, + ElementType::i64 +}; + +// Need to validate the Kernel corner cases. +static const std::vector output_shapes_nightly = { + {1}, {2}, {3}, {2, 2}, {5}, {2, 3}, {7}, {2, 2, 2}, {3, 3}, {2, 5}, {11}, {2, 3, 2}, {13}, {2, 7}, {3, 5}, + {4, 4}, {1, 17}, {2, 9}, {19}, {4, 5}, {21}, {11, 2}, {23, 1}, {4, 2, 3}, {5, 5}, {26}, {1, 27}, {14, 2}, + {29}, {10, 3}, {31}, {2, 8, 2}, {33}, {17, 2}, {5, 7}, {2, 3, 2, 3}, {37}, {2, 19}, {2, 20}, {41}, {42}, + {43}, {22, 2}, {3, 5, 3}, {5, 2, 5}, {1, 3, 1, 17, 1}, {26, 2}, {53}, {54}, {55}, {56}, {57}, {58}, {59}, + {2, 32}, {99}, {127}, {128}, {129}, {199}, {255}, {499}, {997}, {1753}, {2899} +}; + +INSTANTIATE_TEST_SUITE_P(nightly_Param, RandomUniformLayerTestCPU, + ::testing::Combine( + ::testing::ValuesIn(output_shapes_nightly), + ::testing::Values(std::tuple{-31, 17}), + ::testing::Values(ElementType::i32), + ::testing::ValuesIn(output_prc_nightly), + ::testing::Values(3), + ::testing::Values(1), + ::testing::Values(true, false), + ::testing::Values(true, false), + ::testing::Values(true, false), + ::testing::Values(emptyCPUSpec), + ::testing::Values(empty_plugin_config)), + RandomUniformLayerTestCPU::getTestCaseName); + +} // namespace RandomUniform +} // namespace CPULayerTestsDefinitions diff --git a/src/plugins/intel_cpu/tests/functional/test_utils/cpu_test_utils.hpp b/src/plugins/intel_cpu/tests/functional/test_utils/cpu_test_utils.hpp index d8deddfebe5..fff65f9e1c4 100644 --- a/src/plugins/intel_cpu/tests/functional/test_utils/cpu_test_utils.hpp +++ b/src/plugins/intel_cpu/tests/functional/test_utils/cpu_test_utils.hpp @@ -170,6 +170,7 @@ protected: // common parameters const auto emptyCPUSpec = CPUSpecificParams{{}, {}, {}, {}}; const std::map cpuEmptyPluginConfig; +const ov::AnyMap empty_plugin_config{}; const std::map cpuFP32PluginConfig = { { InferenceEngine::PluginConfigParams::KEY_ENFORCE_BF16, InferenceEngine::PluginConfigParams::NO } }; const std::map cpuBF16PluginConfig = diff --git a/src/tests/test_utils/common_test_utils/include/common_test_utils/common_utils.hpp b/src/tests/test_utils/common_test_utils/include/common_test_utils/common_utils.hpp index cc45a47d779..0bd9f4845d4 100644 --- a/src/tests/test_utils/common_test_utils/include/common_test_utils/common_utils.hpp +++ b/src/tests/test_utils/common_test_utils/include/common_test_utils/common_utils.hpp @@ -115,6 +115,10 @@ inline std::string set2str(const std::set& set) { return std::string("()"); } +inline std::string bool2str(const bool val) { + return val ? "True" : "False"; +} + template std::vector> combineParams(const std::map>& keyValueSets) { std::vector> resVec;