diff --git a/src/plugins/intel_cpu/src/cpu_types.cpp b/src/plugins/intel_cpu/src/cpu_types.cpp index e3a4c21142b..3a6335d0648 100644 --- a/src/plugins/intel_cpu/src/cpu_types.cpp +++ b/src/plugins/intel_cpu/src/cpu_types.cpp @@ -187,7 +187,7 @@ static const TypeToNameMap& get_type_to_name_tbl() { {"Ceiling", Type::Math}, {"Cos", Type::Math}, {"Cosh", Type::Math}, - {"Floor", Type::Math}, + {"Floor", Type::Eltwise}, {"HardSigmoid", Type::Math}, {"If", Type::If}, {"Neg", Type::Math}, @@ -394,6 +394,7 @@ std::string algToString(const Algorithm alg) { CASE(EltwiseMultiply); CASE(EltwiseSubtract); CASE(EltwiseDivide); + CASE(EltwiseFloor); CASE(EltwiseFloorMod); CASE(EltwiseMod); CASE(EltwiseMaximum); diff --git a/src/plugins/intel_cpu/src/cpu_types.h b/src/plugins/intel_cpu/src/cpu_types.h index 5593e6728bb..24704d7f7b9 100644 --- a/src/plugins/intel_cpu/src/cpu_types.h +++ b/src/plugins/intel_cpu/src/cpu_types.h @@ -153,6 +153,7 @@ enum class Algorithm { EltwiseMultiply, EltwiseSubtract, EltwiseDivide, + EltwiseFloor, EltwiseFloorMod, EltwiseMod, EltwiseMaximum, diff --git a/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.cpp b/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.cpp index dbe951a4dd3..1e35fe009cc 100644 --- a/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.cpp +++ b/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.cpp @@ -479,6 +479,42 @@ std::set> jit_exp_emitter::get_supported_precisions(c return {{element::f32}}; } +/// Floor /// +jit_floor_emitter::jit_floor_emitter(dnnl::impl::cpu::aarch64::jit_generator* host, + dnnl::impl::cpu::aarch64::cpu_isa_t host_isa, + const std::shared_ptr& node) + : jit_emitter(host, host_isa, node, get_arithmetic_binary_exec_precision(node)) { +} + +jit_floor_emitter::jit_floor_emitter(dnnl::impl::cpu::aarch64::jit_generator* host, + dnnl::impl::cpu::aarch64::cpu_isa_t host_isa, + const ov::element::Type exec_prc) : jit_emitter(host, host_isa, exec_prc) { +} + +size_t jit_floor_emitter::get_inputs_count() const { return 1; } + +void jit_floor_emitter::emit_impl(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const { + if (host_isa_ == dnnl::impl::cpu::aarch64::asimd) { + emit_isa(in_vec_idxs, out_vec_idxs); + } else { + OV_CPU_JIT_EMITTER_THROW("Can't create jit eltwise kernel"); + } +} + +template +void jit_floor_emitter::emit_isa(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const { + OV_CPU_JIT_EMITTER_ASSERT(exec_prc_ == ov::element::f32, "unsupported precision: " + exec_prc_.to_string()); + + using TReg = typename dnnl::impl::cpu::aarch64::cpu_isa_traits::TReg; + TReg src = TReg(in_vec_idxs[0]); + TReg dst = TReg(out_vec_idxs[0]); + h->frintm(dst.s, src.s); +} + +std::set> jit_floor_emitter::get_supported_precisions(const std::shared_ptr& node) { + return {{element::f32}}; +} + /// GELU_ERF /// jit_gelu_erf_emitter::jit_gelu_erf_emitter(dnnl::impl::cpu::aarch64::jit_generator* host, dnnl::impl::cpu::aarch64::cpu_isa_t host_isa, diff --git a/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.hpp b/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.hpp index 1b46d5c2b76..24fd1dde37e 100644 --- a/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.hpp +++ b/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.hpp @@ -193,6 +193,27 @@ private: void emit_isa(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const; }; +class jit_floor_emitter : public jit_emitter { +public: + jit_floor_emitter(dnnl::impl::cpu::aarch64::jit_generator *host, + dnnl::impl::cpu::aarch64::cpu_isa_t host_isa, + const ov::element::Type exec_prc = ov::element::f32); + + jit_floor_emitter(dnnl::impl::cpu::aarch64::jit_generator *host, + dnnl::impl::cpu::aarch64::cpu_isa_t host_isa, + const std::shared_ptr& node); + + size_t get_inputs_count() const override; + + static std::set> get_supported_precisions(const std::shared_ptr& node = nullptr); + +private: + void emit_impl(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const override; + + template + void emit_isa(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const; +}; + class jit_gelu_erf_emitter : public jit_emitter { public: jit_gelu_erf_emitter(dnnl::impl::cpu::aarch64::jit_generator* host, diff --git a/src/plugins/intel_cpu/src/nodes/eltwise.cpp b/src/plugins/intel_cpu/src/nodes/eltwise.cpp index 0e7a75f8925..7005cc88515 100644 --- a/src/plugins/intel_cpu/src/nodes/eltwise.cpp +++ b/src/plugins/intel_cpu/src/nodes/eltwise.cpp @@ -255,6 +255,7 @@ std::set> eltwise_precision_helper::get_supported_pre OV_CASE(Algorithm::EltwiseSubtract, jit_subtract_emitter), OV_CASE(Algorithm::EltwiseMultiply, jit_multiply_emitter), OV_CASE(Algorithm::EltwiseDivide, jit_divide_emitter), + OV_CASE(Algorithm::EltwiseFloor, jit_floor_emitter), OV_CASE(Algorithm::EltwiseFloorMod, jit_floor_mod_emitter), OV_CASE(Algorithm::EltwiseMod, jit_mod_emitter), OV_CASE(Algorithm::EltwiseMaximum, jit_maximum_emitter), @@ -633,6 +634,7 @@ private: OV_CASE(Algorithm::EltwiseSubtract, jit_subtract_emitter), OV_CASE(Algorithm::EltwiseMultiply, jit_multiply_emitter), OV_CASE(Algorithm::EltwiseDivide, jit_divide_emitter), + OV_CASE(Algorithm::EltwiseFloor, jit_floor_emitter), OV_CASE(Algorithm::EltwiseFloorMod, jit_floor_mod_emitter), OV_CASE(Algorithm::EltwiseMod, jit_mod_emitter), OV_CASE(Algorithm::EltwiseMaximum, jit_maximum_emitter), @@ -1082,6 +1084,9 @@ const std::map& Eltwise::getIn {ov::op::v1::Mod::get_type_info_static(), [](const std::shared_ptr& op, Eltwise& node) { node.algorithm = Algorithm::EltwiseMod; }}, + {ov::op::v0::Floor::get_type_info_static(), [](const std::shared_ptr& op, Eltwise& node) { + node.algorithm = Algorithm::EltwiseFloor; + }}, {ov::op::v1::FloorMod::get_type_info_static(), [](const std::shared_ptr& op, Eltwise& node) { node.algorithm = Algorithm::EltwiseFloorMod; }}, @@ -1875,6 +1880,7 @@ public: case Algorithm::EltwiseSubtract: *dst_ptr_f = src_f[0] - src_f[1]; break; case Algorithm::EltwiseMultiply: *dst_ptr_f = src_f[0] * src_f[1]; break; case Algorithm::EltwiseDivide: *dst_ptr_f = src_f[0] / src_f[1]; break; + case Algorithm::EltwiseFloor: *dst_ptr_f = floorf(src_f[0]); break; case Algorithm::EltwiseFloorMod: *dst_ptr_f = src_f[0] - floorf(src_f[0] / src_f[1]) * src_f[1]; break; case Algorithm::EltwiseMod: *dst_ptr_f = src_f[0] - truncf(src_f[0] / src_f[1]) * src_f[1]; break; case Algorithm::EltwiseMaximum: *dst_ptr_f = std::max(src_f[0], src_f[1]); break; @@ -2072,6 +2078,7 @@ size_t Eltwise::getOpInputsNum() const { case Algorithm::EltwiseRelu: case Algorithm::EltwiseGeluErf: case Algorithm::EltwiseGeluTanh: + case Algorithm::EltwiseFloor: case Algorithm::EltwiseElu: case Algorithm::EltwiseTanh: case Algorithm::EltwiseSigmoid: diff --git a/src/plugins/intel_cpu/src/nodes/executors/aarch64/jit_eltwise.cpp b/src/plugins/intel_cpu/src/nodes/executors/aarch64/jit_eltwise.cpp index 72458c3d292..ac3674c9309 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/aarch64/jit_eltwise.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/aarch64/jit_eltwise.cpp @@ -25,6 +25,7 @@ bool JitEltwiseExecutor::isSupported( Algorithm::EltwiseElu, Algorithm::EltwiseEqual, Algorithm::EltwiseExp, + Algorithm::EltwiseFloor, Algorithm::EltwiseGeluErf, Algorithm::EltwiseGeluTanh, Algorithm::EltwiseHswish, @@ -74,9 +75,8 @@ bool JitEltwiseExecutor::isSupported( }; const std::set supported_precisions = - (algorithm == Algorithm::EltwiseDivide) ? - // Divide operation doesn't support int32 tensor inference in fp32 precision. - // As result Divide operation supports fp16 and fp32 only. + // Divide and Floor (issue #138629) operations are supported for fp32 and fp16 only. + ((algorithm == Algorithm::EltwiseDivide) || (algorithm == Algorithm::EltwiseFloor)) ? std::set { ov::element::f16, ov::element::f32 } : std::set { ov::element::f16, diff --git a/src/plugins/intel_cpu/src/nodes/kernels/aarch64/jit_uni_eltwise_generic.cpp b/src/plugins/intel_cpu/src/nodes/kernels/aarch64/jit_uni_eltwise_generic.cpp index 8093f0617eb..10ae54d1fb7 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/aarch64/jit_uni_eltwise_generic.cpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/aarch64/jit_uni_eltwise_generic.cpp @@ -636,6 +636,7 @@ std::shared_ptr jit_uni_eltwise_generic::create_eltwise_emitte OV_CASE(Algorithm::EltwiseElu, ov::intel_cpu::aarch64::jit_elu_emitter), OV_CASE(Algorithm::EltwiseEqual, ov::intel_cpu::aarch64::jit_equal_emitter), OV_CASE(Algorithm::EltwiseExp, ov::intel_cpu::aarch64::jit_exp_emitter), + OV_CASE(Algorithm::EltwiseFloor, ov::intel_cpu::aarch64::jit_floor_emitter), OV_CASE(Algorithm::EltwiseHswish, ov::intel_cpu::aarch64::jit_hswish_emitter), OV_CASE(Algorithm::EltwiseMaximum, ov::intel_cpu::aarch64::jit_maximum_emitter), OV_CASE(Algorithm::EltwiseMinimum, ov::intel_cpu::aarch64::jit_minimum_emitter), @@ -805,6 +806,7 @@ std::set> eltwise_precision_helper::get_supported_pre OV_CASE(Algorithm::EltwiseElu, jit_elu_emitter), OV_CASE(Algorithm::EltwiseEqual, jit_equal_emitter), OV_CASE(Algorithm::EltwiseExp, jit_exp_emitter), + OV_CASE(Algorithm::EltwiseFloor, jit_floor_emitter), OV_CASE(Algorithm::EltwiseGeluErf, jit_gelu_erf_emitter), OV_CASE(Algorithm::EltwiseGeluTanh, jit_gelu_tanh_emitter), OV_CASE(Algorithm::EltwiseHswish, jit_hswish_emitter), diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/activation.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/activation.cpp index f5034d71a8e..47f747afe60 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/activation.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/activation.cpp @@ -159,6 +159,7 @@ std::string ActivationLayerCPUTest::getPrimitiveType(const utils::ActivationType ((activation_type == utils::ActivationTypes::Clamp) || (activation_type == utils::ActivationTypes::Elu) || (activation_type == utils::ActivationTypes::Exp) || + (activation_type == utils::ActivationTypes::Floor) || (activation_type == utils::ActivationTypes::HSwish) || (activation_type == utils::ActivationTypes::HardSigmoid) || (activation_type == utils::ActivationTypes::Mish) || @@ -176,6 +177,9 @@ std::string ActivationLayerCPUTest::getPrimitiveType(const utils::ActivationType return ""; } #endif + if (activation_type == utils::ActivationTypes::Floor) { + return "ref"; + } return "acl"; #else return CPUTestsBase::getPrimitiveType(); @@ -202,6 +206,7 @@ const std::map>>& activat {Exp, {{}}}, {Clamp, {{-2.0f, 2.0f}}}, {Elu, {{0.1f}}}, + {Floor, {{}}}, {Swish, {{0.1f}}}, {HSwish, {{}}}, {PReLu, {{-0.01f}}}, diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp index 7164fdb2cc7..0d55c708f94 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp @@ -317,6 +317,9 @@ std::vector disabledTestPatterns() { R"(.*smoke_LoopForCommon/LoopLayerCPUTest.CompareWithRefs/Input0_IS=\[1..10.1.1..10\]_.*_Input1_IS=\[1..8.1.1..8\]_.*_Input2_IS=\[1..10.\?.1..10\]_TS=.*_types=0_0_1_trip_count_type=.*_trip_count=(1|5)_exec_cond=1_netType=i8.*)", }; + // fp32 floor for bf16 models: conversion issue + retVector.emplace_back(R"(.*smoke.*ActivationLayerCPUTest.*CompareWithRefs/Floor_.*netPRC=bf16.*)"); + #if defined(OPENVINO_ARCH_X86) retVector.emplace_back(R"(.*DetectionOutputLayerTest.*)"); // WIP: plugin cannot be loaded for some reason