diff --git a/src/plugins/intel_cpu/src/nodes/fake_quantize.cpp b/src/plugins/intel_cpu/src/nodes/fake_quantize.cpp index b0ec6f70a41..06203699ba2 100644 --- a/src/plugins/intel_cpu/src/nodes/fake_quantize.cpp +++ b/src/plugins/intel_cpu/src/nodes/fake_quantize.cpp @@ -23,6 +23,7 @@ #include "memory_desc/dnnl_blocked_memory_desc.h" #include "common/cpu_memcpy.h" #include +#include #include #include "utils/ngraph_utils.hpp" @@ -295,6 +296,45 @@ private: bool do_rounding = true; bool do_dequantization = true; + inline void load_broadcasted_vectors_only(size_t idx) { + const auto &broadcasted = jqp_.broadcasted; + if (broadcasted[static_cast(FQ_add_input_type::CROP_LOW)]) uni_vbroadcastss(vmm_crop_low(idx), ptr[reg_crop_low]); + if (broadcasted[static_cast(FQ_add_input_type::CROP_HIGH)]) uni_vbroadcastss(vmm_crop_high(idx), ptr[reg_crop_high]); + if (broadcasted[static_cast(FQ_add_input_type::INPUT_SCALE)]) uni_vbroadcastss(vmm_input_scale(idx), ptr[reg_input_scale]); + if (broadcasted[static_cast(FQ_add_input_type::INPUT_SHIFT)]) uni_vbroadcastss(vmm_input_shift(idx), ptr[reg_input_shift]); + if (do_dequantization) { + if (broadcasted[static_cast(FQ_add_input_type::OUTPUT_SCALE)]) uni_vbroadcastss(vmm_output_scale(idx), ptr[reg_output_scale]); + if (broadcasted[static_cast(FQ_add_input_type::OUTPUT_SHIFT)]) uni_vbroadcastss(vmm_output_shift(idx), ptr[reg_output_shift]); + } + } + + template + inline void load_not_broadcasted_vectors_only(size_t idx, size_t offset) { + const auto &broadcasted = jqp_.broadcasted; + if (!broadcasted[static_cast(FQ_add_input_type::CROP_LOW)]) uni_vmovups(T(vmm_crop_low(idx).getIdx()), ptr[reg_crop_low + offset]); + if (!broadcasted[static_cast(FQ_add_input_type::CROP_HIGH)]) uni_vmovups(T(vmm_crop_high(idx).getIdx()), ptr[reg_crop_high + offset]); + if (!broadcasted[static_cast(FQ_add_input_type::INPUT_SCALE)]) uni_vmovups(T(vmm_input_scale(idx).getIdx()), ptr[reg_input_scale + offset]); + if (!broadcasted[static_cast(FQ_add_input_type::INPUT_SHIFT)]) uni_vmovups(T(vmm_input_shift(idx).getIdx()), ptr[reg_input_shift + offset]); + if (do_dequantization) { + if (!broadcasted[static_cast(FQ_add_input_type::OUTPUT_SCALE)]) + uni_vmovups(T(vmm_output_scale(idx).getIdx()), ptr[reg_output_scale + offset]); + if (!broadcasted[static_cast(FQ_add_input_type::OUTPUT_SHIFT)]) + uni_vmovups(T(vmm_output_shift(idx).getIdx()), ptr[reg_output_shift + offset]); + } + } + + inline void increase_ptrs_if_not_broadcasted(size_t offset) { + const auto &broadcasted = jqp_.broadcasted; + if (!broadcasted[static_cast(FQ_add_input_type::CROP_LOW)]) add(reg_crop_low, offset); + if (!broadcasted[static_cast(FQ_add_input_type::CROP_HIGH)]) add(reg_crop_high, offset); + if (!broadcasted[static_cast(FQ_add_input_type::INPUT_SCALE)]) add(reg_input_scale, offset); + if (!broadcasted[static_cast(FQ_add_input_type::INPUT_SHIFT)]) add(reg_input_shift, offset); + if (do_dequantization) { + if (!broadcasted[static_cast(FQ_add_input_type::OUTPUT_SCALE)]) add(reg_output_scale, offset); + if (!broadcasted[static_cast(FQ_add_input_type::OUTPUT_SHIFT)]) add(reg_output_shift, offset); + } + } + inline void compute_planar() { int src_type_size = jqp_.src_prc.size(); int dst_type_size = jqp_.dst_prc.size(); @@ -430,10 +470,10 @@ private: if (isa == cpu::x64::avx512_core) uni_vpxor(vmm_zero, vmm_zero, vmm_zero); - int simd_w = isa == cpu::x64::avx512_core ? 16 : 8; - int tail8_simd_w = 8; - int tail4_simd_w = 4; - int repeats = isa == cpu::x64::sse41 ? 2 : 1; + constexpr unsigned simd_w = isa == cpu::x64::avx512_core ? 16 : 8; + constexpr unsigned tail8_simd_w = 8; + constexpr unsigned tail4_simd_w = 4; + constexpr unsigned repeats = isa == cpu::x64::sse41 ? 2 : 1; Label main_loop_label; Label tail_blk8_label; @@ -446,18 +486,15 @@ private: Label tail_loop_label; Label exit_label; + for (int i = 0; i < repeats; i++) { + load_broadcasted_vectors_only(i); + } + cmp(reg_block_size, simd_w); jl(simd_w == 16 ? tail_blk8_label : tail_blk4_label, T_NEAR); for (int i = 0; i < repeats; i++) { - uni_vmovups(vmm_crop_low(i), ptr[reg_crop_low + i * (simd_w / 2) * sizeof(float)]); - uni_vmovups(vmm_crop_high(i), ptr[reg_crop_high + i * (simd_w / 2) * sizeof(float)]); - uni_vmovups(vmm_input_scale(i), ptr[reg_input_scale + i * (simd_w / 2) * sizeof(float)]); - uni_vmovups(vmm_input_shift(i), ptr[reg_input_shift + i * (simd_w / 2) * sizeof(float)]); - if (do_dequantization) { - uni_vmovups(vmm_output_scale(i), ptr[reg_output_scale + i * (simd_w / 2) * sizeof(float)]); - uni_vmovups(vmm_output_shift(i), ptr[reg_output_shift + i * (simd_w / 2) * sizeof(float)]); - } + load_not_broadcasted_vectors_only(i, i * (simd_w / 2) * sizeof(float)); } L(main_loop_label); { @@ -493,14 +530,7 @@ private: mov(aux_reg_from, reg_from); mov(reg_work_amount, ptr[param + GET_OFF(work_amount)]); - uni_vmovups(ymm_crop_low(0), ptr[reg_crop_low]); - uni_vmovups(ymm_crop_high(0), ptr[reg_crop_high]); - uni_vmovups(ymm_input_scale(0), ptr[reg_input_scale]); - uni_vmovups(ymm_input_shift(0), ptr[reg_input_shift]); - if (do_dequantization) { - uni_vmovups(ymm_output_scale(0), ptr[reg_output_scale]); - uni_vmovups(ymm_output_shift(0), ptr[reg_output_shift]); - } + load_not_broadcasted_vectors_only(0, 0); L(tail_blk8_loop_label); { cmp(reg_work_amount, 0); @@ -527,14 +557,7 @@ private: add(reg_from, tail8_simd_w * src_type_size); add(reg_to, tail8_simd_w * dst_type_size); - add(reg_crop_low, tail8_simd_w * wei_type_size); - add(reg_crop_high, tail8_simd_w * wei_type_size); - add(reg_input_scale, tail8_simd_w * wei_type_size); - add(reg_input_shift, tail8_simd_w * wei_type_size); - if (do_dequantization) { - add(reg_output_scale, tail8_simd_w * wei_type_size); - add(reg_output_shift, tail8_simd_w * wei_type_size); - } + increase_ptrs_if_not_broadcasted(tail8_simd_w * wei_type_size); sub(reg_block_size, tail8_simd_w); } @@ -547,14 +570,7 @@ private: mov(aux_reg_from, reg_from); mov(reg_work_amount, ptr[param + GET_OFF(work_amount)]); - uni_vmovups(xmm_crop_low(0), ptr[reg_crop_low]); - uni_vmovups(xmm_crop_high(0), ptr[reg_crop_high]); - uni_vmovups(xmm_input_scale(0), ptr[reg_input_scale]); - uni_vmovups(xmm_input_shift(0), ptr[reg_input_shift]); - if (do_dequantization) { - uni_vmovups(xmm_output_scale(0), ptr[reg_output_scale]); - uni_vmovups(xmm_output_shift(0), ptr[reg_output_shift]); - } + load_not_broadcasted_vectors_only(0, 0); L(tail_blk4_loop_label); { cmp(reg_work_amount, 0); @@ -581,14 +597,8 @@ private: add(reg_from, tail4_simd_w * src_type_size); add(reg_to, tail4_simd_w * dst_type_size); - add(reg_crop_low, tail4_simd_w * wei_type_size); - add(reg_crop_high, tail4_simd_w * wei_type_size); - add(reg_input_scale, tail4_simd_w * wei_type_size); - add(reg_input_shift, tail4_simd_w * wei_type_size); - if (do_dequantization) { - add(reg_output_scale, tail4_simd_w * wei_type_size); - add(reg_output_shift, tail4_simd_w * wei_type_size); - } + increase_ptrs_if_not_broadcasted(tail4_simd_w * wei_type_size); + sub(reg_block_size, tail4_simd_w); L(tail_label); @@ -602,28 +612,52 @@ private: L(tail_loop_label); { cmp(reg_work_amount, 0); jle(exit_label, T_NEAR); + Label end_unroll; - for (int i = 0; i < jqp_.c % tail4_simd_w; i++) { - uni_vmovss(xmm_crop_low(0), ptr[reg_crop_low + i * wei_type_size]); - uni_vmovss(xmm_crop_high(0), ptr[reg_crop_high + i * wei_type_size]); - uni_vmovss(xmm_input_scale(0), ptr[reg_input_scale + i * wei_type_size]); - uni_vmovss(xmm_input_shift(0), ptr[reg_input_shift + i * wei_type_size]); - if (do_dequantization) { - uni_vmovss(xmm_output_scale(0), ptr[reg_output_scale + i * wei_type_size]); - uni_vmovss(xmm_output_shift(0), ptr[reg_output_shift + i * wei_type_size]); + auto tail_unroll = [&](size_t iter) { + const auto &broadcasted = jqp_.broadcasted; + for (int i = 0; i < iter; i++) { + if (!broadcasted[static_cast(FQ_add_input_type::CROP_LOW)]) + uni_vmovss(xmm_crop_low(0), ptr[reg_crop_low + i * wei_type_size]); + if (!broadcasted[static_cast(FQ_add_input_type::CROP_HIGH)]) + uni_vmovss(xmm_crop_high(0), ptr[reg_crop_high + i * wei_type_size]); + if (!broadcasted[static_cast(FQ_add_input_type::INPUT_SCALE)]) + uni_vmovss(xmm_input_scale(0), ptr[reg_input_scale + i * wei_type_size]); + if (!broadcasted[static_cast(FQ_add_input_type::INPUT_SHIFT)]) + uni_vmovss(xmm_input_shift(0), ptr[reg_input_shift + i * wei_type_size]); + if (do_dequantization) { + if (!broadcasted[static_cast(FQ_add_input_type::OUTPUT_SCALE)]) + uni_vmovss(xmm_output_scale(0), ptr[reg_output_scale + i * wei_type_size]); + if (!broadcasted[static_cast(FQ_add_input_type::OUTPUT_SHIFT)]) + uni_vmovss(xmm_output_shift(0), ptr[reg_output_shift + i * wei_type_size]); + } + + load_scalar(xmm_val(0), ptr[aux_reg_from + i * src_type_size], jqp_.src_prc); + + uni_vminps(xmm_val(0), xmm_val(0), xmm_crop_high(0)); + uni_vmaxps(xmm_val(0), xmm_val(0), xmm_crop_low(0)); + uni_vfmadd213ps(xmm_val(0), xmm_input_scale(0), xmm_input_shift(0)); + if (do_rounding) uni_vroundps(xmm_val(0), xmm_val(0), 0); + if (do_dequantization) uni_vfmadd213ps(xmm_val(0), xmm_output_scale(0), xmm_output_shift(0)); + + store_scalar(ptr[aux_reg_to + i * dst_type_size], xmm_val(0), jqp_.dst_prc); } + jmp(end_unroll, T_NEAR); + }; - load_scalar(xmm_val(0), ptr[aux_reg_from + i * src_type_size], jqp_.src_prc); - - uni_vminps(xmm_val(0), xmm_val(0), xmm_crop_high(0)); - uni_vmaxps(xmm_val(0), xmm_val(0), xmm_crop_low(0)); - uni_vfmadd213ps(xmm_val(0), xmm_input_scale(0), xmm_input_shift(0)); - if (do_rounding) uni_vroundps(xmm_val(0), xmm_val(0), 0); - if (do_dequantization) uni_vfmadd213ps(xmm_val(0), xmm_output_scale(0), xmm_output_shift(0)); - - store_scalar(ptr[aux_reg_to + i * dst_type_size], xmm_val(0), jqp_.dst_prc); + std::array unroll_labels; + for (size_t i = 1; i < tail4_simd_w; ++i) { + cmp(reg_block_size, i); + je(unroll_labels[i], T_NEAR); } + for (size_t i = 1; i < tail4_simd_w; ++i) { + L(unroll_labels[i]); + tail_unroll(i); + } + + L(end_unroll); + dec(reg_work_amount); add(aux_reg_from, reg_src_step); add(aux_reg_to, reg_dst_step); @@ -895,29 +929,39 @@ namespace { struct FakeQuantKey { jit_quantize_params jqp; size_t hash() const { + using namespace dnnl::impl::primitive_hashing; size_t seed = 0; - seed = hash_combine(seed, jqp.c); seed = hash_combine(seed, jqp.is_planar); seed = hash_combine(seed, jqp.src_prc.getPrecVal()); seed = hash_combine(seed, jqp.wei_prc.getPrecVal()); seed = hash_combine(seed, jqp.dst_prc.getPrecVal()); - seed = dnnl::impl::primitive_hashing::get_vector_hash(seed, jqp.s_str); - seed = dnnl::impl::primitive_hashing::get_vector_hash(seed, jqp.d_str); seed = hash_combine(seed, jqp.op_type); + if (jqp.op_type == Algorithm::FQBinarization) { + seed = hash_combine(seed, jqp.c); + } else { + seed = hash_combine(seed, jqp.broadcasted); + } return seed; } bool operator==(const FakeQuantKey& rhs) const { - bool result = jqp.c == rhs.jqp.c && jqp.is_planar == rhs.jqp.is_planar && jqp.src_prc == rhs.jqp.src_prc && + bool result = jqp.is_planar == rhs.jqp.is_planar && jqp.src_prc == rhs.jqp.src_prc && jqp.wei_prc == rhs.jqp.wei_prc && jqp.dst_prc == rhs.jqp.dst_prc && - jqp.op_type == rhs.jqp.op_type && jqp.s_str == rhs.jqp.s_str && jqp.d_str == rhs.jqp.d_str; + jqp.op_type == rhs.jqp.op_type; + if (result) { + if (jqp.op_type == Algorithm::FQBinarization) { + result = result && jqp.c == rhs.jqp.c; + } else { + result = result && jqp.broadcasted == rhs.jqp.broadcasted; + } + } return result; } }; } // namespace FakeQuantize::FakeQuantize(const std::shared_ptr& op, const GraphContext::CPtr context) : - Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + Node(op, context, PassThroughShapeInferFactory()) { std::string errorMessage; if (isSupportedOperation(op, errorMessage)) { algorithm = Algorithm::FQCommon; @@ -1096,6 +1140,13 @@ FakeQuantize::FakeQuantize(const std::shared_ptr& op, const GraphC outputScaleSize = outputScale.size(); outputShiftSize = outputShift.size(); + broadcasted[static_cast(FQ_add_input_type::CROP_LOW)] = cropLowSize == 1; + broadcasted[static_cast(FQ_add_input_type::CROP_HIGH)] = cropHighSize == 1; + broadcasted[static_cast(FQ_add_input_type::INPUT_SCALE)] = inputScaleSize == 1; + broadcasted[static_cast(FQ_add_input_type::INPUT_SHIFT)] = inputShiftSize == 1; + broadcasted[static_cast(FQ_add_input_type::OUTPUT_SCALE)] = outputScaleSize == 1; + broadcasted[static_cast(FQ_add_input_type::OUTPUT_SHIFT)] = outputShiftSize == 1; + if (everyone_is(1u, cropLowSize, cropHighSize, inputScaleSize, inputShiftSize, outputScaleSize, outputShiftSize)) broadcastingPolicy = PerTensor; else if (one_of(1u, cropLowSize, cropHighSize, inputScaleSize, inputShiftSize, outputScaleSize, outputShiftSize)) @@ -1285,7 +1336,15 @@ void FakeQuantize::initSupportedPrimitiveDescriptors() { } } - for (auto& fmt : getDataFormats()) { + std::vector dataFormats; + // reference implementation supports only planar format + if (impl_type == impl_desc_type::ref) { + dataFormats.push_back(LayoutType::ncsp); + } else { + dataFormats = getDataFormats(); + } + + for (auto& fmt : dataFormats) { NodeConfig config; config.dynBatchSupport = true; for (size_t i = 0; i < getParentEdges().size(); i++) { @@ -1315,32 +1374,33 @@ void FakeQuantize::initSupportedPrimitiveDescriptors() { } bool FakeQuantize::needPrepareParams() const { - auto selectedPrimitiveDescriptor = getSelectedPrimitiveDescriptor(); - if (!selectedPrimitiveDescriptor) - IE_THROW() << "CPU quantize node with name '" << getName() << "' doesn't have primitive descriptors."; + if (isBinarization()) { + auto selectedPrimitiveDescriptor = getSelectedPrimitiveDescriptor(); + if (!selectedPrimitiveDescriptor) + IE_THROW() << "CPU quantize node with name '" << getName() << "' doesn't have primitive descriptors."; - if (internalBlobMemory.empty() || (selectedPrimitiveDescriptor->getImplementationType() != impl_desc_type::ref && inputShapesModified())) { - return true; + if (internalBlobMemory.empty() || (selectedPrimitiveDescriptor->getImplementationType() != impl_desc_type::ref && inputShapesModified())) { + return true; + } + + const auto axisSize = getParentEdgesAtPort(0)[0]->getMemory().getStaticDims()[getAxis()]; + const auto newPaddedSize = rnd_up(axisSize, 16); + const auto currPaddedSize = rnd_up(currentAxisSize, 16); + + return newPaddedSize != currPaddedSize || ((isInputLowBroadcasted || isOutputHighBroadcasted) && axisSize != currentAxisSize); } - - const auto axisSize = getParentEdgesAtPort(0)[0]->getMemory().getStaticDims()[getAxis()]; - const auto newPaddedSize = rnd_up(axisSize, 16); - const auto currPaddedSize = rnd_up(currentAxisSize, 16); - - return newPaddedSize != currPaddedSize || (isBinarization() && (isInputLowBroadcasted || isOutputHighBroadcasted) && - axisSize != currentAxisSize); + return false; } void FakeQuantize::prepareParams() { - const size_t axisSize = getParentEdgesAtPort(0)[0]->getMemory().GetShape().getStaticDims()[getAxis()]; - const size_t newPaddedSize = rnd_up(axisSize, 16); - IE_ASSERT(newPaddedSize != 0); + if (isBinarization()) { + const size_t axisSize = getParentEdgeAt(0)->getMemory().GetShape().getStaticDims()[getAxis()]; + const size_t newPaddedSize = rnd_up(axisSize, 16); + IE_ASSERT(newPaddedSize != 0); - if (internalBlobMemory.empty() || newPaddedSize != rnd_up(currentAxisSize, 16) || - (isBinarization() && (isInputLowBroadcasted || isOutputHighBroadcasted) && axisSize != currentAxisSize)) { - DnnlBlockedMemoryDesc weightsDataDesc(Shape(VectorDims{newPaddedSize}), memory::data_type::f32, memory::format_tag::x); - - if (isBinarization()) { + if (internalBlobMemory.empty() || newPaddedSize != rnd_up(currentAxisSize, 16) || + ((isInputLowBroadcasted || isOutputHighBroadcasted) && axisSize != currentAxisSize)) { + DnnlBlockedMemoryDesc weightsDataDesc(Shape(VectorDims{newPaddedSize}), memory::data_type::f32, memory::format_tag::x); constexpr size_t numBinFqIntBlob = 2; bool needUpdThr = false, needUpdMask = false; if (isInputLowBroadcasted && axisSize != currentAxisSize) { @@ -1376,65 +1436,49 @@ void FakeQuantize::prepareParams() { internalBlobMemory[1] = binarizationMaskDataMem; } } - } else if (levels != 2) { - constexpr size_t numFqIntBlob = 6; - - auto pushInternalBlob = [&](std::vector& data, size_t idx) { - auto memory = std::make_shared(getEngine()); - bool needOverwrite = getInputShapeAtPort(0).getDims()[getAxis()] == Shape::UNDEFINED_DIM && data.size() == 1; - if (needOverwrite) { - memory->Create(weightsDataDesc); - float *ptr = reinterpret_cast(memory->GetPtr()); - std::fill(ptr, ptr + newPaddedSize, data[0]); - } else { - if (data.size() == 1) { - data.resize(newPaddedSize, data[0]); - } else { - data.resize(newPaddedSize); - } - memory->Create(weightsDataDesc, &data[0]); - } - - if (internalBlobMemory.size() != numFqIntBlob) { - internalBlobMemory.push_back(memory); - } else if (needOverwrite) { - internalBlobMemory[idx] = memory; - } - }; - - pushInternalBlob(cropLow, 0); - pushInternalBlob(cropHigh, 1); - pushInternalBlob(inputScale, 2); - pushInternalBlob(inputShift, 3); - pushInternalBlob(outputScale, 4); - pushInternalBlob(outputShift, 5); - } else { - IE_THROW() << "Can't fill internal blob for FakeQuantize node with name: " << getName(); } + currentAxisSize = axisSize; } - currentAxisSize = axisSize; +} +void FakeQuantize::createPrimitive() { + Node::createPrimitive(); auto selectedPrimitiveDescriptor = getSelectedPrimitiveDescriptor(); if (!selectedPrimitiveDescriptor) IE_THROW() << "CPU quantize node with name '" << getName() << "' doesn't have primitive descriptors."; if (selectedPrimitiveDescriptor->getImplementationType() != impl_desc_type::ref) { const auto& config = getSelectedPrimitiveDescriptor()->getConfig(); - const auto& inDims = getParentEdgesAtPort(0)[0]->getMemory().getStaticDims(); + //Form FakeQuanKey FakeQuantKey key = {}; - key.jqp.c = inDims.size() > 1 ? inDims[1] : 1; key.jqp.src_prc = config.inConfs[0].getMemDesc()->getPrecision(); key.jqp.wei_prc = Precision::FP32; key.jqp.dst_prc = config.outConfs[0].getMemDesc()->getPrecision(); - auto srcDesc = getParentEdgeAt(0)->getMemory().GetDescWithType(); - key.jqp.s_str = srcDesc->getStrides(); - auto dstDesc = getChildEdgeAt(0)->getMemory().GetDescWithType(); + const auto &srcMemory = getParentEdgeAt(0)->getMemory(); + const auto &srcDesc = srcMemory.getDesc(); - key.jqp.d_str = dstDesc->getStrides(); - key.jqp.is_planar = srcDesc->hasLayoutType(LayoutType::ncsp) && one_of(srcDesc->getShape().getRank(), 3u, 4u, 5u); + key.jqp.is_planar = srcDesc.hasLayoutType(LayoutType::ncsp) && one_of(srcDesc.getShape().getRank(), 3, 4, 5); key.jqp.op_type = getAlgorithm(); + if (isBinarization()) { + const auto &inDims = srcMemory.getStaticDims(); + key.jqp.c = inDims.size() > 1 ? inDims[1] : 1; + } else { + // in case of blocked layout we need to extend vectors to prevent read from unallocated memory + size_t paddedSize = srcDesc.hasLayoutType(LayoutType::nCsp16c) ? 16 : srcDesc.hasLayoutType(LayoutType::nCsp8c) ? 8 : 1; + if (paddedSize != 1) { + if (!broadcasted[static_cast(FQ_add_input_type::CROP_LOW)]) cropLow.resize(rnd_up(cropLow.size(), paddedSize)); + if (!broadcasted[static_cast(FQ_add_input_type::CROP_HIGH)]) cropHigh.resize(rnd_up(cropHigh.size(), paddedSize)); + if (!broadcasted[static_cast(FQ_add_input_type::INPUT_SCALE)]) inputScale.resize(rnd_up(inputScale.size(), paddedSize)); + if (!broadcasted[static_cast(FQ_add_input_type::INPUT_SHIFT)]) inputShift.resize(rnd_up(inputShift.size(), paddedSize)); + if (!broadcasted[static_cast(FQ_add_input_type::OUTPUT_SCALE)]) outputScale.resize(rnd_up(outputScale.size(), paddedSize)); + if (!broadcasted[static_cast(FQ_add_input_type::OUTPUT_SHIFT)]) outputShift.resize(rnd_up(outputShift.size(), paddedSize)); + } + + key.jqp.broadcasted = broadcasted; + } + auto cache = context->getParamsCache(); auto buildExecutor = [](const FakeQuantKey& key) { return std::make_shared(key.jqp); @@ -1513,13 +1557,6 @@ void FakeQuantize::executeReference() { } else { auto dst = reinterpret_cast(dstMemory->GetPtr()); - auto crop_low = reinterpret_cast(internalBlobMemory[0]->GetData()); - auto crop_high = reinterpret_cast(internalBlobMemory[1]->GetData()); - auto input_scale = reinterpret_cast(internalBlobMemory[2]->GetData()); - auto input_shift = reinterpret_cast(internalBlobMemory[3]->GetData()); - auto output_scale = reinterpret_cast(internalBlobMemory[4]->GetData()); - auto output_shift = reinterpret_cast(internalBlobMemory[5]->GetData()); - parallel_nd(N, C, D, H, W, [&](dim_t n, dim_t c, dim_t d, dim_t h, dim_t w) { size_t src_off = srcDims.size() == 5 ? n * s_str[0] + c * s_str[1] + d * s_str[2] + h * s_str[3] + w * s_str[4] : @@ -1534,12 +1571,12 @@ void FakeQuantize::executeReference() { float src_val = src[src_off]; int wei_idx = getAxis() == 0 ? n : c; - float cl = crop_low[wei_idx]; - float ch = crop_high[wei_idx]; - float isc = input_scale[wei_idx]; - float ish = input_shift[wei_idx]; - float osc = output_scale[wei_idx]; - float osh = output_shift[wei_idx]; + float cl = broadcasted[static_cast(FQ_add_input_type::CROP_LOW)] ? cropLow[0] : cropLow[wei_idx]; + float ch = broadcasted[static_cast(FQ_add_input_type::CROP_HIGH)] ? cropHigh[0] : cropHigh[wei_idx]; + float isc = broadcasted[static_cast(FQ_add_input_type::INPUT_SCALE)] ? inputScale[0] : inputScale[wei_idx]; + float ish = broadcasted[static_cast(FQ_add_input_type::INPUT_SHIFT)] ? inputShift[0] : inputShift[wei_idx]; + float osc = broadcasted[static_cast(FQ_add_input_type::OUTPUT_SCALE)] ? outputScale[0] : outputScale[wei_idx]; + float osh = broadcasted[static_cast(FQ_add_input_type::OUTPUT_SHIFT)] ? outputShift[0] : outputShift[wei_idx]; float dst_val = nstl::min(ch, nstl::max(cl, src_val)); dst_val = dst_val * isc + ish; @@ -1562,7 +1599,7 @@ void FakeQuantize::executeReference() { } void FakeQuantize::executeBinarization(const std::unique_ptr &pKernel) const { - auto &srcMemory = getParentEdgeAt(0)->getMemoryPtr(); + const auto &srcMemory = getParentEdgeAt(0)->getMemoryPtr(); auto &dstMemory = getChildEdgeAt(0)->getMemoryPtr(); auto src = reinterpret_cast(srcMemory->GetPtr()); @@ -1573,8 +1610,8 @@ void FakeQuantize::executeBinarization(const std::unique_ptrgetStaticDims(); - const auto &jqp = pKernel->jqp_; - std::vector s_str = jqp.s_str; + auto srcMemDesc = srcMemory->GetDescWithType(); + std::vector s_str = srcMemDesc->getStrides(); size_t tmp = s_str[s_str.size() - 1]; for (int i = s_str.size() - 1; i > 1; i--) { s_str[i] = s_str[i - 1]; @@ -1608,13 +1645,6 @@ void FakeQuantize::executeQuantization(const std::unique_ptr(srcMemory->GetPtr()); auto dst = reinterpret_cast(dstMemory->GetPtr()); - auto crop_low = reinterpret_cast(internalBlobMemory[0]->GetData()); - auto crop_high = reinterpret_cast(internalBlobMemory[1]->GetData()); - auto input_scale = reinterpret_cast(internalBlobMemory[2]->GetData()); - auto input_shift = reinterpret_cast(internalBlobMemory[3]->GetData()); - auto output_scale = reinterpret_cast(internalBlobMemory[4]->GetData()); - auto output_shift = reinterpret_cast(internalBlobMemory[5]->GetData()); - auto& srcDesc = srcMemory->getDesc(); auto srcDims = srcDesc.getShape().getStaticDims(); @@ -1626,7 +1656,8 @@ void FakeQuantize::executeQuantization(const std::unique_ptrGetDescWithType(); + auto s_str = srcMemDesc->getStrides(); if (is_blk_format) { s_str[1] /= blk_size; @@ -1657,12 +1688,12 @@ void FakeQuantize::executeQuantization(const std::unique_ptr(FQ_add_input_type::CROP_LOW)] ? &cropLow[0] : &cropLow[c]; + arg.crop_high = broadcasted[static_cast(FQ_add_input_type::CROP_HIGH)] ? &cropHigh[0] : &cropHigh[c]; + arg.input_scale = broadcasted[static_cast(FQ_add_input_type::INPUT_SCALE)] ? &inputScale[0] : &inputScale[c]; + arg.input_shift = broadcasted[static_cast(FQ_add_input_type::INPUT_SHIFT)] ? &inputShift[0] : &inputShift[c]; + arg.output_scale = broadcasted[static_cast(FQ_add_input_type::OUTPUT_SCALE)] ? &outputScale[0] : &outputScale[c]; + arg.output_shift = broadcasted[static_cast(FQ_add_input_type::OUTPUT_SHIFT)] ? &outputShift[0] : &outputShift[c]; arg.src_step = (size_t) blk_size * src_type_size; arg.dst_step = (size_t) blk_size * dst_type_size; @@ -1687,12 +1718,12 @@ void FakeQuantize::executeQuantization(const std::unique_ptr(FQ_add_input_type::CROP_LOW)] ? &cropLow[0] : &cropLow[c]; + arg.crop_high = broadcasted[static_cast(FQ_add_input_type::CROP_HIGH)] ? &cropHigh[0] : &cropHigh[c]; + arg.input_scale = broadcasted[static_cast(FQ_add_input_type::INPUT_SCALE)] ? &inputScale[0] : &inputScale[c]; + arg.input_shift = broadcasted[static_cast(FQ_add_input_type::INPUT_SHIFT)] ? &inputShift[0] : &inputShift[c]; + arg.output_scale = broadcasted[static_cast(FQ_add_input_type::OUTPUT_SCALE)] ? &outputScale[0] : &outputScale[c]; + arg.output_shift = broadcasted[static_cast(FQ_add_input_type::OUTPUT_SHIFT)] ? &outputShift[0] : &outputShift[c]; arg.src_step = is_blk_format ? (size_t) blk_size * src_type_size : (size_t) C * src_type_size; arg.dst_step = is_blk_format ? (size_t) blk_size * dst_type_size : (size_t) C * dst_type_size; @@ -1715,12 +1746,12 @@ void FakeQuantize::executeQuantization(const std::unique_ptr(FQ_add_input_type::CROP_LOW)] ? &cropLow[0] : &cropLow[c]; + arg.crop_high = broadcasted[static_cast(FQ_add_input_type::CROP_HIGH)] ? &cropHigh[0] : &cropHigh[c]; + arg.input_scale = broadcasted[static_cast(FQ_add_input_type::INPUT_SCALE)] ? &inputScale[0] : &inputScale[c]; + arg.input_shift = broadcasted[static_cast(FQ_add_input_type::INPUT_SHIFT)] ? &inputShift[0] : &inputShift[c]; + arg.output_scale = broadcasted[static_cast(FQ_add_input_type::OUTPUT_SCALE)] ? &outputScale[0] : &outputScale[c]; + arg.output_shift = broadcasted[static_cast(FQ_add_input_type::OUTPUT_SHIFT)] ? &outputShift[0] : &outputShift[c]; arg.src_step = is_blk_format ? (size_t) blk_size * src_type_size : (size_t) C * src_type_size; arg.dst_step = is_blk_format ? (size_t) blk_size * dst_type_size : (size_t) C * dst_type_size; @@ -1737,11 +1768,7 @@ void FakeQuantize::executeDynamicImpl(dnnl::stream strm) { } void FakeQuantize::execute(dnnl::stream strm) { - auto selectedPrimitiveDescriptor = getSelectedPrimitiveDescriptor(); - if (!selectedPrimitiveDescriptor) - IE_THROW() << "CPU quantize node with name '" << getName() << "' doesn't have primitive descriptors."; - - if (selectedPrimitiveDescriptor->getImplementationType() != impl_desc_type::ref) { + if (getSelectedPrimitiveDescriptor()->getImplementationType() != impl_desc_type::ref) { execPtr->exec(*this); } else { executeReference(); diff --git a/src/plugins/intel_cpu/src/nodes/fake_quantize.h b/src/plugins/intel_cpu/src/nodes/fake_quantize.h index f97947e7f2c..47a7c13a8b0 100644 --- a/src/plugins/intel_cpu/src/nodes/fake_quantize.h +++ b/src/plugins/intel_cpu/src/nodes/fake_quantize.h @@ -17,18 +17,27 @@ namespace ov { namespace intel_cpu { namespace node { +enum class FQ_add_input_type { + CROP_LOW, + CROP_HIGH, + INPUT_SCALE, + INPUT_SHIFT, + OUTPUT_SCALE, + OUTPUT_SHIFT, + INPUTS_SIZE +}; + struct jit_quantize_params { - int c; bool is_planar; InferenceEngine::Precision src_prc; InferenceEngine::Precision wei_prc; InferenceEngine::Precision dst_prc; - std::vector s_str; - std::vector d_str; - Algorithm op_type; + + int c; // need only for binarization + std::bitset(FQ_add_input_type::INPUTS_SIZE)> broadcasted; // need only for quantization }; struct jit_quantize_call_args { @@ -82,6 +91,7 @@ public: bool needPrepareParams() const override; void prepareParams() override; + void createPrimitive() override; const float* getBinarizationTresholdsPtr() const { return &binarizationThresholds[0]; } const float* getBinarizationOutputMaskPtr() const { return reinterpret_cast(&binarizationOutputMask[0]); } @@ -240,6 +250,8 @@ private: size_t outputScaleSize; size_t outputShiftSize; + std::bitset(FQ_add_input_type::INPUTS_SIZE)> broadcasted; + std::vector fqScales; // version based lazy evaluation, any parameter change increases parameterVersion diff --git a/src/plugins/intel_cpu/tests/functional/subgraph_tests/src/fq_caching.cpp b/src/plugins/intel_cpu/tests/functional/subgraph_tests/src/fq_caching.cpp new file mode 100644 index 00000000000..37784337820 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/subgraph_tests/src/fq_caching.cpp @@ -0,0 +1,529 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +// Motivation: +// In a dynamic scenario, depending on the input shapes for the current node, we can either generate a new jit kernel or get an existing one from the cache. +// But the current single layer tests do not allow checking the case when the same kernel can be used for different nodes. +// This subgraph test contains 2 FQ nodes and allows us to check this case. + +// ------------------------------------ ------------------------------------ +// | Input 0 | | Input 1 | +// ------------------------------------ ------------------------------------ +// | | +// ------------------------------------ ------------------------------------ +// | FakeQuantize 0 | | FakeQuantize 1 | +// ------------------------------------ ------------------------------------ +// | | +// | ------------------------------------ +// | |Reshape (if !reshapeShape.empty())| +// | ------------------------------------ +// | | +// ---------------------------------------------------------------------------- +// | Concat | +// ---------------------------------------------------------------------------- +// | +// -------- +// |Output| +// -------- + +#include +#include +#include +#include "test_utils/cpu_test_utils.hpp" +#include "cpp_interfaces/interface/ie_internal_plugin_config.hpp" + +using namespace CPUTestUtils; +using namespace ov::test; +using namespace InferenceEngine; +using namespace ngraph; + +namespace CPUSubgraphTestsDefinitions { + +using InputShapesTuple = std::tuple< + std::vector, // fq dynamic data shapes + std::vector>, // fq range input shapes + std::vector // reshape shape +>; + +using FqSpecificParams = std::tuple, // output low + std::vector, // output high + size_t>; // levels + +typedef std::tuple< + InputShapesTuple, // fq input shapes and reshape shape + FqSpecificParams, // fq specific params + std::pair, std::vector>, // il and ih values + CPUSpecificParams, + std::map // Additional config (disable snippets or no) +> FakeQuantizeCacheTestParams; + +class FakeQuantizeCacheTest : public testing::WithParamInterface, + virtual public SubgraphBaseTest, public CPUTestsBase { +public: + static std::string getTestCaseName(const testing::TestParamInfo &obj) { + InputShapesTuple inputShapesTuple; + FqSpecificParams fqParams; + std::pair, std::vector> inputRangesValues; + CPUSpecificParams cpuParams; + std::map additionalConfig; + std::tie(inputShapesTuple, fqParams, inputRangesValues, cpuParams, additionalConfig) = obj.param; + + std::vector shapes; + std::vector> ranges; + std::vector reshapeShape; + std::tie(shapes, ranges, reshapeShape) = inputShapesTuple; + + int64_t inDataLowBounds, inDataHighBounds; + std::vector inputLow, inputHigh, outputLow, outputHigh; + size_t levels; + inputLow = inputRangesValues.first; + inputHigh = inputRangesValues.second; + std::tie(inDataLowBounds, inDataHighBounds, outputLow, outputHigh, levels) = fqParams; + + std::ostringstream results; + + for (int i = 0; i < shapes.size(); i++) { + results << "FQ" << i << "_IS=(" << CommonTestUtils::partialShape2str({shapes[i].first}) << ")_"; + results << "TS="; + for (const auto& shape : shapes[i].second) { + results << "(" << CommonTestUtils::vec2str(shape) << ")_"; + } + results << "RS="; + for (const auto& range : ranges[i]) { + results << "(" << CommonTestUtils::vec2str(range) << ")_"; + } + } + if (!reshapeShape.empty()) { + results << "ReshapeShape=(" << CommonTestUtils::vec2str(reshapeShape) << ")_"; + } + + results << "LOW_BOUNDS=" << inDataLowBounds << "_"; + results << "HIGH_BOUNDS=" << inDataHighBounds << "_"; + results << "IL=" << CommonTestUtils::vec2str(inputLow) << "_"; + results << "IH=" << CommonTestUtils::vec2str(inputHigh) << "_"; + results << "OL=" << CommonTestUtils::vec2str(outputLow) << "_"; + results << "OH=" << CommonTestUtils::vec2str(outputHigh) << "_"; + results << "LEVELS=" << levels; + + results << CPUTestsBase::getTestCaseName(cpuParams); + + if (!additionalConfig.empty()) { + results << "_PluginConf"; + for (auto& item : additionalConfig) { + results << "_" << item.first << "=" << item.second; + } + } + + return results.str(); + } + +protected: + void SetUp() override { + abs_threshold = 0.01f; + + InputShapesTuple inputShapesTuple; + FqSpecificParams fqParams; + std::pair, std::vector> inputRangesValues; + CPUSpecificParams cpuParams; + std::map additionalConfig; + std::tie(inputShapesTuple, fqParams, inputRangesValues, + cpuParams, additionalConfig) = this->GetParam(); + + std::vector shapesVec; + std::vector> rangesVec; + std::vector reshapeShape; + std::tie(shapesVec, rangesVec, reshapeShape) = inputShapesTuple; + + std::tie(inFmts, outFmts, priority, selectedType) = cpuParams; + + configuration.insert(additionalConfig.begin(), additionalConfig.end()); + targetDevice = CommonTestUtils::DEVICE_CPU; + + init_input_shapes(shapesVec); + + size_t levels; + std::vector> rangesBounds(RANGES_INPUT_NUMBER); + rangesBounds[0] = inputRangesValues.first; + rangesBounds[1] = inputRangesValues.second; + std::tie(inDataLowBounds, inDataHighBounds, rangesBounds[2], rangesBounds[3], levels) = fqParams; + + ParameterVector ngraphParam; + std::vector> ngraphInputs; + + auto ngInPrec = element::Type_t::f32; + + for (size_t i = 0; i < inputDynamicShapes.size(); i++) { + ngraphParam.push_back(std::make_shared(ngInPrec, inputDynamicShapes[i])); + ngraphInputs.push_back(ngraphParam.back()); + } + + auto makeFQ = [&](int i) { + auto extendData = [](const std::vector &data, size_t newSize) { + std::vector extendedData(newSize); + size_t oldSize = data.size(); + for (size_t i = 0; i < newSize; i++) { + extendedData[i] = data[i % oldSize]; + } + return extendedData; + }; + + auto ranges = rangesVec[i]; + + auto il = builder::makeConstant(ngInPrec, ranges[0], extendData(rangesBounds[0], + std::accumulate(ranges[0].begin(), ranges[0].end(), 1, std::multiplies()))); + auto ih = builder::makeConstant(ngInPrec, ranges[1], extendData(rangesBounds[1], + std::accumulate(ranges[1].begin(), ranges[1].end(), 1, std::multiplies()))); + auto ol = builder::makeConstant(ngInPrec, ranges[2], extendData(rangesBounds[2], + std::accumulate(ranges[2].begin(), ranges[2].end(), 1, std::multiplies()))); + auto oh = builder::makeConstant(ngInPrec, ranges[3], extendData(rangesBounds[3], + std::accumulate(ranges[3].begin(), ranges[3].end(), 1, std::multiplies()))); + + auto fqNode = std::make_shared(ngraphParam[i], il, ih, ol, oh, levels); + fqNode->get_rt_info() = getCPUInfo(); + return fqNode; + }; + + std::shared_ptr lastNode0 = makeFQ(0); + std::shared_ptr lastNode1 = makeFQ(1); + + if (!reshapeShape.empty()) { + auto reshapeConstNode = builder::makeConstant(::element::Type(::element::Type_t::i32), + {reshapeShape.size()}, reshapeShape); + lastNode1 = std::make_shared(lastNode1, reshapeConstNode, false); + } + auto concat = builder::makeConcat({lastNode0, lastNode1}, 0); + + if (selectedType.empty()) { + selectedType = getPrimitiveType() + "_FP32"; + } + + function = std::make_shared(concat, ngraphParam, "fq_cache"); + } + + void generate_inputs(const std::vector& targetInputStaticShapes) override { + inputs.clear(); + const auto& funcInputs = function->inputs(); + for (int i = 0; i < funcInputs.size(); ++i) { + const auto& funcInput = funcInputs[i]; + ov::Tensor tensor; + tensor = ov::test::utils::create_and_fill_tensor(funcInput.get_element_type(), + targetInputStaticShapes[i], + inDataHighBounds - inDataLowBounds, + inDataLowBounds); + inputs.insert({funcInput.get_node_shared_ptr(), tensor}); + } + } + +private: + const size_t RANGES_INPUT_NUMBER = 4; + + int64_t inDataLowBounds, inDataHighBounds; +}; + +TEST_P(FakeQuantizeCacheTest, CompareWithRefs) { + run(); + + CheckPluginRelatedResults(compiledModel, "FakeQuantize"); +} + +namespace { + +const std::vector levels = {256}; + +int64_t dataLowBounds{-10}, dataHighBounds{10}; + +const std::vector, std::vector>> inputRanges = { + {{0.0f, 1.0f}, {5.0f, 6.0f}}, +}; + +const std::vector outputLow{5.0f, 6.0f}, outputHigh{25.0f, 31.0f}; + +const auto specificParams = ::testing::Combine(::testing::Values(dataLowBounds), + ::testing::Values(dataHighBounds), + ::testing::Values(outputLow), + ::testing::Values(outputHigh), + ::testing::ValuesIn(levels)); + +const std::map emptyConfig = {}; +const std::map disableSnippets = { + {PluginConfigInternalParams::KEY_SNIPPETS_MODE, PluginConfigInternalParams::DISABLE}}; + + +// 3D +std::vector cpuParams_3D = { + CPUSpecificParams({ncw}, {ncw}, {}, {}), +}; + +std::vector inputShapes_3D = { + { + // fq dynamic data shapes + { + // input0 + {{-1, -1, 43}, {{1, 10, 43}, {1, 20, 43}, {1, 10, 43}, {1, 20, 43}}}, + // input1 + {{-1, -1, 43}, {{1, 10, 43}, {1, 20, 43}, {1, 10, 43}, {1, 20, 43}}}, + }, + // fq range input shapes + { + // input0 + {{1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}}, // miss + // input1 + {{1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}}, // hit + }, + // reshape shape + {}, + }, + { + // fq dynamic data shapes + { + // input0 + {{-1, 10, -1}, {{1, 10, 22}, {1, 10, 44}, {1, 10, 22}, {1, 10, 44}}}, + // input1 + {{-1, 10, -1}, {{1, 10, 22}, {1, 10, 44}, {1, 10, 22}, {1, 10, 44}}}, + }, + // fq range input shapes + { + // input0 + {{1, 10, 1}, {1, 10, 1}, {1, 10, 1}, {1, 10, 1}}, // miss + // input1 + {{1, 10, 1}, {1, 10, 1}, {1, 10, 1}, {1, 10, 1}}, // hit + }, + // reshape shape + {}, + }, + { + // fq dynamic data shapes + { + // input0 + {{-1, 10, -1}, {{1, 10, 22}, {1, 10, 44}, {1, 10, 22}, {1, 10, 44}}}, + // input1 + {{-1, 10, -1}, {{1, 10, 22}, {1, 10, 44}, {1, 10, 22}, {1, 10, 44}}}, + }, + // fq range input shapes + { + // input0 + {{1, 10, 1}, {1, 10, 1}, {1, 10, 1}, {1, 10, 1}}, // miss + // input1 + {{1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}}, // miss + }, + // reshape shape + {}, + }, + { + // fq dynamic data shapes + { + // input0 + {{-1, 10, -1}, {{1, 10, 22}, {1, 10, 22}}}, + // input1 + {{-1, 20, -1}, {{1, 20, 22}, {1, 20, 22}}}, + }, + // fq range input shapes + { + // input0 + {{1, 10, 1}, {1, 10, 1}, {1, 10, 1}, {1, 10, 1}}, // miss + // input1 + {{1, 20, 1}, {1, 20, 1}, {1, 20, 1}, {1, 20, 1}}, // hit + }, + // reshape shape + {-1, 10, 22}, + }, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_FakeQuantizeCache_3D, FakeQuantizeCacheTest, + ::testing::Combine( + ::testing::ValuesIn(inputShapes_3D), + specificParams, + ::testing::ValuesIn(inputRanges), + ::testing::ValuesIn(filterCPUSpecificParams(cpuParams_3D)), + ::testing::Values(disableSnippets)), + FakeQuantizeCacheTest::getTestCaseName); + + +// 4D +std::vector cpuParams_4D = { + CPUSpecificParams({nchw}, {nchw}, {}, {}), + CPUSpecificParams({nhwc}, {nhwc}, {}, {}), + CPUSpecificParams({nChw16c}, {nChw16c}, {}, {}), +}; + +std::vector inputShapes_4D = { + { + // fq dynamic data shapes + { + // input0 + {{-1, -1, -1, 43}, {{1, 17, 3, 43}, {1, 34, 3, 43}, {1, 17, 3, 43}, {1, 34, 3, 43}}}, + // input1 + {{-1, -1, -1, 43}, {{1, 17, 3, 43}, {1, 34, 3, 43}, {1, 17, 3, 43}, {1, 34, 3, 43}}}, + }, + // fq range input shapes + { + // input0 + {{1, 1, 1, 1}, {1, 1, 1, 1}, {1, 1, 1, 1}, {1, 1, 1, 1}}, // miss + // input1 + {{1, 1, 1, 1}, {1, 1, 1, 1}, {1, 1, 1, 1}, {1, 1, 1, 1}}, // hit + }, + // reshape shape + {}, + }, + { + // fq dynamic data shapes + { + // input0 + {{-1, 47, -1, -1}, {{1, 47, 2, 22}, {1, 47, 3, 33}, {1, 47, 2, 22}, {1, 47, 3, 33}}}, + // input1 + {{-1, 47, -1, -1}, {{1, 47, 2, 22}, {1, 47, 3, 33}, {1, 47, 2, 22}, {1, 47, 3, 33}}}, + }, + // fq range input shapes + { + // input0 + {{1, 47, 1, 1}, {1, 47, 1, 1}, {1, 47, 1, 1}, {1, 47, 1, 1}}, // miss + // input1 + {{1, 47, 1, 1}, {1, 47, 1, 1}, {1, 47, 1, 1}, {1, 47, 1, 1}}, // hit + }, + // reshape shape + {}, + }, + { + // fq dynamic data shapes + { + // input0 + {{-1, 47, -1, -1}, {{1, 47, 2, 22}, {1, 47, 3, 33}, {1, 47, 2, 22}, {1, 47, 3, 33}}}, + // input1 + {{-1, 47, -1, -1}, {{1, 47, 2, 22}, {1, 47, 3, 33}, {1, 47, 2, 22}, {1, 47, 3, 33}}}, + }, + // fq range input shapes + { + // input0 + {{1, 47, 1, 1}, {1, 47, 1, 1}, {1, 47, 1, 1}, {1, 47, 1, 1}}, // miss + // input1 + {{1, 1, 1, 1}, {1, 1, 1, 1}, {1, 1, 1, 1}, {1, 1, 1, 1}}, // miss + }, + // reshape shape + {}, + }, + { + // fq dynamic data shapes + { + // input0 + {{-1, 17, -1, -1}, {{1, 17, 2, 22}, {1, 17, 2, 22}}}, + // input1 + {{-1, 34, -1, -1}, {{1, 34, 2, 22}, {1, 34, 2, 22}}}, + }, + // fq range input shapes + { + // input0 + {{1, 17, 1, 1}, {1, 17, 1, 1}, {1, 17, 1, 1}, {1, 17, 1, 1}}, // miss + // input1 + {{1, 34, 1, 1}, {1, 34, 1, 1}, {1, 34, 1, 1}, {1, 34, 1, 1}}, // hit + }, + // reshape shape + {-1, 17, 2, 22}, + }, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_FakeQuantizeCache_4D, FakeQuantizeCacheTest, + ::testing::Combine( + ::testing::ValuesIn(inputShapes_4D), + specificParams, + ::testing::ValuesIn(inputRanges), + ::testing::ValuesIn(filterCPUSpecificParams(cpuParams_4D)), + ::testing::Values(disableSnippets)), + FakeQuantizeCacheTest::getTestCaseName); + + +// 5D +std::vector cpuParams_5D = { + CPUSpecificParams({ncdhw}, {ncdhw}, {}, {}), + CPUSpecificParams({ndhwc}, {ndhwc}, {}, {}), + CPUSpecificParams({nCdhw16c}, {nCdhw16c}, {}, {}), +}; + +std::vector inputShapes_5D = { + { + // fq dynamic data shapes + { + // input0 + {{-1, -1, -1, -1, 43}, {{1, 17, 2, 3, 43}, {1, 34, 2, 3, 43}, {1, 17, 2, 3, 43}, {1, 34, 2, 3, 43}}}, + // input1 + {{-1, -1, -1, -1, 43}, {{1, 17, 2, 3, 43}, {1, 34, 2, 3, 43}, {1, 17, 2, 3, 43}, {1, 34, 2, 3, 43}}}, + }, + // fq range input shapes + { + // input0 + {{1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}}, // miss + // input1 + {{1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}}, // hit + }, + // reshape shape + {}, + }, + { + // fq dynamic data shapes + { + // input0 + {{-1, 47, -1, -1, -1}, {{1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}, {1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}}}, + // input1 + {{-1, 47, -1, -1, -1}, {{1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}, {1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}}}, + }, + // fq range input shapes + { + // input0 + {{1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}}, // miss + // input1 + {{1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}}, // hit + }, + // reshape shape + {}, + }, + { + // fq dynamic data shapes + { + // input0 + {{-1, 47, -1, -1, -1}, {{1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}, {1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}}}, + // input1 + {{-1, 47, -1, -1, -1}, {{1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}, {1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}}}, + }, + // fq range input shapes + { + // input0 + {{1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}}, // miss + // input1 + {{1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}}, // miss + }, + // reshape shape + {}, + }, + { + // fq dynamic data shapes + { + // input0 + {{-1, 17, -1, -1, -1}, {{1, 17, 2, 3, 22}, {1, 17, 2, 3, 22}}}, + // input1 + {{-1, 34, -1, -1, -1}, {{1, 34, 2, 3, 22}, {1, 34, 2, 3, 22}}}, + }, + // fq range input shapes + { + // input0 + {{1, 17, 1, 1, 1}, {1, 17, 1, 1, 1}, {1, 17, 1, 1, 1}, {1, 17, 1, 1, 1}}, // miss + // input1 + {{1, 34, 1, 1, 1}, {1, 34, 1, 1, 1}, {1, 34, 1, 1, 1}, {1, 34, 1, 1, 1}}, // hit + }, + // reshape shape + {-1, 17, 2, 3, 22}, + }, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_FakeQuantizeCache_5D, FakeQuantizeCacheTest, + ::testing::Combine( + ::testing::ValuesIn(inputShapes_5D), + specificParams, + ::testing::ValuesIn(inputRanges), + ::testing::ValuesIn(filterCPUSpecificParams(cpuParams_5D)), + ::testing::Values(disableSnippets)), + FakeQuantizeCacheTest::getTestCaseName); + +} // namespace +} // namespace CPUSubgraphTestsDefinitions