[CPU] FQ shape agnostic kernel (#16585)
This commit is contained in:
Anton Voronov
GitHub
parent
d7f70b647b
commit
b64cbff10b
|
|
@ -23,6 +23,7 @@
|
|||
#include "memory_desc/dnnl_blocked_memory_desc.h"
|
||||
#include "common/cpu_memcpy.h"
|
||||
#include <common/primitive_hashing_utils.hpp>
|
||||
#include <utils/shape_inference/shape_inference_pass_through.hpp>
|
||||
|
||||
#include <ngraph/opsets/opset1.hpp>
|
||||
#include "utils/ngraph_utils.hpp"
|
||||
|
|
@ -295,6 +296,45 @@ private:
|
|||
bool do_rounding = true;
|
||||
bool do_dequantization = true;
|
||||
|
||||
inline void load_broadcasted_vectors_only(size_t idx) {
|
||||
const auto &broadcasted = jqp_.broadcasted;
|
||||
if (broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_LOW)]) uni_vbroadcastss(vmm_crop_low(idx), ptr[reg_crop_low]);
|
||||
if (broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_HIGH)]) uni_vbroadcastss(vmm_crop_high(idx), ptr[reg_crop_high]);
|
||||
if (broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SCALE)]) uni_vbroadcastss(vmm_input_scale(idx), ptr[reg_input_scale]);
|
||||
if (broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SHIFT)]) uni_vbroadcastss(vmm_input_shift(idx), ptr[reg_input_shift]);
|
||||
if (do_dequantization) {
|
||||
if (broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SCALE)]) uni_vbroadcastss(vmm_output_scale(idx), ptr[reg_output_scale]);
|
||||
if (broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SHIFT)]) uni_vbroadcastss(vmm_output_shift(idx), ptr[reg_output_shift]);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline void load_not_broadcasted_vectors_only(size_t idx, size_t offset) {
|
||||
const auto &broadcasted = jqp_.broadcasted;
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_LOW)]) uni_vmovups(T(vmm_crop_low(idx).getIdx()), ptr[reg_crop_low + offset]);
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_HIGH)]) uni_vmovups(T(vmm_crop_high(idx).getIdx()), ptr[reg_crop_high + offset]);
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SCALE)]) uni_vmovups(T(vmm_input_scale(idx).getIdx()), ptr[reg_input_scale + offset]);
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SHIFT)]) uni_vmovups(T(vmm_input_shift(idx).getIdx()), ptr[reg_input_shift + offset]);
|
||||
if (do_dequantization) {
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SCALE)])
|
||||
uni_vmovups(T(vmm_output_scale(idx).getIdx()), ptr[reg_output_scale + offset]);
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SHIFT)])
|
||||
uni_vmovups(T(vmm_output_shift(idx).getIdx()), ptr[reg_output_shift + offset]);
|
||||
}
|
||||
}
|
||||
|
||||
inline void increase_ptrs_if_not_broadcasted(size_t offset) {
|
||||
const auto &broadcasted = jqp_.broadcasted;
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_LOW)]) add(reg_crop_low, offset);
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_HIGH)]) add(reg_crop_high, offset);
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SCALE)]) add(reg_input_scale, offset);
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SHIFT)]) add(reg_input_shift, offset);
|
||||
if (do_dequantization) {
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SCALE)]) add(reg_output_scale, offset);
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SHIFT)]) add(reg_output_shift, offset);
|
||||
}
|
||||
}
|
||||
|
||||
inline void compute_planar() {
|
||||
int src_type_size = jqp_.src_prc.size();
|
||||
int dst_type_size = jqp_.dst_prc.size();
|
||||
|
|
@ -430,10 +470,10 @@ private:
|
|||
if (isa == cpu::x64::avx512_core)
|
||||
uni_vpxor(vmm_zero, vmm_zero, vmm_zero);
|
||||
|
||||
int simd_w = isa == cpu::x64::avx512_core ? 16 : 8;
|
||||
int tail8_simd_w = 8;
|
||||
int tail4_simd_w = 4;
|
||||
int repeats = isa == cpu::x64::sse41 ? 2 : 1;
|
||||
constexpr unsigned simd_w = isa == cpu::x64::avx512_core ? 16 : 8;
|
||||
constexpr unsigned tail8_simd_w = 8;
|
||||
constexpr unsigned tail4_simd_w = 4;
|
||||
constexpr unsigned repeats = isa == cpu::x64::sse41 ? 2 : 1;
|
||||
|
||||
Label main_loop_label;
|
||||
Label tail_blk8_label;
|
||||
|
|
@ -446,18 +486,15 @@ private:
|
|||
Label tail_loop_label;
|
||||
Label exit_label;
|
||||
|
||||
for (int i = 0; i < repeats; i++) {
|
||||
load_broadcasted_vectors_only(i);
|
||||
}
|
||||
|
||||
cmp(reg_block_size, simd_w);
|
||||
jl(simd_w == 16 ? tail_blk8_label : tail_blk4_label, T_NEAR);
|
||||
|
||||
for (int i = 0; i < repeats; i++) {
|
||||
uni_vmovups(vmm_crop_low(i), ptr[reg_crop_low + i * (simd_w / 2) * sizeof(float)]);
|
||||
uni_vmovups(vmm_crop_high(i), ptr[reg_crop_high + i * (simd_w / 2) * sizeof(float)]);
|
||||
uni_vmovups(vmm_input_scale(i), ptr[reg_input_scale + i * (simd_w / 2) * sizeof(float)]);
|
||||
uni_vmovups(vmm_input_shift(i), ptr[reg_input_shift + i * (simd_w / 2) * sizeof(float)]);
|
||||
if (do_dequantization) {
|
||||
uni_vmovups(vmm_output_scale(i), ptr[reg_output_scale + i * (simd_w / 2) * sizeof(float)]);
|
||||
uni_vmovups(vmm_output_shift(i), ptr[reg_output_shift + i * (simd_w / 2) * sizeof(float)]);
|
||||
}
|
||||
load_not_broadcasted_vectors_only<Vmm>(i, i * (simd_w / 2) * sizeof(float));
|
||||
}
|
||||
|
||||
L(main_loop_label); {
|
||||
|
|
@ -493,14 +530,7 @@ private:
|
|||
mov(aux_reg_from, reg_from);
|
||||
mov(reg_work_amount, ptr[param + GET_OFF(work_amount)]);
|
||||
|
||||
uni_vmovups(ymm_crop_low(0), ptr[reg_crop_low]);
|
||||
uni_vmovups(ymm_crop_high(0), ptr[reg_crop_high]);
|
||||
uni_vmovups(ymm_input_scale(0), ptr[reg_input_scale]);
|
||||
uni_vmovups(ymm_input_shift(0), ptr[reg_input_shift]);
|
||||
if (do_dequantization) {
|
||||
uni_vmovups(ymm_output_scale(0), ptr[reg_output_scale]);
|
||||
uni_vmovups(ymm_output_shift(0), ptr[reg_output_shift]);
|
||||
}
|
||||
load_not_broadcasted_vectors_only<Ymm>(0, 0);
|
||||
|
||||
L(tail_blk8_loop_label); {
|
||||
cmp(reg_work_amount, 0);
|
||||
|
|
@ -527,14 +557,7 @@ private:
|
|||
|
||||
add(reg_from, tail8_simd_w * src_type_size);
|
||||
add(reg_to, tail8_simd_w * dst_type_size);
|
||||
add(reg_crop_low, tail8_simd_w * wei_type_size);
|
||||
add(reg_crop_high, tail8_simd_w * wei_type_size);
|
||||
add(reg_input_scale, tail8_simd_w * wei_type_size);
|
||||
add(reg_input_shift, tail8_simd_w * wei_type_size);
|
||||
if (do_dequantization) {
|
||||
add(reg_output_scale, tail8_simd_w * wei_type_size);
|
||||
add(reg_output_shift, tail8_simd_w * wei_type_size);
|
||||
}
|
||||
increase_ptrs_if_not_broadcasted(tail8_simd_w * wei_type_size);
|
||||
sub(reg_block_size, tail8_simd_w);
|
||||
}
|
||||
|
||||
|
|
@ -547,14 +570,7 @@ private:
|
|||
mov(aux_reg_from, reg_from);
|
||||
mov(reg_work_amount, ptr[param + GET_OFF(work_amount)]);
|
||||
|
||||
uni_vmovups(xmm_crop_low(0), ptr[reg_crop_low]);
|
||||
uni_vmovups(xmm_crop_high(0), ptr[reg_crop_high]);
|
||||
uni_vmovups(xmm_input_scale(0), ptr[reg_input_scale]);
|
||||
uni_vmovups(xmm_input_shift(0), ptr[reg_input_shift]);
|
||||
if (do_dequantization) {
|
||||
uni_vmovups(xmm_output_scale(0), ptr[reg_output_scale]);
|
||||
uni_vmovups(xmm_output_shift(0), ptr[reg_output_shift]);
|
||||
}
|
||||
load_not_broadcasted_vectors_only<Xmm>(0, 0);
|
||||
|
||||
L(tail_blk4_loop_label); {
|
||||
cmp(reg_work_amount, 0);
|
||||
|
|
@ -581,14 +597,8 @@ private:
|
|||
|
||||
add(reg_from, tail4_simd_w * src_type_size);
|
||||
add(reg_to, tail4_simd_w * dst_type_size);
|
||||
add(reg_crop_low, tail4_simd_w * wei_type_size);
|
||||
add(reg_crop_high, tail4_simd_w * wei_type_size);
|
||||
add(reg_input_scale, tail4_simd_w * wei_type_size);
|
||||
add(reg_input_shift, tail4_simd_w * wei_type_size);
|
||||
if (do_dequantization) {
|
||||
add(reg_output_scale, tail4_simd_w * wei_type_size);
|
||||
add(reg_output_shift, tail4_simd_w * wei_type_size);
|
||||
}
|
||||
increase_ptrs_if_not_broadcasted(tail4_simd_w * wei_type_size);
|
||||
sub(reg_block_size, tail4_simd_w);
|
||||
|
||||
L(tail_label);
|
||||
|
||||
|
|
@ -602,28 +612,52 @@ private:
|
|||
L(tail_loop_label); {
|
||||
cmp(reg_work_amount, 0);
|
||||
jle(exit_label, T_NEAR);
|
||||
Label end_unroll;
|
||||
|
||||
for (int i = 0; i < jqp_.c % tail4_simd_w; i++) {
|
||||
uni_vmovss(xmm_crop_low(0), ptr[reg_crop_low + i * wei_type_size]);
|
||||
uni_vmovss(xmm_crop_high(0), ptr[reg_crop_high + i * wei_type_size]);
|
||||
uni_vmovss(xmm_input_scale(0), ptr[reg_input_scale + i * wei_type_size]);
|
||||
uni_vmovss(xmm_input_shift(0), ptr[reg_input_shift + i * wei_type_size]);
|
||||
if (do_dequantization) {
|
||||
uni_vmovss(xmm_output_scale(0), ptr[reg_output_scale + i * wei_type_size]);
|
||||
uni_vmovss(xmm_output_shift(0), ptr[reg_output_shift + i * wei_type_size]);
|
||||
auto tail_unroll = [&](size_t iter) {
|
||||
const auto &broadcasted = jqp_.broadcasted;
|
||||
for (int i = 0; i < iter; i++) {
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_LOW)])
|
||||
uni_vmovss(xmm_crop_low(0), ptr[reg_crop_low + i * wei_type_size]);
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_HIGH)])
|
||||
uni_vmovss(xmm_crop_high(0), ptr[reg_crop_high + i * wei_type_size]);
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SCALE)])
|
||||
uni_vmovss(xmm_input_scale(0), ptr[reg_input_scale + i * wei_type_size]);
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SHIFT)])
|
||||
uni_vmovss(xmm_input_shift(0), ptr[reg_input_shift + i * wei_type_size]);
|
||||
if (do_dequantization) {
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SCALE)])
|
||||
uni_vmovss(xmm_output_scale(0), ptr[reg_output_scale + i * wei_type_size]);
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SHIFT)])
|
||||
uni_vmovss(xmm_output_shift(0), ptr[reg_output_shift + i * wei_type_size]);
|
||||
}
|
||||
|
||||
load_scalar(xmm_val(0), ptr[aux_reg_from + i * src_type_size], jqp_.src_prc);
|
||||
|
||||
uni_vminps(xmm_val(0), xmm_val(0), xmm_crop_high(0));
|
||||
uni_vmaxps(xmm_val(0), xmm_val(0), xmm_crop_low(0));
|
||||
uni_vfmadd213ps(xmm_val(0), xmm_input_scale(0), xmm_input_shift(0));
|
||||
if (do_rounding) uni_vroundps(xmm_val(0), xmm_val(0), 0);
|
||||
if (do_dequantization) uni_vfmadd213ps(xmm_val(0), xmm_output_scale(0), xmm_output_shift(0));
|
||||
|
||||
store_scalar(ptr[aux_reg_to + i * dst_type_size], xmm_val(0), jqp_.dst_prc);
|
||||
}
|
||||
jmp(end_unroll, T_NEAR);
|
||||
};
|
||||
|
||||
load_scalar(xmm_val(0), ptr[aux_reg_from + i * src_type_size], jqp_.src_prc);
|
||||
|
||||
uni_vminps(xmm_val(0), xmm_val(0), xmm_crop_high(0));
|
||||
uni_vmaxps(xmm_val(0), xmm_val(0), xmm_crop_low(0));
|
||||
uni_vfmadd213ps(xmm_val(0), xmm_input_scale(0), xmm_input_shift(0));
|
||||
if (do_rounding) uni_vroundps(xmm_val(0), xmm_val(0), 0);
|
||||
if (do_dequantization) uni_vfmadd213ps(xmm_val(0), xmm_output_scale(0), xmm_output_shift(0));
|
||||
|
||||
store_scalar(ptr[aux_reg_to + i * dst_type_size], xmm_val(0), jqp_.dst_prc);
|
||||
std::array<Label, tail4_simd_w> unroll_labels;
|
||||
for (size_t i = 1; i < tail4_simd_w; ++i) {
|
||||
cmp(reg_block_size, i);
|
||||
je(unroll_labels[i], T_NEAR);
|
||||
}
|
||||
|
||||
for (size_t i = 1; i < tail4_simd_w; ++i) {
|
||||
L(unroll_labels[i]);
|
||||
tail_unroll(i);
|
||||
}
|
||||
|
||||
L(end_unroll);
|
||||
|
||||
dec(reg_work_amount);
|
||||
add(aux_reg_from, reg_src_step);
|
||||
add(aux_reg_to, reg_dst_step);
|
||||
|
|
@ -895,29 +929,39 @@ namespace {
|
|||
struct FakeQuantKey {
|
||||
jit_quantize_params jqp;
|
||||
size_t hash() const {
|
||||
using namespace dnnl::impl::primitive_hashing;
|
||||
size_t seed = 0;
|
||||
seed = hash_combine(seed, jqp.c);
|
||||
seed = hash_combine(seed, jqp.is_planar);
|
||||
seed = hash_combine(seed, jqp.src_prc.getPrecVal());
|
||||
seed = hash_combine(seed, jqp.wei_prc.getPrecVal());
|
||||
seed = hash_combine(seed, jqp.dst_prc.getPrecVal());
|
||||
seed = dnnl::impl::primitive_hashing::get_vector_hash(seed, jqp.s_str);
|
||||
seed = dnnl::impl::primitive_hashing::get_vector_hash(seed, jqp.d_str);
|
||||
seed = hash_combine(seed, jqp.op_type);
|
||||
if (jqp.op_type == Algorithm::FQBinarization) {
|
||||
seed = hash_combine(seed, jqp.c);
|
||||
} else {
|
||||
seed = hash_combine(seed, jqp.broadcasted);
|
||||
}
|
||||
return seed;
|
||||
}
|
||||
|
||||
bool operator==(const FakeQuantKey& rhs) const {
|
||||
bool result = jqp.c == rhs.jqp.c && jqp.is_planar == rhs.jqp.is_planar && jqp.src_prc == rhs.jqp.src_prc &&
|
||||
bool result = jqp.is_planar == rhs.jqp.is_planar && jqp.src_prc == rhs.jqp.src_prc &&
|
||||
jqp.wei_prc == rhs.jqp.wei_prc && jqp.dst_prc == rhs.jqp.dst_prc &&
|
||||
jqp.op_type == rhs.jqp.op_type && jqp.s_str == rhs.jqp.s_str && jqp.d_str == rhs.jqp.d_str;
|
||||
jqp.op_type == rhs.jqp.op_type;
|
||||
if (result) {
|
||||
if (jqp.op_type == Algorithm::FQBinarization) {
|
||||
result = result && jqp.c == rhs.jqp.c;
|
||||
} else {
|
||||
result = result && jqp.broadcasted == rhs.jqp.broadcasted;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
FakeQuantize::FakeQuantize(const std::shared_ptr<ngraph::Node>& op, const GraphContext::CPtr context) :
|
||||
Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) {
|
||||
Node(op, context, PassThroughShapeInferFactory()) {
|
||||
std::string errorMessage;
|
||||
if (isSupportedOperation(op, errorMessage)) {
|
||||
algorithm = Algorithm::FQCommon;
|
||||
|
|
@ -1096,6 +1140,13 @@ FakeQuantize::FakeQuantize(const std::shared_ptr<ngraph::Node>& op, const GraphC
|
|||
outputScaleSize = outputScale.size();
|
||||
outputShiftSize = outputShift.size();
|
||||
|
||||
broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_LOW)] = cropLowSize == 1;
|
||||
broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_HIGH)] = cropHighSize == 1;
|
||||
broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SCALE)] = inputScaleSize == 1;
|
||||
broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SHIFT)] = inputShiftSize == 1;
|
||||
broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SCALE)] = outputScaleSize == 1;
|
||||
broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SHIFT)] = outputShiftSize == 1;
|
||||
|
||||
if (everyone_is(1u, cropLowSize, cropHighSize, inputScaleSize, inputShiftSize, outputScaleSize, outputShiftSize))
|
||||
broadcastingPolicy = PerTensor;
|
||||
else if (one_of(1u, cropLowSize, cropHighSize, inputScaleSize, inputShiftSize, outputScaleSize, outputShiftSize))
|
||||
|
|
@ -1285,7 +1336,15 @@ void FakeQuantize::initSupportedPrimitiveDescriptors() {
|
|||
}
|
||||
}
|
||||
|
||||
for (auto& fmt : getDataFormats()) {
|
||||
std::vector<LayoutType> dataFormats;
|
||||
// reference implementation supports only planar format
|
||||
if (impl_type == impl_desc_type::ref) {
|
||||
dataFormats.push_back(LayoutType::ncsp);
|
||||
} else {
|
||||
dataFormats = getDataFormats();
|
||||
}
|
||||
|
||||
for (auto& fmt : dataFormats) {
|
||||
NodeConfig config;
|
||||
config.dynBatchSupport = true;
|
||||
for (size_t i = 0; i < getParentEdges().size(); i++) {
|
||||
|
|
@ -1315,32 +1374,33 @@ void FakeQuantize::initSupportedPrimitiveDescriptors() {
|
|||
}
|
||||
|
||||
bool FakeQuantize::needPrepareParams() const {
|
||||
auto selectedPrimitiveDescriptor = getSelectedPrimitiveDescriptor();
|
||||
if (!selectedPrimitiveDescriptor)
|
||||
IE_THROW() << "CPU quantize node with name '" << getName() << "' doesn't have primitive descriptors.";
|
||||
if (isBinarization()) {
|
||||
auto selectedPrimitiveDescriptor = getSelectedPrimitiveDescriptor();
|
||||
if (!selectedPrimitiveDescriptor)
|
||||
IE_THROW() << "CPU quantize node with name '" << getName() << "' doesn't have primitive descriptors.";
|
||||
|
||||
if (internalBlobMemory.empty() || (selectedPrimitiveDescriptor->getImplementationType() != impl_desc_type::ref && inputShapesModified())) {
|
||||
return true;
|
||||
if (internalBlobMemory.empty() || (selectedPrimitiveDescriptor->getImplementationType() != impl_desc_type::ref && inputShapesModified())) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const auto axisSize = getParentEdgesAtPort(0)[0]->getMemory().getStaticDims()[getAxis()];
|
||||
const auto newPaddedSize = rnd_up(axisSize, 16);
|
||||
const auto currPaddedSize = rnd_up(currentAxisSize, 16);
|
||||
|
||||
return newPaddedSize != currPaddedSize || ((isInputLowBroadcasted || isOutputHighBroadcasted) && axisSize != currentAxisSize);
|
||||
}
|
||||
|
||||
const auto axisSize = getParentEdgesAtPort(0)[0]->getMemory().getStaticDims()[getAxis()];
|
||||
const auto newPaddedSize = rnd_up(axisSize, 16);
|
||||
const auto currPaddedSize = rnd_up(currentAxisSize, 16);
|
||||
|
||||
return newPaddedSize != currPaddedSize || (isBinarization() && (isInputLowBroadcasted || isOutputHighBroadcasted) &&
|
||||
axisSize != currentAxisSize);
|
||||
return false;
|
||||
}
|
||||
|
||||
void FakeQuantize::prepareParams() {
|
||||
const size_t axisSize = getParentEdgesAtPort(0)[0]->getMemory().GetShape().getStaticDims()[getAxis()];
|
||||
const size_t newPaddedSize = rnd_up(axisSize, 16);
|
||||
IE_ASSERT(newPaddedSize != 0);
|
||||
if (isBinarization()) {
|
||||
const size_t axisSize = getParentEdgeAt(0)->getMemory().GetShape().getStaticDims()[getAxis()];
|
||||
const size_t newPaddedSize = rnd_up(axisSize, 16);
|
||||
IE_ASSERT(newPaddedSize != 0);
|
||||
|
||||
if (internalBlobMemory.empty() || newPaddedSize != rnd_up(currentAxisSize, 16) ||
|
||||
(isBinarization() && (isInputLowBroadcasted || isOutputHighBroadcasted) && axisSize != currentAxisSize)) {
|
||||
DnnlBlockedMemoryDesc weightsDataDesc(Shape(VectorDims{newPaddedSize}), memory::data_type::f32, memory::format_tag::x);
|
||||
|
||||
if (isBinarization()) {
|
||||
if (internalBlobMemory.empty() || newPaddedSize != rnd_up(currentAxisSize, 16) ||
|
||||
((isInputLowBroadcasted || isOutputHighBroadcasted) && axisSize != currentAxisSize)) {
|
||||
DnnlBlockedMemoryDesc weightsDataDesc(Shape(VectorDims{newPaddedSize}), memory::data_type::f32, memory::format_tag::x);
|
||||
constexpr size_t numBinFqIntBlob = 2;
|
||||
bool needUpdThr = false, needUpdMask = false;
|
||||
if (isInputLowBroadcasted && axisSize != currentAxisSize) {
|
||||
|
|
@ -1376,65 +1436,49 @@ void FakeQuantize::prepareParams() {
|
|||
internalBlobMemory[1] = binarizationMaskDataMem;
|
||||
}
|
||||
}
|
||||
} else if (levels != 2) {
|
||||
constexpr size_t numFqIntBlob = 6;
|
||||
|
||||
auto pushInternalBlob = [&](std::vector<float>& data, size_t idx) {
|
||||
auto memory = std::make_shared<Memory>(getEngine());
|
||||
bool needOverwrite = getInputShapeAtPort(0).getDims()[getAxis()] == Shape::UNDEFINED_DIM && data.size() == 1;
|
||||
if (needOverwrite) {
|
||||
memory->Create(weightsDataDesc);
|
||||
float *ptr = reinterpret_cast<float *>(memory->GetPtr());
|
||||
std::fill(ptr, ptr + newPaddedSize, data[0]);
|
||||
} else {
|
||||
if (data.size() == 1) {
|
||||
data.resize(newPaddedSize, data[0]);
|
||||
} else {
|
||||
data.resize(newPaddedSize);
|
||||
}
|
||||
memory->Create(weightsDataDesc, &data[0]);
|
||||
}
|
||||
|
||||
if (internalBlobMemory.size() != numFqIntBlob) {
|
||||
internalBlobMemory.push_back(memory);
|
||||
} else if (needOverwrite) {
|
||||
internalBlobMemory[idx] = memory;
|
||||
}
|
||||
};
|
||||
|
||||
pushInternalBlob(cropLow, 0);
|
||||
pushInternalBlob(cropHigh, 1);
|
||||
pushInternalBlob(inputScale, 2);
|
||||
pushInternalBlob(inputShift, 3);
|
||||
pushInternalBlob(outputScale, 4);
|
||||
pushInternalBlob(outputShift, 5);
|
||||
} else {
|
||||
IE_THROW() << "Can't fill internal blob for FakeQuantize node with name: " << getName();
|
||||
}
|
||||
currentAxisSize = axisSize;
|
||||
}
|
||||
currentAxisSize = axisSize;
|
||||
}
|
||||
|
||||
void FakeQuantize::createPrimitive() {
|
||||
Node::createPrimitive();
|
||||
auto selectedPrimitiveDescriptor = getSelectedPrimitiveDescriptor();
|
||||
if (!selectedPrimitiveDescriptor)
|
||||
IE_THROW() << "CPU quantize node with name '" << getName() << "' doesn't have primitive descriptors.";
|
||||
if (selectedPrimitiveDescriptor->getImplementationType() != impl_desc_type::ref) {
|
||||
const auto& config = getSelectedPrimitiveDescriptor()->getConfig();
|
||||
const auto& inDims = getParentEdgesAtPort(0)[0]->getMemory().getStaticDims();
|
||||
|
||||
//Form FakeQuanKey
|
||||
FakeQuantKey key = {};
|
||||
key.jqp.c = inDims.size() > 1 ? inDims[1] : 1;
|
||||
key.jqp.src_prc = config.inConfs[0].getMemDesc()->getPrecision();
|
||||
key.jqp.wei_prc = Precision::FP32;
|
||||
key.jqp.dst_prc = config.outConfs[0].getMemDesc()->getPrecision();
|
||||
|
||||
auto srcDesc = getParentEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>();
|
||||
key.jqp.s_str = srcDesc->getStrides();
|
||||
auto dstDesc = getChildEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>();
|
||||
const auto &srcMemory = getParentEdgeAt(0)->getMemory();
|
||||
const auto &srcDesc = srcMemory.getDesc();
|
||||
|
||||
key.jqp.d_str = dstDesc->getStrides();
|
||||
key.jqp.is_planar = srcDesc->hasLayoutType(LayoutType::ncsp) && one_of(srcDesc->getShape().getRank(), 3u, 4u, 5u);
|
||||
key.jqp.is_planar = srcDesc.hasLayoutType(LayoutType::ncsp) && one_of(srcDesc.getShape().getRank(), 3, 4, 5);
|
||||
key.jqp.op_type = getAlgorithm();
|
||||
|
||||
if (isBinarization()) {
|
||||
const auto &inDims = srcMemory.getStaticDims();
|
||||
key.jqp.c = inDims.size() > 1 ? inDims[1] : 1;
|
||||
} else {
|
||||
// in case of blocked layout we need to extend vectors to prevent read from unallocated memory
|
||||
size_t paddedSize = srcDesc.hasLayoutType(LayoutType::nCsp16c) ? 16 : srcDesc.hasLayoutType(LayoutType::nCsp8c) ? 8 : 1;
|
||||
if (paddedSize != 1) {
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_LOW)]) cropLow.resize(rnd_up(cropLow.size(), paddedSize));
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_HIGH)]) cropHigh.resize(rnd_up(cropHigh.size(), paddedSize));
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SCALE)]) inputScale.resize(rnd_up(inputScale.size(), paddedSize));
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SHIFT)]) inputShift.resize(rnd_up(inputShift.size(), paddedSize));
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SCALE)]) outputScale.resize(rnd_up(outputScale.size(), paddedSize));
|
||||
if (!broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SHIFT)]) outputShift.resize(rnd_up(outputShift.size(), paddedSize));
|
||||
}
|
||||
|
||||
key.jqp.broadcasted = broadcasted;
|
||||
}
|
||||
|
||||
auto cache = context->getParamsCache();
|
||||
auto buildExecutor = [](const FakeQuantKey& key) {
|
||||
return std::make_shared<FakeQuantizeJitExecutor>(key.jqp);
|
||||
|
|
@ -1513,13 +1557,6 @@ void FakeQuantize::executeReference() {
|
|||
} else {
|
||||
auto dst = reinterpret_cast<float *>(dstMemory->GetPtr());
|
||||
|
||||
auto crop_low = reinterpret_cast<const float*>(internalBlobMemory[0]->GetData());
|
||||
auto crop_high = reinterpret_cast<const float*>(internalBlobMemory[1]->GetData());
|
||||
auto input_scale = reinterpret_cast<const float*>(internalBlobMemory[2]->GetData());
|
||||
auto input_shift = reinterpret_cast<const float*>(internalBlobMemory[3]->GetData());
|
||||
auto output_scale = reinterpret_cast<const float*>(internalBlobMemory[4]->GetData());
|
||||
auto output_shift = reinterpret_cast<const float*>(internalBlobMemory[5]->GetData());
|
||||
|
||||
parallel_nd(N, C, D, H, W, [&](dim_t n, dim_t c, dim_t d, dim_t h, dim_t w) {
|
||||
size_t src_off = srcDims.size() == 5 ?
|
||||
n * s_str[0] + c * s_str[1] + d * s_str[2] + h * s_str[3] + w * s_str[4] :
|
||||
|
|
@ -1534,12 +1571,12 @@ void FakeQuantize::executeReference() {
|
|||
float src_val = src[src_off];
|
||||
|
||||
int wei_idx = getAxis() == 0 ? n : c;
|
||||
float cl = crop_low[wei_idx];
|
||||
float ch = crop_high[wei_idx];
|
||||
float isc = input_scale[wei_idx];
|
||||
float ish = input_shift[wei_idx];
|
||||
float osc = output_scale[wei_idx];
|
||||
float osh = output_shift[wei_idx];
|
||||
float cl = broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_LOW)] ? cropLow[0] : cropLow[wei_idx];
|
||||
float ch = broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_HIGH)] ? cropHigh[0] : cropHigh[wei_idx];
|
||||
float isc = broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SCALE)] ? inputScale[0] : inputScale[wei_idx];
|
||||
float ish = broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SHIFT)] ? inputShift[0] : inputShift[wei_idx];
|
||||
float osc = broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SCALE)] ? outputScale[0] : outputScale[wei_idx];
|
||||
float osh = broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SHIFT)] ? outputShift[0] : outputShift[wei_idx];
|
||||
|
||||
float dst_val = nstl::min(ch, nstl::max(cl, src_val));
|
||||
dst_val = dst_val * isc + ish;
|
||||
|
|
@ -1562,7 +1599,7 @@ void FakeQuantize::executeReference() {
|
|||
}
|
||||
|
||||
void FakeQuantize::executeBinarization(const std::unique_ptr<jit_uni_quantize_kernel> &pKernel) const {
|
||||
auto &srcMemory = getParentEdgeAt(0)->getMemoryPtr();
|
||||
const auto &srcMemory = getParentEdgeAt(0)->getMemoryPtr();
|
||||
auto &dstMemory = getChildEdgeAt(0)->getMemoryPtr();
|
||||
|
||||
auto src = reinterpret_cast<const uint8_t *>(srcMemory->GetPtr());
|
||||
|
|
@ -1573,8 +1610,8 @@ void FakeQuantize::executeBinarization(const std::unique_ptr<jit_uni_quantize_ke
|
|||
|
||||
auto src_dims = srcMemory->getStaticDims();
|
||||
|
||||
const auto &jqp = pKernel->jqp_;
|
||||
std::vector<size_t> s_str = jqp.s_str;
|
||||
auto srcMemDesc = srcMemory->GetDescWithType<BlockedMemoryDesc>();
|
||||
std::vector<size_t> s_str = srcMemDesc->getStrides();
|
||||
size_t tmp = s_str[s_str.size() - 1];
|
||||
for (int i = s_str.size() - 1; i > 1; i--) {
|
||||
s_str[i] = s_str[i - 1];
|
||||
|
|
@ -1608,13 +1645,6 @@ void FakeQuantize::executeQuantization(const std::unique_ptr<jit_uni_quantize_ke
|
|||
auto src = reinterpret_cast<const uint8_t *>(srcMemory->GetPtr());
|
||||
auto dst = reinterpret_cast<uint8_t *>(dstMemory->GetPtr());
|
||||
|
||||
auto crop_low = reinterpret_cast<const float*>(internalBlobMemory[0]->GetData());
|
||||
auto crop_high = reinterpret_cast<const float*>(internalBlobMemory[1]->GetData());
|
||||
auto input_scale = reinterpret_cast<const float*>(internalBlobMemory[2]->GetData());
|
||||
auto input_shift = reinterpret_cast<const float*>(internalBlobMemory[3]->GetData());
|
||||
auto output_scale = reinterpret_cast<const float*>(internalBlobMemory[4]->GetData());
|
||||
auto output_shift = reinterpret_cast<const float*>(internalBlobMemory[5]->GetData());
|
||||
|
||||
auto& srcDesc = srcMemory->getDesc();
|
||||
auto srcDims = srcDesc.getShape().getStaticDims();
|
||||
|
||||
|
|
@ -1626,7 +1656,8 @@ void FakeQuantize::executeQuantization(const std::unique_ptr<jit_uni_quantize_ke
|
|||
auto src_type_size = jqp.src_prc.size();
|
||||
auto dst_type_size = jqp.dst_prc.size();
|
||||
|
||||
auto s_str = jqp.s_str;
|
||||
auto srcMemDesc = srcMemory->GetDescWithType<BlockedMemoryDesc>();
|
||||
auto s_str = srcMemDesc->getStrides();
|
||||
|
||||
if (is_blk_format) {
|
||||
s_str[1] /= blk_size;
|
||||
|
|
@ -1657,12 +1688,12 @@ void FakeQuantize::executeQuantization(const std::unique_ptr<jit_uni_quantize_ke
|
|||
|
||||
arg.from = &src[data_off * src_type_size];
|
||||
arg.to = &dst[data_off * dst_type_size];
|
||||
arg.crop_low = &crop_low[c];
|
||||
arg.crop_high = &crop_high[c];
|
||||
arg.input_scale = &input_scale[c];
|
||||
arg.input_shift = &input_shift[c];
|
||||
arg.output_scale = &output_scale[c];
|
||||
arg.output_shift = &output_shift[c];
|
||||
arg.crop_low = broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_LOW)] ? &cropLow[0] : &cropLow[c];
|
||||
arg.crop_high = broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_HIGH)] ? &cropHigh[0] : &cropHigh[c];
|
||||
arg.input_scale = broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SCALE)] ? &inputScale[0] : &inputScale[c];
|
||||
arg.input_shift = broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SHIFT)] ? &inputShift[0] : &inputShift[c];
|
||||
arg.output_scale = broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SCALE)] ? &outputScale[0] : &outputScale[c];
|
||||
arg.output_shift = broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SHIFT)] ? &outputShift[0] : &outputShift[c];
|
||||
|
||||
arg.src_step = (size_t) blk_size * src_type_size;
|
||||
arg.dst_step = (size_t) blk_size * dst_type_size;
|
||||
|
|
@ -1687,12 +1718,12 @@ void FakeQuantize::executeQuantization(const std::unique_ptr<jit_uni_quantize_ke
|
|||
|
||||
arg.from = &src[data_off * src_type_size];
|
||||
arg.to = &dst[data_off * dst_type_size];
|
||||
arg.crop_low = &crop_low[c];
|
||||
arg.crop_high = &crop_high[c];
|
||||
arg.input_scale = &input_scale[c];
|
||||
arg.input_shift = &input_shift[c];
|
||||
arg.output_scale = &output_scale[c];
|
||||
arg.output_shift = &output_shift[c];
|
||||
arg.crop_low = broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_LOW)] ? &cropLow[0] : &cropLow[c];
|
||||
arg.crop_high = broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_HIGH)] ? &cropHigh[0] : &cropHigh[c];
|
||||
arg.input_scale = broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SCALE)] ? &inputScale[0] : &inputScale[c];
|
||||
arg.input_shift = broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SHIFT)] ? &inputShift[0] : &inputShift[c];
|
||||
arg.output_scale = broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SCALE)] ? &outputScale[0] : &outputScale[c];
|
||||
arg.output_shift = broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SHIFT)] ? &outputShift[0] : &outputShift[c];
|
||||
|
||||
arg.src_step = is_blk_format ? (size_t) blk_size * src_type_size : (size_t) C * src_type_size;
|
||||
arg.dst_step = is_blk_format ? (size_t) blk_size * dst_type_size : (size_t) C * dst_type_size;
|
||||
|
|
@ -1715,12 +1746,12 @@ void FakeQuantize::executeQuantization(const std::unique_ptr<jit_uni_quantize_ke
|
|||
|
||||
arg.from = &src[data_off * src_type_size];
|
||||
arg.to = &dst[data_off * dst_type_size];
|
||||
arg.crop_low = &crop_low[c];
|
||||
arg.crop_high = &crop_high[c];
|
||||
arg.input_scale = &input_scale[c];
|
||||
arg.input_shift = &input_shift[c];
|
||||
arg.output_scale = &output_scale[c];
|
||||
arg.output_shift = &output_shift[c];
|
||||
arg.crop_low = broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_LOW)] ? &cropLow[0] : &cropLow[c];
|
||||
arg.crop_high = broadcasted[static_cast<size_t>(FQ_add_input_type::CROP_HIGH)] ? &cropHigh[0] : &cropHigh[c];
|
||||
arg.input_scale = broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SCALE)] ? &inputScale[0] : &inputScale[c];
|
||||
arg.input_shift = broadcasted[static_cast<size_t>(FQ_add_input_type::INPUT_SHIFT)] ? &inputShift[0] : &inputShift[c];
|
||||
arg.output_scale = broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SCALE)] ? &outputScale[0] : &outputScale[c];
|
||||
arg.output_shift = broadcasted[static_cast<size_t>(FQ_add_input_type::OUTPUT_SHIFT)] ? &outputShift[0] : &outputShift[c];
|
||||
|
||||
arg.src_step = is_blk_format ? (size_t) blk_size * src_type_size : (size_t) C * src_type_size;
|
||||
arg.dst_step = is_blk_format ? (size_t) blk_size * dst_type_size : (size_t) C * dst_type_size;
|
||||
|
|
@ -1737,11 +1768,7 @@ void FakeQuantize::executeDynamicImpl(dnnl::stream strm) {
|
|||
}
|
||||
|
||||
void FakeQuantize::execute(dnnl::stream strm) {
|
||||
auto selectedPrimitiveDescriptor = getSelectedPrimitiveDescriptor();
|
||||
if (!selectedPrimitiveDescriptor)
|
||||
IE_THROW() << "CPU quantize node with name '" << getName() << "' doesn't have primitive descriptors.";
|
||||
|
||||
if (selectedPrimitiveDescriptor->getImplementationType() != impl_desc_type::ref) {
|
||||
if (getSelectedPrimitiveDescriptor()->getImplementationType() != impl_desc_type::ref) {
|
||||
execPtr->exec(*this);
|
||||
} else {
|
||||
executeReference();
|
||||
|
|
|
|||
|
|
@ -17,18 +17,27 @@ namespace ov {
|
|||
namespace intel_cpu {
|
||||
namespace node {
|
||||
|
||||
enum class FQ_add_input_type {
|
||||
CROP_LOW,
|
||||
CROP_HIGH,
|
||||
INPUT_SCALE,
|
||||
INPUT_SHIFT,
|
||||
OUTPUT_SCALE,
|
||||
OUTPUT_SHIFT,
|
||||
INPUTS_SIZE
|
||||
};
|
||||
|
||||
struct jit_quantize_params {
|
||||
int c;
|
||||
bool is_planar;
|
||||
|
||||
InferenceEngine::Precision src_prc;
|
||||
InferenceEngine::Precision wei_prc;
|
||||
InferenceEngine::Precision dst_prc;
|
||||
|
||||
std::vector<size_t> s_str;
|
||||
std::vector<size_t> d_str;
|
||||
|
||||
Algorithm op_type;
|
||||
|
||||
int c; // need only for binarization
|
||||
std::bitset<static_cast<size_t>(FQ_add_input_type::INPUTS_SIZE)> broadcasted; // need only for quantization
|
||||
};
|
||||
|
||||
struct jit_quantize_call_args {
|
||||
|
|
@ -82,6 +91,7 @@ public:
|
|||
|
||||
bool needPrepareParams() const override;
|
||||
void prepareParams() override;
|
||||
void createPrimitive() override;
|
||||
|
||||
const float* getBinarizationTresholdsPtr() const { return &binarizationThresholds[0]; }
|
||||
const float* getBinarizationOutputMaskPtr() const { return reinterpret_cast<const float*>(&binarizationOutputMask[0]); }
|
||||
|
|
@ -240,6 +250,8 @@ private:
|
|||
size_t outputScaleSize;
|
||||
size_t outputShiftSize;
|
||||
|
||||
std::bitset<static_cast<size_t>(FQ_add_input_type::INPUTS_SIZE)> broadcasted;
|
||||
|
||||
std::vector<float> fqScales;
|
||||
|
||||
// version based lazy evaluation, any parameter change increases parameterVersion
|
||||
|
|
|
|||
|
|
@ -0,0 +1,529 @@
|
|||
// Copyright (C) 2023 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
// Motivation:
|
||||
// In a dynamic scenario, depending on the input shapes for the current node, we can either generate a new jit kernel or get an existing one from the cache.
|
||||
// But the current single layer tests do not allow checking the case when the same kernel can be used for different nodes.
|
||||
// This subgraph test contains 2 FQ nodes and allows us to check this case.
|
||||
|
||||
// ------------------------------------ ------------------------------------
|
||||
// | Input 0 | | Input 1 |
|
||||
// ------------------------------------ ------------------------------------
|
||||
// | |
|
||||
// ------------------------------------ ------------------------------------
|
||||
// | FakeQuantize 0 | | FakeQuantize 1 |
|
||||
// ------------------------------------ ------------------------------------
|
||||
// | |
|
||||
// | ------------------------------------
|
||||
// | |Reshape (if !reshapeShape.empty())|
|
||||
// | ------------------------------------
|
||||
// | |
|
||||
// ----------------------------------------------------------------------------
|
||||
// | Concat |
|
||||
// ----------------------------------------------------------------------------
|
||||
// |
|
||||
// --------
|
||||
// |Output|
|
||||
// --------
|
||||
|
||||
#include <shared_test_classes/base/ov_subgraph.hpp>
|
||||
#include <ngraph_functions/builders.hpp>
|
||||
#include <common_test_utils/ov_tensor_utils.hpp>
|
||||
#include "test_utils/cpu_test_utils.hpp"
|
||||
#include "cpp_interfaces/interface/ie_internal_plugin_config.hpp"
|
||||
|
||||
using namespace CPUTestUtils;
|
||||
using namespace ov::test;
|
||||
using namespace InferenceEngine;
|
||||
using namespace ngraph;
|
||||
|
||||
namespace CPUSubgraphTestsDefinitions {
|
||||
|
||||
using InputShapesTuple = std::tuple<
|
||||
std::vector<InputShape>, // fq dynamic data shapes
|
||||
std::vector<std::vector<SizeVector>>, // fq range input shapes
|
||||
std::vector<int32_t> // reshape shape
|
||||
>;
|
||||
|
||||
using FqSpecificParams = std::tuple<int64_t, // 'data' input low bounds
|
||||
int64_t, // 'data' input high bounds
|
||||
std::vector<float>, // output low
|
||||
std::vector<float>, // output high
|
||||
size_t>; // levels
|
||||
|
||||
typedef std::tuple<
|
||||
InputShapesTuple, // fq input shapes and reshape shape
|
||||
FqSpecificParams, // fq specific params
|
||||
std::pair<std::vector<float>, std::vector<float>>, // il and ih values
|
||||
CPUSpecificParams,
|
||||
std::map<std::string, std::string> // Additional config (disable snippets or no)
|
||||
> FakeQuantizeCacheTestParams;
|
||||
|
||||
class FakeQuantizeCacheTest : public testing::WithParamInterface<FakeQuantizeCacheTestParams>,
|
||||
virtual public SubgraphBaseTest, public CPUTestsBase {
|
||||
public:
|
||||
static std::string getTestCaseName(const testing::TestParamInfo<FakeQuantizeCacheTestParams> &obj) {
|
||||
InputShapesTuple inputShapesTuple;
|
||||
FqSpecificParams fqParams;
|
||||
std::pair<std::vector<float>, std::vector<float>> inputRangesValues;
|
||||
CPUSpecificParams cpuParams;
|
||||
std::map<std::string, std::string> additionalConfig;
|
||||
std::tie(inputShapesTuple, fqParams, inputRangesValues, cpuParams, additionalConfig) = obj.param;
|
||||
|
||||
std::vector<InputShape> shapes;
|
||||
std::vector<std::vector<SizeVector>> ranges;
|
||||
std::vector<int32_t> reshapeShape;
|
||||
std::tie(shapes, ranges, reshapeShape) = inputShapesTuple;
|
||||
|
||||
int64_t inDataLowBounds, inDataHighBounds;
|
||||
std::vector<float> inputLow, inputHigh, outputLow, outputHigh;
|
||||
size_t levels;
|
||||
inputLow = inputRangesValues.first;
|
||||
inputHigh = inputRangesValues.second;
|
||||
std::tie(inDataLowBounds, inDataHighBounds, outputLow, outputHigh, levels) = fqParams;
|
||||
|
||||
std::ostringstream results;
|
||||
|
||||
for (int i = 0; i < shapes.size(); i++) {
|
||||
results << "FQ" << i << "_IS=(" << CommonTestUtils::partialShape2str({shapes[i].first}) << ")_";
|
||||
results << "TS=";
|
||||
for (const auto& shape : shapes[i].second) {
|
||||
results << "(" << CommonTestUtils::vec2str(shape) << ")_";
|
||||
}
|
||||
results << "RS=";
|
||||
for (const auto& range : ranges[i]) {
|
||||
results << "(" << CommonTestUtils::vec2str(range) << ")_";
|
||||
}
|
||||
}
|
||||
if (!reshapeShape.empty()) {
|
||||
results << "ReshapeShape=(" << CommonTestUtils::vec2str(reshapeShape) << ")_";
|
||||
}
|
||||
|
||||
results << "LOW_BOUNDS=" << inDataLowBounds << "_";
|
||||
results << "HIGH_BOUNDS=" << inDataHighBounds << "_";
|
||||
results << "IL=" << CommonTestUtils::vec2str(inputLow) << "_";
|
||||
results << "IH=" << CommonTestUtils::vec2str(inputHigh) << "_";
|
||||
results << "OL=" << CommonTestUtils::vec2str(outputLow) << "_";
|
||||
results << "OH=" << CommonTestUtils::vec2str(outputHigh) << "_";
|
||||
results << "LEVELS=" << levels;
|
||||
|
||||
results << CPUTestsBase::getTestCaseName(cpuParams);
|
||||
|
||||
if (!additionalConfig.empty()) {
|
||||
results << "_PluginConf";
|
||||
for (auto& item : additionalConfig) {
|
||||
results << "_" << item.first << "=" << item.second;
|
||||
}
|
||||
}
|
||||
|
||||
return results.str();
|
||||
}
|
||||
|
||||
protected:
|
||||
void SetUp() override {
|
||||
abs_threshold = 0.01f;
|
||||
|
||||
InputShapesTuple inputShapesTuple;
|
||||
FqSpecificParams fqParams;
|
||||
std::pair<std::vector<float>, std::vector<float>> inputRangesValues;
|
||||
CPUSpecificParams cpuParams;
|
||||
std::map<std::string, std::string> additionalConfig;
|
||||
std::tie(inputShapesTuple, fqParams, inputRangesValues,
|
||||
cpuParams, additionalConfig) = this->GetParam();
|
||||
|
||||
std::vector<InputShape> shapesVec;
|
||||
std::vector<std::vector<SizeVector>> rangesVec;
|
||||
std::vector<int32_t> reshapeShape;
|
||||
std::tie(shapesVec, rangesVec, reshapeShape) = inputShapesTuple;
|
||||
|
||||
std::tie(inFmts, outFmts, priority, selectedType) = cpuParams;
|
||||
|
||||
configuration.insert(additionalConfig.begin(), additionalConfig.end());
|
||||
targetDevice = CommonTestUtils::DEVICE_CPU;
|
||||
|
||||
init_input_shapes(shapesVec);
|
||||
|
||||
size_t levels;
|
||||
std::vector<std::vector<float>> rangesBounds(RANGES_INPUT_NUMBER);
|
||||
rangesBounds[0] = inputRangesValues.first;
|
||||
rangesBounds[1] = inputRangesValues.second;
|
||||
std::tie(inDataLowBounds, inDataHighBounds, rangesBounds[2], rangesBounds[3], levels) = fqParams;
|
||||
|
||||
ParameterVector ngraphParam;
|
||||
std::vector<std::shared_ptr<Node>> ngraphInputs;
|
||||
|
||||
auto ngInPrec = element::Type_t::f32;
|
||||
|
||||
for (size_t i = 0; i < inputDynamicShapes.size(); i++) {
|
||||
ngraphParam.push_back(std::make_shared<opset1::Parameter>(ngInPrec, inputDynamicShapes[i]));
|
||||
ngraphInputs.push_back(ngraphParam.back());
|
||||
}
|
||||
|
||||
auto makeFQ = [&](int i) {
|
||||
auto extendData = [](const std::vector<float> &data, size_t newSize) {
|
||||
std::vector<float> extendedData(newSize);
|
||||
size_t oldSize = data.size();
|
||||
for (size_t i = 0; i < newSize; i++) {
|
||||
extendedData[i] = data[i % oldSize];
|
||||
}
|
||||
return extendedData;
|
||||
};
|
||||
|
||||
auto ranges = rangesVec[i];
|
||||
|
||||
auto il = builder::makeConstant(ngInPrec, ranges[0], extendData(rangesBounds[0],
|
||||
std::accumulate(ranges[0].begin(), ranges[0].end(), 1, std::multiplies<size_t>())));
|
||||
auto ih = builder::makeConstant(ngInPrec, ranges[1], extendData(rangesBounds[1],
|
||||
std::accumulate(ranges[1].begin(), ranges[1].end(), 1, std::multiplies<size_t>())));
|
||||
auto ol = builder::makeConstant(ngInPrec, ranges[2], extendData(rangesBounds[2],
|
||||
std::accumulate(ranges[2].begin(), ranges[2].end(), 1, std::multiplies<size_t>())));
|
||||
auto oh = builder::makeConstant(ngInPrec, ranges[3], extendData(rangesBounds[3],
|
||||
std::accumulate(ranges[3].begin(), ranges[3].end(), 1, std::multiplies<size_t>())));
|
||||
|
||||
auto fqNode = std::make_shared<opset5::FakeQuantize>(ngraphParam[i], il, ih, ol, oh, levels);
|
||||
fqNode->get_rt_info() = getCPUInfo();
|
||||
return fqNode;
|
||||
};
|
||||
|
||||
std::shared_ptr<Node> lastNode0 = makeFQ(0);
|
||||
std::shared_ptr<Node> lastNode1 = makeFQ(1);
|
||||
|
||||
if (!reshapeShape.empty()) {
|
||||
auto reshapeConstNode = builder::makeConstant(::element::Type(::element::Type_t::i32),
|
||||
{reshapeShape.size()}, reshapeShape);
|
||||
lastNode1 = std::make_shared<opset5::Reshape>(lastNode1, reshapeConstNode, false);
|
||||
}
|
||||
auto concat = builder::makeConcat({lastNode0, lastNode1}, 0);
|
||||
|
||||
if (selectedType.empty()) {
|
||||
selectedType = getPrimitiveType() + "_FP32";
|
||||
}
|
||||
|
||||
function = std::make_shared<Function>(concat, ngraphParam, "fq_cache");
|
||||
}
|
||||
|
||||
void generate_inputs(const std::vector<ov::Shape>& targetInputStaticShapes) override {
|
||||
inputs.clear();
|
||||
const auto& funcInputs = function->inputs();
|
||||
for (int i = 0; i < funcInputs.size(); ++i) {
|
||||
const auto& funcInput = funcInputs[i];
|
||||
ov::Tensor tensor;
|
||||
tensor = ov::test::utils::create_and_fill_tensor(funcInput.get_element_type(),
|
||||
targetInputStaticShapes[i],
|
||||
inDataHighBounds - inDataLowBounds,
|
||||
inDataLowBounds);
|
||||
inputs.insert({funcInput.get_node_shared_ptr(), tensor});
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const size_t RANGES_INPUT_NUMBER = 4;
|
||||
|
||||
int64_t inDataLowBounds, inDataHighBounds;
|
||||
};
|
||||
|
||||
TEST_P(FakeQuantizeCacheTest, CompareWithRefs) {
|
||||
run();
|
||||
|
||||
CheckPluginRelatedResults(compiledModel, "FakeQuantize");
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
const std::vector<size_t> levels = {256};
|
||||
|
||||
int64_t dataLowBounds{-10}, dataHighBounds{10};
|
||||
|
||||
const std::vector<std::pair<std::vector<float>, std::vector<float>>> inputRanges = {
|
||||
{{0.0f, 1.0f}, {5.0f, 6.0f}},
|
||||
};
|
||||
|
||||
const std::vector<float> outputLow{5.0f, 6.0f}, outputHigh{25.0f, 31.0f};
|
||||
|
||||
const auto specificParams = ::testing::Combine(::testing::Values(dataLowBounds),
|
||||
::testing::Values(dataHighBounds),
|
||||
::testing::Values(outputLow),
|
||||
::testing::Values(outputHigh),
|
||||
::testing::ValuesIn(levels));
|
||||
|
||||
const std::map<std::string, std::string> emptyConfig = {};
|
||||
const std::map<std::string, std::string> disableSnippets = {
|
||||
{PluginConfigInternalParams::KEY_SNIPPETS_MODE, PluginConfigInternalParams::DISABLE}};
|
||||
|
||||
|
||||
// 3D
|
||||
std::vector<CPUSpecificParams> cpuParams_3D = {
|
||||
CPUSpecificParams({ncw}, {ncw}, {}, {}),
|
||||
};
|
||||
|
||||
std::vector<InputShapesTuple> inputShapes_3D = {
|
||||
{
|
||||
// fq dynamic data shapes
|
||||
{
|
||||
// input0
|
||||
{{-1, -1, 43}, {{1, 10, 43}, {1, 20, 43}, {1, 10, 43}, {1, 20, 43}}},
|
||||
// input1
|
||||
{{-1, -1, 43}, {{1, 10, 43}, {1, 20, 43}, {1, 10, 43}, {1, 20, 43}}},
|
||||
},
|
||||
// fq range input shapes
|
||||
{
|
||||
// input0
|
||||
{{1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}}, // miss
|
||||
// input1
|
||||
{{1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}}, // hit
|
||||
},
|
||||
// reshape shape
|
||||
{},
|
||||
},
|
||||
{
|
||||
// fq dynamic data shapes
|
||||
{
|
||||
// input0
|
||||
{{-1, 10, -1}, {{1, 10, 22}, {1, 10, 44}, {1, 10, 22}, {1, 10, 44}}},
|
||||
// input1
|
||||
{{-1, 10, -1}, {{1, 10, 22}, {1, 10, 44}, {1, 10, 22}, {1, 10, 44}}},
|
||||
},
|
||||
// fq range input shapes
|
||||
{
|
||||
// input0
|
||||
{{1, 10, 1}, {1, 10, 1}, {1, 10, 1}, {1, 10, 1}}, // miss
|
||||
// input1
|
||||
{{1, 10, 1}, {1, 10, 1}, {1, 10, 1}, {1, 10, 1}}, // hit
|
||||
},
|
||||
// reshape shape
|
||||
{},
|
||||
},
|
||||
{
|
||||
// fq dynamic data shapes
|
||||
{
|
||||
// input0
|
||||
{{-1, 10, -1}, {{1, 10, 22}, {1, 10, 44}, {1, 10, 22}, {1, 10, 44}}},
|
||||
// input1
|
||||
{{-1, 10, -1}, {{1, 10, 22}, {1, 10, 44}, {1, 10, 22}, {1, 10, 44}}},
|
||||
},
|
||||
// fq range input shapes
|
||||
{
|
||||
// input0
|
||||
{{1, 10, 1}, {1, 10, 1}, {1, 10, 1}, {1, 10, 1}}, // miss
|
||||
// input1
|
||||
{{1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}}, // miss
|
||||
},
|
||||
// reshape shape
|
||||
{},
|
||||
},
|
||||
{
|
||||
// fq dynamic data shapes
|
||||
{
|
||||
// input0
|
||||
{{-1, 10, -1}, {{1, 10, 22}, {1, 10, 22}}},
|
||||
// input1
|
||||
{{-1, 20, -1}, {{1, 20, 22}, {1, 20, 22}}},
|
||||
},
|
||||
// fq range input shapes
|
||||
{
|
||||
// input0
|
||||
{{1, 10, 1}, {1, 10, 1}, {1, 10, 1}, {1, 10, 1}}, // miss
|
||||
// input1
|
||||
{{1, 20, 1}, {1, 20, 1}, {1, 20, 1}, {1, 20, 1}}, // hit
|
||||
},
|
||||
// reshape shape
|
||||
{-1, 10, 22},
|
||||
},
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(smoke_FakeQuantizeCache_3D, FakeQuantizeCacheTest,
|
||||
::testing::Combine(
|
||||
::testing::ValuesIn(inputShapes_3D),
|
||||
specificParams,
|
||||
::testing::ValuesIn(inputRanges),
|
||||
::testing::ValuesIn(filterCPUSpecificParams(cpuParams_3D)),
|
||||
::testing::Values(disableSnippets)),
|
||||
FakeQuantizeCacheTest::getTestCaseName);
|
||||
|
||||
|
||||
// 4D
|
||||
std::vector<CPUSpecificParams> cpuParams_4D = {
|
||||
CPUSpecificParams({nchw}, {nchw}, {}, {}),
|
||||
CPUSpecificParams({nhwc}, {nhwc}, {}, {}),
|
||||
CPUSpecificParams({nChw16c}, {nChw16c}, {}, {}),
|
||||
};
|
||||
|
||||
std::vector<InputShapesTuple> inputShapes_4D = {
|
||||
{
|
||||
// fq dynamic data shapes
|
||||
{
|
||||
// input0
|
||||
{{-1, -1, -1, 43}, {{1, 17, 3, 43}, {1, 34, 3, 43}, {1, 17, 3, 43}, {1, 34, 3, 43}}},
|
||||
// input1
|
||||
{{-1, -1, -1, 43}, {{1, 17, 3, 43}, {1, 34, 3, 43}, {1, 17, 3, 43}, {1, 34, 3, 43}}},
|
||||
},
|
||||
// fq range input shapes
|
||||
{
|
||||
// input0
|
||||
{{1, 1, 1, 1}, {1, 1, 1, 1}, {1, 1, 1, 1}, {1, 1, 1, 1}}, // miss
|
||||
// input1
|
||||
{{1, 1, 1, 1}, {1, 1, 1, 1}, {1, 1, 1, 1}, {1, 1, 1, 1}}, // hit
|
||||
},
|
||||
// reshape shape
|
||||
{},
|
||||
},
|
||||
{
|
||||
// fq dynamic data shapes
|
||||
{
|
||||
// input0
|
||||
{{-1, 47, -1, -1}, {{1, 47, 2, 22}, {1, 47, 3, 33}, {1, 47, 2, 22}, {1, 47, 3, 33}}},
|
||||
// input1
|
||||
{{-1, 47, -1, -1}, {{1, 47, 2, 22}, {1, 47, 3, 33}, {1, 47, 2, 22}, {1, 47, 3, 33}}},
|
||||
},
|
||||
// fq range input shapes
|
||||
{
|
||||
// input0
|
||||
{{1, 47, 1, 1}, {1, 47, 1, 1}, {1, 47, 1, 1}, {1, 47, 1, 1}}, // miss
|
||||
// input1
|
||||
{{1, 47, 1, 1}, {1, 47, 1, 1}, {1, 47, 1, 1}, {1, 47, 1, 1}}, // hit
|
||||
},
|
||||
// reshape shape
|
||||
{},
|
||||
},
|
||||
{
|
||||
// fq dynamic data shapes
|
||||
{
|
||||
// input0
|
||||
{{-1, 47, -1, -1}, {{1, 47, 2, 22}, {1, 47, 3, 33}, {1, 47, 2, 22}, {1, 47, 3, 33}}},
|
||||
// input1
|
||||
{{-1, 47, -1, -1}, {{1, 47, 2, 22}, {1, 47, 3, 33}, {1, 47, 2, 22}, {1, 47, 3, 33}}},
|
||||
},
|
||||
// fq range input shapes
|
||||
{
|
||||
// input0
|
||||
{{1, 47, 1, 1}, {1, 47, 1, 1}, {1, 47, 1, 1}, {1, 47, 1, 1}}, // miss
|
||||
// input1
|
||||
{{1, 1, 1, 1}, {1, 1, 1, 1}, {1, 1, 1, 1}, {1, 1, 1, 1}}, // miss
|
||||
},
|
||||
// reshape shape
|
||||
{},
|
||||
},
|
||||
{
|
||||
// fq dynamic data shapes
|
||||
{
|
||||
// input0
|
||||
{{-1, 17, -1, -1}, {{1, 17, 2, 22}, {1, 17, 2, 22}}},
|
||||
// input1
|
||||
{{-1, 34, -1, -1}, {{1, 34, 2, 22}, {1, 34, 2, 22}}},
|
||||
},
|
||||
// fq range input shapes
|
||||
{
|
||||
// input0
|
||||
{{1, 17, 1, 1}, {1, 17, 1, 1}, {1, 17, 1, 1}, {1, 17, 1, 1}}, // miss
|
||||
// input1
|
||||
{{1, 34, 1, 1}, {1, 34, 1, 1}, {1, 34, 1, 1}, {1, 34, 1, 1}}, // hit
|
||||
},
|
||||
// reshape shape
|
||||
{-1, 17, 2, 22},
|
||||
},
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(smoke_FakeQuantizeCache_4D, FakeQuantizeCacheTest,
|
||||
::testing::Combine(
|
||||
::testing::ValuesIn(inputShapes_4D),
|
||||
specificParams,
|
||||
::testing::ValuesIn(inputRanges),
|
||||
::testing::ValuesIn(filterCPUSpecificParams(cpuParams_4D)),
|
||||
::testing::Values(disableSnippets)),
|
||||
FakeQuantizeCacheTest::getTestCaseName);
|
||||
|
||||
|
||||
// 5D
|
||||
std::vector<CPUSpecificParams> cpuParams_5D = {
|
||||
CPUSpecificParams({ncdhw}, {ncdhw}, {}, {}),
|
||||
CPUSpecificParams({ndhwc}, {ndhwc}, {}, {}),
|
||||
CPUSpecificParams({nCdhw16c}, {nCdhw16c}, {}, {}),
|
||||
};
|
||||
|
||||
std::vector<InputShapesTuple> inputShapes_5D = {
|
||||
{
|
||||
// fq dynamic data shapes
|
||||
{
|
||||
// input0
|
||||
{{-1, -1, -1, -1, 43}, {{1, 17, 2, 3, 43}, {1, 34, 2, 3, 43}, {1, 17, 2, 3, 43}, {1, 34, 2, 3, 43}}},
|
||||
// input1
|
||||
{{-1, -1, -1, -1, 43}, {{1, 17, 2, 3, 43}, {1, 34, 2, 3, 43}, {1, 17, 2, 3, 43}, {1, 34, 2, 3, 43}}},
|
||||
},
|
||||
// fq range input shapes
|
||||
{
|
||||
// input0
|
||||
{{1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}}, // miss
|
||||
// input1
|
||||
{{1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}}, // hit
|
||||
},
|
||||
// reshape shape
|
||||
{},
|
||||
},
|
||||
{
|
||||
// fq dynamic data shapes
|
||||
{
|
||||
// input0
|
||||
{{-1, 47, -1, -1, -1}, {{1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}, {1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}}},
|
||||
// input1
|
||||
{{-1, 47, -1, -1, -1}, {{1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}, {1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}}},
|
||||
},
|
||||
// fq range input shapes
|
||||
{
|
||||
// input0
|
||||
{{1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}}, // miss
|
||||
// input1
|
||||
{{1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}}, // hit
|
||||
},
|
||||
// reshape shape
|
||||
{},
|
||||
},
|
||||
{
|
||||
// fq dynamic data shapes
|
||||
{
|
||||
// input0
|
||||
{{-1, 47, -1, -1, -1}, {{1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}, {1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}}},
|
||||
// input1
|
||||
{{-1, 47, -1, -1, -1}, {{1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}, {1, 47, 2, 3, 22}, {1, 47, 3, 2, 33}}},
|
||||
},
|
||||
// fq range input shapes
|
||||
{
|
||||
// input0
|
||||
{{1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}, {1, 47, 1, 1, 1}}, // miss
|
||||
// input1
|
||||
{{1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}, {1, 1, 1, 1, 1}}, // miss
|
||||
},
|
||||
// reshape shape
|
||||
{},
|
||||
},
|
||||
{
|
||||
// fq dynamic data shapes
|
||||
{
|
||||
// input0
|
||||
{{-1, 17, -1, -1, -1}, {{1, 17, 2, 3, 22}, {1, 17, 2, 3, 22}}},
|
||||
// input1
|
||||
{{-1, 34, -1, -1, -1}, {{1, 34, 2, 3, 22}, {1, 34, 2, 3, 22}}},
|
||||
},
|
||||
// fq range input shapes
|
||||
{
|
||||
// input0
|
||||
{{1, 17, 1, 1, 1}, {1, 17, 1, 1, 1}, {1, 17, 1, 1, 1}, {1, 17, 1, 1, 1}}, // miss
|
||||
// input1
|
||||
{{1, 34, 1, 1, 1}, {1, 34, 1, 1, 1}, {1, 34, 1, 1, 1}, {1, 34, 1, 1, 1}}, // hit
|
||||
},
|
||||
// reshape shape
|
||||
{-1, 17, 2, 3, 22},
|
||||
},
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(smoke_FakeQuantizeCache_5D, FakeQuantizeCacheTest,
|
||||
::testing::Combine(
|
||||
::testing::ValuesIn(inputShapes_5D),
|
||||
specificParams,
|
||||
::testing::ValuesIn(inputRanges),
|
||||
::testing::ValuesIn(filterCPUSpecificParams(cpuParams_5D)),
|
||||
::testing::Values(disableSnippets)),
|
||||
FakeQuantizeCacheTest::getTestCaseName);
|
||||
|
||||
} // namespace
|
||||
} // namespace CPUSubgraphTestsDefinitions
|
||||
Loading…
Reference in New Issue