diff --git a/inference-engine/src/low_precision_transformations/include/low_precision_transformations/gemm.hpp b/inference-engine/src/low_precision_transformations/include/low_precision_transformations/gemm.hpp index ac5bd226f5c..96980ed031f 100644 --- a/inference-engine/src/low_precision_transformations/include/low_precision_transformations/gemm.hpp +++ b/inference-engine/src/low_precision_transformations/include/low_precision_transformations/gemm.hpp @@ -21,6 +21,8 @@ public: ~GemmTransformation() override {}; bool canBeTransformed(const TransformationContext& context, const CNNLayer& layer) const override; void transform(TransformationContext& context, CNNLayer& layer) const override; + + bool isQuantized(const CNNLayer& layer) const noexcept override; }; IE_SUPPRESS_DEPRECATED_END diff --git a/inference-engine/src/low_precision_transformations/include/low_precision_transformations/weightable_layer_transformation.hpp b/inference-engine/src/low_precision_transformations/include/low_precision_transformations/weightable_layer_transformation.hpp index 2b247380534..34af7c72ab5 100644 --- a/inference-engine/src/low_precision_transformations/include/low_precision_transformations/weightable_layer_transformation.hpp +++ b/inference-engine/src/low_precision_transformations/include/low_precision_transformations/weightable_layer_transformation.hpp @@ -83,6 +83,8 @@ protected: const std::vector& originalWeightsDequantizationShifts, std::vector& dequantizationScales, std::vector& dequantizationShifts) const; + + static bool getDequantizationDimIsSupported(const CNNLayer& weightableLayer); }; typedef std::shared_ptr WeightableLayerTransformationPtr; diff --git a/inference-engine/src/low_precision_transformations/src/fully_connected.cpp b/inference-engine/src/low_precision_transformations/src/fully_connected.cpp index 6da5e9bfbcd..afb1da38e51 100644 --- a/inference-engine/src/low_precision_transformations/src/fully_connected.cpp +++ b/inference-engine/src/low_precision_transformations/src/fully_connected.cpp @@ -25,15 +25,6 @@ using namespace InferenceEngine; using namespace InferenceEngine::details; -bool getDequantizationValuesAreBroadcasted(const CNNLayer& fullyConnected) { - const DataPtr inputData = fullyConnected.insData[0].lock(); - if (inputData == nullptr) { - THROW_IE_LPT_EXCEPTION(fullyConnected) << "input data is absent"; - } - - return inputData->getDims().size() == 3ul; -} - bool FullyConnectedTransformation::canBeTransformed(const TransformationContext& context, const CNNLayer& fullyConnected) const { if (!WeightableLayerTransformation::canBeTransformed(context, fullyConnected)) { return false; @@ -72,7 +63,12 @@ bool FullyConnectedTransformation::canBeTransformed(const TransformationContext& std::vector dequantizationShifts; fillFromDequantizationLayer(*scaleShift, dequantizationScales, dequantizationShifts); - if ((inTensorDims.size() == 3ul) && (!DequantizationDetails::isPerTensor(dequantizationScales, dequantizationShifts))) { + const bool dequantizationDimIsSupported = !getDequantizationDimIsSupported(fullyConnected); + if ((!dequantizationDimIsSupported) && + (!DequantizationDetails::isPerTensor(dequantizationScales, dequantizationShifts) || + // if asymmetric quantization is not supported then no shifts for dequantizationDimIsSupported = false case: + // in this case we can not dequantize with shifts + (!supportAsymmetricQuantization && (dequantizationShifts[0] != 0.f)))) { return false; } @@ -318,7 +314,7 @@ void FullyConnectedTransformation::calculateDequantizationForSymmetric( const auto prevDequantizationScaleBuffer = CNNNetworkHelper::getFloatData(CNNNetworkHelper::getBlob(scaleShift, "weights")); const auto prevDequantizationShiftBuffer = CNNNetworkHelper::getFloatData(CNNNetworkHelper::getBlob(scaleShift, "biases")); - const bool dequantizationValuesAreBroadcasted = getDequantizationValuesAreBroadcasted(fullyConnected); + const bool dequantizationValuesAreBroadcasted = !getDequantizationDimIsSupported(fullyConnected); for (size_t i = 0; i < outputChannelsCount; ++i) { dequantizationScales[i] = prevDequantizationScaleBuffer.get()[0] * @@ -401,7 +397,7 @@ void FullyConnectedTransformation::calculateDequantizationForAsymmetric( THROW_IE_EXCEPTION << "Unexpected layer type to calculate quantization values " << scaleShift->type; } - const bool dequantizationValuesAreBroadcasted = getDequantizationValuesAreBroadcasted(fullyConnected); + const bool dequantizationValuesAreBroadcasted = !getDequantizationDimIsSupported(fullyConnected); dequantizationScales.resize(outputChannelsCount); dequantizationShifts.resize(outputChannelsCount); @@ -412,10 +408,10 @@ void FullyConnectedTransformation::calculateDequantizationForAsymmetric( prevDequantizationScaleBuffer.get()[0] * (originalWeightsDequantizationScales.size() == 0 ? 1.0 : - (originalWeightsDequantizationScales.size() == 1 ? originalWeightsDequantizationScales[0] : originalWeightsDequantizationScales[i])); + originalWeightsDequantizationScales[((originalWeightsDequantizationScales.size() == 1) || dequantizationValuesAreBroadcasted) ? 0 : i]); } - if (CNNNetworkHelper::isQuantizedConstWeights(fullyConnected)) { + if (CNNNetworkHelper::isQuantizedConstWeights(fullyConnected) && (!dequantizationValuesAreBroadcasted)) { const Blob::Ptr weightsBlob = CNNNetworkHelper::getWeights(fullyConnected, roundQuantizedValues); const auto weightsBuffer = CNNNetworkHelper::getFloatData(weightsBlob); const Blob::Ptr biasesBlob = CNNNetworkHelper::getBiases(fullyConnected); @@ -432,7 +428,7 @@ void FullyConnectedTransformation::calculateDequantizationForAsymmetric( for (size_t w = 0; w < inputChannelsCount; ++w) { const float kernel = weightsBuffer.get()[channel * inputChannelsCount + w]; - const float shift = dequantizationValuesAreBroadcasted ? prevDequantizationShiftBuffer.get()[0] : prevDequantizationShiftBuffer.get()[w]; + const float shift = prevDequantizationShiftBuffer.get()[w]; sum1 += kernel * shift * weightsDequantizationScale; sum2 += kernel * dataZeroPoints[w] * weightsDequantizationScale; } diff --git a/inference-engine/src/low_precision_transformations/src/gemm.cpp b/inference-engine/src/low_precision_transformations/src/gemm.cpp index a24c00139e4..1dac79923bb 100644 --- a/inference-engine/src/low_precision_transformations/src/gemm.cpp +++ b/inference-engine/src/low_precision_transformations/src/gemm.cpp @@ -133,3 +133,8 @@ void GemmTransformation::transform(TransformationContext& context, CNNLayer& gem addDequantizationLayer(context, gemm, dequantizationScales, dequantizationShifts); } + +bool GemmTransformation::isQuantized(const CNNLayer& layer) const noexcept { + // weightable layer version overriding + return true; +} diff --git a/inference-engine/src/low_precision_transformations/src/weightable_layer_transformation.cpp b/inference-engine/src/low_precision_transformations/src/weightable_layer_transformation.cpp index 1317c5250ca..b30e3b17f76 100644 --- a/inference-engine/src/low_precision_transformations/src/weightable_layer_transformation.cpp +++ b/inference-engine/src/low_precision_transformations/src/weightable_layer_transformation.cpp @@ -128,6 +128,15 @@ bool WeightableLayerTransformation::isPrecisionPreserved(const CNNLayer& layer) return false; } +bool WeightableLayerTransformation::getDequantizationDimIsSupported(const CNNLayer& fullyConnected) { + const DataPtr inputData = fullyConnected.insData[0].lock(); + if (inputData == nullptr) { + THROW_IE_LPT_EXCEPTION(fullyConnected) << "input data is absent"; + } + + return inputData->getDims().size() != 3ul; +} + void WeightableLayerTransformation::updateLayerBiases( TransformationContext& context, const CNNLayer& weightableLayer, @@ -135,7 +144,17 @@ void WeightableLayerTransformation::updateLayerBiases( std::vector& dequantizationScales, std::vector& dequantizationShifts, std::vector& biasesShifts) const { - if (!std::all_of(dequantizationShifts.begin(), dequantizationShifts.end(), [](float value) { return value == 0.0; })) { + const bool dequantizationShiftsAreZero = std::all_of( + dequantizationShifts.begin(), + dequantizationShifts.end(), + [](float value) { return value == 0.0; }); + + const bool dequantizationDimIsNotSupported = !getDequantizationDimIsSupported(weightableLayer); + CNNLayerPtr biasesLayer = CNNNetworkHelper::getParent(weightableLayer, 2); + + // we need to correct biases if dequantization shifts values are not zero or + // dequantization dimention is not supported (as result dequantization shifts can not be calculated) + if ((dequantizationDimIsNotSupported && (biasesLayer != nullptr)) || (!dequantizationShiftsAreZero)) { const DataPtr insData = weightableLayer.insData[0].lock(); if (insData == nullptr) { THROW_IE_LPT_EXCEPTION(weightableLayer) << "input data is absent"; @@ -144,7 +163,6 @@ void WeightableLayerTransformation::updateLayerBiases( std::shared_ptr biasesBufferPtr; Blob::Ptr biasesBlob; - CNNLayerPtr biasesLayer = CNNNetworkHelper::getParent(weightableLayer, 2); if (biasesLayer == nullptr) { if (weightableLayer.outData.size() != 1ul) { THROW_IE_LPT_EXCEPTION(weightableLayer) << "unexpected output data count " << weightableLayer.outData.size(); diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_fullyconnected_node.cpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_fullyconnected_node.cpp index 4717e067f72..80f0ed0113b 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_fullyconnected_node.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_fullyconnected_node.cpp @@ -209,32 +209,34 @@ void MKLDNNFullyConnectedNode::setPostOps(mkldnn::primitive_attr &attr, bool ini PostOpsIntBlobMemory.push_back(MKLDNNMemoryPtr(new MKLDNNMemory(getEngine()))); PostOpsIntBlobMemory[blob_idx]->Create(depthwiseDims, memory::data_type::f32, memory::format::x); - PostOpsIntBlobMemory[blob_idx]->SetData(memory::data_type::f32, memory::x, - depthwiseLayer->_weights->buffer(), - depthwiseLayer->_weights->size() * - MKLDNNExtensionUtils::sizeOfDataType(memory::data_type::f32)); - + // In case ndims == 3 graph optimizer allows fusing only if all weights values are the same if (depthwiseNode->isBroadcast() || ndims == 3) { - float broadcastValue = static_cast(PostOpsIntBlobMemory[blob_idx]->GetData())[0]; - for (int i = 1; i < PostOpsIntBlobMemory[blob_idx]->GetPrimitiveDescriptor().desc().data.dims[0]; i++) { + float broadcastValue = static_cast(depthwiseLayer->_weights->buffer())[0]; + for (int i = 0; i < PostOpsIntBlobMemory[blob_idx]->GetPrimitiveDescriptor().desc().data.dims[0]; i++) { static_cast(PostOpsIntBlobMemory[blob_idx]->GetData())[i] = broadcastValue; } + } else { + PostOpsIntBlobMemory[blob_idx]->SetData(memory::data_type::f32, memory::x, + depthwiseLayer->_weights->buffer(), + depthwiseLayer->_weights->size() * + MKLDNNExtensionUtils::sizeOfDataType(memory::data_type::f32)); } if (depthwiseNode->getAlgorithm() == depthwise_scale_shift) { PostOpsIntBlobMemory.push_back(MKLDNNMemoryPtr(new MKLDNNMemory(getEngine()))); - PostOpsIntBlobMemory[blob_idx + 1]->Create(depthwiseDims, memory::data_type::f32, - memory::format::x); - PostOpsIntBlobMemory[blob_idx + 1]->SetData(memory::data_type::f32, memory::x, - depthwiseLayer->_biases->buffer(), - depthwiseLayer->_biases->size() * - MKLDNNExtensionUtils::sizeOfDataType(memory::data_type::f32)); + PostOpsIntBlobMemory[blob_idx + 1]->Create(depthwiseDims, memory::data_type::f32, memory::format::x); + // In case ndims == 3 graph optimizer allows fusing only if all biases values are the same if (depthwiseNode->isBroadcast() || ndims == 3) { - float broadcastValue = static_cast(PostOpsIntBlobMemory[blob_idx + 1]->GetData())[0]; - for (int i = 1; i < PostOpsIntBlobMemory[blob_idx + 1]->GetPrimitiveDescriptor().desc().data.dims[0]; i++) { + float broadcastValue = static_cast(depthwiseLayer->_biases->buffer())[0]; + for (int i = 0; i < PostOpsIntBlobMemory[blob_idx + 1]->GetPrimitiveDescriptor().desc().data.dims[0]; i++) { static_cast(PostOpsIntBlobMemory[blob_idx + 1]->GetData())[i] = broadcastValue; } + } else { + PostOpsIntBlobMemory[blob_idx + 1]->SetData(memory::data_type::f32, memory::x, + depthwiseLayer->_biases->buffer(), + depthwiseLayer->_biases->size() * + MKLDNNExtensionUtils::sizeOfDataType(memory::data_type::f32)); } ops.append_depthwise(depthwiseNode->getAlgorithm(),