natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[LPT] BERT with specific biases support & improvement (#968) 

* [LPT] BERT with biases support

* [LPT] Gemm biases and quantization

* [CPU] Fixed FullyConnected + Depthwise node fusing

* [LPT] FullyConnected 3D: symmetric quantization support

* [LPT] FullyConnected 3D: symmetric quantization support fix

* [CPU] Fixed FullyConnected + Depthwise fusing initialization

Co-authored-by: dmitrygo <dmitry.gorokhov@intel.com>
This commit is contained in:
Edward Shogulin 2020-06-19 13:14:20 +03:00 committed by GitHub
parent 5e97a3123f
commit 76af547c17
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 57 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
inference-engine/src/low_precision_transformations/include/low_precision_transformations/gemm.hpp
View File

@ -21,6 +21,8 @@ public:
~GemmTransformation() override {};
bool canBeTransformed(const TransformationContext& context, const CNNLayer& layer) const override;
void transform(TransformationContext& context, CNNLayer& layer) const override;
bool isQuantized(const CNNLayer& layer) const noexcept override;
};
IE_SUPPRESS_DEPRECATED_END

2
inference-engine/src/low_precision_transformations/include/low_precision_transformations/weightable_layer_transformation.hpp
View File

@ -83,6 +83,8 @@ protected:
const std::vector<float>& originalWeightsDequantizationShifts,
std::vector<float>& dequantizationScales,
std::vector<float>& dequantizationShifts) const;
static bool getDequantizationDimIsSupported(const CNNLayer& weightableLayer);
};
typedef std::shared_ptr<WeightableLayerTransformation> WeightableLayerTransformationPtr;

26
inference-engine/src/low_precision_transformations/src/fully_connected.cpp
View File

@ -25,15 +25,6 @@
using namespace InferenceEngine;
using namespace InferenceEngine::details;
bool getDequantizationValuesAreBroadcasted(const CNNLayer& fullyConnected) {
const DataPtr inputData = fullyConnected.insData[0].lock();
if (inputData == nullptr) {
THROW_IE_LPT_EXCEPTION(fullyConnected) << "input data is absent";
}
return inputData->getDims().size() == 3ul;
}
bool FullyConnectedTransformation::canBeTransformed(const TransformationContext& context, const CNNLayer& fullyConnected) const {
if (!WeightableLayerTransformation::canBeTransformed(context, fullyConnected)) {
return false;
@ -72,7 +63,12 @@ bool FullyConnectedTransformation::canBeTransformed(const TransformationContext&
std::vector<float> dequantizationShifts;
fillFromDequantizationLayer(*scaleShift, dequantizationScales, dequantizationShifts);
if ((inTensorDims.size() == 3ul) && (!DequantizationDetails::isPerTensor(dequantizationScales, dequantizationShifts))) {
const bool dequantizationDimIsSupported = !getDequantizationDimIsSupported(fullyConnected);
if ((!dequantizationDimIsSupported) &&
(!DequantizationDetails::isPerTensor(dequantizationScales, dequantizationShifts) ||
// if asymmetric quantization is not supported then no shifts for dequantizationDimIsSupported = false case:
// in this case we can not dequantize with shifts
(!supportAsymmetricQuantization && (dequantizationShifts[0] != 0.f)))) {
return false;
}
@ -318,7 +314,7 @@ void FullyConnectedTransformation::calculateDequantizationForSymmetric(
const auto prevDequantizationScaleBuffer = CNNNetworkHelper::getFloatData(CNNNetworkHelper::getBlob(scaleShift, "weights"));
const auto prevDequantizationShiftBuffer = CNNNetworkHelper::getFloatData(CNNNetworkHelper::getBlob(scaleShift, "biases"));
const bool dequantizationValuesAreBroadcasted = getDequantizationValuesAreBroadcasted(fullyConnected);
const bool dequantizationValuesAreBroadcasted = !getDequantizationDimIsSupported(fullyConnected);
for (size_t i = 0; i < outputChannelsCount; ++i) {
dequantizationScales[i] =
prevDequantizationScaleBuffer.get()[0] *
@ -401,7 +397,7 @@ void FullyConnectedTransformation::calculateDequantizationForAsymmetric(
THROW_IE_EXCEPTION << "Unexpected layer type to calculate quantization values " << scaleShift->type;
}
const bool dequantizationValuesAreBroadcasted = getDequantizationValuesAreBroadcasted(fullyConnected);
const bool dequantizationValuesAreBroadcasted = !getDequantizationDimIsSupported(fullyConnected);
dequantizationScales.resize(outputChannelsCount);
dequantizationShifts.resize(outputChannelsCount);
@ -412,10 +408,10 @@ void FullyConnectedTransformation::calculateDequantizationForAsymmetric(
prevDequantizationScaleBuffer.get()[0] *
(originalWeightsDequantizationScales.size() == 0 ?
1.0 :
(originalWeightsDequantizationScales.size() == 1 ? originalWeightsDequantizationScales[0] : originalWeightsDequantizationScales[i]));
originalWeightsDequantizationScales[((originalWeightsDequantizationScales.size() == 1) || dequantizationValuesAreBroadcasted) ? 0 : i]);
}
if (CNNNetworkHelper::isQuantizedConstWeights(fullyConnected)) {
if (CNNNetworkHelper::isQuantizedConstWeights(fullyConnected) && (!dequantizationValuesAreBroadcasted)) {
const Blob::Ptr weightsBlob = CNNNetworkHelper::getWeights(fullyConnected, roundQuantizedValues);
const auto weightsBuffer = CNNNetworkHelper::getFloatData(weightsBlob);
const Blob::Ptr biasesBlob = CNNNetworkHelper::getBiases(fullyConnected);
@ -432,7 +428,7 @@ void FullyConnectedTransformation::calculateDequantizationForAsymmetric(
for (size_t w = 0; w < inputChannelsCount; ++w) {
const float kernel = weightsBuffer.get()[channel * inputChannelsCount + w];
const float shift = dequantizationValuesAreBroadcasted ? prevDequantizationShiftBuffer.get()[0] : prevDequantizationShiftBuffer.get()[w];
const float shift = prevDequantizationShiftBuffer.get()[w];
sum1 += kernel * shift * weightsDequantizationScale;
sum2 += kernel * dataZeroPoints[w] * weightsDequantizationScale;
}

5
inference-engine/src/low_precision_transformations/src/gemm.cpp
View File

@ -133,3 +133,8 @@ void GemmTransformation::transform(TransformationContext& context, CNNLayer& gem
addDequantizationLayer(context, gemm, dequantizationScales, dequantizationShifts);
}
bool GemmTransformation::isQuantized(const CNNLayer& layer) const noexcept {
// weightable layer version overriding
return true;
}

22
inference-engine/src/low_precision_transformations/src/weightable_layer_transformation.cpp
View File

@ -128,6 +128,15 @@ bool WeightableLayerTransformation::isPrecisionPreserved(const CNNLayer& layer)
return false;
}
bool WeightableLayerTransformation::getDequantizationDimIsSupported(const CNNLayer& fullyConnected) {
const DataPtr inputData = fullyConnected.insData[0].lock();
if (inputData == nullptr) {
THROW_IE_LPT_EXCEPTION(fullyConnected) << "input data is absent";
}
return inputData->getDims().size() != 3ul;
}
void WeightableLayerTransformation::updateLayerBiases(
TransformationContext& context,
const CNNLayer& weightableLayer,
@ -135,7 +144,17 @@ void WeightableLayerTransformation::updateLayerBiases(
std::vector<float>& dequantizationScales,
std::vector<float>& dequantizationShifts,
std::vector<float>& biasesShifts) const {
if (!std::all_of(dequantizationShifts.begin(), dequantizationShifts.end(), [](float value) { return value == 0.0; })) {
const bool dequantizationShiftsAreZero = std::all_of(
dequantizationShifts.begin(),
dequantizationShifts.end(),
[](float value) { return value == 0.0; });
const bool dequantizationDimIsNotSupported = !getDequantizationDimIsSupported(weightableLayer);
CNNLayerPtr biasesLayer = CNNNetworkHelper::getParent(weightableLayer, 2);
// we need to correct biases if dequantization shifts values are not zero or
// dequantization dimention is not supported (as result dequantization shifts can not be calculated)
if ((dequantizationDimIsNotSupported && (biasesLayer != nullptr)) || (!dequantizationShiftsAreZero)) {
const DataPtr insData = weightableLayer.insData[0].lock();
if (insData == nullptr) {
THROW_IE_LPT_EXCEPTION(weightableLayer) << "input data is absent";
@ -144,7 +163,6 @@ void WeightableLayerTransformation::updateLayerBiases(
std::shared_ptr<float> biasesBufferPtr;
Blob::Ptr biasesBlob;
CNNLayerPtr biasesLayer = CNNNetworkHelper::getParent(weightableLayer, 2);
if (biasesLayer == nullptr) {
if (weightableLayer.outData.size() != 1ul) {
THROW_IE_LPT_EXCEPTION(weightableLayer) << "unexpected output data count " << weightableLayer.outData.size();

32
inference-engine/src/mkldnn_plugin/nodes/mkldnn_fullyconnected_node.cpp
View File

@ -209,32 +209,34 @@ void MKLDNNFullyConnectedNode::setPostOps(mkldnn::primitive_attr &attr, bool ini
PostOpsIntBlobMemory.push_back(MKLDNNMemoryPtr(new MKLDNNMemory(getEngine())));
PostOpsIntBlobMemory[blob_idx]->Create(depthwiseDims, memory::data_type::f32, memory::format::x);
PostOpsIntBlobMemory[blob_idx]->SetData(memory::data_type::f32, memory::x,
depthwiseLayer->_weights->buffer(),
depthwiseLayer->_weights->size() *
MKLDNNExtensionUtils::sizeOfDataType(memory::data_type::f32));
// In case ndims == 3 graph optimizer allows fusing only if all weights values are the same
if (depthwiseNode->isBroadcast() || ndims == 3) {
float broadcastValue = static_cast<float *>(PostOpsIntBlobMemory[blob_idx]->GetData())[0];
for (int i = 1; i < PostOpsIntBlobMemory[blob_idx]->GetPrimitiveDescriptor().desc().data.dims[0]; i++) {
float broadcastValue = static_cast<float *>(depthwiseLayer->_weights->buffer())[0];
for (int i = 0; i < PostOpsIntBlobMemory[blob_idx]->GetPrimitiveDescriptor().desc().data.dims[0]; i++) {
static_cast<float *>(PostOpsIntBlobMemory[blob_idx]->GetData())[i] = broadcastValue;
}
} else {
PostOpsIntBlobMemory[blob_idx]->SetData(memory::data_type::f32, memory::x,
depthwiseLayer->_weights->buffer(),
depthwiseLayer->_weights->size() *
MKLDNNExtensionUtils::sizeOfDataType(memory::data_type::f32));
}
if (depthwiseNode->getAlgorithm() == depthwise_scale_shift) {
PostOpsIntBlobMemory.push_back(MKLDNNMemoryPtr(new MKLDNNMemory(getEngine())));
PostOpsIntBlobMemory[blob_idx + 1]->Create(depthwiseDims, memory::data_type::f32,
memory::format::x);
PostOpsIntBlobMemory[blob_idx + 1]->SetData(memory::data_type::f32, memory::x,
depthwiseLayer->_biases->buffer(),
depthwiseLayer->_biases->size() *
MKLDNNExtensionUtils::sizeOfDataType(memory::data_type::f32));
PostOpsIntBlobMemory[blob_idx + 1]->Create(depthwiseDims, memory::data_type::f32, memory::format::x);
// In case ndims == 3 graph optimizer allows fusing only if all biases values are the same
if (depthwiseNode->isBroadcast() || ndims == 3) {
float broadcastValue = static_cast<float *>(PostOpsIntBlobMemory[blob_idx + 1]->GetData())[0];
for (int i = 1; i < PostOpsIntBlobMemory[blob_idx + 1]->GetPrimitiveDescriptor().desc().data.dims[0]; i++) {
float broadcastValue = static_cast<float *>(depthwiseLayer->_biases->buffer())[0];
for (int i = 0; i < PostOpsIntBlobMemory[blob_idx + 1]->GetPrimitiveDescriptor().desc().data.dims[0]; i++) {
static_cast<float *>(PostOpsIntBlobMemory[blob_idx + 1]->GetData())[i] = broadcastValue;
}
} else {
PostOpsIntBlobMemory[blob_idx + 1]->SetData(memory::data_type::f32, memory::x,
depthwiseLayer->_biases->buffer(),
depthwiseLayer->_biases->size() *
MKLDNNExtensionUtils::sizeOfDataType(memory::data_type::f32));
}
ops.append_depthwise(depthwiseNode->getAlgorithm(),