natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add model compression to FP16 weights (#7588) 

* Add model compression to FP16 weights

* Fix build

* Fix build

* Fix build

* Add wrapper over ConvertPrecision

* Add documentation to attributes

* Fix MO IR Reader

* Fix build

* Return DisableDecompressionConvertConstantFolding call in CommonOptimizations

* Temporarily disable old_api map

* Fix TI Convert issue

* Apply review feedback

* Fix build

* Fix build

* Fix build
This commit is contained in:
Maxim Vafin 2021-11-09 19:37:41 +03:00 committed by GitHub
parent a9b9e14bf4
commit 34886b650d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
45 changed files with 986 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
inference-engine/ie_bridges/python/src/openvino/offline_transformations/offline_transformations_api.pyx
View File

@ -31,6 +31,10 @@ def ApplyLowLatencyTransformation(IENetwork network, bool use_const_initializer
C.ApplyLowLatencyTransformation(network.impl, use_const_initializer)
def CompressModelTransformation(IENetwork network):
C.CompressModelTransformation(network.impl)
def ApplyPruningTransformation(IENetwork network):
C.ApplyPruningTransformation(network.impl)

11
inference-engine/ie_bridges/python/src/openvino/offline_transformations/offline_transformations_api_impl.cpp
View File

@ -12,6 +12,8 @@
#include <openvino/pass/make_stateful.hpp>
#include <pot_transformations.hpp>
#include <pruning.hpp>
#include <transformations/common_optimizations/compress_float_constants.hpp>
#include <transformations/common_optimizations/mark_precision_sensitive_subgraphs.hpp>
#include <transformations/common_optimizations/moc_transformations.hpp>
#include <transformations/control_flow/unroll_tensor_iterator.hpp>
#include <transformations/serialize.hpp>
@ -56,6 +58,13 @@ void InferenceEnginePython::GenerateMappingFile(InferenceEnginePython::IENetwork
manager.run_passes(network.actual->getFunction());
}
void InferenceEnginePython::CompressModelTransformation(InferenceEnginePython::IENetwork network) {
ngraph::pass::Manager manager;
manager.register_pass<ov::pass::MarkPrecisionSensitiveSubgraphs>();
manager.register_pass<ov::pass::CompressFloatConstants>();
manager.run_passes(network.actual->getFunction());
}
void InferenceEnginePython::Serialize(InferenceEnginePython::IENetwork network,
std::string path_to_xml,
std::string path_to_bin) {
@ -80,4 +89,4 @@ void InferenceEnginePython::CheckAPI() {
auto reshape = f->get_result()->input_value(0).get_node_shared_ptr();
assert(std::dynamic_pointer_cast<ngraph::opset6::Parameter>(reshape->input_value(0).get_node_shared_ptr()));
assert(std::dynamic_pointer_cast<ngraph::opset6::Constant>(reshape->input_value(1).get_node_shared_ptr()));
}
}

2
inference-engine/ie_bridges/python/src/openvino/offline_transformations/offline_transformations_api_impl.hpp
View File

@ -25,6 +25,8 @@ void ApplyPruningTransformation(InferenceEnginePython::IENetwork network);
void GenerateMappingFile(InferenceEnginePython::IENetwork network, std::string path, bool extract_names);
void CompressModelTransformation(InferenceEnginePython::IENetwork network);
void Serialize(InferenceEnginePython::IENetwork network, std::string path_to_xml, std::string path_to_bin);
void CheckAPI();

2
inference-engine/ie_bridges/python/src/openvino/offline_transformations/offline_transformations_api_impl_defs.pxd
View File

@ -17,6 +17,8 @@ cdef extern from "offline_transformations_api_impl.hpp" namespace "InferenceEngi
cdef void ApplyMakeStatefulTransformation(IENetwork network, map[string, string]& in_out_names)
cdef void ApplyPruningTransformation(IENetwork network)
cdef void CompressModelTransformation(IENetwork network)
cdef void GenerateMappingFile(IENetwork network, string path, bool extract_names)

3
inference-engine/src/cldnn_engine/cldnn_transformations_pipeline.cpp
View File

@ -34,6 +34,7 @@
#include <transformations/common_optimizations/lin_op_sequence_fusion.hpp>
#include <transformations/common_optimizations/weights_dequantize_to_fake_quantize.hpp>
#include "transformations/common_optimizations/convert_quantize_dequantize.hpp"
#include "transformations/common_optimizations/convert_compression_only_to_legacy.hpp"
#include <transformations/op_conversions/convert_depth_to_space.hpp>
#include <transformations/op_conversions/convert_space_to_depth.hpp>
#include <transformations/op_conversions/convert_gelu.hpp>
@ -163,6 +164,8 @@ void TransformationsPipeline::apply(std::shared_ptr<ov::Function> func) {
auto pass_config = manager.get_pass_config();
pass_config->enable<ov::pass::ConvertCompressedOnlyToLegacy>();
// SpaceToDepth/DepthToSpace node implementation supports only equal input/output tensors with rank <= 5
pass_config->set_callback<ngraph::pass::ConvertSpaceToDepth,
ngraph::pass::ConvertDepthToSpace>(

51
inference-engine/src/transformations/include/transformations/common_optimizations/compress_float_constants.hpp Normal file
View File

@ -0,0 +1,51 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "transformations_visibility.hpp"
#include "openvino/pass/graph_rewrite.hpp"
namespace ov {
namespace pass {
class TRANSFORMATIONS_API CompressFloatConstantsImpl;
class TRANSFORMATIONS_API AddOldApiMapToParameters;
class TRANSFORMATIONS_API CompressFloatConstants;
} // namespace pass
} // namespace ov
/**
* @ingroup ie_transformation_common_api
* @brief CompressFloatConstantsImpl transformation replaces FP32/FP64 Constants with FP16 ones.
*/
class ov::pass::CompressFloatConstantsImpl : public ov::pass::MatcherPass {
public:
OPENVINO_RTTI("CompressFloatConstantsImpl", "0");
CompressFloatConstantsImpl();
};
/**
* @ingroup ie_transformation_common_api
* @brief AddOldApiMapToParameters transformation adds OldApiMap to each float input to the model.
*/
class ov::pass::AddOldApiMapToParameters : public ov::pass::MatcherPass {
public:
OPENVINO_RTTI("AddOldApiMapToParameters", "0");
AddOldApiMapToParameters();
};
/**
* @ingroup ie_transformation_common_api
* @brief CompressFloatConstants transformation replaces FP32/FP64 Constants with FP16 ones.
*/
class ov::pass::CompressFloatConstants : public ov::pass::GraphRewrite {
public:
OPENVINO_RTTI("CompressFloatConstants", "0");
CompressFloatConstants() {
add_matcher<ov::pass::CompressFloatConstantsImpl>();
add_matcher<ov::pass::AddOldApiMapToParameters>();
}
};

50
inference-engine/src/transformations/include/transformations/common_optimizations/convert_compression_only_to_legacy.hpp Normal file
View File

@ -0,0 +1,50 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "transformations_visibility.hpp"
#include "openvino/pass/pass.hpp"
#include "openvino/pass/graph_rewrite.hpp"
namespace ov {
namespace pass {
class TRANSFORMATIONS_API ConvertPrecisionCompressedOnly;
class TRANSFORMATIONS_API EnableDecompressionConvertConstantFolding;
class TRANSFORMATIONS_API ConvertCompressedOnlyToLegacy;
} // namespace pass
} // namespace ov
/**
* @ingroup ie_transformation_common_api
* @brief ConvertPrecisionCompressedOnly transformation runs ConvertPrecision transformation for CompressedOnly format.
*/
class ov::pass::ConvertPrecisionCompressedOnly : public ov::pass::FunctionPass {
public:
OPENVINO_RTTI("ConvertPrecisionCompressedOnly", "0");
bool run_on_function(std::shared_ptr<Function> f) override;
};
/**
* @ingroup ie_transformation_common_api
* @brief Enables ConstantFolding for Convert operation in compressed function.
*/
class ov::pass::EnableDecompressionConvertConstantFolding : public ov::pass::MatcherPass {
public:
OPENVINO_RTTI("EnableDecompressionConvertConstantFolding", "0");
EnableDecompressionConvertConstantFolding();
};
/**
* @ingroup ie_transformation_common_api
* @brief ConvertCompressedOnlyToLegacy transformation converts compression only FP16 format to legacy FP16 format.
*/
class ov::pass::ConvertCompressedOnlyToLegacy : public ov::pass::FunctionPass {
public:
OPENVINO_RTTI("ConvertCompressedOnlyToLegacy", "0");
bool run_on_function(std::shared_ptr<Function> f) override;
};

29
inference-engine/src/transformations/include/transformations/common_optimizations/mark_precision_sensitive_subgraphs.hpp Normal file
View File

@ -0,0 +1,29 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "transformations_visibility.hpp"
#include "openvino/pass/pass.hpp"
namespace ov {
namespace pass {
class TRANSFORMATIONS_API MarkPrecisionSensitiveSubgraphs;
} // namespace pass
} // namespace ov
/**
* @ingroup ie_transformation_common_api
* @brief MarkPrecisionSensitiveSubgraphs transformation marks the constants
* inside the subgraph starting from precision-sensitive input and ending at
* the ShapeOf node as disabled for FP16 compression.
*/
class ov::pass::MarkPrecisionSensitiveSubgraphs : public FunctionPass {
public:
OPENVINO_RTTI("MarkPrecisionSensitiveSubgraphs", "0");
bool run_on_function(std::shared_ptr<ov::Function> f) override;
};

26
inference-engine/src/transformations/include/transformations/disable_decompression_convert_constant_folding.hpp Normal file
View File

@ -0,0 +1,26 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "transformations_visibility.hpp"
#include "openvino/pass/graph_rewrite.hpp"
namespace ov {
namespace pass {
class TRANSFORMATIONS_API DisableDecompressionConvertConstantFolding;
} // namespace pass
} // namespace ov
/**
* @ingroup ie_transformation_common_api
* @brief Disables ConstantFolding for Convert operation in compressed function.
*/
class ov::pass::DisableDecompressionConvertConstantFolding : public ov::pass::MatcherPass {
public:
OPENVINO_RTTI("DisableDecompressionConvertConstantFolding", "0");
DisableDecompressionConvertConstantFolding();
};

6
inference-engine/src/transformations/include/transformations/rt_info/attributes.hpp
View File

@ -12,11 +12,13 @@
#include <openvino/core/variant.hpp>
#include <set>
#include <transformations/rt_info/disable_constant_folding.hpp>
#include <transformations/rt_info/disable_fp16_compression.hpp>
#include <transformations/rt_info/fused_names_attribute.hpp>
#include <transformations/rt_info/nms_selected_indices.hpp>
#include <transformations/rt_info/old_api_map_attribute.hpp>
#include <transformations/rt_info/primitives_priority_attribute.hpp>
#include <transformations/rt_info/strides_property.hpp>
#include <transformations/rt_info/decompression.hpp>
#include <transformations_visibility.hpp>
#include <utility>
@ -37,5 +39,5 @@ private:
ngraph::FactoryRegistry<Variant> m_factory_registry;
};
} // namespace pass
} // namespace ov
} // namespace pass
} // namespace ov

42
inference-engine/src/transformations/include/transformations/rt_info/decompression.hpp Normal file
View File

@ -0,0 +1,42 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <assert.h>
#include <functional>
#include <memory>
#include <string>
#include <set>
#include "openvino/core/node.hpp"
#include "openvino/core/variant.hpp"
#include "transformations_visibility.hpp"
namespace ov {
TRANSFORMATIONS_API void mark_as_decompression(const std::shared_ptr<Node>& node);
TRANSFORMATIONS_API void unmark_as_decompression(const std::shared_ptr<Node>& node);
TRANSFORMATIONS_API bool is_decompression(const std::shared_ptr<Node>& node);
/**
* @ingroup ie_runtime_attr_api
* @brief Decompression class represents runtime info attribute that marks operation
* as used as decompression for Compressed Only format.
*/
class TRANSFORMATIONS_API Decompression : public VariantImpl<void> {
public:
OPENVINO_RTTI("decompression", "0");
Decompression() = default;
bool visit_attributes(AttributeVisitor& visitor) override { return true; }
bool is_copyable() const override { return false; }
};
} // namespace ov

34
inference-engine/src/transformations/include/transformations/rt_info/disable_fp16_compression.hpp Normal file
View File

@ -0,0 +1,34 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "openvino/core/node.hpp"
#include "openvino/core/variant.hpp"
#include "transformations_visibility.hpp"
namespace ov {
TRANSFORMATIONS_API void disable_fp16_compression(const std::shared_ptr<Node>& node);
TRANSFORMATIONS_API void enable_fp16_compression(const std::shared_ptr<Node>& node);
TRANSFORMATIONS_API bool fp16_compression_is_disabled(const std::shared_ptr<Node>& node);
/**
* @ingroup ie_runtime_attr_api
* @brief DisableFP16Compression class represents runtime info attribute that marks operation
* as prohibitted to convert to FP16 as part of Compressed Only format.
*/
class TRANSFORMATIONS_API DisableFP16Compression : public VariantImpl<void> {
public:
OPENVINO_RTTI("disable_fp16_compression", "0");
DisableFP16Compression() = default;
bool is_copyable() const override { return false; }
};
} // namespace ov

13
inference-engine/src/transformations/include/transformations/utils/utils.hpp
View File

@ -15,10 +15,12 @@
#include <ngraph/op/constant.hpp>
#include <ngraph/opsets/opset3.hpp>
#include <ngraph/opsets/opset4.hpp>
#include <ngraph/opsets/opset8.hpp>
#include <ngraph/rt_info.hpp>
#include <ngraph/pattern/op/wrap_type.hpp>
#include <ngraph/pass/graph_rewrite.hpp>
#include <transformations/rt_info/attributes.hpp>
namespace ngraph {
namespace op {
@ -49,6 +51,17 @@ bool has_op_with_type(const std::shared_ptr<const ngraph::Function> &function) {
}
return false;
}
inline bool has_decompression_converts(const std::shared_ptr<const ngraph::Function>& function) {
for (const auto& op : function->get_ops()) {
if (std::dynamic_pointer_cast<ngraph::opset8::Convert>(op)) {
if (ov::is_decompression(op))
return true;
}
}
return false;
}
inline std::string create_ie_output_name(const ngraph::Output<const ngraph::Node>& output) {
const auto& prev_layer = output.get_node_shared_ptr();
std::string out_name = prev_layer->get_friendly_name();

6
inference-engine/src/transformations/src/transformations/common_optimizations/common_optimizations.cpp
View File

@ -49,6 +49,7 @@
#include "transformations/common_optimizations/strides_optimization.hpp"
#include "transformations/common_optimizations/convert_nms_gather_path_to_unsigned.hpp"
#include "transformations/common_optimizations/mul_conv_fusion.hpp"
#include "transformations/common_optimizations/convert_compression_only_to_legacy.hpp"
#include "transformations/op_conversions/bidirectional_sequences_decomposition.hpp"
#include "transformations/op_conversions/convert_pad_to_group_conv.hpp"
#include "transformations/op_conversions/convert_divide.hpp"
@ -78,6 +79,7 @@
#include "transformations/op_conversions/gather_normalize_negative_indices.hpp"
#include "transformations/op_conversions/convert_deformable_conv_v8_to_v1.hpp"
#include "transformations/op_conversions/convert_maxpool_downgrade.hpp"
#include "transformations/disable_decompression_convert_constant_folding.hpp"
#include <ngraph/pass/manager.hpp>
#include <ngraph/pass/constant_folding.hpp>
@ -95,10 +97,14 @@ bool ngraph::pass::CommonOptimizations::run_on_function(std::shared_ptr<ngraph::
RUN_ON_FUNCTION_SCOPE(CommonOptimizations);
ngraph::pass::Manager manager(get_pass_config());
manager.register_pass<ov::pass::DisableDecompressionConvertConstantFolding>();
// Disable low_precision_enabled as all plugins handle low-precision sub-graph manually
// before CommonOptimization pipeline execution
manager.register_pass<ngraph::pass::MOCTransformations>(true, false);
manager.register_pass<ov::pass::ConvertCompressedOnlyToLegacy, false>();
// TODO: move to KMB
manager.register_pass<ngraph::pass::WeightsDequantizeToFakeQuantize>();

125
inference-engine/src/transformations/src/transformations/common_optimizations/compress_float_constants.cpp Normal file
View File

@ -0,0 +1,125 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "transformations/common_optimizations/compress_float_constants.hpp"
#include "openvino/opsets/opset8.hpp"
#include "ngraph/rt_info.hpp"
#include "openvino/pass/pattern/op/wrap_type.hpp"
#include "transformations/rt_info/decompression.hpp"
#include "transformations/rt_info/disable_fp16_compression.hpp"
#include "transformations/rt_info/old_api_map_attribute.hpp"
#include "itt.hpp"
namespace {
template <ov::element::Type_t PREC_FROM>
std::shared_ptr<ov::Node> change_constant_precision_to_fp16(std::shared_ptr<ov::opset8::Constant>& constant) {
using src_type = typename ov::element_type_traits<PREC_FROM>::value_type;
const auto* src_data = constant->get_data_ptr<src_type>();
const auto size = ov::shape_size(constant->get_shape());
auto new_constant = std::make_shared<ov::opset8::Constant>(ov::element::f16, constant->get_shape());
auto* dst_data = const_cast<ov::float16*>(reinterpret_cast<const ov::float16*>(new_constant->get_data_ptr()));
if (dst_data == nullptr)
return nullptr;
bool is_overflow = false;
for (size_t i = 0; i < size; ++i) {
if (src_data[i] > std::numeric_limits<ov::float16>::max()) {
dst_data[i] = std::numeric_limits<ov::float16>::max();
is_overflow = true;
} else if (src_data[i] < std::numeric_limits<ov::float16>::lowest()) {
dst_data[i] = std::numeric_limits<ov::float16>::lowest();
is_overflow = true;
} else {
dst_data[i] = static_cast<ov::float16>(src_data[i]);
}
}
if (is_overflow) {
std::cerr << "Warning: One or more of the values of the Constant can't fit in the float16 data type."
" Those values were casted to the nearest limit value, the model can produce incorrect results." << std::endl;
}
return new_constant;
}
} // namespace
ov::pass::CompressFloatConstantsImpl::CompressFloatConstantsImpl() {
MATCHER_SCOPE(CompressFloatConstantsImpl);
auto const_pattern = pattern::wrap_type<opset8::Constant>();
ov::matcher_pass_callback callback = [=](pattern::Matcher& m) {
const auto& pattern_map = m.get_pattern_value_map();
const auto& const_node_pattern = pattern_map.at(const_pattern);
auto const_node = std::dynamic_pointer_cast<ov::opset8::Constant>(
const_node_pattern.get_node_shared_ptr());
if (!const_node)
return false;
if (ov::fp16_compression_is_disabled(const_node))
return false;
auto c_type = const_node->get_element_type();
std::shared_ptr<ov::Node> new_const;
if (c_type == ov::element::f32) {
new_const = change_constant_precision_to_fp16<ov::element::Type_t::f32>(const_node);
} else if (c_type == ov::element::f64) {
new_const = change_constant_precision_to_fp16<ov::element::Type_t::f64>(const_node);
} else {
return false;
}
auto convert = std::make_shared<ov::opset8::Convert>(new_const, const_node->get_element_type());
convert->set_friendly_name(const_node->get_friendly_name());
ngraph::copy_runtime_info(const_node, convert);
ov::mark_as_decompression(convert);
ov::replace_node(const_node, convert);
return true;
};
auto m = std::make_shared<pattern::Matcher>(const_pattern, matcher_name);
this->register_matcher(m, callback);
}
ov::pass::AddOldApiMapToParameters::AddOldApiMapToParameters() {
MATCHER_SCOPE(AddOldApiMapToParameters);
auto param_pattern = pattern::wrap_type<opset8::Parameter>();
ov::matcher_pass_callback callback = [=](pattern::Matcher& m) {
const auto& pattern_map = m.get_pattern_value_map();
auto node = pattern_map.at(param_pattern).get_node_shared_ptr();
auto param_node = std::dynamic_pointer_cast<ov::opset8::Parameter>(node);
if (!param_node)
return false;
auto p_type = param_node->get_element_type();
if (p_type == ov::element::f32 || p_type == ov::element::f64) {
std::vector<uint64_t> order;
if (ov::has_old_api_map(node)) {
auto old_api = ov::get_old_api_map(node).get();
order = old_api.get_order();
} else {
auto p_rank = param_node->get_partial_shape().rank();
if (p_rank.is_static()) {
auto r = p_rank.get_length();
order.resize(r);
std::iota(order.begin(), order.end(), 0);
} else {
return false;
}
}
ov::set_old_api_map(node, ov::OldApiMap(ov::OldApiMapAttr(order, ov::element::Type_t::f16)));
} else {
return false;
}
return true;
};
auto m = std::make_shared<pattern::Matcher>(param_pattern, matcher_name);
this->register_matcher(m, callback);
}

4
inference-engine/src/transformations/src/transformations/common_optimizations/conv_bias_fusion.cpp
View File

@ -99,7 +99,7 @@ bool conv_callback(ngraph::pattern::Matcher &m) {
auto expected_shape = Shape(output_rank, 1);
expected_shape[1] = channel_dim;
if (op::util::check_for_broadcast(expected_shape, const_shape)) {
if (ngraph::op::util::check_for_broadcast(expected_shape, const_shape)) {
return false;
}
@ -107,7 +107,7 @@ bool conv_callback(ngraph::pattern::Matcher &m) {
// the number of weights dimensions.
Output<Node> final_const = m_const;
if (is_scalar_multiplier) {
final_const = op::util::broadcastTo(m_const, expected_shape);
final_const = ngraph::op::util::broadcastTo(m_const, expected_shape);
}
if (final_const.get_shape().size() > 1) {

52
inference-engine/src/transformations/src/transformations/common_optimizations/convert_compression_only_to_legacy.cpp Normal file
View File

@ -0,0 +1,52 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "transformations/common_optimizations/convert_compression_only_to_legacy.hpp"
#include "transformations/convert_precision.hpp"
#include "transformations/utils/utils.hpp"
#include "openvino/opsets/opset8.hpp"
#include "openvino/pass/manager.hpp"
#include "itt.hpp"
using namespace ov;
bool ov::pass::ConvertPrecisionCompressedOnly::run_on_function(std::shared_ptr<ov::Function> f) {
if (ngraph::op::util::has_decompression_converts(f)) {
const precisions_array convert_precision_list{
{ov::element::f32, ov::element::f16}
};
auto convert_precision = ngraph::pass::ConvertPrecision(convert_precision_list);
return convert_precision.run_on_function(f);
}
return false;
}
ov::pass::EnableDecompressionConvertConstantFolding::EnableDecompressionConvertConstantFolding() {
MATCHER_SCOPE(EnableDecompressionConvertConstantFolding);
auto convert = pattern::wrap_type<opset8::Convert>();
ov::matcher_pass_callback callback = [=](pattern::Matcher& m) {
const auto& node = m.get_match_root();
if (!ov::is_decompression(node))
return false;
enable_constant_folding(node);
return true;
};
auto m = std::make_shared<ov::pass::pattern::Matcher>(convert, matcher_name);
this->register_matcher(m, callback);
}
bool ov::pass::ConvertCompressedOnlyToLegacy::run_on_function(std::shared_ptr<ov::Function> f) {
Manager manager(get_pass_config());
manager.register_pass<ov::pass::ConvertPrecisionCompressedOnly>();
manager.register_pass<ov::pass::EnableDecompressionConvertConstantFolding>();
manager.register_pass<ov::pass::ConstantFolding>();
manager.run_passes(f);
return false;
}

73
inference-engine/src/transformations/src/transformations/common_optimizations/mark_precision_sensitive_subgraphs.cpp Normal file
View File

@ -0,0 +1,73 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "transformations/common_optimizations/mark_precision_sensitive_subgraphs.hpp"
#include <memory>
#include <vector>
#include "transformations/utils/utils.hpp"
#include "transformations/rt_info/disable_fp16_compression.hpp"
#include "openvino/opsets/opset1.hpp"
#include "openvino/opsets/opset3.hpp"
#include "openvino/opsets/opset8.hpp"
#include "openvino/pass/pattern/op/wrap_type.hpp"
#include "openvino/op/util/precision_sensitive_attribute.hpp"
using namespace std;
namespace {
void visit_shape_path(const shared_ptr<ov::Node>& node, unordered_set<shared_ptr<ov::Node>>& visited) {
if (!node)
return;
visited.insert(node);
deque<shared_ptr<ov::Node>> nodes{ node };
while (!nodes.empty()) {
auto curr_node = nodes.front();
nodes.pop_front();
// Do not check if already visited
if (ov::is_type<ov::opset1::ShapeOf>(curr_node) || ov::is_type<ov::opset3::ShapeOf>(curr_node)) {
continue;
}
visited.insert(curr_node);
if (ov::is_type<ov::opset8::Constant>(curr_node)) {
ov::disable_fp16_compression(curr_node);
} else {
for (auto& input_value : curr_node->input_values()) {
// continue searching
const auto& input_node = input_value.get_node_shared_ptr();
nodes.push_front(input_node);
}
}
}
}
} // namespace
bool ov::pass::MarkPrecisionSensitiveSubgraphs::run_on_function(std::shared_ptr<ov::Function> f) {
deque<shared_ptr<Node>> nodes;
unordered_set<shared_ptr<Node>> visited;
for (auto& r : f->get_results())
nodes.push_back(r);
for (auto& r : f->get_sinks())
nodes.emplace_back(r);
while (!nodes.empty()) {
auto curr_node = nodes.front();
nodes.pop_front();
if (visited.count(curr_node))
continue;
for (auto& input : curr_node->inputs()) {
if (ov::is_precision_sensitive(input))
visit_shape_path(input.get_source_output().get_node_shared_ptr(), visited);
}
visited.insert(curr_node);
for (auto& input_value : curr_node->input_values()) {
// continue searching
const auto& input_node = input_value.get_node_shared_ptr();
nodes.push_front(input_node);
}
}
return true;
}

27
inference-engine/src/transformations/src/transformations/disable_decompression_convert_constant_folding.cpp Normal file
View File

@ -0,0 +1,27 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "transformations/disable_decompression_convert_constant_folding.hpp"
#include "openvino/opsets/opset8.hpp"
#include "openvino/pass/pattern/op/wrap_type.hpp"
#include "transformations/rt_info/disable_constant_folding.hpp"
#include "transformations/rt_info/decompression.hpp"
#include "itt.hpp"
ov::pass::DisableDecompressionConvertConstantFolding::DisableDecompressionConvertConstantFolding() {
MATCHER_SCOPE(DisableDecompressionConvertConstantFolding);
auto convert = pattern::wrap_type<opset8::Convert>();
ov::matcher_pass_callback callback = [=](pattern::Matcher& m) {
const auto& node = m.get_match_root();
if (!ov::is_decompression(node))
return false;
disable_constant_folding(node);
return true;
};
auto m = std::make_shared<ov::pass::pattern::Matcher>(convert, matcher_name);
this->register_matcher(m, callback);
}

2
inference-engine/src/transformations/src/transformations/rt_info/attributes.cpp
View File

@ -8,10 +8,12 @@ ov::pass::Attributes::Attributes() {
register_factory<VariantWrapper<ngraph::FusedNames>>();
register_factory<PrimitivesPriority>();
register_factory<DisableConstantFolding>();
register_factory<DisableFP16Compression>();
register_factory<NmsSelectedIndices>();
register_factory<StridesPropagation>();
register_factory<OldApiMap>();
register_factory<LayoutAttribute>();
register_factory<Decompression>();
}
ov::Variant* ov::pass::Attributes::create_by_type_info(const ov::DiscreteTypeInfo& type_info) {

20
inference-engine/src/transformations/src/transformations/rt_info/decompression.cpp Normal file
View File

@ -0,0 +1,20 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "transformations/rt_info/decompression.hpp"
void ov::mark_as_decompression(const std::shared_ptr<Node>& node) {
auto& rt_info = node->get_rt_info();
rt_info[Decompression::get_type_info_static()] = std::make_shared<Decompression>();
}
void ov::unmark_as_decompression(const std::shared_ptr<Node>& node) {
auto& rt_info = node->get_rt_info();
rt_info.erase(Decompression::get_type_info_static());
}
bool ov::is_decompression(const std::shared_ptr<Node>& node) {
const auto& rt_info = node->get_rt_info();
return rt_info.count(Decompression::get_type_info_static());
}

20
inference-engine/src/transformations/src/transformations/rt_info/disable_fp16_compression.cpp Normal file
View File

@ -0,0 +1,20 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "transformations/rt_info/disable_fp16_compression.hpp"
void ov::disable_fp16_compression(const std::shared_ptr<Node>& node) {
auto& rt_info = node->get_rt_info();
rt_info[DisableFP16Compression::get_type_info_static()] = std::make_shared<DisableFP16Compression>();
}
void ov::enable_fp16_compression(const std::shared_ptr<Node>& node) {
auto& rt_info = node->get_rt_info();
rt_info.erase(DisableFP16Compression::get_type_info_static());
}
bool ov::fp16_compression_is_disabled(const std::shared_ptr<Node>& node) {
const auto& rt_info = node->get_rt_info();
return rt_info.count(DisableFP16Compression::get_type_info_static());
}

6
inference-engine/tests/functional/inference_engine/ir_serialization/rt_info_serialization.cpp
View File

@ -57,6 +57,7 @@ TEST_F(RTInfoSerializationTest, all_attributes_latest) {
std::make_shared<ov::PrimitivesPriority>("priority");
info[ov::OldApiMap::get_type_info_static()] = std::make_shared<ov::OldApiMap>(
ov::OldApiMapAttr(std::vector<uint64_t>{0, 2, 3, 1}, ngraph::element::Type_t::f32));
info[ov::Decompression::get_type_info_static()] = std::make_shared<ov::Decompression>();
};
std::shared_ptr<ngraph::Function> function;
@ -100,6 +101,11 @@ TEST_F(RTInfoSerializationTest, all_attributes_latest) {
auto old_api_map_attr_val = old_api_map_attr->get();
ASSERT_EQ(old_api_map_attr_val.get_order(), std::vector<uint64_t>({0, 2, 3, 1}));
ASSERT_EQ(old_api_map_attr_val.get_type(), ngraph::element::Type_t::f32);
const std::string& dkey = ov::Decompression::get_type_info_static();
ASSERT_TRUE(info.count(dkey));
auto decompression_attr = std::dynamic_pointer_cast<ov::Decompression>(info.at(dkey));
ASSERT_TRUE(decompression_attr);
};
auto add = f->get_results()[0]->get_input_node_ptr(0);

138
inference-engine/tests/functional/inference_engine/transformations/compress_float_constants_test.cpp Normal file
View File

@ -0,0 +1,138 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <gtest/gtest.h>
#include <string>
#include <memory>
#include "openvino/core/function.hpp"
#include "openvino/opsets/opset8.hpp"
#include "openvino/pass/manager.hpp"
#include "transformations/common_optimizations/compress_float_constants.hpp"
#include "transformations/common_optimizations/mark_precision_sensitive_subgraphs.hpp"
#include "transformations/init_node_info.hpp"
#include "transformations/utils/utils.hpp"
#include "common_test_utils/ngraph_test_utils.hpp"
using namespace testing;
TEST(TransformationTests, CompressConstants_f32) {
std::shared_ptr<ov::Function> f(nullptr), f_ref(nullptr);
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{ 1, 3, 12, 12 });
auto const_weights = ov::opset8::Constant::create(ov::element::f32,
ov::Shape{ 1, 3, 3, 3 },
{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 });
auto conv = std::make_shared<ov::opset8::Convolution>(input,
const_weights,
ov::Strides{1, 1},
ov::CoordinateDiff{0, 0},
ov::CoordinateDiff{0, 0},
ov::Strides{1, 1});
auto const_scales = ov::opset8::Constant::create(ov::element::f32, ov::Shape{ 1 }, { 1.4 });
auto shape = std::make_shared<ov::opset8::ShapeOf>(conv);
auto convert1 = std::make_shared<ov::opset8::Convert>(shape, ov::element::f32);
auto mul = std::make_shared<ov::opset8::Multiply>(convert1, const_scales);
auto convert2 = std::make_shared<ov::opset8::Convert>(mul, ov::element::i32);
auto default_scales_node = ov::opset8::Constant::create(ov::element::f32, ov::Shape{ 4 }, { 1., 1., 1.4, 1.4 });
auto axes_node = ov::opset8::Constant::create(ov::element::i64, ov::Shape{ 4 }, { 0, 1, 2, 3 });
auto interpolate4_attr = ov::opset8::Interpolate::InterpolateAttrs(ov::opset8::Interpolate::InterpolateMode::NEAREST,
ov::opset8::Interpolate::ShapeCalcMode::SIZES, std::vector<size_t>{0, 0, 0, 0}, std::vector<size_t>{0, 0, 0, 0},
ov::opset8::Interpolate::CoordinateTransformMode::ASYMMETRIC, ov::opset8::Interpolate::NearestMode::SIMPLE,
false, -0.75);
auto resize = std::make_shared<ov::opset8::Interpolate>(conv, convert2, default_scales_node, axes_node, interpolate4_attr);
f = std::make_shared<ov::Function>(ov::NodeVector{ resize }, ov::ParameterVector{ input });
ov::pass::Manager manager;
manager.register_pass<ngraph::pass::InitNodeInfo>();
manager.register_pass<ov::pass::MarkPrecisionSensitiveSubgraphs>();
manager.register_pass<ov::pass::CompressFloatConstants>();
manager.run_passes(f);
ASSERT_NO_THROW(check_rt_info(f));
}
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{ 1, 3, 12, 12 });
auto const_weights = ov::opset8::Constant::create(ov::element::f16,
ov::Shape{ 1, 3, 3, 3 },
{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 });
auto convert_ins1 = std::make_shared<ov::opset8::Convert>(const_weights, ov::element::f32);
auto conv = std::make_shared<ov::opset8::Convolution>(input,
convert_ins1,
ov::Strides{ 1, 1 },
ov::CoordinateDiff{ 0, 0 },
ov::CoordinateDiff{ 0, 0 },
ov::Strides{ 1, 1 });
auto const_scales = ov::opset8::Constant::create(ov::element::f32, ov::Shape{ 1 }, { 1.4 });
auto shape = std::make_shared<ov::opset8::ShapeOf>(conv);
auto convert1 = std::make_shared<ov::opset8::Convert>(shape, ov::element::f32);
auto mul = std::make_shared<ov::opset8::Multiply>(convert1, const_scales);
auto convert2 = std::make_shared<ov::opset8::Convert>(mul, ov::element::i32);
auto default_scales_node = ov::opset8::Constant::create(ov::element::f32, ov::Shape{ 4 }, { 1., 1., 1.4, 1.4 });
auto axes_node = ov::opset8::Constant::create(ov::element::i64, ov::Shape{ 4 }, { 0, 1, 2, 3 });
auto interpolate4_attr = ov::opset8::Interpolate::InterpolateAttrs(ov::opset8::Interpolate::InterpolateMode::NEAREST,
ov::opset8::Interpolate::ShapeCalcMode::SIZES, std::vector<size_t>{0, 0, 0, 0}, std::vector<size_t>{0, 0, 0, 0},
ov::opset8::Interpolate::CoordinateTransformMode::ASYMMETRIC, ov::opset8::Interpolate::NearestMode::SIMPLE,
false, -0.75);
auto resize = std::make_shared<ov::opset8::Interpolate>(conv, convert2, default_scales_node, axes_node, interpolate4_attr);
f_ref = std::make_shared<ov::Function>(ov::NodeVector{ resize }, ov::ParameterVector{ input });
}
auto res = compare_functions(f, f_ref, true);
ASSERT_TRUE(res.first) << res.second;
}
TEST(TransformationTests, CompressConstants_f64) {
std::shared_ptr<ov::Function> f(nullptr), f_ref(nullptr);
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f64, ov::Shape{ 1, 3, 12, 12 });
auto const_weights = ov::opset8::Constant::create(ov::element::f64,
ov::Shape{ 1, 3, 3, 3 },
{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 });
auto conv = std::make_shared<ov::opset8::Convolution>(input,
const_weights,
ov::Strides{ 1, 1 },
ov::CoordinateDiff{ 0, 0 },
ov::CoordinateDiff{ 0, 0 },
ov::Strides{ 1, 1 });
f = std::make_shared<ov::Function>(ov::NodeVector{ conv }, ov::ParameterVector{ input });
ov::pass::Manager manager;
manager.register_pass<ngraph::pass::InitNodeInfo>();
manager.register_pass<ov::pass::MarkPrecisionSensitiveSubgraphs>();
manager.register_pass<ov::pass::CompressFloatConstants>();
manager.run_passes(f);
ASSERT_NO_THROW(check_rt_info(f));
}
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f64, ov::Shape{ 1, 3, 12, 12 });
auto const_weights = ov::opset8::Constant::create(ov::element::f16,
ov::Shape{ 1, 3, 3, 3 },
{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 });
auto convert_ins1 = std::make_shared<ov::opset8::Convert>(const_weights, ov::element::f64);
auto conv = std::make_shared<ov::opset8::Convolution>(input,
convert_ins1,
ov::Strides{ 1, 1 },
ov::CoordinateDiff{ 0, 0 },
ov::CoordinateDiff{ 0, 0 },
ov::Strides{ 1, 1 });
f_ref = std::make_shared<ov::Function>(ov::NodeVector{ conv }, ov::ParameterVector{ input });
}
auto res = compare_functions(f, f_ref, true);
ASSERT_TRUE(res.first) << res.second;
}

106
inference-engine/tests/functional/inference_engine/transformations/convert_compression_only_to_legacy_test.cpp Normal file
View File

@ -0,0 +1,106 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <gtest/gtest.h>
#include <string>
#include <memory>
#include "openvino/core/function.hpp"
#include "openvino/opsets/opset8.hpp"
#include "openvino/pass/manager.hpp"
#include "transformations/common_optimizations/convert_compression_only_to_legacy.hpp"
#include "transformations/rt_info/decompression.hpp"
#include "transformations/init_node_info.hpp"
#include "transformations/utils/utils.hpp"
#include "common_test_utils/ngraph_test_utils.hpp"
using namespace testing;
TEST(TransformationTests, ConvertCompressionOnlyToLegacy) {
std::shared_ptr<ov::Function> f(nullptr), f_ref(nullptr);
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{ 1, 3, 12, 12 });
auto const_weights = ov::opset8::Constant::create(ov::element::f16,
ov::Shape{ 1, 3, 3, 3 },
{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 });
auto convert_ins1 = std::make_shared<ov::opset8::Convert>(const_weights, ov::element::f32);
ov::mark_as_decompression(convert_ins1);
auto conv = std::make_shared<ov::opset8::Convolution>(input,
convert_ins1,
ov::Strides{ 1, 1 },
ov::CoordinateDiff{ 0, 0 },
ov::CoordinateDiff{ 0, 0 },
ov::Strides{ 1, 1 });
f = std::make_shared<ov::Function>(ov::NodeVector{ conv }, ov::ParameterVector{ input });
ov::pass::Manager manager;
manager.register_pass<ngraph::pass::InitNodeInfo>();
manager.register_pass<ov::pass::ConvertCompressedOnlyToLegacy>();
manager.run_passes(f);
ASSERT_NO_THROW(check_rt_info(f));
}
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f16, ov::Shape{ 1, 3, 12, 12 });
auto const_weights = ov::opset8::Constant::create(ov::element::f16,
ov::Shape{ 1, 3, 3, 3 },
{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 });
auto conv = std::make_shared<ov::opset8::Convolution>(input,
const_weights,
ov::Strides{ 1, 1 },
ov::CoordinateDiff{ 0, 0 },
ov::CoordinateDiff{ 0, 0 },
ov::Strides{ 1, 1 });
f_ref = std::make_shared<ov::Function>(ov::NodeVector{ conv }, ov::ParameterVector{ input });
}
auto res = compare_functions(f, f_ref, true);
ASSERT_TRUE(res.first) << res.second;
}
TEST(TransformationTests, ConvertCompressionOnlyToLegacyNoConvertion) {
std::shared_ptr<ov::Function> f(nullptr), f_ref(nullptr);
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{ 1, 3, 12, 12 });
auto const_weights = ov::opset8::Constant::create(ov::element::f32,
ov::Shape{ 1, 3, 3, 3 },
{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 });
auto conv = std::make_shared<ov::opset8::Convolution>(input,
const_weights,
ov::Strides{ 1, 1 },
ov::CoordinateDiff{ 0, 0 },
ov::CoordinateDiff{ 0, 0 },
ov::Strides{ 1, 1 });
f = std::make_shared<ov::Function>(ov::NodeVector{ conv }, ov::ParameterVector{ input });
ov::pass::Manager manager;
manager.register_pass<ngraph::pass::InitNodeInfo>();
manager.register_pass<ov::pass::ConvertCompressedOnlyToLegacy>();
manager.run_passes(f);
ASSERT_NO_THROW(check_rt_info(f));
}
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{ 1, 3, 12, 12 });
auto const_weights = ov::opset8::Constant::create(ov::element::f32,
ov::Shape{ 1, 3, 3, 3 },
{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 });
auto conv = std::make_shared<ov::opset8::Convolution>(input,
const_weights,
ov::Strides{ 1, 1 },
ov::CoordinateDiff{ 0, 0 },
ov::CoordinateDiff{ 0, 0 },
ov::Strides{ 1, 1 });
f_ref = std::make_shared<ov::Function>(ov::NodeVector{ conv }, ov::ParameterVector{ input });
}
auto res = compare_functions(f, f_ref, true);
ASSERT_TRUE(res.first) << res.second;
}

12
model-optimizer/mo/back/offline_transformations.py
View File

@ -33,8 +33,11 @@ def apply_moc_transformations(net: object):
from openvino.offline_transformations import ApplyMOCTransformations # pylint: disable=import-error,no-name-in-module
ApplyMOCTransformations(net, False)
def compress_model(net:object):
from openvino.offline_transformations import CompressModelTransformation # pylint: disable=import-error,no-name-in-module
CompressModelTransformation(net)
def apply_offline_transformations(input_model: str, framework: str, transforms: list):
def apply_offline_transformations(input_model: str, framework: str, transforms: list, compress_fp16=False):
# This variable is only needed by GenerateMappingFile transformation
# to produce correct mapping
extract_names = framework in ['tf', 'mxnet', 'kaldi']
@ -58,6 +61,10 @@ def apply_offline_transformations(input_model: str, framework: str, transforms:
apply_user_transformations(net, transforms)
apply_moc_transformations(net)
if compress_fp16:
compress_model(net)
Serialize(net, str(input_model + ".xml").encode('utf-8'), (input_model + ".bin").encode('utf-8'))
path_to_mapping = input_model + ".mapping"
GenerateMappingFile(net, path_to_mapping.encode('utf-8'), extract_names)
@ -68,6 +75,7 @@ if __name__ == "__main__":
parser.add_argument("--input_model")
parser.add_argument("--framework")
parser.add_argument("--transform")
parser.add_argument("--compress_fp16", action='store_true')
args = parser.parse_args()
apply_offline_transformations(args.input_model, args.framework, parse_transform(args.transform))
apply_offline_transformations(args.input_model, args.framework, parse_transform(args.transform), args.compress_fp16)

15
model-optimizer/mo/main.py
View File

@ -205,6 +205,12 @@ def arguments_post_parsing(argv: argparse.Namespace):
except Exception as e:
raise_ie_not_found()
if 'data_type' in argv and argv.data_type in ['FP16', 'half']:
argv.data_type = 'FP32'
argv.compress_fp16 = True
else:
argv.compress_fp16 = False
# This is just to check that transform key is valid and transformations are available
check_available_transforms(parse_transform(argv.transform))
@ -355,10 +361,15 @@ def emit_ir(graph: Graph, argv: argparse.Namespace):
if not argv.legacy_ir_generation:
path_to_offline_transformations = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'back',
'offline_transformations.py')
status = subprocess.run([sys.executable, path_to_offline_transformations,
cmd = [sys.executable, path_to_offline_transformations,
"--input_model", orig_model_name,
"--framework", argv.framework,
"--transform", argv.transform], env=os.environ)
"--transform", argv.transform]
if argv.compress_fp16:
cmd += ["--compress_fp16"]
# restore data_type cmd parameter
argv.data_type = 'FP16'
status = subprocess.run(cmd, env=os.environ)
return_code = status.returncode
except Exception as e:
return_code = "failed"

4
model-optimizer/mo/moc_frontend/serialize.py
View File

@ -19,6 +19,10 @@ def moc_emit_ir(ngraph_function: Function, argv: argparse.Namespace):
apply_user_transformations(network, parse_transform(argv.transform))
apply_moc_transformations(network)
if argv.compress_fp16:
from mo.back.offline_transformations import compress_model
compress_model(network)
orig_model_name = os.path.normpath(os.path.join(output_dir, argv.model_name))
network.serialize(orig_model_name + ".xml", orig_model_name + ".bin")

13
model-optimizer/mo/pipeline/common.py
View File

@ -173,18 +173,19 @@ def convert_inputs_of_specific_ops(graph: Graph):
def prepare_emit_ir(graph: Graph, data_type: str, output_dir: str, output_model_name: str,
mean_data: [list, None] = None, input_names: list = None, meta_info: dict = None,
use_temporary_path=False):
use_temporary_path=False, used_by_ir_reader=False):
if input_names is None:
input_names = []
if meta_info is None:
meta_info = {}
graph.strict_mode = False
# convert Parameter data types
convert_data_type.convert_parameters_data_type(graph, data_type)
# convert blobs (usually weights and biases)
for sub_graph in [graph] + collect_sub_graphs(graph):
convert_data_type.convert_blobs(sub_graph, data_type)
if not used_by_ir_reader:
# convert Parameter data types
convert_data_type.convert_parameters_data_type(graph, data_type)
# convert blobs (usually weights and biases)
for sub_graph in [graph] + collect_sub_graphs(graph):
convert_data_type.convert_blobs(sub_graph, data_type)
# restore data type for specific inputs/outputs of specific ops to the data types required by nGraph
for_graph_and_each_sub_graph_recursively(graph, convert_inputs_of_specific_ops)

2
model-optimizer/mo/utils/ir_reader/restore_graph.py
View File

@ -88,4 +88,4 @@ def save_restored_graph(graph: Graph, path: str, meta_data, name=None):
for_graph_and_each_sub_graph_recursively(graph, RemoveConstOps().find_and_replace_pattern)
for_graph_and_each_sub_graph_recursively(graph, CreateConstNodesReplacement().find_and_replace_pattern)
prepare_emit_ir(graph, data_type, path, name, meta_info=meta_data)
prepare_emit_ir(graph, data_type, path, name, meta_info=meta_data, used_by_ir_reader=True)

8
ngraph/core/include/openvino/core/variant.hpp
View File

@ -58,6 +58,14 @@ protected:
value_type m_value;
};
template <>
class VariantImpl<void> : public Variant {
public:
using value_type = void;
VariantImpl() = default;
};
extern template class OPENVINO_API VariantImpl<std::string>;
extern template class OPENVINO_API VariantImpl<int64_t>;
extern template class OPENVINO_API VariantImpl<bool>;

35
ngraph/core/include/openvino/op/util/precision_sensitive_attribute.hpp Normal file
View File

@ -0,0 +1,35 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "openvino/core/core_visibility.hpp"
#include "openvino/core/node.hpp"
#include "openvino/core/variant.hpp"
namespace ov {
void OPENVINO_API mark_as_precision_sensitive(ov::Input<ov::Node> node_input);
void OPENVINO_API unmark_as_precision_sensitive(ov::Input<ov::Node> node_input);
bool OPENVINO_API is_precision_sensitive(const ov::Input<ov::Node>& node_input);
/**
* @brief PrecisionSensitive class represents runtime info attribute that marks
* input to an operation as a precision sensitive and disables compression to FP16
* of the subgraph before this input.
*/
class OPENVINO_API PrecisionSensitive : public VariantImpl<void> {
public:
OPENVINO_RTTI("precision_sensitive", "0");
PrecisionSensitive() = default;
bool is_copyable() const override {
return false;
}
};
} // namespace ov

4
ngraph/core/src/op/batch_to_space.cpp
View File

@ -19,6 +19,7 @@
#include "ngraph/runtime/reference/strided_slice.hpp"
#include "ngraph/shape.hpp"
#include "ngraph/slice_plan.hpp"
#include "openvino/op/util/precision_sensitive_attribute.hpp"
using namespace std;
using namespace ngraph;
@ -30,6 +31,9 @@ ngraph::op::v1::BatchToSpace::BatchToSpace(const ngraph::Output<ngraph::Node>& d
const ngraph::Output<ngraph::Node>& crops_begin,
const ngraph::Output<ngraph::Node>& crops_end)
: Op({data, block_shape, crops_begin, crops_end}) {
ov::mark_as_precision_sensitive(input(1));
ov::mark_as_precision_sensitive(input(2));
ov::mark_as_precision_sensitive(input(3));
constructor_validate_and_infer_types();
}

2
ngraph/core/src/op/convolution.cpp
View File

@ -12,6 +12,7 @@
#include "ngraph/op/reshape.hpp"
#include "ngraph/util.hpp"
#include "ngraph/validation_util.hpp"
#include "openvino/op/util/precision_sensitive_attribute.hpp"
using namespace std;
using namespace ngraph;
@ -117,6 +118,7 @@ op::v1::ConvolutionBackpropData::ConvolutionBackpropData(const Output<Node>& dat
m_pads_end(pads_end),
m_auto_pad(auto_pad),
m_output_padding(output_padding) {
ov::mark_as_precision_sensitive(input(2));
constructor_validate_and_infer_types();
}

2
ngraph/core/src/op/group_conv.cpp
View File

@ -14,6 +14,7 @@
#include "ngraph/op/convolution.hpp"
#include "ngraph/op/reshape.hpp"
#include "ngraph/validation_util.hpp"
#include "openvino/op/util/precision_sensitive_attribute.hpp"
using namespace std;
using namespace ngraph;
@ -277,6 +278,7 @@ op::v1::GroupConvolutionBackpropData::GroupConvolutionBackpropData(const Output<
m_pads_end(pads_end),
m_auto_pad(auto_pad),
m_output_padding(output_padding) {
ov::mark_as_precision_sensitive(input(2));
constructor_validate_and_infer_types();
}

4
ngraph/core/src/op/interpolate.cpp
View File

@ -13,6 +13,7 @@
#include "itt.hpp"
#include "ngraph/op/constant.hpp"
#include "ngraph/runtime/reference/interpolate.hpp"
#include "openvino/op/util/precision_sensitive_attribute.hpp"
using namespace std;
using namespace ngraph;
@ -22,6 +23,7 @@ BWDCMP_RTTI_DEFINITION(op::v0::Interpolate);
op::v0::Interpolate::Interpolate(const Output<Node>& image, const Output<Node>& output_shape, const Attributes& attrs)
: Op({image, output_shape}),
m_attrs(attrs) {
ov::mark_as_precision_sensitive(input(1));
constructor_validate_and_infer_types();
}
@ -99,6 +101,8 @@ op::v4::Interpolate::Interpolate(const Output<Node>& image,
const op::v4::Interpolate::InterpolateAttrs& attrs)
: Op({image, output_shape, scales, axes}),
m_attrs(attrs) {
ov::mark_as_precision_sensitive(input(1));
ov::mark_as_precision_sensitive(input(2));
constructor_validate_and_infer_types();
}

2
ngraph/core/src/op/one_hot.cpp
View File

@ -9,6 +9,7 @@
#include "ngraph/op/util/op_types.hpp"
#include "ngraph/runtime/reference/one_hot.hpp"
#include "ngraph/validation_util.hpp"
#include "openvino/op/util/precision_sensitive_attribute.hpp"
using namespace std;
using namespace ngraph;
@ -22,6 +23,7 @@ op::v1::OneHot::OneHot(const Output<Node>& indices,
int64_t axis)
: Op({indices, depth, on_value, off_value}),
m_axis(axis) {
ov::mark_as_precision_sensitive(input(1));
constructor_validate_and_infer_types();
}

5
ngraph/core/src/op/pad.cpp
View File

@ -13,6 +13,7 @@
#include "ngraph/op/constant.hpp"
#include "ngraph/op/util/op_types.hpp"
#include "ngraph/runtime/reference/pad.hpp"
#include "openvino/op/util/precision_sensitive_attribute.hpp"
using namespace std;
using namespace ngraph;
@ -26,6 +27,8 @@ op::v1::Pad::Pad(const Output<Node>& arg,
PadMode pad_mode)
: Op({arg, pads_begin, pads_end, arg_pad_value}),
m_pad_mode{pad_mode} {
ov::mark_as_precision_sensitive(input(1));
ov::mark_as_precision_sensitive(input(2));
constructor_validate_and_infer_types();
}
@ -35,6 +38,8 @@ op::v1::Pad::Pad(const Output<Node>& arg,
PadMode pad_mode)
: Op({arg, pads_begin, pads_end, op::v0::Constant::create(arg.get_element_type(), ov::Shape{}, {0})}),
m_pad_mode{pad_mode} {
ov::mark_as_precision_sensitive(input(1));
ov::mark_as_precision_sensitive(input(2));
constructor_validate_and_infer_types();
}

2
ngraph/core/src/op/reshape.cpp
View File

@ -11,6 +11,7 @@
#include "ngraph/op/constant.hpp"
#include "ngraph/runtime/opt_kernel/reshape.hpp"
#include "ngraph/runtime/reference/reshape.hpp"
#include "openvino/op/util/precision_sensitive_attribute.hpp"
using namespace std;
using namespace ngraph;
@ -44,6 +45,7 @@ BWDCMP_RTTI_DEFINITION(op::v1::Reshape);
op::v1::Reshape::Reshape(const Output<Node>& arg, const Output<Node>& shape_pattern, bool zero_flag)
: Op({arg, shape_pattern}),
m_special_zero(zero_flag) {
ov::mark_as_precision_sensitive(input(1));
constructor_validate_and_infer_types();
}

4
ngraph/core/src/op/space_to_batch.cpp
View File

@ -17,6 +17,7 @@
#include "ngraph/runtime/opt_kernel/reshape.hpp"
#include "ngraph/runtime/reference/pad.hpp"
#include "ngraph/shape.hpp"
#include "openvino/op/util/precision_sensitive_attribute.hpp"
using namespace std;
using namespace ngraph;
@ -28,6 +29,9 @@ ngraph::op::v1::SpaceToBatch::SpaceToBatch(const ngraph::Output<ngraph::Node>& d
const ngraph::Output<ngraph::Node>& pads_begin,
const ngraph::Output<ngraph::Node>& pads_end)
: Op({data, block_shape, pads_begin, pads_end}) {
ov::mark_as_precision_sensitive(input(1));
ov::mark_as_precision_sensitive(input(2));
ov::mark_as_precision_sensitive(input(3));
constructor_validate_and_infer_types();
}

4
ngraph/core/src/op/strided_slice.cpp
View File

@ -19,6 +19,7 @@
#include "ngraph/type/element_type_traits.hpp"
#include "ngraph/util.hpp"
#include "ngraph/validation_util.hpp"
#include "openvino/op/util/precision_sensitive_attribute.hpp"
using namespace std;
using namespace ngraph;
@ -40,6 +41,9 @@ op::v1::StridedSlice::StridedSlice(const Output<Node>& data,
m_new_axis_mask{new_axis_mask},
m_shrink_axis_mask{shrink_axis_mask},
m_ellipsis_mask{ellipsis_mask} {
ov::mark_as_precision_sensitive(input(1));
ov::mark_as_precision_sensitive(input(2));
ov::mark_as_precision_sensitive(input(3));
constructor_validate_and_infer_types();
}

2
ngraph/core/src/op/tile.cpp
View File

@ -9,6 +9,7 @@
#include "itt.hpp"
#include "ngraph/op/constant.hpp"
#include "ngraph/runtime/reference/tile.hpp"
#include "openvino/op/util/precision_sensitive_attribute.hpp"
using namespace std;
using namespace ngraph;
@ -16,6 +17,7 @@ using namespace ngraph;
BWDCMP_RTTI_DEFINITION(op::v0::Tile);
op::v0::Tile::Tile(const Output<Node>& data, const Output<Node>& repeats) : Op({data, repeats}) {
ov::mark_as_precision_sensitive(input(1));
constructor_validate_and_infer_types();
}

3
ngraph/core/src/op/topk.cpp
View File

@ -15,6 +15,7 @@
#include "ngraph/runtime/reference/topk.hpp"
#include "ngraph/shape.hpp"
#include "ngraph/validation_util.hpp"
#include "openvino/op/util/precision_sensitive_attribute.hpp"
using namespace std;
using namespace ngraph;
@ -154,6 +155,7 @@ op::v1::TopK::TopK(const Output<Node>& data,
m_mode{as_enum<Mode>(mode)},
m_sort{as_enum<SortType>(sort)},
m_index_element_type{index_element_type} {
ov::mark_as_precision_sensitive(input(1));
constructor_validate_and_infer_types();
}
@ -169,6 +171,7 @@ op::v1::TopK::TopK(const Output<Node>& data,
m_mode{mode},
m_sort{sort},
m_index_element_type{index_element_type} {
ov::mark_as_precision_sensitive(input(1));
constructor_validate_and_infer_types();
}

9
ngraph/core/src/op/util/broadcast_base.cpp
View File

@ -14,6 +14,7 @@
#include "ngraph/op/util/op_types.hpp"
#include "ngraph/partial_shape.hpp"
#include "ngraph/runtime/reference/broadcast.hpp"
#include "openvino/op/util/precision_sensitive_attribute.hpp"
using namespace std;
@ -24,13 +25,17 @@ ov::op::util::BroadcastBase::BroadcastBase(const Output<Node>& arg,
const Output<Node>& axes_mapping,
const BroadcastModeSpec& broadcast_mode)
: Op({arg, target_shape, axes_mapping}),
m_mode{broadcast_mode} {}
m_mode{broadcast_mode} {
ov::mark_as_precision_sensitive(input(1));
}
ov::op::util::BroadcastBase::BroadcastBase(const Output<Node>& arg,
const Output<Node>& target_shape,
const BroadcastModeSpec& broadcast_mode)
: Op({arg, target_shape}),
m_mode{broadcast_mode} {}
m_mode{broadcast_mode} {
ov::mark_as_precision_sensitive(input(1));
}
ov::PartialShape ov::op::util::BroadcastBase::get_result_shape_pdpd(const PartialShape& arg0_shape,
const PartialShape& target_pshape,

20
ngraph/core/src/op/util/precision_sensitive_attribute.cpp Normal file
View File

@ -0,0 +1,20 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "openvino/op/util/precision_sensitive_attribute.hpp"
void ov::mark_as_precision_sensitive(ov::Input<ov::Node> node_input) {
auto& rt_info = node_input.get_rt_info();
rt_info[PrecisionSensitive::get_type_info_static()] = std::make_shared<PrecisionSensitive>();
}
void ov::unmark_as_precision_sensitive(ov::Input<ov::Node> node_input) {
auto& rt_info = node_input.get_rt_info();
rt_info.erase(PrecisionSensitive::get_type_info_static());
}
bool ov::is_precision_sensitive(const ov::Input<ov::Node>& node_input) {
const auto& rt_info = node_input.get_rt_info();
return rt_info.count(PrecisionSensitive::get_type_info_static());
}