natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

DeformableConv v8: reference implementation (#6514) 

* DeformableConv v8:reference implementation

* ngraph codestyle

* update reference iplementation

* Disable tests on GPU

* fix unit tests

* Update DeformConv single layer tests

* fix serialization tests
This commit is contained in:
Ivan Tikhonov 2021-07-19 06:15:34 +03:00 committed by GitHub
parent bc36425381
commit 697c52abfe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 3367 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp
View File

@ -293,6 +293,7 @@ static void Transformation(CNNNetwork& clonedNetwork, const Config& conf) {
pass_config->disable<ngraph::pass::WeightsDequantizeToFakeQuantize>();
pass_config->disable<ngraph::pass::SimplifyCTCGreedyDecoderSeqLen>();
pass_config->disable<ngraph::pass::ConvertGather7ToGather1>();
pass_config->disable<ngraph::pass::ConvertDeformableConv8To1>();
pass_config->enable<ngraph::pass::ConvertInterpolate1ToInterpolate4>();
pass_config->enable<ngraph::pass::ConvertGather1ToGather7>();

10
inference-engine/tests/functional/inference_engine/serialization/single_layer/deformable_convolution.cpp
View File

@ -25,6 +25,8 @@ const std::vector<std::vector<size_t>> dilations = {{1, 1}};
const std::vector<size_t> groups = {1};
const std::vector<size_t> defor_groups = {1};
const std::vector<size_t> numOutChannels = {1};
const std::vector<bool> with_bilinear_interpolation_pad = { false, true };
const std::vector<bool> with_modulated_scalar = { false, true };
const auto conv2DParams_ExplicitPadding = ::testing::Combine(
::testing::ValuesIn(offsets), ::testing::ValuesIn(filters),
@ -32,7 +34,9 @@ const auto conv2DParams_ExplicitPadding = ::testing::Combine(
::testing::ValuesIn(padEnds), ::testing::ValuesIn(dilations),
::testing::ValuesIn(groups), ::testing::ValuesIn(defor_groups),
::testing::ValuesIn(numOutChannels),
::testing::Values(ngraph::op::PadType::EXPLICIT));
::testing::Values(ngraph::op::PadType::EXPLICIT),
::testing::ValuesIn(with_bilinear_interpolation_pad),
::testing::ValuesIn(with_modulated_scalar));
const auto conv2DParams_AutoPadValid = ::testing::Combine(
::testing::ValuesIn(offsets), ::testing::ValuesIn(filters),
::testing::ValuesIn(strides),
@ -40,7 +44,9 @@ const auto conv2DParams_AutoPadValid = ::testing::Combine(
::testing::Values(std::vector<ptrdiff_t>({0, 0})),
::testing::ValuesIn(dilations), ::testing::ValuesIn(groups),
::testing::ValuesIn(defor_groups), ::testing::ValuesIn(numOutChannels),
::testing::Values(ngraph::op::PadType::VALID));
::testing::Values(ngraph::op::PadType::VALID),
::testing::ValuesIn(with_bilinear_interpolation_pad),
::testing::ValuesIn(with_modulated_scalar));
INSTANTIATE_TEST_SUITE_P(
smoke_DeformableConvolution2D_Serialization_ExplicitPadding, DeformableConvolutionLayerTest,

43
inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/deformable_convolution.cpp
View File

@ -1,14 +1,10 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <vector>
#include "common_test_utils/test_constants.hpp"
#include "single_layer_tests/deformable_convolution.hpp"
using namespace LayerTestsDefinitions;
namespace {
const std::vector<InferenceEngine::Precision> netPrecisions = {
@ -16,18 +12,21 @@ const std::vector<InferenceEngine::Precision> netPrecisions = {
InferenceEngine::Precision::I32, InferenceEngine::Precision::I16};
/* ============= 2D DeformableConvolution ============= */
const std::vector<std::vector<size_t>> deformable_vals = {{1, 18, 28, 28}};
const std::vector<std::vector<size_t>> kernels = {{1, 1, 3, 3}};
const std::vector<std::vector<size_t>> deformable_vals = {{1, 16, 2, 2}};
const std::vector<std::vector<size_t>> kernels = {{2, 2, 2, 2}};
const std::vector<std::vector<size_t>> strides = {{1, 1}};
const std::vector<std::vector<ptrdiff_t>> padBegins = {{0, 0}};
const std::vector<std::vector<ptrdiff_t>> padEnds ={{0, 0}};
const std::vector<std::vector<size_t>> dilations = {{1, 1}};
const std::vector<size_t> groups = {1};
const std::vector<size_t> defor_groups = {1};
const std::vector<size_t> defor_groups = {2};
const std::vector<size_t> numOutChannels = {1, 5};
const std::vector<size_t> multiple_defor_groups = {4};
const std::vector<std::vector<size_t>> deform_vals = {{1, 200, 220, 220}};
const std::vector<std::vector<size_t>> kernel = {{64, 4, 5, 5}};
const std::vector<std::vector<size_t>> kernel = {{64, 16, 5, 5}};
const std::vector<bool> with_bilinear_interpolation_pad = { false, true };
const std::vector<bool> with_modulated_scalar = { false, true };
const auto deformableConv2DParams_ExplicitPadding = ::testing::Combine(
::testing::ValuesIn(deformable_vals),
@ -35,7 +34,9 @@ const auto deformableConv2DParams_ExplicitPadding = ::testing::Combine(
::testing::ValuesIn(padBegins), ::testing::ValuesIn(padEnds),
::testing::ValuesIn(dilations), ::testing::ValuesIn(groups),
::testing::ValuesIn(defor_groups), ::testing::ValuesIn(numOutChannels),
::testing::Values(ngraph::op::PadType::EXPLICIT));
::testing::Values(ngraph::op::PadType::EXPLICIT), ::testing::ValuesIn(with_bilinear_interpolation_pad),
::testing::ValuesIn(with_modulated_scalar));
const auto deformableConv2DParams_AutoPadValid = ::testing::Combine(
::testing::ValuesIn(deformable_vals),
::testing::ValuesIn(kernels), ::testing::ValuesIn(strides),
@ -43,7 +44,9 @@ const auto deformableConv2DParams_AutoPadValid = ::testing::Combine(
::testing::Values(std::vector<ptrdiff_t>({0, 0})),
::testing::ValuesIn(dilations), ::testing::ValuesIn(groups),
::testing::ValuesIn(defor_groups), ::testing::ValuesIn(numOutChannels),
::testing::Values(ngraph::op::PadType::VALID));
::testing::Values(ngraph::op::PadType::VALID),
::testing::ValuesIn(with_bilinear_interpolation_pad),
::testing::ValuesIn(with_modulated_scalar));
const auto deformableConv2DParams_DeformableGroups_AutoPadExplicit = ::testing::Combine(
::testing::ValuesIn(deform_vals),
@ -52,7 +55,9 @@ const auto deformableConv2DParams_DeformableGroups_AutoPadExplicit = ::testing::
::testing::Values(std::vector<ptrdiff_t>({0, 0})),
::testing::ValuesIn(dilations), ::testing::ValuesIn(groups),
::testing::ValuesIn(multiple_defor_groups), ::testing::ValuesIn(numOutChannels),
::testing::Values(ngraph::op::PadType::EXPLICIT));
::testing::Values(ngraph::op::PadType::EXPLICIT),
::testing::ValuesIn(with_bilinear_interpolation_pad),
::testing::ValuesIn(with_modulated_scalar));
INSTANTIATE_TEST_SUITE_P(
smoke_DeformableConvolution2D_ExplicitPadding, DeformableConvolutionLayerTest,
@ -62,7 +67,7 @@ INSTANTIATE_TEST_SUITE_P(
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(std::vector<size_t>({1, 1, 30, 30})),
::testing::Values(std::vector<size_t>({1, 2, 3, 3})),
::testing::Values(CommonTestUtils::DEVICE_CPU)),
DeformableConvolutionLayerTest::getTestCaseName);
@ -74,19 +79,20 @@ INSTANTIATE_TEST_SUITE_P(
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(std::vector<size_t>({1, 1, 30, 30})),
::testing::Values(std::vector<size_t>({1, 2, 3, 3})),
::testing::Values(CommonTestUtils::DEVICE_CPU)),
DeformableConvolutionLayerTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(
smoke_DeformableConvolution2D_DeformableGroups_ExplicitPadding, DeformableConvolutionLayerTest,
::testing::Combine(
deformableConv2DParams_DeformableGroups_AutoPadExplicit, ::testing::ValuesIn(netPrecisions),
deformableConv2DParams_DeformableGroups_AutoPadExplicit,
::testing::ValuesIn(netPrecisions),
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(std::vector<size_t>({1, 4, 224, 224})),
::testing::Values(std::vector<size_t>({1, 16, 224, 224})),
::testing::Values(CommonTestUtils::DEVICE_CPU)),
DeformableConvolutionLayerTest::getTestCaseName);
@ -101,12 +107,15 @@ const auto deformableConv2DParams_SingleTestCase = ::testing::Combine(
::testing::ValuesIn(padBegins), ::testing::ValuesIn(padEnds),
::testing::ValuesIn(dilations), ::testing::ValuesIn(groups),
::testing::ValuesIn(single_deform_groups), ::testing::ValuesIn(numOutChannels),
::testing::Values(ngraph::op::PadType::EXPLICIT));
::testing::Values(ngraph::op::PadType::EXPLICIT),
::testing::ValuesIn(with_bilinear_interpolation_pad),
::testing::ValuesIn(with_modulated_scalar));
INSTANTIATE_TEST_SUITE_P(
smoke_DeformableConvolution2D_SingleTestCase, DeformableConvolutionLayerTest,
::testing::Combine(
deformableConv2DParams_SingleTestCase, ::testing::ValuesIn(netPrecisions),
deformableConv2DParams_SingleTestCase,
::testing::ValuesIn(netPrecisions),
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Layout::ANY),

4
inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/single_layer/deformable_convolution.hpp
View File

@ -26,7 +26,9 @@ typedef std::tuple<
size_t, // Groups
size_t, // Deformable groups
size_t, // Num out channels
ngraph::op::PadType // Padding type
ngraph::op::PadType, // Padding type
bool, // Bilinear interpolation pad
bool // Modulation
> deformableConvSpecificParams;
typedef std::tuple<
deformableConvSpecificParams,

62
inference-engine/tests/functional/shared_test_classes/src/single_layer/deformable_convolution.cpp
View File

@ -1,11 +1,9 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "shared_test_classes/single_layer/deformable_convolution.hpp"
namespace LayerTestsDefinitions {
std::string DeformableConvolutionLayerTest::getTestCaseName(testing::TestParamInfo<deformableConvLayerTestParamsSet> obj) {
deformableConvSpecificParams convParams;
InferenceEngine::Precision netPrecision;
@ -14,12 +12,14 @@ std::string DeformableConvolutionLayerTest::getTestCaseName(testing::TestParamIn
InferenceEngine::SizeVector inputShapes;
std::string targetDevice;
std::tie(convParams, netPrecision, inPrc, outPrc, inLayout, outLayout, inputShapes, targetDevice) =
obj.param;
obj.param;
ngraph::op::PadType padType;
InferenceEngine::SizeVector offsets, filter, stride, dilation;
std::vector<ptrdiff_t> padBegin, padEnd;
size_t groups, deformable_groups, convOutChannels;
std::tie(offsets, filter, stride, padBegin, padEnd, dilation, groups, deformable_groups, convOutChannels, padType) = convParams;
bool with_bilinear_interpolation_pad, with_modulation;
std::tie(offsets, filter, stride, padBegin, padEnd, dilation, groups, deformable_groups, convOutChannels, padType,
with_bilinear_interpolation_pad, with_modulation) = convParams;
std::ostringstream result;
result << "IS=" << CommonTestUtils::vec2str(inputShapes) << "_";
@ -33,6 +33,8 @@ std::string DeformableConvolutionLayerTest::getTestCaseName(testing::TestParamIn
result << "DG=" << deformable_groups << "_";
result << "O=" << convOutChannels << "_";
result << "AP=" << padType << "_";
result << "BI_PAD=" << with_bilinear_interpolation_pad << "_";
result << "MODULATION=" << with_modulation << "_";
result << "netPRC=" << netPrecision.name() << "_";
result << "inPRC=" << inPrc.name() << "_";
result << "outPRC=" << outPrc.name() << "_";
@ -43,29 +45,32 @@ std::string DeformableConvolutionLayerTest::getTestCaseName(testing::TestParamIn
}
InferenceEngine::Blob::Ptr DeformableConvolutionLayerTest::GenerateInput(const InferenceEngine::InputInfo &info) const {
InferenceEngine::Blob::Ptr blobPtr;
const std::string& name = info.name();
if (name == "a_data") {
blobPtr = LayerTestsUtils::LayerTestsCommon::GenerateInput(info);
} else if (name == "b_offset_vals") {
blobPtr = FuncTestUtils::createAndFillBlobFloat(info.getTensorDesc(), 2, 0, 10);
} else if (name == "c_filter_vals") {
blobPtr = LayerTestsUtils::LayerTestsCommon::GenerateInput(info);
}
return blobPtr;
InferenceEngine::Blob::Ptr blobPtr;
const std::string& name = info.name();
if (name == "a_data") {
blobPtr = LayerTestsUtils::LayerTestsCommon::GenerateInput(info);
} else if (name == "b_offset_vals") {
blobPtr = FuncTestUtils::createAndFillBlobFloat(info.getTensorDesc(), 2, 0, 10);
} else if (name == "c_filter_vals") {
blobPtr = LayerTestsUtils::LayerTestsCommon::GenerateInput(info);
} else if (name == "c_modulation_scalars") {
blobPtr = FuncTestUtils::createAndFillBlobFloat(info.getTensorDesc(), 1, 0, 20);
}
return blobPtr;
}
void DeformableConvolutionLayerTest::SetUp() {
deformableConvSpecificParams convParams;
std::vector<size_t> inputShape;
InferenceEngine::Precision netPrecision;
std::tie(convParams, netPrecision, inPrc, outPrc, inLayout, outLayout, inputShape, targetDevice) =
this->GetParam();
this->GetParam();
ngraph::op::PadType padType;
InferenceEngine::SizeVector offsets, filter, stride, dilation;
std::vector<ptrdiff_t> padBegin, padEnd;
size_t groups, deformable_groups, convOutChannels;
std::tie(offsets, filter, stride, padBegin, padEnd, dilation, groups, deformable_groups, convOutChannels, padType) = convParams;
bool with_bilinear_interpolation_pad, with_modulation;
std::tie(offsets, filter, stride, padBegin, padEnd, dilation, groups, deformable_groups, convOutChannels, padType,
with_bilinear_interpolation_pad, with_modulation) = convParams;
auto ngPrc = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(netPrecision);
auto params = ngraph::builder::makeParams(ngPrc, {inputShape, offsets, filter});
auto paramOuts = ngraph::helpers::convert2OutputVector(
@ -76,9 +81,24 @@ void DeformableConvolutionLayerTest::SetUp() {
offset_vals->set_friendly_name("b_offset_vals");
auto filter_vals = std::make_shared<ngraph::op::Parameter>(ngPrc, ngraph::Shape(filter));
filter_vals->set_friendly_name("c_filter_vals");
auto deformable_conv = std::make_shared<ngraph::opset1::DeformableConvolution>(data, offset_vals, filter_vals,
stride, padBegin, padEnd, dilation, padType, groups, deformable_groups);
ngraph::ParameterVector parameters{data, offset_vals, filter_vals};
std::shared_ptr<ngraph::Node> deformable_conv;
if (with_modulation) {
auto modulation_shape = ngraph::Shape(offsets);
modulation_shape[1] = offsets[1] / 2;
auto modulation_scalars = std::make_shared<ngraph::op::Parameter>(ngPrc, modulation_shape);
modulation_scalars->set_friendly_name("c_modulation_scalars");
deformable_conv = std::make_shared<ngraph::op::v8::DeformableConvolution>(data, offset_vals, filter_vals, modulation_scalars, stride, padBegin,
padEnd, dilation, padType, groups, deformable_groups,
with_bilinear_interpolation_pad);
parameters.push_back(modulation_scalars);
} else {
deformable_conv = std::make_shared<ngraph::op::v8::DeformableConvolution>(data, offset_vals, filter_vals, stride, padBegin, padEnd, dilation,
padType, groups, deformable_groups, with_bilinear_interpolation_pad);
}
ngraph::ResultVector results{std::make_shared<ngraph::opset1::Result>(deformable_conv)};
function = std::make_shared<ngraph::Function>(results, ngraph::ParameterVector{data, offset_vals, filter_vals}, "deformable_convolution");
function = std::make_shared<ngraph::Function>(results, parameters, "deformable_convolution");
}
} // namespace LayerTestsDefinitions
} // namespace LayerTestsDefinitions

5
ngraph/core/include/ngraph/op/deformable_convolution.hpp
View File

@ -160,6 +160,11 @@ namespace ngraph
void validate_and_infer_types() override;
bool evaluate(const HostTensorVector& outputs,
const HostTensorVector& inputs) const override;
bool has_evaluate() const override;
std::shared_ptr<Node>
clone_with_new_inputs(const OutputVector& new_args) const override;

148
ngraph/core/reference/include/ngraph/runtime/reference/deformable_convolution.hpp
View File

@ -17,6 +17,7 @@ namespace ngraph
inline void validate_deformable_convolution_params(const Shape& in_shape,
const Shape& o_shape,
const Shape& f_shape,
const Shape& m_shape,
const Shape& out_shape,
const Strides& strides,
const Strides& dilations,
@ -29,6 +30,7 @@ namespace ngraph
NGRAPH_CHECK(in_shape.size() == 4, "Unsupported input rank: ", in_shape);
NGRAPH_CHECK(o_shape.size() == 4, "Unsupported offset rank: ", o_shape);
NGRAPH_CHECK(f_shape.size() == 4, "Unsupported kernel rank: ", f_shape);
NGRAPH_CHECK(m_shape.size() == 4, "Unsupported mask rank: ", m_shape);
NGRAPH_CHECK(in_shape[1] % groups == 0,
"Input channels of data batch input must be evenly divisible by "
@ -53,14 +55,20 @@ namespace ngraph
const Shape f_spatial_shape{std::next(f_shape.begin(), 2), std::end(f_shape)};
const Shape o_spatial_shape{std::next(o_shape.begin(), 2), std::end(o_shape)};
const Shape m_spatial_shape{std::next(m_shape.begin(), 2), std::end(m_shape)};
const Shape out_spatial_shape{std::next(out_shape.begin(), 2),
std::end(out_shape)};
NGRAPH_CHECK(o_shape[1] == deformable_groups * shape_size(f_spatial_shape) * 2,
"The channels dimension of offsets input is not "
"compatible with filters and 'deformable group' attribute");
NGRAPH_CHECK(m_shape[1] == deformable_groups * shape_size(f_spatial_shape),
"The channels dimension of mask input is not "
"compatible with filters and 'deformable group' attribute");
NGRAPH_CHECK(out_spatial_shape == o_spatial_shape,
"Spatial dimensions of output and offsets values must be equal");
NGRAPH_CHECK(out_spatial_shape == m_spatial_shape,
"Spatial dimensions of output and mask values must be equal");
}
inline Shape shape_reduce(const Shape& s) { return Shape(++s.begin(), s.end()); }
@ -76,20 +84,37 @@ namespace ngraph
const float x_idx,
const float y_idx,
const int x_size,
const int y_size)
const int y_size,
const bool use_pad)
{
const int x1 = std::max(static_cast<int>(std::floor(x_idx)), 0);
const int x2 = std::min(static_cast<int>(std::ceil(x_idx)), x_size - 1);
const int y1 = std::max(static_cast<int>(std::floor(y_idx)), 0);
const int y2 = std::min(static_cast<int>(std::ceil(y_idx)), y_size - 1);
const int y1 = use_pad ? static_cast<int>(std::floor(y_idx))
: std::max(static_cast<int>(std::floor(y_idx)), 0);
const int x1 = use_pad ? static_cast<int>(std::floor(x_idx))
: std::max(static_cast<int>(std::floor(x_idx)), 0);
const int y2 =
use_pad ? y1 + 1 : std::min(static_cast<int>(std::ceil(y_idx)), y_size - 1);
const int x2 =
use_pad ? x1 + 1 : std::min(static_cast<int>(std::ceil(x_idx)), x_size - 1);
const float distX = x_idx - x1;
const float distY = y_idx - y1;
const float value11 = data[y1 * x_size + x1];
const float value12 = data[y2 * x_size + x1];
const float value21 = data[y1 * x_size + x2];
const float value22 = data[y2 * x_size + x2];
float value11 = 0;
if (y1 >= 0 && x1 >= 0)
value11 = data[y1 * x_size + x1];
float value21 = 0;
if (y1 >= 0 && x2 < x_size)
value21 = data[y1 * x_size + x2];
float value12 = 0;
if (y2 < y_size && x1 >= 0)
value12 = data[y2 * x_size + x1];
float value22 = 0;
if (y2 < y_size && x2 < x_size)
value22 = data[y2 * x_size + x2];
const float value = (1 - distX) * (1 - distY) * value11 +
(1 - distX) * distY * value12 +
@ -105,10 +130,13 @@ namespace ngraph
const Shape& offset_shape,
const T* filter,
const Shape& filter_shape,
const T* mask,
const Shape& mask_shape,
T* out,
size_t group_idx,
int64_t groups,
int64_t deformable_groups)
int64_t deformable_groups,
bool bilinear_interpolation_pad)
{
const int input_size_y = batch_shape[1];
const int input_size_x = batch_shape[2];
@ -124,6 +152,8 @@ namespace ngraph
const int offsets_size = shape_size(offset_shape);
const int offsets_spatial_size = shape_size(shape_reduce(offset_shape));
const int filter_channels_count = filter_shape[0];
const int mask_size = shape_size(mask_shape);
const int mask_spatial_size = shape_size(shape_reduce(mask_shape));
int out_idx = 0;
for (int i_y = -p.pads_begin[0];
@ -146,29 +176,45 @@ namespace ngraph
{
for (int f_x = 0; f_x < filter_size_x; ++f_x)
{
T y_offset = offsets[deformable_group_idx * offsets_size +
(f_y * filter_size_x + f_x) * 2 *
offsets_spatial_size +
out_idx];
T x_offset = offsets[deformable_group_idx * offsets_size +
((f_y * filter_size_x + f_x) * 2 + 1) *
offsets_spatial_size +
out_idx];
int f_buf_idx = (f_y * filter_size_x) + f_x;
T y_offset =
offsets[deformable_group_idx * offsets_size +
f_buf_idx * 2 * offsets_spatial_size + out_idx];
T x_offset =
offsets[deformable_group_idx * offsets_size +
(f_buf_idx * 2 + 1) * offsets_spatial_size +
out_idx];
T rel_i_y = i_y + (f_y * p.dilation[0]) + y_offset;
T rel_i_x = i_x + (f_x * p.dilation[1]) + x_offset;
bool padding = !(in_range(rel_i_x, {0, input_size_x}) &&
in_range(rel_i_y, {0, input_size_y}));
bool padding;
if (bilinear_interpolation_pad)
{
padding =
!((static_cast<int>(rel_i_x) > -1 &&
static_cast<int>(rel_i_x) < input_size_x) &&
(static_cast<int>(rel_i_y) > -1 &&
static_cast<int>(rel_i_y) < input_size_y));
}
else
{
padding = !(in_range(rel_i_x, {0, input_size_x}) &&
in_range(rel_i_y, {0, input_size_y}));
}
if (padding)
continue;
int f_buf_idx = (f_y * filter_size_x) + f_x;
T mask_scalar =
mask[deformable_group_idx * mask_size +
f_buf_idx * mask_spatial_size + out_idx];
sum += bilinear_interpolation(input_channel,
rel_i_x,
rel_i_y,
input_size_x,
input_size_y) *
filter_channel[f_buf_idx];
input_size_y,
bilinear_interpolation_pad) *
filter_channel[f_buf_idx] * mask_scalar;
}
}
input_channel += input_channel_size;
@ -180,21 +226,25 @@ namespace ngraph
}
} // namespace def_conv_impl
template <typename T>
void deformable_convolution(const T* in,
const T* offsets,
const T* filters,
const T* mask,
T* out,
const Shape& in_shape,
const Shape& o_shape,
const Shape& f_shape,
const Shape& m_shape,
const Shape& out_shape,
const Strides& strides,
const Strides& dilation,
const CoordinateDiff& pads_begin,
const CoordinateDiff& pads_end,
const int64_t groups,
const int64_t deformable_groups)
const int64_t deformable_groups,
const bool bilinear_interpolation_pad)
{
using namespace def_conv_impl;
@ -202,6 +252,7 @@ namespace ngraph
validate_deformable_convolution_params(in_shape,
o_shape,
f_shape,
m_shape,
out_shape,
strides,
dilation,
@ -227,12 +278,17 @@ namespace ngraph
const Shape group_filter_shape = shape_reduce(f_shape);
const size_t group_filter_size = shape_size(group_filter_shape);
const Shape group_mask_shape =
shape_scale(shape_reduce(m_shape), deformable_groups);
const size_t group_mask_batch_size = shape_size(shape_reduce(m_shape));
const size_t out_ch_size = shape_size(shape_reduce(shape_reduce(out_shape)));
for (size_t batch_idx = 0; batch_idx < batches_count; ++batch_idx)
{
const T* group_filters = filters;
const T* group_offsets = offsets;
const T* group_mask = mask;
for (size_t group_idx = 0; group_idx < groups_count; ++group_idx)
{
for (size_t f_idx = 0; f_idx < group_filters_count; ++f_idx)
@ -244,18 +300,60 @@ namespace ngraph
group_offset_shape,
group_filters,
group_filter_shape,
group_mask,
group_mask_shape,
out,
group_idx,
groups,
deformable_groups);
deformable_groups,
bilinear_interpolation_pad);
group_filters += group_filter_size;
out += out_ch_size;
}
in += group_in_size;
}
offsets += group_offset_batch_size;
mask += group_mask_batch_size;
}
}
template <typename T>
void deformable_convolution(const T* in,
const T* offsets,
const T* filters,
T* out,
const Shape& in_shape,
const Shape& o_shape,
const Shape& f_shape,
const Shape& out_shape,
const Strides& strides,
const Strides& dilation,
const CoordinateDiff& pads_begin,
const CoordinateDiff& pads_end,
const int64_t groups,
const int64_t deformable_groups,
const bool bilinear_interpolation_pad = false)
{
Shape m_shape = {o_shape[0], o_shape[1] / 2, o_shape[2], o_shape[3]};
std::vector<T> mask(ngraph::shape_size(m_shape), 1);
deformable_convolution(in,
offsets,
filters,
mask.data(),
out,
in_shape,
o_shape,
f_shape,
m_shape,
out_shape,
strides,
dilation,
pads_begin,
pads_end,
groups,
deformable_groups,
bilinear_interpolation_pad);
}
} // namespace reference
} // namespace runtime
} // namespace ngraph

157
ngraph/core/src/op/deformable_convolution.cpp
View File

@ -6,6 +6,7 @@
#include "itt.hpp"
#include "ngraph/axis_vector.hpp"
#include "ngraph/coordinate_diff.hpp"
#include "ngraph/runtime/reference/deformable_convolution.hpp"
#include "ngraph/util.hpp"
#include "ngraph/validation_util.hpp"
@ -195,6 +196,162 @@ std::shared_ptr<Node>
}
}
namespace deformable_convolution
{
template <element::Type_t ET>
inline bool evaluate(const HostTensorVector& inputs,
const HostTensorPtr& out,
const Strides& strides,
const CoordinateDiff& pads_begin,
const CoordinateDiff& pads_end,
const Strides& dilations,
const ngraph::op::PadType& auto_pad,
const int64_t group,
const int64_t deformable_group,
const bool use_bilinear_interpolation_padding)
{
using T = typename element_type_traits<ET>::value_type;
if (inputs.size() == 3)
{
runtime::reference::deformable_convolution<T>(inputs[0]->get_data_ptr<ET>(),
inputs[1]->get_data_ptr<ET>(),
inputs[2]->get_data_ptr<ET>(),
out->get_data_ptr<ET>(),
inputs[0]->get_shape(),
inputs[1]->get_shape(),
inputs[2]->get_shape(),
out->get_shape(),
strides,
dilations,
pads_begin,
pads_end,
group,
deformable_group,
use_bilinear_interpolation_padding);
}
else if (inputs.size() == 4)
{
runtime::reference::deformable_convolution<T>(inputs[0]->get_data_ptr<ET>(),
inputs[1]->get_data_ptr<ET>(),
inputs[2]->get_data_ptr<ET>(),
inputs[3]->get_data_ptr<ET>(),
out->get_data_ptr<ET>(),
inputs[0]->get_shape(),
inputs[1]->get_shape(),
inputs[2]->get_shape(),
inputs[3]->get_shape(),
out->get_shape(),
strides,
dilations,
pads_begin,
pads_end,
group,
deformable_group,
use_bilinear_interpolation_padding);
}
return true;
}
bool evaluate_deformable_convolution(const HostTensorVector& inputs,
const HostTensorPtr& out,
const Strides& strides,
const Strides& dilations,
const CoordinateDiff& pads_begin,
const CoordinateDiff& pads_end,
const ngraph::op::PadType& auto_pad,
const int64_t group,
const int64_t deformable_group,
const bool use_bilinear_interpolation_padding)
{
bool rc = true;
switch (inputs[0]->get_element_type())
{
NGRAPH_TYPE_CASE(evaluate_deformable_convolution,
f32,
inputs,
out,
strides,
pads_begin,
pads_end,
dilations,
auto_pad,
group,
deformable_group,
use_bilinear_interpolation_padding);
NGRAPH_TYPE_CASE(evaluate_deformable_convolution,
f16,
inputs,
out,
strides,
pads_begin,
pads_end,
dilations,
auto_pad,
group,
deformable_group,
use_bilinear_interpolation_padding);
NGRAPH_TYPE_CASE(evaluate_deformable_convolution,
i32,
inputs,
out,
strides,
pads_begin,
pads_end,
dilations,
auto_pad,
group,
deformable_group,
use_bilinear_interpolation_padding);
NGRAPH_TYPE_CASE(evaluate_deformable_convolution,
i16,
inputs,
out,
strides,
pads_begin,
pads_end,
dilations,
auto_pad,
group,
deformable_group,
use_bilinear_interpolation_padding);
default: rc = false; break;
}
return rc;
}
} // namespace deformable_convolution
bool op::v8::DeformableConvolution::evaluate(const HostTensorVector& outputs,
const HostTensorVector& inputs) const
{
NGRAPH_OP_SCOPE(DeformableConvolution_v8_evaluate);
deformable_convolution::evaluate_deformable_convolution(inputs,
outputs[0],
get_strides(),
get_dilations(),
get_pads_begin(),
get_pads_end(),
get_auto_pad(),
get_group(),
get_deformable_group(),
get_bilinear_interpolation_pad());
return true;
}
bool op::v8::DeformableConvolution::has_evaluate() const
{
NGRAPH_OP_SCOPE(DeformableConvolution_v8_has_evaluate);
switch (get_input_element_type(0))
{
case ngraph::element::f16:
case ngraph::element::i16:
case ngraph::element::i32:
case ngraph::element::f32: return true;
default: break;
}
return false;
}
op::v1::DeformableConvolution::DeformableConvolution(const Output<Node>& arg,
const Output<Node>& offsets,
const Output<Node>& filters,

1
ngraph/test/CMakeLists.txt
View File

@ -404,6 +404,7 @@ set(MULTI_TEST_SRC
backend/dft.in.cpp
backend/divide.in.cpp
backend/deformable_convolution.in.cpp
backend/deformable_convolution_opset8.in.cpp
backend/depth_to_space.in.cpp
backend/dyn_reshape.in.cpp
backend/experimental_detectron_generate_proposals.in.cpp

2940
ngraph/test/backend/deformable_convolution_opset8.in.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

8
ngraph/test/runtime/ie/unit_test.manifest
View File

@ -1609,6 +1609,14 @@ IE_CPU.deformable_convolution_2D_integral_offsets_groups_and_deforgroups
IE_CPU.deformable_convolution_2D_real_offsets_groups_basic
IE_CPU.deformable_convolution_2D_real_offsets_groups_complex
IE_CPU.deformable_convolution_2D_real_offsets_groups_and_deforgroups
# No plugin support for DeformableConvolution v8
IE_GPU.deformable_convolution_opset8_2D_v8_zeroed_offsets_default_mask
IE_GPU.deformable_convolution_opset8_2D_real_offsets_groups_and_deforgroups_mask
IE_GPU.deformable_convolution_opset8_2D_real_offsets_groups_and_deforgroups_mask_2
IE_GPU.deformable_convolution_opset8_2D_neg_offsets_groups_and_deforgroups_mask
# results missmatch, ticket: 59600
IE_GPU.deformable_convolution_2D_integral_offsets_groups_and_deforgroups
IE_GPU.deformable_convolution_opset8_2D_integral_offsets_groups_and_deforgroups
# No plugin support for AdaptiveAvgPool and AdaptiveMaxPool
adaptive_avg_pool_1d

54
ngraph/test/runtime/interpreter/evaluates_map.cpp
View File

@ -345,6 +345,60 @@ namespace
return true;
}
template <element::Type_t ET>
bool evaluate(const shared_ptr<op::v8::DeformableConvolution>& op,
const HostTensorVector& outputs,
const HostTensorVector& inputs) {
const auto in_data_ptr = inputs[0]->get_data_ptr<ET>();
const auto offset_data_ptr = inputs[1]->get_data_ptr<ET>();
const auto filter_data_ptr = inputs[2]->get_data_ptr<ET>();
auto out_data_ptr = outputs[0]->get_data_ptr<ET>();
const auto& out_shape = outputs[0]->get_shape();
const auto& in_shape = inputs[0]->get_shape();
const auto& offset_shape = inputs[1]->get_shape();
const auto& filter_shape = inputs[2]->get_shape();
if (inputs.size() == 3) {
runtime::reference::deformable_convolution<typename element_type_traits<ET>::value_type>(
in_data_ptr,
offset_data_ptr,
filter_data_ptr,
out_data_ptr,
in_shape,
offset_shape,
filter_shape,
out_shape,
op->get_strides(),
op->get_dilations(),
op->get_pads_begin(),
op->get_pads_end(),
op->get_group(),
op->get_deformable_group(),
op->get_bilinear_interpolation_pad());
} else {
const auto mask_data_ptr = inputs[3]->get_data_ptr<ET>();
const auto& mask_shape = inputs[3]->get_shape();
runtime::reference::deformable_convolution<typename element_type_traits<ET>::value_type>(
in_data_ptr,
offset_data_ptr,
filter_data_ptr,
mask_data_ptr,
out_data_ptr,
in_shape,
offset_shape,
filter_shape,
mask_shape,
out_shape,
op->get_strides(),
op->get_dilations(),
op->get_pads_begin(),
op->get_pads_end(),
op->get_group(),
op->get_deformable_group(),
op->get_bilinear_interpolation_pad());
}
return true;
}
template <element::Type_t ET>
bool evaluate(const shared_ptr<op::v1::DeformableConvolution>& op,
const HostTensorVector& outputs,