natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Preprocessing: convert color RGBX/BGRX -> RGB/BGR implementation (#8755)

This commit is contained in:
Mikhail Nosov 2021-11-23 10:29:00 +03:00 committed by GitHub
parent 3b88682159
commit 7545af07d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 216 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
docs/template_plugin/tests/functional/subgraph_reference/preprocess.cpp
View File

@ -71,26 +71,24 @@ static std::shared_ptr<Function> create_simple_function(element::Type type, cons
return std::make_shared<ov::Function>(ResultVector{res}, ParameterVector{data1});
}
static std::shared_ptr<Function> create_2inputs(element::Type type, const PartialShape& shape) {
auto data1 = std::make_shared<op::v0::Parameter>(type, shape);
data1->set_friendly_name("input1");
data1->get_output_tensor(0).set_names({"tensor_input1"});
auto c1 = op::v0::Constant::create(type, {1}, {0});
auto op1 = std::make_shared<op::v1::Add>(data1, c1);
op1->set_friendly_name("Add01");
auto data2 = std::make_shared<op::v0::Parameter>(type, shape);
data2->get_output_tensor(0).set_names({"tensor_input2"});
data2->set_friendly_name("input2");
auto c2 = op::v0::Constant::create(type, {1}, {0});
auto op2 = std::make_shared<op::v1::Add>(data2, c2);
op2->set_friendly_name("Add02");
auto res1 = std::make_shared<op::v0::Result>(op1);
res1->set_friendly_name("Result1");
res1->get_output_tensor(0).set_names({"tensor_output1"});
auto res2 = std::make_shared<op::v0::Result>(op2);
res2->set_friendly_name("Result2");
res2->get_output_tensor(0).set_names({"tensor_output2"});
return std::make_shared<ov::Function>(ResultVector{res1, res2}, ParameterVector{data1, data2});
template <int N>
static std::shared_ptr<Function> create_n_inputs(element::Type type, const PartialShape& shape) {
auto params = ParameterVector();
auto results = ResultVector();
for (int i = 1; i <= N; i++) {
auto param = std::make_shared<op::v0::Parameter>(type, shape);
param->set_friendly_name("input" + std::to_string(i));
param->get_output_tensor(0).set_names({"tensor_input" + std::to_string(i)});
auto c1 = op::v0::Constant::create(type, {1}, {0});
auto op1 = std::make_shared<op::v1::Add>(param, c1);
op1->set_friendly_name("Add" + std::to_string(i));
auto res1 = std::make_shared<op::v0::Result>(op1);
res1->set_friendly_name("Result" + std::to_string(i));
res1->get_output_tensor(0).set_names({"tensor_output" + std::to_string(i)});
results.push_back(res1);
params.push_back(param);
}
return std::make_shared<ov::Function>(results, params);
}
static RefPreprocessParams simple_mean_scale() {
@ -242,7 +240,7 @@ static RefPreprocessParams test_lvalue() {
static RefPreprocessParams test_2_inputs_basic() {
RefPreprocessParams res("test_2_inputs_basic");
res.function = []() {
auto f = create_2inputs(element::f32, Shape{1, 3, 1, 1});
auto f = create_n_inputs<2>(element::f32, Shape{1, 3, 1, 1});
f = PrePostProcessor(f).input(InputInfo(0)
.preprocess(
PreProcessSteps()
@ -837,7 +835,7 @@ static RefPreprocessParams convert_color_i420_single_plane() {
static RefPreprocessParams postprocess_2_inputs_basic() {
RefPreprocessParams res("postprocess_2_inputs_basic");
res.function = []() {
auto f = create_2inputs(element::f32, Shape{1, 3, 1, 2});
auto f = create_n_inputs<2>(element::f32, Shape{1, 3, 1, 2});
f = PrePostProcessor(f)
.output(OutputInfo("tensor_output1")
.network(OutputNetworkInfo().set_layout("NCHW"))
@ -901,7 +899,7 @@ static RefPreprocessParams post_convert_layout_by_dims_multi() {
static RefPreprocessParams pre_and_post_processing() {
RefPreprocessParams res("pre_and_post_processing");
res.function = []() {
auto f = create_2inputs(element::f32, Shape{1, 3, 1, 2});
auto f = create_n_inputs<2>(element::f32, Shape{1, 3, 1, 2});
f = PrePostProcessor(f)
.input(InputInfo(0)
.tensor(InputTensorInfo().set_element_type(element::u8))
@ -970,6 +968,43 @@ static RefPreprocessParams reverse_channels_nchw() {
return res;
}
static RefPreprocessParams color_cut_last_channel() {
RefPreprocessParams res("color_cut_last_channel");
auto input_tensor = Tensor(Shape{1, 2, 2, 4}, element::f32, std::vector<float>{1, 2, 3, 4,
5, 6, 7, 8,
3, 4, 5, 6,
6, 7, 8, 9});
auto exp_3_channels = Tensor(Shape{1, 2, 2, 3}, element::f32, std::vector<float>{1, 2, 3,
5, 6, 7,
3, 4, 5,
6, 7, 8});
auto inv_3_channels = Tensor(Shape{1, 2, 2, 3}, element::f32, std::vector<float>{3, 2, 1,
7, 6, 5,
5, 4, 3,
8, 7, 6});
res.function = []() {
auto f = create_n_inputs<4>(element::f32, Shape{1, 2, 2, 3});
auto prep = PrePostProcessor(f);
prep.input(0).tensor().set_color_format(ColorFormat::RGBX);
prep.input(0).preprocess().convert_color(ColorFormat::RGB);
prep.input(1).tensor().set_color_format(ColorFormat::RGBX);
prep.input(1).preprocess().convert_color(ColorFormat::BGR);
prep.input(2).tensor().set_color_format(ColorFormat::BGRX);
prep.input(2).preprocess().convert_color(ColorFormat::BGR);
prep.input(3).tensor().set_color_format(ColorFormat::BGRX);
prep.input(3).preprocess().convert_color(ColorFormat::RGB);
return prep.build();
};
res.inputs = std::vector<Tensor>{input_tensor, input_tensor, input_tensor, input_tensor};
res.expected = std::vector<Tensor>{exp_3_channels, inv_3_channels, exp_3_channels, inv_3_channels};
return res;
}
static RefPreprocessParams reverse_channels_dyn_layout() {
RefPreprocessParams res("reverse_channels_dyn_layout");
res.function = []() {
@ -1057,6 +1092,7 @@ std::vector<RefPreprocessParams> allPreprocessTests() {
pre_and_post_processing(),
rgb_to_bgr(),
bgr_to_rgb(),
color_cut_last_channel(),
reverse_channels_nchw(),
reverse_channels_dyn_layout(),
reverse_dyn_shape(),

60
docs/template_plugin/tests/functional/subgraph_reference/preprocess_legacy.cpp
View File

@ -107,6 +107,66 @@ TEST_F(ReferencePreprocessLegacyTest, resize) {
Exec();
}
TEST_F(ReferencePreprocessLegacyTest, bgrx_to_bgr) {
const int h = 32;
const int w = 32;
auto rgbx_input = std::vector<uint8_t>(h * w * 4, 0);
for (auto i = 0; i < h * w * 4; i++) {
rgbx_input[i] = i % 256;
}
function = create_simple_function(element::f32, Shape{1, 3, h, w});
auto f2 = create_simple_function(element::f32, Shape{1, 3, h, w});
legacy_network = InferenceEngine::CNNNetwork(f2);
auto p = PrePostProcessor(function);
auto& input = p.input();
input.tensor().set_color_format(ColorFormat::BGRX).set_element_type(element::u8);
input.preprocess().convert_color(ColorFormat::BGR);
input.network().set_layout("NCHW");
function = p.build();
inputData.emplace_back(element::u8, Shape{1, h, w, 4}, rgbx_input.data());
InferenceEngine::TensorDesc rgbx_plane_desc(InferenceEngine::Precision::U8,
{1, 4, h, w},
InferenceEngine::Layout::NHWC);
legacy_network.getInputsInfo().begin()->second->setLayout(InferenceEngine::NHWC);
auto &preProcess = legacy_network.getInputsInfo().begin()->second->getPreProcess();
preProcess.setColorFormat(InferenceEngine::ColorFormat::BGRX);
legacy_input_blobs["input1"] = InferenceEngine::make_shared_blob<uint8_t>(rgbx_plane_desc, rgbx_input.data());
Exec();
}
TEST_F(ReferencePreprocessLegacyTest, rgbx_to_bgr) {
const int h = 32;
const int w = 32;
auto rgbx_input = std::vector<uint8_t>(h * w * 4, 0);
for (auto i = 0; i < h * w * 4; i++) {
rgbx_input[i] = i % 256;
}
function = create_simple_function(element::f32, Shape{1, 3, h, w});
auto f2 = create_simple_function(element::f32, Shape{1, 3, h, w});
legacy_network = InferenceEngine::CNNNetwork(f2);
auto p = PrePostProcessor(function);
auto& input = p.input();
input.tensor().set_color_format(ColorFormat::RGBX).set_element_type(element::u8);
input.preprocess().convert_color(ColorFormat::BGR);
input.network().set_layout("NCHW");
function = p.build();
inputData.emplace_back(element::u8, Shape{1, h, w, 4}, rgbx_input.data());
InferenceEngine::TensorDesc rgbx_plane_desc(InferenceEngine::Precision::U8,
{1, 4, h, w},
InferenceEngine::Layout::NHWC);
legacy_network.getInputsInfo().begin()->second->setLayout(InferenceEngine::NHWC);
auto &preProcess = legacy_network.getInputsInfo().begin()->second->getPreProcess();
preProcess.setColorFormat(InferenceEngine::ColorFormat::RGBX);
legacy_input_blobs["input1"] = InferenceEngine::make_shared_blob<uint8_t>(rgbx_plane_desc, rgbx_input.data());
Exec();
}
class ConvertNV12WithLegacyTest: public ReferencePreprocessLegacyTest {
public:
// Create OV20 function with pre-processing + legacy network + reference NV12 inputs

1
inference-engine/tests/functional/plugin/gpu/shared_tests_instances/subgraph_tests/preprocess.cpp
View File

@ -35,6 +35,7 @@ inline std::vector<preprocess_func> GPU_smoke_preprocess_functions() {
preprocess_func(cvt_color_nv12_cvt_layout_resize, "cvt_color_nv12_cvt_layout_resize", 1.f),
preprocess_func(cvt_color_i420_to_rgb_single_plane, "cvt_color_i420_to_rgb_single_plane", 1.f),
preprocess_func(cvt_color_i420_to_bgr_three_planes, "cvt_color_i420_to_bgr_three_planes", 1.f),
preprocess_func(cvt_color_bgrx_to_bgr, "cvt_color_bgrx_to_bgr", 0.01f),
};
}

22
inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/preprocess/preprocess_builders.hpp
View File

@ -385,6 +385,17 @@ inline std::shared_ptr<Function> cvt_color_i420_to_bgr_three_planes() {
return p.build();
}
inline std::shared_ptr<Function> cvt_color_bgrx_to_bgr() {
using namespace ov::preprocess;
auto function = create_preprocess_2inputs(element::f32, PartialShape{1, 32, 32, 3});
auto p = PrePostProcessor(function);
p.input(0).tensor().set_color_format(ColorFormat::BGRX);
p.input(0).preprocess().convert_color(ColorFormat::BGR);
p.input(1).tensor().set_color_format(ColorFormat::RGBX);
p.input(1).preprocess().convert_color(ColorFormat::BGR);
return p.build();
}
inline std::vector<preprocess_func> generic_preprocess_functions() {
return std::vector<preprocess_func> {
preprocess_func(mean_only, "mean_only", 0.01f),
@ -408,11 +419,12 @@ inline std::vector<preprocess_func> generic_preprocess_functions() {
preprocess_func(convert_layout_by_dims, "convert_layout_by_dims", 0.01f),
preprocess_func(resize_and_convert_layout, "resize_and_convert_layout", 0.01f),
preprocess_func(resize_and_convert_layout_i8, "resize_and_convert_layout_i8", 0.01f),
preprocess_func(cvt_color_nv12_to_rgb_single_plane, "cvt_color_nv12_to_rgb_single_plane", 2.f),
preprocess_func(cvt_color_nv12_to_bgr_two_planes, "cvt_color_nv12_to_bgr_two_planes", 2.f),
preprocess_func(cvt_color_nv12_cvt_layout_resize, "cvt_color_nv12_cvt_layout_resize", 2.f),
preprocess_func(cvt_color_i420_to_rgb_single_plane, "cvt_color_i420_to_rgb_single_plane", 2.f),
preprocess_func(cvt_color_i420_to_bgr_three_planes, "cvt_color_i420_to_bgr_three_planes", 2.f),
preprocess_func(cvt_color_nv12_to_rgb_single_plane, "cvt_color_nv12_to_rgb_single_plane", 1.f),
preprocess_func(cvt_color_nv12_to_bgr_two_planes, "cvt_color_nv12_to_bgr_two_planes", 1.f),
preprocess_func(cvt_color_nv12_cvt_layout_resize, "cvt_color_nv12_cvt_layout_resize", 1.f),
preprocess_func(cvt_color_i420_to_rgb_single_plane, "cvt_color_i420_to_rgb_single_plane", 1.f),
preprocess_func(cvt_color_i420_to_bgr_three_planes, "cvt_color_i420_to_bgr_three_planes", 1.f),
preprocess_func(cvt_color_bgrx_to_bgr, "cvt_color_bgrx_to_bgr", 0.01f),
};
}

6
ngraph/core/include/openvino/core/preprocess/color_format.hpp
View File

@ -17,7 +17,11 @@ enum class ColorFormat {
/// \brief Image in I420 format represented as separate tensors for Y, U and V planes.
I420_THREE_PLANES,
RGB,
BGR
BGR,
/// \brief Image in RGBX interleaved format (4 channels)
RGBX,
/// \brief Image in BGRX interleaved format (4 channels)
BGRX
};
} // namespace preprocess

4
ngraph/core/src/preprocess/color_utils.cpp
View File

@ -23,6 +23,10 @@ std::unique_ptr<ColorFormatInfo> ColorFormatInfo::get(ColorFormat format) {
case ColorFormat::BGR:
res.reset(new ColorFormatNHWC(format));
break;
case ColorFormat::RGBX:
case ColorFormat::BGRX:
res.reset(new ColorFormatInfo_RGBX_Base(format));
break;
default:
res.reset(new ColorFormatInfo(format));
break;

21
ngraph/core/src/preprocess/color_utils.hpp
View File

@ -37,6 +37,12 @@ inline std::string color_format_name(ColorFormat format) {
case ColorFormat::I420_SINGLE_PLANE:
name = "I420 (single plane)";
break;
case ColorFormat::RGBX:
name = "RGBX";
break;
case ColorFormat::BGRX:
name = "BGRX";
break;
default:
name = "Unknown";
break;
@ -162,5 +168,20 @@ protected:
}
};
class ColorFormatInfo_RGBX_Base : public ColorFormatNHWC {
public:
explicit ColorFormatInfo_RGBX_Base(ColorFormat format) : ColorFormatNHWC(format) {}
protected:
PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const override {
PartialShape result = image_shape;
if (image_shape.rank().is_static() && image_shape.rank().get_length() == 4) {
result[3] = 4;
return result;
}
return result;
}
};
} // namespace preprocess
} // namespace ov

46
ngraph/core/src/preprocess/preprocess_steps_impl.cpp
View File

@ -5,11 +5,11 @@
#include "preprocess_steps_impl.hpp"
#include "color_utils.hpp"
#include "ngraph/opsets/opset1.hpp"
#include "openvino/core/node.hpp"
#include "openvino/core/shape.hpp"
#include "openvino/op/nv12_to_bgr.hpp"
#include "openvino/op/nv12_to_rgb.hpp"
#include "openvino/opsets/opset8.hpp"
namespace ov {
namespace preprocess {
@ -287,6 +287,32 @@ void PreStepsList::add_convert_color_impl(const ColorFormat& dst_format) {
context.color_format() = dst_format;
return res;
}
if (context.color_format() == ColorFormat::RGBX) {
if (dst_format == ColorFormat::RGB) {
auto res = cut_last_channel(nodes, function, context);
context.color_format() = dst_format;
return res;
} else if (dst_format == ColorFormat::BGR) {
auto cut = cut_last_channel(nodes, function, context);
auto reverse = reverse_channels(std::get<0>(cut), function, context);
bool updated = std::get<1>(cut) | std::get<1>(reverse);
context.color_format() = dst_format;
return std::make_tuple(std::get<0>(reverse), updated);
}
}
if (context.color_format() == ColorFormat::BGRX) {
if (dst_format == ColorFormat::BGR) {
auto res = cut_last_channel(nodes, function, context);
context.color_format() = dst_format;
return res;
} else if (dst_format == ColorFormat::RGB) {
auto cut = cut_last_channel(nodes, function, context);
auto reverse = reverse_channels(std::get<0>(cut), function, context);
bool updated = std::get<1>(cut) | std::get<1>(reverse);
context.color_format() = dst_format;
return std::make_tuple(std::get<0>(reverse), updated);
}
}
OPENVINO_ASSERT(false,
"Source color format '",
color_format_name(context.color_format()),
@ -334,6 +360,24 @@ std::tuple<std::vector<Output<Node>>, bool> PreStepsList::reverse_channels(const
return std::make_tuple(std::vector<Output<Node>>{convert}, false);
}
std::tuple<std::vector<Output<Node>>, bool> PreStepsList::cut_last_channel(const std::vector<Output<Node>>& nodes,
const std::shared_ptr<Function>& function,
PreprocessingContext& context) {
OPENVINO_ASSERT(nodes.size() == 1, "Internal error: can't cut X channel for multi-plane inputs");
OPENVINO_ASSERT(ov::layout::has_channels(context.layout()),
"Layout ",
context.layout().to_string(),
" doesn't have `channels` dimension");
auto channels_idx = ov::layout::channels_idx(context.layout());
auto start = opset8::Constant::create(element::i32, {1}, {0});
auto stop = opset8::Constant::create(element::i32, {1}, {-1}); // Everything except last channel
auto step = opset8::Constant::create(element::i32, {1}, {1});
auto axis = opset8::Constant::create(element::i32, {1}, {channels_idx}); // E.g. 3
auto slice = std::make_shared<ov::op::v8::Slice>(nodes[0], start, stop, step, axis);
return std::make_tuple(std::vector<Output<Node>>{slice}, false);
}
//------------- Post processing ------
void PostStepsList::add_convert_impl(const element::Type& type) {
m_actions.emplace_back([type](const Output<Node>& node, PostprocessingContext& ctxt) {

4
ngraph/core/src/preprocess/preprocess_steps_impl.hpp
View File

@ -191,6 +191,10 @@ private:
const std::shared_ptr<Function>& function,
PreprocessingContext& context);
static std::tuple<std::vector<Output<Node>>, bool> cut_last_channel(const std::vector<Output<Node>>& nodes,
const std::shared_ptr<Function>& function,
PreprocessingContext& context);
private:
std::list<InternalPreprocessOp> m_actions;
std::list<std::vector<uint64_t>> m_layout_converts;