natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Removed compile_tool and benchmark_app_legacy from OpenVINO repo (#18350)

This commit is contained in:
Anastasiya(Asya) Pronina 2023-07-04 17:35:51 +02:00 committed by GitHub
parent 642c69115d
commit 84d35ab6f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 7 additions and 4054 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.ci/azure/linux_onnxruntime.yml
View File

@ -131,7 +131,6 @@ jobs:
-DENABLE_CPPLINT=OFF
-DENABLE_PROFILING_ITT=OFF
-DENABLE_SAMPLES=OFF
-DENABLE_COMPILE_TOOL=OFF
-DENABLE_OV_TF_FRONTEND=OFF
-DENABLE_OV_PADDLE_FRONTEND=OFF
-DENABLE_OV_PYTORCH_FRONTEND=OFF

2
CMakeLists.txt
View File

@ -124,7 +124,7 @@ endif()
include(thirdparty/dependencies.cmake)
add_subdirectory(src)
if(ENABLE_SAMPLES OR ENABLE_TESTS OR ENABLE_COMPILE_TOOL)
if(ENABLE_SAMPLES OR ENABLE_TESTS)
add_subdirectory(samples)
endif()

4
cmake/extra_modules.cmake
View File

@ -15,7 +15,7 @@ function(ie_generate_dev_package_config)
add_custom_target(ie_dev_targets DEPENDS ${all_dev_targets})
set(PATH_VARS "OpenVINO_SOURCE_DIR")
if(ENABLE_SAMPLES OR ENABLE_COMPILE_TOOL OR ENABLE_TESTS)
if(ENABLE_SAMPLES OR ENABLE_TESTS)
list(APPEND PATH_VARS "gflags_BINARY_DIR")
# if we've found system gflags
if(gflags_DIR)
@ -52,7 +52,7 @@ function(ov_generate_dev_package_config)
add_custom_target(ov_dev_targets DEPENDS ${all_dev_targets})
set(PATH_VARS "OpenVINO_SOURCE_DIR")
if(ENABLE_SAMPLES OR ENABLE_COMPILE_TOOL OR ENABLE_TESTS)
if(ENABLE_SAMPLES OR ENABLE_TESTS)
list(APPEND PATH_VARS "gflags_BINARY_DIR")
# if we've found system gflags
if(gflags_DIR)

2
cmake/features.cmake
View File

@ -12,8 +12,6 @@ ie_dependent_option (ENABLE_ARM_COMPUTE_CMAKE "Enable ARM Compute build via cmak
ie_option (ENABLE_TESTS "unit, behavior and functional tests" OFF)
ie_option (ENABLE_COMPILE_TOOL "Enables compile_tool" ON)
ie_option (ENABLE_STRICT_DEPENDENCIES "Skip configuring \"convinient\" dependencies for efficient parallel builds" ON)
if(X86_64)

3
samples/cpp/benchmark_app/main.cpp
View File

@ -95,8 +95,7 @@ bool parse_and_check_command_line(int argc, char* argv[]) {
bool isPrecisionSet = !(FLAGS_ip.empty() && FLAGS_op.empty() && FLAGS_iop.empty());
if (isNetworkCompiled && isPrecisionSet) {
std::string err = std::string("Cannot set precision for a compiled model. ") +
std::string("Please re-compile your model with required precision "
"using compile_tool");
std::string("Please re-compile your model with required precision.");
throw std::logic_error(err);
}

5
scripts/setupvars/setupvars.bat
View File

@ -60,11 +60,6 @@ if exist %INTEL_OPENVINO_DIR%\runtime\3rdparty\tbb (
)
)
:: Compile tool
if exist %INTEL_OPENVINO_DIR%\tools\compile_tool (
set "PATH=%INTEL_OPENVINO_DIR%\tools\compile_tool;%PATH%"
)
:: Add libs dirs to the PATH
set "PATH=%OPENVINO_LIB_PATHS%;%PATH%"

4
scripts/setupvars/setupvars.sh
View File

@ -79,10 +79,6 @@ if [ -e "$INSTALLDIR/runtime" ]; then
fi
fi
if [ -e "$INSTALLDIR/tools/compile_tool" ]; then
export LD_LIBRARY_PATH=$INSTALLDIR/tools/compile_tool${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
fi
# OpenCV environment
if [ -f "$INSTALLDIR/opencv/setupvars.sh" ]; then
# shellcheck source=/dev/null

4
thirdparty/dependencies.cmake vendored
View File

@ -362,7 +362,7 @@ endif()
# Gflags
#
if(ENABLE_SAMPLES OR ENABLE_COMPILE_TOOL OR ENABLE_TESTS)
if(ENABLE_SAMPLES OR ENABLE_TESTS)
if(CMAKE_TOOLCHAIN_FILE MATCHES "vcpkg " OR DEFINED VCPKG_VERBOSE OR
CMAKE_TOOLCHAIN_FILE MATCHES "conan_toolchain" OR DEFINED CONAN_EXPORTED)
# vcpkg contains only libs compiled with threads
@ -628,7 +628,7 @@ endif()
if(CPACK_GENERATOR MATCHES "^(DEB|RPM|CONDA-FORGE|BREW|CONAN|VCPKG)$")
# These libraries are dependencies for openvino-samples package
if(ENABLE_SAMPLES OR ENABLE_COMPILE_TOOL OR ENABLE_TESTS)
if(ENABLE_SAMPLES OR ENABLE_TESTS)
if(NOT gflags_FOUND AND CPACK_GENERATOR MATCHES "^(DEB|RPM)$")
message(FATAL_ERROR "gflags must be used as a ${CPACK_GENERATOR} package. Install libgflags-dev / gflags-devel")
endif()

12
tools/CMakeLists.txt
View File

@ -1,18 +1,6 @@
# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
# C++ tools
#
if(ENABLE_COMPILE_TOOL)
add_subdirectory(compile_tool)
endif()
if(ENABLE_SAMPLES)
add_subdirectory(legacy/benchmark_app)
endif()
#
# Python tools
#

3
tools/benchmark_tool/openvino/tools/benchmark/main.py
View File

@ -46,8 +46,7 @@ def parse_and_check_command_line():
if is_network_compiled and is_precisiton_set:
raise Exception("Cannot set precision for a compiled model. " \
"Please re-compile your model with required precision " \
"using compile_tool")
"Please re-compile your model with required precision.")
return args, is_network_compiled

52
tools/compile_tool/CMakeLists.txt
View File

@ -1,52 +0,0 @@
# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
set(TARGET_NAME compile_tool)
add_definitions(-DIN_OV_COMPONENT)
file(GLOB SRCS
${CMAKE_CURRENT_SOURCE_DIR}/*.cpp
)
add_executable(${TARGET_NAME} ${SRCS})
if (CMAKE_COMPILER_IS_GNUCXX)
target_compile_options(${TARGET_NAME} PRIVATE -Wall)
endif()
# for cross-compilation with gflags
find_package(Threads REQUIRED)
target_link_libraries(${TARGET_NAME} PRIVATE
inference_engine
inference_engine_plugin_api
gflags
ie_samples_utils
Threads::Threads
)
set_target_properties(${TARGET_NAME} PROPERTIES
COMPILE_PDB_NAME ${TARGET_NAME}
FOLDER tools
)
add_cpplint_target(${TARGET_NAME}_cpplint FOR_TARGETS ${TARGET_NAME})
# install
ov_cpack_add_component(${OV_CPACK_COMP_CORE_TOOLS}
HIDDEN
DEPENDS ${OV_CPACK_COMP_CORE})
if(CPACK_GENERATOR MATCHES "^(DEB|RPM)$")
install(TARGETS compile_tool
RUNTIME DESTINATION ${OV_CPACK_TOOLSDIR}
COMPONENT ${OV_CPACK_COMP_CORE_TOOLS}
${OV_CPACK_COMP_CORE_TOOLS_EXCLUDE_ALL})
else()
install(TARGETS compile_tool
RUNTIME DESTINATION ${OV_CPACK_TOOLSDIR}/compile_tool
COMPONENT ${OV_CPACK_COMP_CORE_TOOLS}
${OV_CPACK_COMP_CORE_TOOLS_EXCLUDE_ALL})
endif()

743
tools/compile_tool/main.cpp
View File

@ -1,743 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <cstdlib>
#include <iostream>
#include <fstream>
#include <algorithm>
#include <chrono>
#include <unordered_map>
#include <map>
#include <vector>
#include <string>
#include <gflags/gflags.h>
#include "inference_engine.hpp"
#include "openvino/openvino.hpp"
static constexpr char help_message[] =
"Optional. Print the usage message.";
static constexpr char model_message[] =
"Required. Path to the XML model.";
static constexpr char targetDeviceMessage[] =
"Required. Specify a target device for which executable network will be compiled.\n"
" Use \"-d HETERO:<comma-separated_devices_list>\" format to specify HETERO plugin.\n"
" Use \"-d MULTI:<comma-separated_devices_list>\" format to specify MULTI plugin.\n"
" The application looks for a suitable plugin for the specified device.";
static constexpr char output_message[] =
"Optional. Path to the output file. Default value: \"<model_xml_file>.blob\".";
static constexpr char log_level_message[] =
"Optional. Log level for InferenceEngine library.";
static constexpr char config_message[] =
"Optional. Path to the configuration file.";
static constexpr char inputs_precision_message[] =
"Optional. Specifies precision for all input layers of the network.";
static constexpr char outputs_precision_message[] =
"Optional. Specifies precision for all output layers of the network.";
static constexpr char iop_message[] =
"Optional. Specifies precision for input and output layers by name.\n"
" Example: -iop \"input:FP16, output:FP16\".\n"
" Notice that quotes are required.\n"
" Overwrites precision from ip and op options for specified layers.";
static constexpr char inputs_layout_message[] =
"Optional. Specifies layout for all input layers of the network.";
static constexpr char outputs_layout_message[] =
"Optional. Specifies layout for all output layers of the network.";
static constexpr char iol_message[] =
"Optional. Specifies layout for input and output layers by name.\n"
" Example: -iol \"input:NCHW, output:NHWC\".\n"
" Notice that quotes are required.\n"
" Overwrites layout from il and ol options for specified layers.";
static constexpr char inputs_model_layout_message[] =
"Optional. Specifies model layout for all input layers of the network.";
static constexpr char outputs_model_layout_message[] =
"Optional. Specifies model layout for all output layers of the network.";
static constexpr char ioml_message[] =
"Optional. Specifies model layout for input and output tensors by name.\n"
" Example: -ionl \"input:NCHW, output:NHWC\".\n"
" Notice that quotes are required.\n"
" Overwrites layout from il and ol options for specified layers.";
static constexpr char api1_message[] =
"Optional. Compile model to legacy format for usage in Inference Engine API,\n"
" by default compiles to OV 2.0 API";
DEFINE_bool(h, false, help_message);
DEFINE_string(m, "", model_message);
DEFINE_string(d, "", targetDeviceMessage);
DEFINE_string(o, "", output_message);
DEFINE_string(log_level, "", log_level_message);
DEFINE_string(c, "", config_message);
DEFINE_string(ip, "", inputs_precision_message);
DEFINE_string(op, "", outputs_precision_message);
DEFINE_string(iop, "", iop_message);
DEFINE_string(il, "", inputs_layout_message);
DEFINE_string(ol, "", outputs_layout_message);
DEFINE_string(iol, "", iol_message);
DEFINE_string(iml, "", inputs_model_layout_message);
DEFINE_string(oml, "", outputs_model_layout_message);
DEFINE_string(ioml, "", ioml_message);
DEFINE_bool(ov_api_1_0, false, api1_message);
namespace {
std::vector<std::string> splitStringList(const std::string& str, char delim) {
if (str.empty())
return {};
std::istringstream istr(str);
std::vector<std::string> result;
std::string elem;
while (std::getline(istr, elem, delim)) {
if (elem.empty()) {
continue;
}
result.emplace_back(std::move(elem));
}
return result;
}
std::map<std::string, std::string> parseArgMap(std::string argMap) {
argMap.erase(std::remove_if(argMap.begin(), argMap.end(), ::isspace), argMap.end());
const auto pairs = splitStringList(argMap, ',');
std::map<std::string, std::string> parsedMap;
for (auto&& pair : pairs) {
const auto lastDelimPos = pair.find_last_of(':');
auto key = pair.substr(0, lastDelimPos);
auto value = pair.substr(lastDelimPos + 1);
if (lastDelimPos == std::string::npos || key.empty() || value.empty()) {
throw std::invalid_argument("Invalid key/value pair " + pair + ". Expected <layer_name>:<value>");
}
parsedMap[std::move(key)] = std::move(value);
}
return parsedMap;
}
using supported_precisions_t = std::unordered_map<std::string, InferenceEngine::Precision>;
InferenceEngine::Precision getPrecision(std::string value, const supported_precisions_t& supported_precisions) {
std::transform(value.begin(), value.end(), value.begin(), ::toupper);
const auto precision = supported_precisions.find(value);
if (precision == supported_precisions.end()) {
throw std::logic_error("\"" + value + "\"" + " is not a valid precision");
}
return precision->second;
}
InferenceEngine::Precision getPrecision(const std::string& value) {
static const supported_precisions_t supported_precisions = {
{"FP32", InferenceEngine::Precision::FP32}, {"f32", InferenceEngine::Precision::FP32},
{"FP16", InferenceEngine::Precision::FP16}, {"f16", InferenceEngine::Precision::FP16},
{"BF16", InferenceEngine::Precision::BF16}, {"bf16", InferenceEngine::Precision::BF16},
{"U64", InferenceEngine::Precision::U64}, {"u64", InferenceEngine::Precision::U64},
{"I64", InferenceEngine::Precision::I64}, {"i64", InferenceEngine::Precision::I64},
{"U32", InferenceEngine::Precision::U32}, {"u32", InferenceEngine::Precision::U32},
{"I32", InferenceEngine::Precision::I32}, {"i32", InferenceEngine::Precision::I32},
{"U16", InferenceEngine::Precision::U16}, {"u16", InferenceEngine::Precision::U16},
{"I16", InferenceEngine::Precision::I16}, {"i16", InferenceEngine::Precision::I16},
{"U8", InferenceEngine::Precision::U8}, {"u8", InferenceEngine::Precision::U8},
{"I8", InferenceEngine::Precision::I8}, {"i8", InferenceEngine::Precision::I8},
{"BOOL", InferenceEngine::Precision::BOOL}, {"boolean", InferenceEngine::Precision::BOOL},
};
return getPrecision(value, supported_precisions);
}
void setPrecisions(const InferenceEngine::CNNNetwork& network, const std::string& iop) {
const auto user_precisions_map = parseArgMap(iop);
auto inputs = network.getInputsInfo();
auto outputs = network.getOutputsInfo();
for (auto&& item : user_precisions_map) {
const auto& layer_name = item.first;
const auto& user_precision = item.second;
const auto input = inputs.find(layer_name);
const auto output = outputs.find(layer_name);
if (input != inputs.end()) {
input->second->setPrecision(getPrecision(user_precision));
} else if (output != outputs.end()) {
output->second->setPrecision(getPrecision(user_precision));
} else {
throw std::logic_error(layer_name + " is not an input neither output");
}
}
}
} //namespace
void processPrecision(InferenceEngine::CNNNetwork& network,
const std::string& ip,
const std::string& op,
const std::string& iop) {
if (!ip.empty()) {
const auto user_precision = getPrecision(ip);
for (auto&& layer : network.getInputsInfo()) {
layer.second->setPrecision(user_precision);
}
}
if (!op.empty()) {
auto user_precision = getPrecision(op);
for (auto&& layer : network.getOutputsInfo()) {
layer.second->setPrecision(user_precision);
}
}
if (!iop.empty()) {
setPrecisions(network, iop);
}
}
using supported_layouts_t = std::unordered_map<std::string, InferenceEngine::Layout>;
using matchLayoutToDims_t = std::unordered_map<size_t, size_t>;
InferenceEngine::Layout getLayout(std::string value, const supported_layouts_t& supported_layouts) {
std::transform(value.begin(), value.end(), value.begin(), ::toupper);
const auto layout = supported_layouts.find(value);
if (layout == supported_layouts.end()) {
throw std::logic_error("\"" + value + "\"" + " is not a valid layout");
}
return layout->second;
}
InferenceEngine::Layout getLayout(const std::string& value) {
static const supported_layouts_t supported_layouts = {
{"NCDHW", InferenceEngine::Layout::NCDHW},
{"NDHWC", InferenceEngine::Layout::NDHWC},
{"NCHW", InferenceEngine::Layout::NCHW},
{"NHWC", InferenceEngine::Layout::NHWC},
{"CHW", InferenceEngine::Layout::CHW},
{"HWC", InferenceEngine::Layout::HWC},
{"NC", InferenceEngine::Layout::NC},
{"C", InferenceEngine::Layout::C},
};
return getLayout(value, supported_layouts);
}
bool isMatchLayoutToDims(InferenceEngine::Layout layout, size_t dimension) {
static const matchLayoutToDims_t matchLayoutToDims = { {static_cast<size_t>(InferenceEngine::Layout::NCDHW), 5},
{static_cast<size_t>(InferenceEngine::Layout::NDHWC), 5},
{static_cast<size_t>(InferenceEngine::Layout::NCHW), 4},
{static_cast<size_t>(InferenceEngine::Layout::NHWC), 4},
{static_cast<size_t>(InferenceEngine::Layout::CHW), 3},
{static_cast<size_t>(InferenceEngine::Layout::NC), 2},
{static_cast<size_t>(InferenceEngine::Layout::C), 1} };
const auto dims = matchLayoutToDims.find(static_cast<size_t>(layout));
if (dims == matchLayoutToDims.end()) {
throw std::logic_error("Layout is not valid.");
}
return dimension == dims->second;
}
void setLayouts(const InferenceEngine::CNNNetwork& network, const std::string iol) {
const auto user_layouts_map = parseArgMap(iol);
auto inputs = network.getInputsInfo();
auto outputs = network.getOutputsInfo();
for (auto&& item : user_layouts_map) {
const auto& layer_name = item.first;
const auto& user_layout = getLayout(item.second);
const auto input = inputs.find(layer_name);
const auto output = outputs.find(layer_name);
if (input != inputs.end()) {
if (!isMatchLayoutToDims(user_layout, input->second->getTensorDesc().getDims().size())) {
throw std::logic_error(item.second + " layout is not applicable to " + layer_name);
}
input->second->setLayout(user_layout);
} else if (output != outputs.end()) {
if (!isMatchLayoutToDims(user_layout, output->second->getTensorDesc().getDims().size())) {
throw std::logic_error(item.second + " layout is not applicable to " + layer_name);
}
output->second->setLayout(user_layout);
} else {
throw std::logic_error(layer_name + " is not an input neither output");
}
}
}
void processLayout(InferenceEngine::CNNNetwork& network,
const std::string& il,
const std::string& ol,
const std::string& iol) {
if (!il.empty()) {
const auto layout = getLayout(il);
for (auto&& layer : network.getInputsInfo()) {
if (isMatchLayoutToDims(layout, layer.second->getTensorDesc().getDims().size())) {
layer.second->setLayout(layout);
}
}
}
if (!ol.empty()) {
const auto layout = getLayout(ol);
for (auto&& layer : network.getOutputsInfo()) {
if (isMatchLayoutToDims(layout, layer.second->getTensorDesc().getDims().size())) {
layer.second->setLayout(layout);
}
}
}
if (!iol.empty()) {
setLayouts(network, iol);
}
}
using supported_type_t = std::unordered_map<std::string, ov::element::Type>;
ov::element::Type getType(std::string value, const supported_type_t& supported_precisions) {
std::transform(value.begin(), value.end(), value.begin(), ::toupper);
const auto precision = supported_precisions.find(value);
if (precision == supported_precisions.end()) {
throw std::logic_error("\"" + value + "\"" + " is not a valid precision");
}
return precision->second;
}
ov::element::Type getType(const std::string& value) {
static const supported_type_t supported_types = {
{"FP32", ov::element::f32}, {"f32", ov::element::f32}, {"FP16", ov::element::f16},
{"f16", ov::element::f16}, {"BF16", ov::element::bf16}, {"bf16", ov::element::bf16},
{"U64", ov::element::u64}, {"u64", ov::element::u64}, {"I64", ov::element::i64},
{"i64", ov::element::i64}, {"U32", ov::element::u32}, {"u32", ov::element::u32},
{"I32", ov::element::i32}, {"i32", ov::element::i32}, {"U16", ov::element::u16},
{"u16", ov::element::u16}, {"I16", ov::element::i16}, {"i16", ov::element::i16},
{"U8", ov::element::u8}, {"u8", ov::element::u8}, {"I8", ov::element::i8},
{"i8", ov::element::i8}, {"BOOL", ov::element::boolean}, {"boolean", ov::element::boolean},
};
return getType(value, supported_types);
}
bool isFP32(const ov::element::Type& type) {
return type == ov::element::f32;
}
static void setDefaultIO(ov::preprocess::PrePostProcessor& preprocessor,
const std::vector<ov::Output<ov::Node>>& inputs,
const std::vector<ov::Output<ov::Node>>& outputs) {
const bool isVPUX = FLAGS_d.find("VPUX") != std::string::npos;
if (isVPUX) {
for (size_t i = 0; i < inputs.size(); i++) {
preprocessor.input(i).tensor().set_element_type(ov::element::u8);
}
for (size_t i = 0; i < outputs.size(); i++) {
preprocessor.output(i).tensor().set_element_type(ov::element::f32);
}
}
}
void configurePrePostProcessing(std::shared_ptr<ov::Model>& model,
const std::string& ip,
const std::string& op,
const std::string& iop,
const std::string& il,
const std::string& ol,
const std::string& iol,
const std::string& iml,
const std::string& oml,
const std::string& ioml) {
auto preprocessor = ov::preprocess::PrePostProcessor(model);
const auto inputs = model->inputs();
const auto outputs = model->outputs();
setDefaultIO(preprocessor, inputs, outputs);
if (!ip.empty()) {
auto type = getType(ip);
for (size_t i = 0; i < inputs.size(); i++) {
preprocessor.input(i).tensor().set_element_type(type);
}
}
if (!op.empty()) {
auto type = getType(op);
for (size_t i = 0; i < outputs.size(); i++) {
preprocessor.output(i).tensor().set_element_type(type);
}
}
if (!iop.empty()) {
const auto user_precisions_map = parseArgMap(iop);
for (auto&& item : user_precisions_map) {
const auto& tensor_name = item.first;
const auto type = getType(item.second);
bool tensorFound = false;
for (size_t i = 0; i < inputs.size(); i++) {
if (inputs[i].get_names().count(tensor_name)) {
preprocessor.input(i).tensor().set_element_type(type);
tensorFound = true;
break;
}
}
if (!tensorFound) {
for (size_t i = 0; i < outputs.size(); i++) {
if (outputs[i].get_names().count(tensor_name)) {
preprocessor.output(i).tensor().set_element_type(type);
tensorFound = true;
break;
}
}
}
OPENVINO_ASSERT(tensorFound, "Model doesn't have input/output with tensor name: ", tensor_name);
}
}
if (!il.empty()) {
for (size_t i = 0; i < inputs.size(); i++) {
preprocessor.input(i).tensor().set_layout(ov::Layout(il));
}
}
if (!ol.empty()) {
for (size_t i = 0; i < outputs.size(); i++) {
preprocessor.output(i).tensor().set_layout(ov::Layout(ol));
}
}
if (!iol.empty()) {
const auto user_precisions_map = parseArgMap(iol);
for (auto&& item : user_precisions_map) {
const auto& tensor_name = item.first;
bool tensorFound = false;
for (size_t i = 0; i < inputs.size(); i++) {
if (inputs[i].get_names().count(tensor_name)) {
preprocessor.input(i).tensor().set_layout(ov::Layout(item.second));
tensorFound = true;
break;
}
}
if (!tensorFound) {
for (size_t i = 0; i < outputs.size(); i++) {
if (outputs[i].get_names().count(tensor_name)) {
preprocessor.output(i).tensor().set_layout(ov::Layout(item.second));
tensorFound = true;
break;
}
}
}
OPENVINO_ASSERT(tensorFound, "Model doesn't have input/output with tensor name: ", tensor_name);
}
}
if (!iml.empty()) {
for (size_t i = 0; i < inputs.size(); i++) {
preprocessor.input(i).model().set_layout(ov::Layout(iml));
}
}
if (!oml.empty()) {
for (size_t i = 0; i < outputs.size(); i++) {
preprocessor.output(i).model().set_layout(ov::Layout(oml));
}
}
if (!ioml.empty()) {
const auto user_precisions_map = parseArgMap(ioml);
for (auto&& item : user_precisions_map) {
const auto& tensor_name = item.first;
bool tensorFound = false;
for (size_t i = 0; i < inputs.size(); i++) {
if (inputs[i].get_names().count(tensor_name)) {
preprocessor.input(i).model().set_layout(ov::Layout(item.second));
tensorFound = true;
break;
}
}
if (!tensorFound) {
for (size_t i = 0; i < outputs.size(); i++) {
if (outputs[i].get_names().count(tensor_name)) {
preprocessor.output(i).model().set_layout(ov::Layout(item.second));
tensorFound = true;
break;
}
}
}
OPENVINO_ASSERT(tensorFound, "Model doesn't have input/output with tensor name: ", tensor_name);
}
}
model = preprocessor.build();
}
void printInputAndOutputsInfo(const InferenceEngine::CNNNetwork& network) {
std::cout << "Network inputs:" << std::endl;
for (auto&& layer : network.getInputsInfo()) {
std::cout << " " << layer.first << " : " << layer.second->getPrecision() << " / "
<< layer.second->getLayout() << std::endl;
}
std::cout << "Network outputs:" << std::endl;
for (auto&& layer : network.getOutputsInfo()) {
std::cout << " " << layer.first << " : " << layer.second->getPrecision() << " / "
<< layer.second->getLayout() << std::endl;
}
}
void printInputAndOutputsInfoShort(const ov::Model& network) {
std::cout << "Network inputs:" << std::endl;
for (auto&& param : network.get_parameters()) {
auto l = param->get_layout();
std::cout << " " << param->get_friendly_name() << " : " << param->get_element_type() << " / "
<< param->get_layout().to_string() << std::endl;
}
std::cout << "Network outputs:" << std::endl;
for (auto&& result : network.get_results()) {
std::cout << " " << result->get_friendly_name() << " : " << result->get_element_type() << " / "
<< result->get_layout().to_string() << std::endl;
}
}
inline std::string fileNameNoExt(const std::string& filepath) {
auto pos = filepath.rfind('.');
if (pos == std::string::npos)
return filepath;
return filepath.substr(0, pos);
}
static void showUsage() {
std::cout << "compile_tool [OPTIONS]" << std::endl;
std::cout << std::endl;
std::cout << " Common options: " << std::endl;
std::cout << " -h " << help_message << std::endl;
std::cout << " -m <value> " << model_message << std::endl;
std::cout << " -d <value> " << targetDeviceMessage << std::endl;
std::cout << " -o <value> " << output_message << std::endl;
std::cout << " -c <value> " << config_message << std::endl;
std::cout << " -ip <value> " << inputs_precision_message << std::endl;
std::cout << " -op <value> " << outputs_precision_message << std::endl;
std::cout << " -iop \"<value>\" " << iop_message << std::endl;
std::cout << " -il <value> " << inputs_layout_message << std::endl;
std::cout << " -ol <value> " << outputs_layout_message << std::endl;
std::cout << " -iol \"<value>\" " << iol_message << std::endl;
std::cout << " -iml <value> " << inputs_model_layout_message << std::endl;
std::cout << " -oml <value> " << outputs_model_layout_message << std::endl;
std::cout << " -ioml \"<value>\" " << ioml_message << std::endl;
std::cout << " -ov_api_1_0 " << api1_message << std::endl;
std::cout << std::endl;
}
static bool parseCommandLine(int* argc, char*** argv) {
gflags::ParseCommandLineNonHelpFlags(argc, argv, true);
if (FLAGS_h) {
showUsage();
return false;
}
if (FLAGS_m.empty()) {
throw std::invalid_argument("Path to model xml file is required");
}
if (FLAGS_d.empty()) {
throw std::invalid_argument("Target device name is required");
}
if (1 < *argc) {
std::stringstream message;
message << "Unknown arguments: ";
for (auto arg = 1; arg < *argc; arg++) {
message << (*argv)[arg];
if (arg < *argc) {
message << " ";
}
}
throw std::invalid_argument(message.str());
}
return true;
}
static std::map<std::string, std::string> parseConfigFile(char comment = '#') {
std::map<std::string, std::string> config;
std::ifstream file(FLAGS_c);
if (file.is_open()) {
std::string option;
while (std::getline(file, option)) {
if (option.empty() || option[0] == comment) {
continue;
}
size_t spacePos = option.find(' ');
OPENVINO_ASSERT(spacePos != std::string::npos,
"Failed to find a space separator in "
"provided plugin config option: " +
option);
std::string key = option.substr(0, spacePos);
std::string value = option.substr(spacePos + 1);
config[key] = value;
}
}
return config;
}
bool isFP16(InferenceEngine::Precision precision) {
return precision == InferenceEngine::Precision::FP16;
}
bool isFP32(InferenceEngine::Precision precision) {
return precision == InferenceEngine::Precision::FP32;
}
bool isFloat(InferenceEngine::Precision precision) {
return isFP16(precision) || isFP32(precision);
}
static void setDefaultIO(InferenceEngine::CNNNetwork& network) {
const bool isVPUX = FLAGS_d.find("VPUX") != std::string::npos;
if (isVPUX) {
const InferenceEngine::Precision u8 = InferenceEngine::Precision::U8;
const InferenceEngine::Precision fp32 = InferenceEngine::Precision::FP32;
for (auto&& layer : network.getInputsInfo()) {
layer.second->setPrecision(u8);
}
for (auto&& layer : network.getOutputsInfo()) {
layer.second->setPrecision(fp32);
}
}
}
std::string getFileNameFromPath(const std::string& path,
#if defined(_WIN32)
const std::string& sep = "\\") {
#else
const std::string& sep = "/") {
#endif
const auto pos = path.rfind(sep);
if (std::string::npos == pos) {
return path;
} else {
return path.substr(pos + 1);
}
}
using TimeDiff = std::chrono::milliseconds;
int main(int argc, char* argv[]) {
TimeDiff loadNetworkTimeElapsed {0};
try {
const auto& version = ov::get_openvino_version();
std::cout << version.description << " version ......... ";
std::cout << OPENVINO_VERSION_MAJOR << "." << OPENVINO_VERSION_MINOR << "." << OPENVINO_VERSION_PATCH << std::endl;
std::cout << "Build ........... ";
std::cout << version.buildNumber << std::endl;
if (!parseCommandLine(&argc, &argv)) {
return EXIT_SUCCESS;
}
if (FLAGS_ov_api_1_0) {
InferenceEngine::Core ie;
if (!FLAGS_log_level.empty()) {
ie.SetConfig({{CONFIG_KEY(LOG_LEVEL), FLAGS_log_level}}, FLAGS_d);
}
auto network = ie.ReadNetwork(FLAGS_m);
setDefaultIO(network);
processPrecision(network, FLAGS_ip, FLAGS_op, FLAGS_iop);
processLayout(network, FLAGS_il, FLAGS_ol, FLAGS_iol);
printInputAndOutputsInfo(network);
auto timeBeforeLoadNetwork = std::chrono::steady_clock::now();
auto executableNetwork = ie.LoadNetwork(network, FLAGS_d, parseConfigFile());
loadNetworkTimeElapsed = std::chrono::duration_cast<TimeDiff>(std::chrono::steady_clock::now() - timeBeforeLoadNetwork);
std::string outputName = FLAGS_o;
if (outputName.empty()) {
outputName = getFileNameFromPath(fileNameNoExt(FLAGS_m)) + ".blob";
}
std::ofstream outputFile{outputName, std::ios::out | std::ios::binary};
if (!outputFile.is_open()) {
std::cout << "Output file " << outputName << " can't be opened for writing" << std::endl;
return EXIT_FAILURE;
} else {
executableNetwork.Export(outputFile);
}
} else {
ov::Core core;
if (!FLAGS_log_level.empty()) {
ov::log::Level level;
std::stringstream{FLAGS_log_level} >> level;
core.set_property(FLAGS_d, ov::log::level(level));
}
auto model = core.read_model(FLAGS_m);
configurePrePostProcessing(model, FLAGS_ip, FLAGS_op, FLAGS_iop, FLAGS_il, FLAGS_ol, FLAGS_iol, FLAGS_iml, FLAGS_oml, FLAGS_ioml);
printInputAndOutputsInfoShort(*model);
auto timeBeforeLoadNetwork = std::chrono::steady_clock::now();
auto configs = parseConfigFile();
auto compiledModel = core.compile_model(model, FLAGS_d, {configs.begin(), configs.end()});
loadNetworkTimeElapsed = std::chrono::duration_cast<TimeDiff>(std::chrono::steady_clock::now() - timeBeforeLoadNetwork);
std::string outputName = FLAGS_o;
if (outputName.empty()) {
outputName = getFileNameFromPath(fileNameNoExt(FLAGS_m)) + ".blob";
}
std::ofstream outputFile{outputName, std::ios::out | std::ios::binary};
if (!outputFile.is_open()) {
std::cout << "Output file " << outputName << " can't be opened for writing" << std::endl;
return EXIT_FAILURE;
} else {
compiledModel.export_model(outputFile);
}
}
} catch (const std::exception& error) {
std::cerr << error.what() << std::endl;
return EXIT_FAILURE;
} catch (...) {
std::cerr << "Unknown/internal exception happened." << std::endl;
return EXIT_FAILURE;
}
std::cout << "Done. LoadNetwork time elapsed: " << loadNetworkTimeElapsed.count() << " ms" << std::endl;
return EXIT_SUCCESS;
}

42
tools/legacy/benchmark_app/CMakeLists.txt
View File

@ -1,42 +0,0 @@
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
set(TARGET_NAME benchmark_app_legacy)
add_definitions(-DIN_OV_COMPONENT)
file (GLOB SRC ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
file (GLOB HDR ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp)
if(OpenVINO_SOURCE_DIR)
set(OpenVINO_DIR "${CMAKE_BINARY_DIR}")
endif()
source_group("src" FILES ${SRC})
source_group("include" FILES ${HDR})
add_executable(${TARGET_NAME} ${SRC} ${HDR})
find_package(OpenVINO REQUIRED COMPONENTS Runtime)
set_target_properties(${TARGET_NAME} PROPERTIES
COMPILE_PDB_NAME ${TARGET_NAME}
FOLDER tools
)
if (CMAKE_COMPILER_IS_GNUCXX)
target_compile_options(${TARGET_NAME} PRIVATE -Wall)
endif()
# for cross-compilation with gflags
find_package(Threads REQUIRED)
target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime format_reader gflags Threads::Threads)
find_package(OpenCV QUIET COMPONENTS core)
if(NOT OpenCV_FOUND)
message(WARNING "OpenCV is disabled or not found, ${TARGET_NAME} will be built without OpenCV support. Set OpenCV_DIR")
else()
target_compile_definitions(${TARGET_NAME} PRIVATE USE_OPENCV)
target_link_libraries(${TARGET_NAME} PRIVATE opencv_core)
endif()

183
tools/legacy/benchmark_app/README.md
View File

@ -1,183 +0,0 @@
# Benchmark C++ Tool
This topic demonstrates how to use the Benchmark C++ Tool to estimate deep learning inference performance on supported devices. Performance can be measured for two inference modes: synchronous (latency-oriented) and asynchronous (throughput-oriented).
> **NOTE**: This topic describes usage of C++ implementation of the Benchmark Tool. For the Python* implementation, refer to [Benchmark Python* Tool](../../benchmark_tool/README.md).
## How It Works
Upon start-up, the application reads command-line parameters and loads a network and images/binary files to the Inference Engine plugin, which is chosen depending on a specified device. The number of infer requests and execution approach depend on the mode defined with the `-api` command-line parameter.
> **NOTE**: By default, Inference Engine samples, tools and demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with `--reverse_input_channels` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of [Converting a Model to Intermediate Representation (IR)](../../../docs/MO_DG/prepare_model/convert_model/Converting_Model.md).
If you run the application in the synchronous mode, it creates one infer request and executes the `Infer` method.
If you run the application in the asynchronous mode, it creates as many infer requests as specified in the `-nireq` command-line parameter and executes the `StartAsync` method for each of them. If `-nireq` is not set, the application will use the default value for specified device.
A number of execution steps is defined by one of the following parameters:
* Number of iterations specified with the `-niter` command-line argument
* Time duration specified with the `-t` command-line argument
* Both of them (execution will continue until both conditions are met)
* Predefined duration if `-niter` and `-t` are not specified. Predefined duration value depends on a device.
During the execution, the application collects latency for each executed infer request.
Reported latency value is calculated as a median value of all collected latencies. Reported throughput value is reported
in frames per second (FPS) and calculated as a derivative from:
* Reported latency in the Sync mode
* The total execution time in the Async mode
Throughput value also depends on batch size.
The application also collects per-layer Performance Measurement (PM) counters for each executed infer request if you
enable statistics dumping by setting the `-report_type` parameter to one of the possible values:
* `no_counters` report includes configuration options specified, resulting FPS and latency.
* `average_counters` report extends the `no_counters` report and additionally includes average PM counters values for each layer from the network.
* `detailed_counters` report extends the `average_counters` report and additionally includes per-layer PM counters and latency for each executed infer request.
Depending on the type, the report is stored to `benchmark_no_counters_report.csv`, `benchmark_average_counters_report.csv`,
or `benchmark_detailed_counters_report.csv` file located in the path specified in `-report_folder`.
The application also saves executable graph information serialized to an XML file if you specify a path to it with the
`-exec_graph_path` parameter.
## Building
This tools can be built as part of OpenVINO, during standard building process. More information about building OpenVINO can be found here[Build OpenVINO Inference Engine](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md)
## Run the Tool
Note that the benchmark_app usually produces optimal performance for any device out of the box.
**So in most cases you don't need to play the app options explicitly and the plain device name is enough**, for example, for CPU:
```sh
./benchmark_app -m <model> -i <input> -d CPU
```
It still may be sub-optimal for some cases, especially for very small networks. For all devices, including the [MULTI device](../../../docs/OV_Runtime_UG/multi_device.md) it is preferable to use the FP16 IR for the model. If latency of the CPU inference on the multi-socket machines is of concern.
These, as well as other topics are explained in the [Performance Optimization Guide](../../../docs/optimization_guide/dldt_deployment_optimization_guide.md).
Running the application with the `-h` option yields the following usage message:
```
./benchmark_app -h
InferenceEngine:
API version ............ <version>
Build .................. <number>
[ INFO ] Parsing input parameters
benchmark_app [OPTION]
Options:
-h, --help Print a usage message
-m "<path>" Required. Path to an .xml/.onnx/.prototxt file with a trained model or to a .blob files with a trained compiled model.
-i "<path>" Optional. Path to a folder with images and/or binaries or to specific image or binary file.
-d "<device>" Optional. Specify a target device to infer on (the list of available devices is shown below). Default value is CPU.
Use "-d HETERO:<comma-separated_devices_list>" format to specify HETERO plugin.
Use "-d MULTI:<comma-separated_devices_list>" format to specify MULTI plugin.
The application looks for a suitable plugin for the specified device.
-l "<absolute_path>" Required for CPU custom layers. Absolute path to a shared library with the kernels implementations.
Or
-c "<absolute_path>" Required for GPU custom kernels. Absolute path to an .xml file with the kernels description.
-api "<sync/async>" Optional. Enable Sync/Async API. Default value is "async".
-niter "<integer>" Optional. Number of iterations. If not specified, the number of iterations is calculated depending on a device.
-nireq "<integer>" Optional. Number of infer requests. Default value is determined automatically for a device.
-b "<integer>" Optional. Batch size value. If not specified, the batch size value is determined from Intermediate Representation.
-stream_output Optional. Print progress as a plain text. When specified, an interactive progress bar is replaced with a multiline output.
-t Optional. Time, in seconds, to execute topology.
-progress Optional. Show progress bar (can affect performance measurement). Default values is "false".
-shape Optional. Set shape for input. For example, "input1[1,3,224,224],input2[1,4]" or "[1,3,224,224]" in case of one input size.
-layout Optional. Prompts how network layouts should be treated by application. For example, "input1[NCHW],input2[NC]" or "[NCHW]" in case of one input size.
-cache_dir "<path>" Optional. Enables caching of loaded models to specified directory.
-load_from_file Optional. Loads model from file directly without ReadNetwork.
CPU-specific performance options:
-nstreams "<integer>" Optional. Number of streams to use for inference on the CPU, GPU devices
(for HETERO and MULTI device cases use format <device1>:<nstreams1>,<device2>:<nstreams2> or just <nstreams>).
Default value is determined automatically for a device.
Please note that although the automatic selection usually provides a reasonable performance,
it still may be non-optimal for some cases, especially for very small networks.
Also, using nstreams>1 is inherently throughput-oriented option, while for the best-latency
estimations the number of streams should be set to 1.
-nthreads "<integer>" Optional. Number of threads to use for inference on the CPU (including HETERO and MULTI cases).
-enforcebf16="<true/false>" Optional. By default floating point operations execution in bfloat16 precision are enforced if supported by platform.
-pin "YES"/"HYBRID_AWARE"/"NUMA"/"NO"
Optional. Explicit inference threads binding options (leave empty to let the OpenVINO to make a choice):
enabling threads->cores pinning ("YES", which is already default for a conventional CPU),
letting the runtime to decide on the threads->different core types ("HYBRID_AWARE", which is default on the hybrid CPUs)
threads->(NUMA)nodes ("NUMA") or
completely disable ("NO") CPU inference threads pinning.
-ip "U8"/"FP16"/"FP32" Optional. Specifies precision for all input layers of the network.
-op "U8"/"FP16"/"FP32" Optional. Specifies precision for all output layers of the network.
-iop Optional. Specifies precision for input and output layers by name. Example: -iop "input:FP16, output:FP16". Notice that quotes are required. Overwrites precision from ip and op options for specified layers.
Statistics dumping options:
-report_type "<type>" Optional. Enable collecting statistics report. "no_counters" report contains configuration options specified, resulting FPS and latency. "average_counters" report extends "no_counters" report and additionally includes average PM counters values for each layer from the network. "detailed_counters" report extends "average_counters" report and additionally includes per-layer PM counters and latency for each executed infer request.
-report_folder Optional. Path to a folder where statistics report is stored.
-exec_graph_path Optional. Path to a file where to store executable graph information serialized.
-pc Optional. Report performance counters.
-dump_config Optional. Path to XML/YAML/JSON file to dump IE parameters, which were set by application.
-load_config Optional. Path to XML/YAML/JSON file to load custom IE parameters. Please note, command line parameters have higher priority then parameters from configuration file.
```
Running the application with the empty list of options yields the usage message given above and an error message.
Application supports topologies with one or more inputs. If a topology is not data-sensitive, you can skip the input parameter. In this case, inputs are filled with random values.
If a model has only image input(s), please provide a folder with images or a path to an image as input.
If a model has some specific input(s) (not images), please prepare a binary file(s) that is filled with data of appropriate precision and provide a path to them as input.
If a model has mixed input types, input folder should contain all required files. Image inputs are filled with image files one by one. Binary inputs are filled with binary inputs one by one.
To run the tool, you can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader).
> **NOTE**: Before running the tool with a trained model, make sure the model is converted to the Inference Engine format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md).
>
> The sample accepts models in ONNX format (.onnx) that do not require preprocessing.
## Examples of Running the Tool
This section provides step-by-step instructions on how to run the Benchmark Tool with the `googlenet-v1` public model on CPU or GPU devices. As an input, the `car.png` file from the `<INSTALL_DIR>/samples/scripts/` directory is used.
> **NOTE**: The Internet access is required to execute the following steps successfully. If you have access to the Internet through the proxy server only, please make sure that it is configured in your OS environment.
1. Download the model. Install openvino-dev package into Python virtual environment from PyPi and run omz_downloader:
```sh
omz_downloader --name googlenet-v1 -o <models_dir>
2. Convert the model to the Inference Engine IR format. Run the Model Optimizer using the `mo` command with the path to the model, model format (which must be FP32 for CPU and FPG) and output directory to generate the IR files:
```sh
mo --input_model <models_dir>/public/googlenet-v1/googlenet-v1.caffemodel
```
3. Run the tool with specifying the `<INSTALL_DIR>/samples/scripts/car.png` file as an input image, the IR of the `googlenet-v1` model and a device to perform inference on. The following commands demonstrate running the Benchmark Tool in the asynchronous mode on CPU and GPU devices:
* On CPU:
```sh
./benchmark_app -m <ir_dir>/googlenet-v1.xml -i <INSTALL_DIR>/samples/scripts/car.png -d CPU -api async --progress true
```
* On GPU:
```sh
./benchmark_app -m <ir_dir>/googlenet-v1.xml -i <INSTALL_DIR>/samples/scripts/car.png -d GPU -api async --progress true
```
The application outputs the number of executed iterations, total duration of execution, latency, and throughput.
Additionally, if you set the `-report_type` parameter, the application outputs statistics report. If you set the `-pc` parameter, the application outputs performance counters. If you set `-exec_graph_path`, the application reports executable graph information serialized. All measurements including per-layer PM counters are reported in milliseconds.
Below are fragments of sample output:
```
[Step 10/11] Measuring performance (Start inference asynchronously, 4 inference requests using 4 streams for CPU, limits: 60000 ms duration)
[ INFO ] BENCHMARK IS IN INFERENCE ONLY MODE.
[ INFO ] Input blobs will be filled once before performance measurements.
[ INFO ] First inference took 26.26 ms
Progress: [................... ] 99% done
[Step 11/11] Dumping statistics report
[ INFO ] Count: 6640 iterations
[ INFO ] Duration: 60039.70 ms
[ INFO ] Latency:
[ INFO ] Median: 35.36 ms
[ INFO ] Avg: 36.12 ms
[ INFO ] Min: 18.55 ms
[ INFO ] Max: 88.96 ms
[ INFO ] Throughput: 110.59 FPS
```
## See Also
* [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md)
* [Model Downloader](@ref omz_tools_downloader)

216
tools/legacy/benchmark_app/args_helper.cpp
View File

@ -1,216 +0,0 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "args_helper.hpp"
#include <gflags/gflags.h>
#include <sys/stat.h>
#include <iostream>
#include "slog.hpp"
#ifdef _WIN32
#include "w_dirent.h"
#else
#include <dirent.h>
#endif
/**
* @brief Checks input file argument and add it to files vector
* @param files reference to vector to store file names
* @param arg file or folder name
* @return none
*/
void readInputFilesArguments(std::vector<std::string>& files, const std::string& arg) {
struct stat sb;
if (stat(arg.c_str(), &sb) != 0) {
slog::warn << "File " << arg << " cannot be opened!" << slog::endl;
return;
}
if (S_ISDIR(sb.st_mode)) {
struct CloseDir {
void operator()(DIR* d) const noexcept {
if (d) {
closedir(d);
}
}
};
using Dir = std::unique_ptr<DIR, CloseDir>;
Dir dp(opendir(arg.c_str()));
if (dp == nullptr) {
slog::warn << "Directory " << arg << " cannot be opened!" << slog::endl;
return;
}
struct dirent* ep;
while (nullptr != (ep = readdir(dp.get()))) {
std::string fileName = ep->d_name;
if (fileName == "." || fileName == "..")
continue;
files.push_back(arg + "/" + ep->d_name);
}
} else {
files.push_back(arg);
}
}
/**
* @brief This function find -i key in input args. It's necessary to process multiple values for
* single key
* @param files reference to vector
* @return none.
*/
void parseInputFilesArguments(std::vector<std::string>& files) {
std::vector<std::string> args = gflags::GetArgvs();
auto args_it = begin(args);
const auto is_image_arg = [](const std::string& s) {
return s == "-i" || s == "--images";
};
const auto is_arg = [](const std::string& s) {
return s.front() == '-';
};
while (args_it != args.end()) {
const auto img_start = std::find_if(args_it, end(args), is_image_arg);
if (img_start == end(args)) {
break;
}
const auto img_begin = std::next(img_start);
const auto img_end = std::find_if(img_begin, end(args), is_arg);
for (auto img = img_begin; img != img_end; ++img) {
readInputFilesArguments(files, *img);
}
args_it = img_end;
}
if (files.empty()) {
return;
}
size_t max_files = 20;
if (files.size() < max_files) {
slog::info << "Files were added: " << files.size() << slog::endl;
for (const auto& filePath : files) {
slog::info << " " << filePath << slog::endl;
}
} else {
slog::info << "Files were added: " << files.size() << ". Too many to display each of them." << slog::endl;
}
}
namespace {
std::vector<std::string> splitStringList(const std::string& str, char delim) {
if (str.empty())
return {};
std::istringstream istr(str);
std::vector<std::string> result;
std::string elem;
while (std::getline(istr, elem, delim)) {
if (elem.empty()) {
continue;
}
result.emplace_back(std::move(elem));
}
return result;
}
std::map<std::string, std::string> parseArgMap(std::string argMap) {
argMap.erase(std::remove_if(argMap.begin(), argMap.end(), ::isspace), argMap.end());
const auto pairs = splitStringList(argMap, ',');
std::map<std::string, std::string> parsedMap;
for (auto&& pair : pairs) {
const auto keyValue = splitStringList(pair, ':');
if (keyValue.size() != 2) {
throw std::invalid_argument("Invalid key/value pair " + pair + ". Expected <layer_name>:<value>");
}
parsedMap[keyValue[0]] = keyValue[1];
}
return parsedMap;
}
using supported_precisions_t = std::unordered_map<std::string, InferenceEngine::Precision>;
InferenceEngine::Precision getPrecision(std::string value, const supported_precisions_t& supported_precisions) {
std::transform(value.begin(), value.end(), value.begin(), ::toupper);
const auto precision = supported_precisions.find(value);
if (precision == supported_precisions.end()) {
throw std::logic_error("\"" + value + "\"" + " is not a valid precision");
}
return precision->second;
}
InferenceEngine::Precision getPrecision(const std::string& value) {
static const supported_precisions_t supported_precisions = {
{"FP32", InferenceEngine::Precision::FP32}, {"FP16", InferenceEngine::Precision::FP16}, {"BF16", InferenceEngine::Precision::BF16},
{"U64", InferenceEngine::Precision::U64}, {"I64", InferenceEngine::Precision::I64}, {"U32", InferenceEngine::Precision::U32},
{"I32", InferenceEngine::Precision::I32}, {"U16", InferenceEngine::Precision::U16}, {"I16", InferenceEngine::Precision::I16},
{"U8", InferenceEngine::Precision::U8}, {"I8", InferenceEngine::Precision::I8}, {"BOOL", InferenceEngine::Precision::BOOL},
};
return getPrecision(value, supported_precisions);
}
void setPrecisions(const InferenceEngine::CNNNetwork& network, const std::string& iop) {
const auto user_precisions_map = parseArgMap(iop);
auto inputs = network.getInputsInfo();
auto outputs = network.getOutputsInfo();
for (auto&& item : user_precisions_map) {
const auto& layer_name = item.first;
const auto& user_precision = item.second;
const auto input = inputs.find(layer_name);
const auto output = outputs.find(layer_name);
if (input != inputs.end()) {
input->second->setPrecision(getPrecision(user_precision));
} else if (output != outputs.end()) {
output->second->setPrecision(getPrecision(user_precision));
} else {
throw std::logic_error(layer_name + " is not an input neither output");
}
}
}
} // namespace
void processPrecision(InferenceEngine::CNNNetwork& network, const std::string& ip, const std::string& op, const std::string& iop) {
if (!ip.empty()) {
const auto user_precision = getPrecision(ip);
for (auto&& layer : network.getInputsInfo()) {
layer.second->setPrecision(user_precision);
}
}
if (!op.empty()) {
auto user_precision = getPrecision(op);
for (auto&& layer : network.getOutputsInfo()) {
layer.second->setPrecision(user_precision);
}
}
if (!iop.empty()) {
setPrecisions(network, iop);
}
}
void printInputAndOutputsInfo(const InferenceEngine::CNNNetwork& network) {
std::cout << "Network inputs:" << std::endl;
for (auto&& layer : network.getInputsInfo()) {
std::cout << " " << layer.first << " : " << layer.second->getPrecision() << " / " << layer.second->getLayout() << std::endl;
}
std::cout << "Network outputs:" << std::endl;
for (auto&& layer : network.getOutputsInfo()) {
std::cout << " " << layer.first << " : " << layer.second->getPrecision() << " / " << layer.second->getLayout() << std::endl;
}
}

32
tools/legacy/benchmark_app/args_helper.hpp
View File

@ -1,32 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
/**
* @brief a header file with common samples functionality
* @file args_helper.hpp
*/
#pragma once
#include <inference_engine.hpp>
#include <string>
#include <vector>
/**
* @brief This function checks input args and existence of specified files in a given folder
* @param arg path to a file to be checked for existence
* @return files updated vector of verified input files
*/
void readInputFilesArguments(std::vector<std::string>& files, const std::string& arg);
/**
* @brief This function find -i/--images key in input args
* It's necessary to process multiple values for single key
* @return files updated vector of verified input files
*/
void parseInputFilesArguments(std::vector<std::string>& files);
void processPrecision(InferenceEngine::CNNNetwork& network, const std::string& ip, const std::string& op, const std::string& iop);
void printInputAndOutputsInfo(const InferenceEngine::CNNNetwork& network);

299
tools/legacy/benchmark_app/benchmark_app.hpp
View File

@ -1,299 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <gflags/gflags.h>
#include <iostream>
#include <string>
#include <vector>
/// @brief message for help argument
static const char help_message[] = "Print a usage message";
/// @brief message for images argument
static const char input_message[] = "Optional. Path to a folder with images and/or binaries or to specific image or binary file.";
/// @brief message for model argument
static const char model_message[] = "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to a .blob files with "
"a trained compiled model.";
/// @brief message for execution mode
static const char api_message[] = "Optional. Enable Sync/Async API. Default value is \"async\".";
/// @brief message for assigning cnn calculation to device
static const char target_device_message[] = "Optional. Specify a target device to infer on (the list of available devices is shown below). "
"Default value is CPU. Use \"-d HETERO:<comma-separated_devices_list>\" format to specify "
"HETERO plugin. "
"Use \"-d MULTI:<comma-separated_devices_list>\" format to specify MULTI plugin. "
"The application looks for a suitable plugin for the specified device.";
/// @brief message for iterations count
static const char iterations_count_message[] = "Optional. Number of iterations. "
"If not specified, the number of iterations is calculated depending on a device.";
/// @brief message for requests count
static const char infer_requests_count_message[] = "Optional. Number of infer requests. Default value is determined automatically for device.";
/// @brief message for execution time
static const char execution_time_message[] = "Optional. Time in seconds to execute topology.";
/// @brief message for #threads for CPU inference
static const char infer_num_threads_message[] = "Optional. Number of threads to use for inference on the CPU "
"(including HETERO and MULTI cases).";
/// @brief message for #streams for CPU inference
static const char infer_num_streams_message[] = "Optional. Number of streams to use for inference on the CPU or GPU devices "
"(for HETERO and MULTI device cases use format <dev1>:<nstreams1>,<dev2>:<nstreams2> or just "
"<nstreams>). "
"Default value is determined automatically for a device.Please note that although the "
"automatic selection "
"usually provides a reasonable performance, it still may be non - optimal for some cases, "
"especially for "
"very small networks. See sample's README for more details. "
"Also, using nstreams>1 is inherently throughput-oriented option, "
"while for the best-latency estimations the number of streams should be set to 1.";
/// @brief message for enforcing of BF16 execution where it is possible
static const char enforce_bf16_message[] = "Optional. By default floating point operations execution in bfloat16 precision are enforced "
"if supported by platform.\n"
" 'true' - enable bfloat16 regardless of platform support\n"
" 'false' - disable bfloat16 regardless of platform support";
/// @brief message for user library argument
static const char custom_cpu_library_message[] = "Required for CPU custom layers. Absolute path to a shared library with the kernels "
"implementations.";
/// @brief message for clDNN custom kernels desc
static const char custom_cldnn_message[] = "Required for GPU custom kernels. Absolute path to an .xml file with the kernels description.";
static const char batch_size_message[] = "Optional. Batch size value. If not specified, the batch size value is determined from "
"Intermediate Representation.";
// @brief message for CPU threads pinning option
static const char infer_threads_pinning_message[] =
"Optional. Explicit inference threads binding options (leave empty to let the OpenVINO to make a choice):\n"
"\t\t\t\tenabling threads->cores pinning(\"YES\", which is already default for any conventional CPU), \n"
"\t\t\t\tletting the runtime to decide on the threads->different core types(\"HYBRID_AWARE\", which is default on the hybrid CPUs) \n"
"\t\t\t\tthreads->(NUMA)nodes(\"NUMA\") or \n"
"\t\t\t\tcompletely disable(\"NO\") CPU inference threads pinning";
// @brief message for stream_output option
static const char stream_output_message[] = "Optional. Print progress as a plain text. When specified, an interactive progress bar is "
"replaced with a "
"multiline output.";
// @brief message for report_type option
static const char report_type_message[] = "Optional. Enable collecting statistics report. \"no_counters\" report contains "
"configuration options specified, resulting FPS and latency. \"average_counters\" "
"report extends \"no_counters\" report and additionally includes average PM "
"counters values for each layer from the network. \"detailed_counters\" report "
"extends \"average_counters\" report and additionally includes per-layer PM "
"counters and latency for each executed infer request.";
// @brief message for report_folder option
static const char report_folder_message[] = "Optional. Path to a folder where statistics report is stored.";
// @brief message for exec_graph_path option
static const char exec_graph_path_message[] = "Optional. Path to a file where to store executable graph information serialized.";
// @brief message for progress bar option
static const char progress_message[] = "Optional. Show progress bar (can affect performance measurement). Default values is "
"\"false\".";
// @brief message for performance counters option
static const char pc_message[] = "Optional. Report performance counters.";
#ifdef USE_OPENCV
// @brief message for load config option
static const char load_config_message[] = "Optional. Path to XML/YAML/JSON file to load custom IE parameters."
" Please note, command line parameters have higher priority then parameters from configuration "
"file.";
// @brief message for dump config option
static const char dump_config_message[] = "Optional. Path to XML/YAML/JSON file to dump IE parameters, which were set by application.";
#endif
static const char shape_message[] = "Optional. Set shape for input. For example, \"input1[1,3,224,224],input2[1,4]\" or "
"\"[1,3,224,224]\""
" in case of one input size.";
static const char layout_message[] = "Optional. Prompts how network layouts should be treated by application. "
"For example, \"input1[NCHW],input2[NC]\" or \"[NCHW]\" in case of one input size.";
// @brief message for enabling caching
static const char cache_dir_message[] = "Optional. Enables caching of loaded models to specified directory. "
"List of devices which support caching is shown at the end of this message.";
// @brief message for single load network
static const char load_from_file_message[] = "Optional. Loads model from file directly without ReadNetwork."
"All CNNNetwork options (like re-shape) will be ignored";
// @brief message for quantization bits
static const char gna_qb_message[] = "Optional. Weight bits for quantization: 8 or 16 (default)";
static constexpr char inputs_precision_message[] = "Optional. Specifies precision for all input layers of the network.";
static constexpr char outputs_precision_message[] = "Optional. Specifies precision for all output layers of the network.";
static constexpr char iop_message[] = "Optional. Specifies precision for input and output layers by name.\n"
" Example: -iop \"input:FP16, output:FP16\".\n"
" Notice that quotes are required.\n"
" Overwrites precision from ip and op options for "
"specified layers.";
/// @brief Define flag for showing help message <br>
DEFINE_bool(h, false, help_message);
/// @brief Declare flag for showing help message <br>
DECLARE_bool(help);
/// @brief Define parameter for set image file <br>
/// i or mif is a required parameter
DEFINE_string(i, "", input_message);
/// @brief Define parameter for set model file <br>
/// It is a required parameter
DEFINE_string(m, "", model_message);
/// @brief Define execution mode
DEFINE_string(api, "async", api_message);
/// @brief device the target device to infer on <br>
DEFINE_string(d, "CPU", target_device_message);
/// @brief Absolute path to CPU library with user layers <br>
/// It is a required parameter
DEFINE_string(l, "", custom_cpu_library_message);
/// @brief Define parameter for clDNN custom kernels path <br>
/// Default is ./lib
DEFINE_string(c, "", custom_cldnn_message);
/// @brief Iterations count (default 0)
/// Sync mode: iterations count
/// Async mode: StartAsync counts
DEFINE_uint64(niter, 0, iterations_count_message);
/// @brief Time to execute topology in seconds
DEFINE_uint64(t, 0, execution_time_message);
/// @brief Number of infer requests in parallel
DEFINE_uint64(nireq, 0, infer_requests_count_message);
/// @brief Number of threads to use for inference on the CPU in throughput mode (also affects Hetero
/// cases)
DEFINE_uint64(nthreads, 0, infer_num_threads_message);
/// @brief Number of streams to use for inference on the CPU (also affects Hetero cases)
DEFINE_string(nstreams, "", infer_num_streams_message);
/// @brief Enforces bf16 execution with bfloat16 precision on systems having this capability
DEFINE_bool(enforcebf16, false, enforce_bf16_message);
/// @brief Define parameter for batch size <br>
/// Default is 0 (that means don't specify)
DEFINE_uint64(b, 0, batch_size_message);
// @brief Enable plugin messages
DEFINE_string(pin, "", infer_threads_pinning_message);
/// @brief Enables multiline text output instead of progress bar
DEFINE_bool(stream_output, false, stream_output_message);
/// @brief Enables statistics report collecting
DEFINE_string(report_type, "", report_type_message);
/// @brief Path to a folder where statistics report is stored
DEFINE_string(report_folder, "", report_folder_message);
/// @brief Path to a file where to store executable graph information serialized
DEFINE_string(exec_graph_path, "", exec_graph_path_message);
/// @brief Define flag for showing progress bar <br>
DEFINE_bool(progress, false, progress_message);
/// @brief Define flag for showing performance counters <br>
DEFINE_bool(pc, false, pc_message);
#ifdef USE_OPENCV
/// @brief Define flag for loading configuration file <br>
DEFINE_string(load_config, "", load_config_message);
/// @brief Define flag for dumping configuration file <br>
DEFINE_string(dump_config, "", dump_config_message);
#endif
/// @brief Define flag for input shape <br>
DEFINE_string(shape, "", shape_message);
/// @brief Define flag for layout shape <br>
DEFINE_string(layout, "", layout_message);
/// @brief Define flag for quantization bits (default 16)
DEFINE_int32(qb, 16, gna_qb_message);
/// @brief Specify precision for all input layers of the network
DEFINE_string(ip, "", inputs_precision_message);
/// @brief Specify precision for all ouput layers of the network
DEFINE_string(op, "", outputs_precision_message);
/// @brief Specify precision for input and output layers by name.\n"
/// Example: -iop \"input:FP16, output:FP16\".\n"
/// Notice that quotes are required.\n"
/// Overwrites layout from ip and op options for specified layers.";
DEFINE_string(iop, "", iop_message);
/// @brief Define parameter for cache model dir <br>
DEFINE_string(cache_dir, "", cache_dir_message);
/// @brief Define flag for load network from model file by name without ReadNetwork <br>
DEFINE_bool(load_from_file, false, load_from_file_message);
/**
* @brief This function show a help message
*/
static void showUsage() {
std::cout << std::endl;
std::cout << "benchmark_app [OPTION]" << std::endl;
std::cout << "Options:" << std::endl;
std::cout << std::endl;
std::cout << " -h, --help " << help_message << std::endl;
std::cout << " -m \"<path>\" " << model_message << std::endl;
std::cout << " -i \"<path>\" " << input_message << std::endl;
std::cout << " -d \"<device>\" " << target_device_message << std::endl;
std::cout << " -l \"<absolute_path>\" " << custom_cpu_library_message << std::endl;
std::cout << " Or" << std::endl;
std::cout << " -c \"<absolute_path>\" " << custom_cldnn_message << std::endl;
std::cout << " -api \"<sync/async>\" " << api_message << std::endl;
std::cout << " -niter \"<integer>\" " << iterations_count_message << std::endl;
std::cout << " -nireq \"<integer>\" " << infer_requests_count_message << std::endl;
std::cout << " -b \"<integer>\" " << batch_size_message << std::endl;
std::cout << " -stream_output " << stream_output_message << std::endl;
std::cout << " -t " << execution_time_message << std::endl;
std::cout << " -progress " << progress_message << std::endl;
std::cout << " -shape " << shape_message << std::endl;
std::cout << " -layout " << layout_message << std::endl;
std::cout << " -cache_dir \"<path>\" " << cache_dir_message << std::endl;
std::cout << " -load_from_file " << load_from_file_message << std::endl;
std::cout << std::endl << " device-specific performance options:" << std::endl;
std::cout << " -nstreams \"<integer>\" " << infer_num_streams_message << std::endl;
std::cout << " -nthreads \"<integer>\" " << infer_num_threads_message << std::endl;
std::cout << " -enforcebf16=<true/false> " << enforce_bf16_message << std::endl;
std::cout << " -pin \"YES\"/\"HYBRID_AWARE\"/\"NO\"/\"NUMA\" " << infer_threads_pinning_message << std::endl;
std::cout << std::endl << " Statistics dumping options:" << std::endl;
std::cout << " -report_type \"<type>\" " << report_type_message << std::endl;
std::cout << " -report_folder " << report_folder_message << std::endl;
std::cout << " -exec_graph_path " << exec_graph_path_message << std::endl;
std::cout << " -pc " << pc_message << std::endl;
#ifdef USE_OPENCV
std::cout << " -dump_config " << dump_config_message << std::endl;
std::cout << " -load_config " << load_config_message << std::endl;
#endif
std::cout << " -qb " << gna_qb_message << std::endl;
std::cout << " -ip <value> " << inputs_precision_message << std::endl;
std::cout << " -op <value> " << outputs_precision_message << std::endl;
std::cout << " -iop \"<value>\" " << iop_message << std::endl;
}

168
tools/legacy/benchmark_app/common.hpp
View File

@ -1,168 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
/**
* @brief a header file with common samples functionality
* @file common.hpp
*/
#pragma once
#include <algorithm>
#include <fstream>
#include <functional>
#include <inference_engine.hpp>
#include <iomanip>
#include <iostream>
#include <limits>
#include <list>
#include <map>
#include <random>
#include <string>
#include <utility>
#include <vector>
#ifndef UNUSED
#if defined(_MSC_VER) && !defined(__clang__)
#define UNUSED
#else
#define UNUSED __attribute__((unused))
#endif
#endif
\
/**
* @brief Get extension from filename
* @param filename - name of the file which extension should be extracted
* @return string with extracted file extension
*/
inline std::string fileExt(const std::string& filename) {
auto pos = filename.rfind('.');
if (pos == std::string::npos)
return "";
return filename.substr(pos + 1);
}
inline std::ostream& operator<<(std::ostream& os, const InferenceEngine::Version& version) {
os << "\t" << version.description << " version ......... ";
os << IE_VERSION_MAJOR << "." << IE_VERSION_MINOR << "." << IE_VERSION_PATCH;
os << "\n\tBuild ........... ";
os << version.buildNumber;
return os;
}
inline std::ostream& operator<<(std::ostream& os, const InferenceEngine::Version* version) {
if (nullptr != version) {
os << std::endl << *version;
}
return os;
}
inline std::ostream& operator<<(std::ostream& os, const std::map<std::string, InferenceEngine::Version>& versions) {
for (auto&& version : versions) {
os << "\t" << version.first << std::endl;
os << version.second << std::endl;
}
return os;
}
static std::vector<std::pair<std::string, InferenceEngine::InferenceEngineProfileInfo>> perfCountersSorted(
std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> perfMap) {
using perfItem = std::pair<std::string, InferenceEngine::InferenceEngineProfileInfo>;
std::vector<perfItem> sorted;
for (auto& kvp : perfMap)
sorted.push_back(kvp);
std::stable_sort(sorted.begin(), sorted.end(), [](const perfItem& l, const perfItem& r) {
return l.second.execution_index < r.second.execution_index;
});
return sorted;
}
static UNUSED void printPerformanceCounts(const std::map<std::string, InferenceEngine::InferenceEngineProfileInfo>& performanceMap, std::ostream& stream,
std::string deviceName, bool bshowHeader = true) {
std::ios::fmtflags fmt(stream.flags());
long long totalTime = 0;
// Print performance counts
if (bshowHeader) {
stream << std::endl << "performance counts:" << std::endl << std::endl;
}
auto performanceMapSorted = perfCountersSorted(performanceMap);
for (const auto& it : performanceMapSorted) {
std::string toPrint(it.first);
const int maxLayerName = 30;
if (it.first.length() >= maxLayerName) {
toPrint = it.first.substr(0, maxLayerName - 4);
toPrint += "...";
}
stream << std::setw(maxLayerName) << std::left << toPrint;
switch (it.second.status) {
case InferenceEngine::InferenceEngineProfileInfo::EXECUTED:
stream << std::setw(15) << std::left << "EXECUTED";
break;
case InferenceEngine::InferenceEngineProfileInfo::NOT_RUN:
stream << std::setw(15) << std::left << "NOT_RUN";
break;
case InferenceEngine::InferenceEngineProfileInfo::OPTIMIZED_OUT:
stream << std::setw(15) << std::left << "OPTIMIZED_OUT";
break;
}
stream << std::setw(30) << std::left << "layerType: " + std::string(it.second.layer_type) + " ";
stream << std::setw(20) << std::left << "realTime: " + std::to_string(it.second.realTime_uSec);
stream << std::setw(20) << std::left << "cpu: " + std::to_string(it.second.cpu_uSec);
stream << " execType: " << it.second.exec_type << std::endl;
if (it.second.realTime_uSec > 0) {
totalTime += it.second.realTime_uSec;
}
}
stream << std::setw(20) << std::left << "Total time: " + std::to_string(totalTime) << " microseconds" << std::endl;
std::cout << std::endl;
std::cout << "Full device name: " << deviceName << std::endl;
std::cout << std::endl;
stream.flags(fmt);
}
static UNUSED void printPerformanceCounts(InferenceEngine::InferRequest request, std::ostream& stream, std::string deviceName, bool bshowHeader = true) {
auto performanceMap = request.GetPerformanceCounts();
printPerformanceCounts(performanceMap, stream, deviceName, bshowHeader);
}
inline std::string getFullDeviceName(std::map<std::string, std::string>& devicesMap, std::string device) {
std::map<std::string, std::string>::iterator it = devicesMap.find(device);
if (it != devicesMap.end()) {
return it->second;
} else {
return "";
}
}
inline std::string getFullDeviceName(InferenceEngine::Core& ie, std::string device) {
InferenceEngine::Parameter p;
try {
p = ie.GetMetric(device, METRIC_KEY(FULL_DEVICE_NAME));
return p.as<std::string>();
} catch (InferenceEngine::Exception&) {
return "";
}
}
inline void showAvailableDevices() {
InferenceEngine::Core ie;
std::vector<std::string> devices = ie.GetAvailableDevices();
std::cout << std::endl;
std::cout << "Available target devices:";
for (const auto& device : devices) {
std::cout << " " << device;
}
std::cout << std::endl;
}

107
tools/legacy/benchmark_app/console_progress.hpp
View File

@ -1,107 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <cstdio>
#include <iomanip>
#include <sstream>
/**
* @class ConsoleProgress
* @brief A ConsoleProgress class provides functionality for printing progress dynamics
*/
class ConsoleProgress {
static const size_t DEFAULT_DETALIZATION = 20;
static const size_t DEFAULT_PERCENT_TO_UPDATE_PROGRESS = 1;
size_t total;
size_t cur_progress = 0;
size_t prev_progress = 0;
bool stream_output;
size_t detalization;
size_t percent_to_update;
public:
/**
* @brief A constructor of ConsoleProgress class
* @param _total - maximum value that is correspondent to 100%
* @param _detalization - number of symbols(.) to use to represent progress
*/
explicit ConsoleProgress(size_t _total,
bool _stream_output = false,
size_t _percent_to_update = DEFAULT_PERCENT_TO_UPDATE_PROGRESS,
size_t _detalization = DEFAULT_DETALIZATION)
: total(_total),
detalization(_detalization),
percent_to_update(_percent_to_update) {
stream_output = _stream_output;
if (total == 0) {
total = 1;
}
}
/**
* @brief Shows progress with current data. Progress is shown from the beginning of the current
* line.
*/
void showProgress() const {
std::stringstream strm;
if (!stream_output) {
strm << '\r';
}
strm << "Progress: [";
size_t i = 0;
for (; i < detalization * cur_progress / total; i++) {
strm << ".";
}
for (; i < detalization; i++) {
strm << " ";
}
strm << "] " << std::setw(3) << 100 * cur_progress / total << "% done";
if (stream_output) {
strm << std::endl;
}
std::fputs(strm.str().c_str(), stdout);
std::fflush(stdout);
}
/**
* @brief Updates current value and progressbar
*/
void updateProgress() {
if (cur_progress > total)
cur_progress = total;
size_t prev_percent = 100 * prev_progress / total;
size_t cur_percent = 100 * cur_progress / total;
if (prev_progress == 0 || cur_progress == total || prev_percent + percent_to_update <= cur_percent) {
showProgress();
prev_progress = cur_progress;
}
}
/**
* @brief Adds value to currently represented and redraw progressbar
* @param add - value to add
*/
void addProgress(int add) {
if (add < 0 && -add > static_cast<int>(cur_progress)) {
add = -static_cast<int>(cur_progress);
}
cur_progress += add;
updateProgress();
}
/**
* @brief Output end line.
* @return
*/
void finish() {
std::stringstream strm;
strm << std::endl;
std::fputs(strm.str().c_str(), stdout);
std::fflush(stdout);
}
};

98
tools/legacy/benchmark_app/csv_dumper.hpp
View File

@ -1,98 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <time.h>
#include <fstream>
#include <iostream>
#include "slog.hpp"
#include <sstream>
#include <string>
/**
* @class CsvDumper
* @brief A CsvDumper class provides functionality for dumping the values in CSV files
*/
class CsvDumper {
std::ofstream file;
std::string filename;
bool canDump = true;
char delimiter = ';';
std::string generateFilename() {
std::stringstream filename;
filename << "dumpfile-";
filename << time(nullptr);
filename << ".csv";
return filename.str();
}
public:
/**
* @brief A constructor. Disables dumping in case dump file cannot be created
* @param enabled - True if dumping is enabled by default.
* @param name - name of file to dump to. File won't be created if first parameter is false.
*/
explicit CsvDumper(bool enabled = true, const std::string& name = ""): canDump(enabled) {
if (!canDump) {
return;
}
filename = (name == "" ? generateFilename() : name);
file.open(filename, std::ios::out);
if (!file) {
slog::warn << "Cannot create dump file! Disabling dump." << slog::endl;
canDump = false;
}
}
/**
* @brief Sets a delimiter to use in csv file
* @param c - Delimiter char
* @return
*/
void setDelimiter(char c) {
delimiter = c;
}
/**
* @brief Overloads operator to organize streaming values to file. Does nothing if dumping is
* disabled Adds delimiter at the end of value provided
* @param add - value to add to dump
* @return reference to same object
*/
template <class T>
CsvDumper& operator<<(const T& add) {
if (canDump) {
file << add << delimiter;
}
return *this;
}
/**
* @brief Finishes line in dump file. Does nothing if dumping is disabled
*/
void endLine() {
if (canDump) {
file << "\n";
}
}
/**
* @brief Gets information if dump is enabled.
* @return true if dump is enabled and file was successfully created
*/
bool dumpEnabled() {
return canDump;
}
/**
* @brief Gets name of a dump file
* @return name of a dump file
*/
std::string getFilename() const {
return filename;
}
};

146
tools/legacy/benchmark_app/infer_request_wrap.hpp
View File

@ -1,146 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <algorithm>
#include <chrono>
#include <condition_variable>
#include <functional>
#include <inference_engine.hpp>
#include <map>
#include <memory>
#include <mutex>
#include <queue>
#include <string>
#include <vector>
#include "statistics_report.hpp"
typedef std::chrono::high_resolution_clock Time;
typedef std::chrono::nanoseconds ns;
typedef std::function<void(size_t id, const double latency)> QueueCallbackFunction;
/// @brief Wrapper class for InferenceEngine::InferRequest. Handles asynchronous callbacks and calculates execution time.
class InferReqWrap final {
public:
using Ptr = std::shared_ptr<InferReqWrap>;
~InferReqWrap() = default;
explicit InferReqWrap(InferenceEngine::ExecutableNetwork& net, size_t id, QueueCallbackFunction callbackQueue)
: _request(net.CreateInferRequest()), _id(id), _callbackQueue(callbackQueue) {
_request.SetCompletionCallback([&]() {
_endTime = Time::now();
_callbackQueue(_id, getExecutionTimeInMilliseconds());
});
}
void startAsync() {
_startTime = Time::now();
_request.StartAsync();
}
void wait() {
_request.Wait(InferenceEngine::InferRequest::RESULT_READY);
}
void infer() {
_startTime = Time::now();
_request.Infer();
_endTime = Time::now();
_callbackQueue(_id, getExecutionTimeInMilliseconds());
}
std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> getPerformanceCounts() {
return _request.GetPerformanceCounts();
}
InferenceEngine::Blob::Ptr getBlob(const std::string& name) {
return _request.GetBlob(name);
}
double getExecutionTimeInMilliseconds() const {
auto execTime = std::chrono::duration_cast<ns>(_endTime - _startTime);
return static_cast<double>(execTime.count()) * 0.000001;
}
private:
InferenceEngine::InferRequest _request;
Time::time_point _startTime;
Time::time_point _endTime;
size_t _id;
QueueCallbackFunction _callbackQueue;
};
class InferRequestsQueue final {
public:
InferRequestsQueue(InferenceEngine::ExecutableNetwork& net, size_t nireq) {
for (size_t id = 0; id < nireq; id++) {
requests.push_back(
std::make_shared<InferReqWrap>(net, id, std::bind(&InferRequestsQueue::putIdleRequest, this, std::placeholders::_1, std::placeholders::_2)));
_idleIds.push(id);
}
resetTimes();
}
~InferRequestsQueue() {
// Inference Request guarantee that it will wait for all asynchronous internal tasks in destructor
// So it should be released before any context that the request can use inside internal asynchronous tasks
// For example all members of InferRequestsQueue would be destroyed before `requests` vector
// So requests can try to use this members from `putIdleRequest()` that would be called from request callback
// To avoid this we should move this vector declaration after all members declaration or just clear it manually in destructor
requests.clear();
}
void resetTimes() {
_startTime = Time::time_point::max();
_endTime = Time::time_point::min();
_latencies.clear();
}
double getDurationInMilliseconds() {
return std::chrono::duration_cast<ns>(_endTime - _startTime).count() * 0.000001;
}
void putIdleRequest(size_t id, const double latency) {
std::unique_lock<std::mutex> lock(_mutex);
_latencies.push_back(latency);
_idleIds.push(id);
_endTime = std::max(Time::now(), _endTime);
_cv.notify_one();
}
InferReqWrap::Ptr getIdleRequest() {
std::unique_lock<std::mutex> lock(_mutex);
_cv.wait(lock, [this] {
return _idleIds.size() > 0;
});
auto request = requests.at(_idleIds.front());
_idleIds.pop();
_startTime = std::min(Time::now(), _startTime);
return request;
}
void waitAll() {
std::unique_lock<std::mutex> lock(_mutex);
_cv.wait(lock, [this] {
return _idleIds.size() == requests.size();
});
}
std::vector<double> getLatencies() {
return _latencies;
}
std::vector<InferReqWrap::Ptr> requests;
private:
std::queue<size_t> _idleIds;
std::mutex _mutex;
std::condition_variable _cv;
Time::time_point _startTime;
Time::time_point _endTime;
std::vector<double> _latencies;
};

356
tools/legacy/benchmark_app/inputs_filling.cpp
View File

@ -1,356 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "inputs_filling.hpp"
#include <format_reader_ptr.h>
#include <algorithm>
#include <memory>
#include "slog.hpp"
#include <string>
#include <utility>
#include <vector>
using namespace InferenceEngine;
#ifdef USE_OPENCV
static const std::vector<std::string> supported_image_extensions = {"bmp", "dib", "jpeg", "jpg", "jpe", "jp2", "png",
"pbm", "pgm", "ppm", "sr", "ras", "tiff", "tif"};
#else
static const std::vector<std::string> supported_image_extensions = {"bmp"};
#endif
static const std::vector<std::string> supported_binary_extensions = {"bin"};
std::vector<std::string> filterFilesByExtensions(const std::vector<std::string>& filePaths, const std::vector<std::string>& extensions) {
std::vector<std::string> filtered;
auto getExtension = [](const std::string& name) {
auto extensionPosition = name.rfind('.', name.size());
return extensionPosition == std::string::npos ? "" : name.substr(extensionPosition + 1, name.size() - 1);
};
for (auto& filePath : filePaths) {
auto extension = getExtension(filePath);
std::transform(extension.begin(), extension.end(), extension.begin(), ::tolower);
if (std::find(extensions.begin(), extensions.end(), extension) != extensions.end()) {
filtered.push_back(filePath);
}
}
return filtered;
}
template <typename T>
void fillBlobImage(Blob::Ptr& inputBlob, const std::vector<std::string>& filePaths, const size_t& batchSize, const benchmark_app::InputInfo& app_info,
const size_t& requestId, const size_t& inputId, const size_t& inputSize) {
MemoryBlob::Ptr minput = as<MemoryBlob>(inputBlob);
if (!minput) {
IE_THROW() << "We expect inputBlob to be inherited from MemoryBlob in "
"fillBlobImage, "
<< "but by fact we were not able to cast inputBlob to MemoryBlob";
}
// locked memory holder should be alive all time while access to its buffer
// happens
auto minputHolder = minput->wmap();
auto inputBlobData = minputHolder.as<T*>();
/** Collect images data ptrs **/
std::vector<std::shared_ptr<uint8_t>> vreader;
vreader.reserve(batchSize);
for (size_t i = 0ULL, inputIndex = requestId * batchSize * inputSize + inputId; i < batchSize; i++, inputIndex += inputSize) {
inputIndex %= filePaths.size();
slog::info << "Prepare image " << filePaths[inputIndex] << slog::endl;
FormatReader::ReaderPtr reader(filePaths[inputIndex].c_str());
if (reader.get() == nullptr) {
slog::warn << "Image " << filePaths[inputIndex] << " cannot be read!" << slog::endl << slog::endl;
continue;
}
/** Getting image data **/
std::shared_ptr<uint8_t> imageData(reader->getData(app_info.width(), app_info.height()));
if (imageData) {
vreader.push_back(imageData);
}
}
/** Fill input tensor with images. First b channel, then g and r channels **/
const size_t numChannels = app_info.channels();
const size_t width = app_info.width();
const size_t height = app_info.height();
/** Iterate over all input images **/
for (size_t imageId = 0; imageId < vreader.size(); ++imageId) {
/** Iterate over all width **/
for (size_t w = 0; w < app_info.width(); ++w) {
/** Iterate over all height **/
for (size_t h = 0; h < app_info.height(); ++h) {
/** Iterate over all channels **/
for (size_t ch = 0; ch < numChannels; ++ch) {
/** [images stride + channels stride + pixel id ] all in
* bytes **/
size_t offset = imageId * numChannels * width * height + (((app_info.layout == "NCHW") || (app_info.layout == "CHW"))
? (ch * width * height + h * width + w)
: (h * width * numChannels + w * numChannels + ch));
inputBlobData[offset] = static_cast<T>(vreader.at(imageId).get()[h * width * numChannels + w * numChannels + ch]);
}
}
}
}
}
template <typename T>
void fillBlobBinary(Blob::Ptr& inputBlob, const std::vector<std::string>& filePaths, const size_t& batchSize, const size_t& requestId, const size_t& inputId,
const size_t& inputSize) {
MemoryBlob::Ptr minput = as<MemoryBlob>(inputBlob);
if (!minput) {
IE_THROW() << "We expect inputBlob to be inherited from MemoryBlob in "
"fillBlobBinary, "
<< "but by fact we were not able to cast inputBlob to MemoryBlob";
}
// locked memory holder should be alive all time while access to its buffer
// happens
auto minputHolder = minput->wmap();
auto inputBlobData = minputHolder.as<char*>();
for (size_t i = 0ULL, inputIndex = requestId * batchSize * inputSize + inputId; i < batchSize; i++, inputIndex += inputSize) {
inputIndex %= filePaths.size();
slog::info << "Prepare binary file " << filePaths[inputIndex] << slog::endl;
std::ifstream binaryFile(filePaths[inputIndex], std::ios_base::binary | std::ios_base::ate);
if (!binaryFile) {
IE_THROW() << "Cannot open " << filePaths[inputIndex];
}
auto fileSize = static_cast<std::size_t>(binaryFile.tellg());
binaryFile.seekg(0, std::ios_base::beg);
if (!binaryFile.good()) {
IE_THROW() << "Can not read " << filePaths[inputIndex];
}
auto inputSize = inputBlob->size() * sizeof(T) / batchSize;
if (fileSize != inputSize) {
IE_THROW() << "File " << filePaths[inputIndex] << " contains " << std::to_string(fileSize)
<< " bytes "
"but the network expects "
<< std::to_string(inputSize);
}
binaryFile.read(&inputBlobData[i * inputSize], inputSize);
}
}
template <typename T>
using uniformDistribution =
typename std::conditional<std::is_floating_point<T>::value, std::uniform_real_distribution<T>,
typename std::conditional<std::is_integral<T>::value, std::uniform_int_distribution<T>, void>::type>::type;
template <typename T, typename T2>
void fillBlobRandom(Blob::Ptr& inputBlob, T rand_min = std::numeric_limits<uint8_t>::min(), T rand_max = std::numeric_limits<uint8_t>::max()) {
MemoryBlob::Ptr minput = as<MemoryBlob>(inputBlob);
if (!minput) {
IE_THROW() << "We expect inputBlob to be inherited from MemoryBlob in "
"fillBlobRandom, "
<< "but by fact we were not able to cast inputBlob to MemoryBlob";
}
// locked memory holder should be alive all time while access to its buffer
// happens
auto minputHolder = minput->wmap();
auto inputBlobData = minputHolder.as<T*>();
std::mt19937 gen(0);
uniformDistribution<T2> distribution(rand_min, rand_max);
for (size_t i = 0; i < inputBlob->size(); i++) {
inputBlobData[i] = static_cast<T>(distribution(gen));
}
}
template <typename T>
void fillBlobImInfo(Blob::Ptr& inputBlob, const size_t& batchSize, std::pair<size_t, size_t> image_size) {
MemoryBlob::Ptr minput = as<MemoryBlob>(inputBlob);
if (!minput) {
IE_THROW() << "We expect inputBlob to be inherited from MemoryBlob in "
"fillBlobImInfo, "
<< "but by fact we were not able to cast inputBlob to MemoryBlob";
}
// locked memory holder should be alive all time while access to its buffer
// happens
auto minputHolder = minput->wmap();
auto inputBlobData = minputHolder.as<T*>();
for (size_t b = 0; b < batchSize; b++) {
size_t iminfoSize = inputBlob->size() / batchSize;
for (size_t i = 0; i < iminfoSize; i++) {
size_t index = b * iminfoSize + i;
if (0 == i)
inputBlobData[index] = static_cast<T>(image_size.first);
else if (1 == i)
inputBlobData[index] = static_cast<T>(image_size.second);
else
inputBlobData[index] = 1;
}
}
}
void fillBlobs(const std::vector<std::string>& inputFiles, const size_t& batchSize, benchmark_app::InputsInfo& app_inputs_info,
std::vector<InferReqWrap::Ptr> requests) {
std::vector<std::pair<size_t, size_t>> input_image_sizes;
for (auto& item : app_inputs_info) {
if (item.second.isImage()) {
input_image_sizes.push_back(std::make_pair(item.second.width(), item.second.height()));
}
slog::info << "Network input '" << item.first << "' precision " << item.second.precision << ", dimensions (" << item.second.layout << "): ";
for (const auto& i : item.second.shape) {
slog::info << i << " ";
}
slog::info << slog::endl;
}
size_t imageInputCount = input_image_sizes.size();
size_t binaryInputCount = app_inputs_info.size() - imageInputCount;
std::vector<std::string> binaryFiles;
std::vector<std::string> imageFiles;
if (inputFiles.empty()) {
slog::warn << "No input files were given: all inputs will be filled with "
"random values!"
<< slog::endl;
} else {
binaryFiles = filterFilesByExtensions(inputFiles, supported_binary_extensions);
std::sort(std::begin(binaryFiles), std::end(binaryFiles));
auto binaryToBeUsed = binaryInputCount * batchSize * requests.size();
if (binaryToBeUsed > 0 && binaryFiles.empty()) {
std::stringstream ss;
for (auto& ext : supported_binary_extensions) {
if (!ss.str().empty()) {
ss << ", ";
}
ss << ext;
}
slog::warn << "No supported binary inputs found! Please check your file "
"extensions: "
<< ss.str() << slog::endl;
} else if (binaryToBeUsed > binaryFiles.size()) {
slog::warn << "Some binary input files will be duplicated: " << binaryToBeUsed << " files are required but only " << binaryFiles.size()
<< " are provided" << slog::endl;
} else if (binaryToBeUsed < binaryFiles.size()) {
slog::warn << "Some binary input files will be ignored: only " << binaryToBeUsed << " are required from " << binaryFiles.size() << slog::endl;
}
imageFiles = filterFilesByExtensions(inputFiles, supported_image_extensions);
std::sort(std::begin(imageFiles), std::end(imageFiles));
auto imagesToBeUsed = imageInputCount * batchSize * requests.size();
if (imagesToBeUsed > 0 && imageFiles.empty()) {
std::stringstream ss;
for (auto& ext : supported_image_extensions) {
if (!ss.str().empty()) {
ss << ", ";
}
ss << ext;
}
slog::warn << "No supported image inputs found! Please check your file "
"extensions: "
<< ss.str() << slog::endl;
} else if (imagesToBeUsed > imageFiles.size()) {
slog::warn << "Some image input files will be duplicated: " << imagesToBeUsed << " files are required but only " << imageFiles.size()
<< " are provided" << slog::endl;
} else if (imagesToBeUsed < imageFiles.size()) {
slog::warn << "Some image input files will be ignored: only " << imagesToBeUsed << " are required from " << imageFiles.size() << slog::endl;
}
}
for (size_t requestId = 0; requestId < requests.size(); requestId++) {
slog::info << "Infer Request " << requestId << " filling" << slog::endl;
size_t imageInputId = 0;
size_t binaryInputId = 0;
for (auto& item : app_inputs_info) {
Blob::Ptr inputBlob = requests.at(requestId)->getBlob(item.first);
auto app_info = app_inputs_info.at(item.first);
auto precision = app_info.precision;
if (app_info.isImage()) {
if (!imageFiles.empty()) {
// Fill with Images
if (precision == InferenceEngine::Precision::FP32) {
fillBlobImage<float>(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount);
} else if (precision == InferenceEngine::Precision::FP16) {
fillBlobImage<short>(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount);
} else if (precision == InferenceEngine::Precision::I32) {
fillBlobImage<int32_t>(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount);
} else if (precision == InferenceEngine::Precision::I64) {
fillBlobImage<int64_t>(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount);
} else if (precision == InferenceEngine::Precision::U8) {
fillBlobImage<uint8_t>(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount);
} else {
IE_THROW() << "Input precision is not supported for " << item.first;
}
continue;
}
} else {
if (!binaryFiles.empty()) {
// Fill with binary files
if (precision == InferenceEngine::Precision::FP32) {
fillBlobBinary<float>(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount);
} else if (precision == InferenceEngine::Precision::FP16) {
fillBlobBinary<short>(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount);
} else if (precision == InferenceEngine::Precision::I32) {
fillBlobBinary<int32_t>(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount);
} else if (precision == InferenceEngine::Precision::I64) {
fillBlobBinary<int64_t>(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount);
} else if ((precision == InferenceEngine::Precision::U8) || (precision == InferenceEngine::Precision::BOOL)) {
fillBlobBinary<uint8_t>(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount);
} else {
IE_THROW() << "Input precision is not supported for " << item.first;
}
continue;
}
if (app_info.isImageInfo() && (input_image_sizes.size() == 1)) {
// Most likely it is image info: fill with image information
auto image_size = input_image_sizes.at(0);
slog::info << "Fill input '" << item.first << "' with image size " << image_size.first << "x" << image_size.second << slog::endl;
if (precision == InferenceEngine::Precision::FP32) {
fillBlobImInfo<float>(inputBlob, batchSize, image_size);
} else if (precision == InferenceEngine::Precision::FP16) {
fillBlobImInfo<short>(inputBlob, batchSize, image_size);
} else if (precision == InferenceEngine::Precision::I32) {
fillBlobImInfo<int32_t>(inputBlob, batchSize, image_size);
} else if (precision == InferenceEngine::Precision::I64) {
fillBlobImInfo<int64_t>(inputBlob, batchSize, image_size);
} else {
IE_THROW() << "Input precision is not supported for image info!";
}
continue;
}
}
// Fill random
slog::info << "Fill input '" << item.first << "' with random values (" << std::string((app_info.isImage() ? "image" : "some binary data"))
<< " is expected)" << slog::endl;
if (precision == InferenceEngine::Precision::FP32) {
fillBlobRandom<float, float>(inputBlob);
} else if (precision == InferenceEngine::Precision::FP16) {
fillBlobRandom<short, short>(inputBlob);
} else if (precision == InferenceEngine::Precision::I32) {
fillBlobRandom<int32_t, int32_t>(inputBlob);
} else if (precision == InferenceEngine::Precision::I64) {
fillBlobRandom<int64_t, int64_t>(inputBlob);
} else if (precision == InferenceEngine::Precision::U8) {
// uniform_int_distribution<uint8_t> is not allowed in the C++17
// standard and vs2017/19
fillBlobRandom<uint8_t, uint32_t>(inputBlob);
} else if (precision == InferenceEngine::Precision::I8) {
// uniform_int_distribution<int8_t> is not allowed in the C++17 standard
// and vs2017/19
fillBlobRandom<int8_t, int32_t>(inputBlob);
} else if (precision == InferenceEngine::Precision::U16) {
fillBlobRandom<uint16_t, uint16_t>(inputBlob);
} else if (precision == InferenceEngine::Precision::I16) {
fillBlobRandom<int16_t, int16_t>(inputBlob);
} else if (precision == InferenceEngine::Precision::BOOL) {
fillBlobRandom<uint8_t, uint32_t>(inputBlob, 0, 1);
} else {
IE_THROW() << "Input precision is not supported for " << item.first;
}
}
}
}

15
tools/legacy/benchmark_app/inputs_filling.hpp
View File

@ -1,15 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <inference_engine.hpp>
#include <string>
#include <vector>
#include "infer_request_wrap.hpp"
#include "utils.hpp"
void fillBlobs(const std::vector<std::string>& inputFiles, const size_t& batchSize, benchmark_app::InputsInfo& app_inputs_info,
std::vector<InferReqWrap::Ptr> requests);

700
tools/legacy/benchmark_app/main.cpp
View File

@ -1,700 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <algorithm>
#include <chrono>
#include <gna/gna_config.hpp>
#include <gpu/gpu_config.hpp>
#include <inference_engine.hpp>
#include <map>
#include <memory>
#include "args_helper.hpp"
#include "common.hpp"
#include "slog.hpp"
#include <string>
#include <utility>
#include <vector>
#include "benchmark_app.hpp"
#include "infer_request_wrap.hpp"
#include "inputs_filling.hpp"
#include "progress_bar.hpp"
#include "statistics_report.hpp"
#include "utils.hpp"
using namespace InferenceEngine;
static const size_t progressBarDefaultTotalCount = 1000;
uint64_t getDurationInMilliseconds(uint64_t duration) {
return duration * 1000LL;
}
uint64_t getDurationInNanoseconds(uint64_t duration) {
return duration * 1000000000LL;
}
bool ParseAndCheckCommandLine(int argc, char* argv[]) {
// ---------------------------Parsing and validating input
// arguments--------------------------------------
slog::info << "Parsing input parameters" << slog::endl;
gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true);
if (FLAGS_help || FLAGS_h) {
showUsage();
showAvailableDevices();
return false;
}
if (FLAGS_m.empty()) {
showUsage();
throw std::logic_error("Model is required but not set. Please set -m option.");
}
if (FLAGS_api != "async" && FLAGS_api != "sync") {
throw std::logic_error("Incorrect API. Please set -api option to `sync` or `async` value.");
}
if (!FLAGS_report_type.empty() && FLAGS_report_type != noCntReport && FLAGS_report_type != averageCntReport && FLAGS_report_type != detailedCntReport) {
std::string err = "only " + std::string(noCntReport) + "/" + std::string(averageCntReport) + "/" + std::string(detailedCntReport) +
" report types are supported (invalid -report_type option value)";
throw std::logic_error(err);
}
if ((FLAGS_report_type == averageCntReport) && ((FLAGS_d.find("MULTI") != std::string::npos))) {
throw std::logic_error("only " + std::string(detailedCntReport) + " report type is supported for MULTI device");
}
bool isNetworkCompiled = fileExt(FLAGS_m) == "blob";
bool isPrecisionSet = !(FLAGS_ip.empty() && FLAGS_op.empty() && FLAGS_iop.empty());
if (isNetworkCompiled && isPrecisionSet) {
std::string err = std::string("Cannot set precision for a compiled network. ") + std::string("Please re-compile your network with required precision "
"using compile_tool");
throw std::logic_error(err);
}
return true;
}
static void next_step(const std::string additional_info = "") {
static size_t step_id = 0;
static const std::map<size_t, std::string> step_names = {{1, "Parsing and validating input arguments"},
{2, "Loading Inference Engine"},
{3, "Setting device configuration"},
{4, "Reading network files"},
{5, "Resizing network to match image sizes and given batch"},
{6, "Configuring input of the model"},
{7, "Loading the model to the device"},
{8, "Setting optimal runtime parameters"},
{9, "Creating infer requests and filling input blobs with images"},
{10, "Measuring performance"},
{11, "Dumping statistics report"}};
step_id++;
if (step_names.count(step_id) == 0)
IE_THROW() << "Step ID " << step_id << " is out of total steps number " << step_names.size();
std::cout << "[Step " << step_id << "/" << step_names.size() << "] " << step_names.at(step_id)
<< (additional_info.empty() ? "" : " (" + additional_info + ")") << std::endl;
}
template <typename T>
T getMedianValue(const std::vector<T>& vec) {
std::vector<T> sortedVec(vec);
std::sort(sortedVec.begin(), sortedVec.end());
return (sortedVec.size() % 2 != 0) ? sortedVec[sortedVec.size() / 2ULL]
: (sortedVec[sortedVec.size() / 2ULL] + sortedVec[sortedVec.size() / 2ULL - 1ULL]) / static_cast<T>(2.0);
}
/**
* @brief The entry point of the benchmark application
*/
int main(int argc, char* argv[]) {
std::shared_ptr<StatisticsReport> statistics;
try {
ExecutableNetwork exeNetwork;
// ----------------- 1. Parsing and validating input arguments
// -------------------------------------------------
next_step();
if (!ParseAndCheckCommandLine(argc, argv)) {
return 0;
}
bool isNetworkCompiled = fileExt(FLAGS_m) == "blob";
if (isNetworkCompiled) {
slog::info << "Network is compiled" << slog::endl;
}
std::vector<gflags::CommandLineFlagInfo> flags;
StatisticsReport::Parameters command_line_arguments;
gflags::GetAllFlags(&flags);
for (auto& flag : flags) {
if (!flag.is_default) {
command_line_arguments.push_back({flag.name, flag.current_value});
}
}
if (!FLAGS_report_type.empty()) {
statistics = std::make_shared<StatisticsReport>(StatisticsReport::Config {FLAGS_report_type, FLAGS_report_folder});
statistics->addParameters(StatisticsReport::Category::COMMAND_LINE_PARAMETERS, command_line_arguments);
}
auto isFlagSetInCommandLine = [&command_line_arguments](const std::string& name) {
return (std::find_if(command_line_arguments.begin(), command_line_arguments.end(), [name](const std::pair<std::string, std::string>& p) {
return p.first == name;
}) != command_line_arguments.end());
};
std::string device_name = FLAGS_d;
// Parse devices
auto devices = parseDevices(device_name);
// Parse nstreams per device
std::map<std::string, std::string> device_nstreams = parseNStreamsValuePerDevice(devices, FLAGS_nstreams);
// Load device config file if specified
std::map<std::string, std::map<std::string, std::string>> config;
#ifdef USE_OPENCV
if (!FLAGS_load_config.empty()) {
load_config(FLAGS_load_config, config);
}
#endif
/** This vector stores paths to the processed images **/
std::vector<std::string> inputFiles;
parseInputFilesArguments(inputFiles);
// ----------------- 2. Loading the Inference Engine
// -----------------------------------------------------------
next_step();
Core ie;
if (FLAGS_d.find("CPU") != std::string::npos && !FLAGS_l.empty()) {
// CPU plugin extensions are loaded as a shared library and passed as a
// pointer to base extension
const auto extension_ptr = std::make_shared<InferenceEngine::Extension>(FLAGS_l);
ie.AddExtension(extension_ptr);
slog::info << "CPU plugin extensions are loaded " << FLAGS_l << slog::endl;
}
// Load clDNN Extensions
if ((FLAGS_d.find("GPU") != std::string::npos) && !FLAGS_c.empty()) {
// Override config if command line parameter is specified
if (!config.count("GPU"))
config["GPU"] = {};
config["GPU"][CONFIG_KEY(CONFIG_FILE)] = FLAGS_c;
}
if (config.count("GPU") && config.at("GPU").count(CONFIG_KEY(CONFIG_FILE))) {
auto ext = config.at("GPU").at(CONFIG_KEY(CONFIG_FILE));
ie.SetConfig({{CONFIG_KEY(CONFIG_FILE), ext}}, "GPU");
slog::info << "GPU extensions is loaded " << ext << slog::endl;
}
slog::info << "InferenceEngine: " << GetInferenceEngineVersion() << slog::endl;
slog::info << "Device info: " << slog::endl;
std::cout << ie.GetVersions(device_name) << std::endl;
// ----------------- 3. Setting device configuration
// -----------------------------------------------------------
next_step();
bool perf_counts = false;
// Update config per device according to command line parameters
for (auto& device : devices) {
if (!config.count(device))
config[device] = {};
std::map<std::string, std::string>& device_config = config.at(device);
// Set performance counter
if (isFlagSetInCommandLine("pc")) {
// set to user defined value
device_config[CONFIG_KEY(PERF_COUNT)] = FLAGS_pc ? CONFIG_VALUE(YES) : CONFIG_VALUE(NO);
} else if (device_config.count(CONFIG_KEY(PERF_COUNT)) && (device_config.at(CONFIG_KEY(PERF_COUNT)) == "YES")) {
slog::warn << "Performance counters for " << device << " device is turned on. To print results use -pc option." << slog::endl;
} else if (FLAGS_report_type == detailedCntReport || FLAGS_report_type == averageCntReport) {
slog::warn << "Turn on performance counters for " << device << " device since report type is " << FLAGS_report_type << "." << slog::endl;
device_config[CONFIG_KEY(PERF_COUNT)] = CONFIG_VALUE(YES);
} else if (!FLAGS_exec_graph_path.empty()) {
slog::warn << "Turn on performance counters for " << device << " device due to execution graph dumping." << slog::endl;
device_config[CONFIG_KEY(PERF_COUNT)] = CONFIG_VALUE(YES);
} else {
// set to default value
device_config[CONFIG_KEY(PERF_COUNT)] = FLAGS_pc ? CONFIG_VALUE(YES) : CONFIG_VALUE(NO);
}
perf_counts = (device_config.at(CONFIG_KEY(PERF_COUNT)) == CONFIG_VALUE(YES)) ? true : perf_counts;
auto setThroughputStreams = [&]() {
const std::string key = device + "_THROUGHPUT_STREAMS";
if (device_nstreams.count(device)) {
// set to user defined value
auto supported_config_keys = ie.GetMetric(device, METRIC_KEY(SUPPORTED_CONFIG_KEYS)).as<std::vector<std::string>>();
if (std::find(supported_config_keys.begin(), supported_config_keys.end(), key) == supported_config_keys.end()) {
throw std::logic_error("Device " + device + " doesn't support config key '" + key + "'! " +
"Please specify -nstreams for correct devices in format "
"<dev1>:<nstreams1>,<dev2>:<nstreams2>" +
" or via configuration file.");
}
device_config[key] = device_nstreams.at(device);
} else if (!device_config.count(key) && (FLAGS_api == "async")) {
slog::warn << "-nstreams default value is determined automatically for " << device
<< " device. "
"Although the automatic selection usually provides a "
"reasonable performance, "
"but it still may be non-optimal for some cases, for more "
"information look at README."
<< slog::endl;
if (std::string::npos == device.find("MYRIAD")) // MYRIAD sets the default number of
// streams implicitly (without _AUTO)
device_config[key] = std::string(device + "_THROUGHPUT_AUTO");
}
if (device_config.count(key))
device_nstreams[device] = device_config.at(key);
};
if (device == "CPU") { // CPU supports few special performance-oriented keys
// limit threading for CPU portion of inference
if (isFlagSetInCommandLine("nthreads"))
device_config[CONFIG_KEY(CPU_THREADS_NUM)] = std::to_string(FLAGS_nthreads);
if (isFlagSetInCommandLine("enforcebf16"))
device_config[CONFIG_KEY(ENFORCE_BF16)] = FLAGS_enforcebf16 ? CONFIG_VALUE(YES) : CONFIG_VALUE(NO);
if (isFlagSetInCommandLine("pin")) {
// set to user defined value
device_config[CONFIG_KEY(CPU_BIND_THREAD)] = FLAGS_pin;
} else if (!device_config.count(CONFIG_KEY(CPU_BIND_THREAD))) {
if ((device_name.find("MULTI") != std::string::npos) && (device_name.find("GPU") != std::string::npos)) {
slog::warn << "Turn off threads pinning for " << device << " device since multi-scenario with GPU device is used." << slog::endl;
device_config[CONFIG_KEY(CPU_BIND_THREAD)] = CONFIG_VALUE(NO);
}
}
// for CPU execution, more throughput-oriented execution via streams
setThroughputStreams();
} else if (device == ("GPU")) {
// for GPU execution, more throughput-oriented execution via streams
setThroughputStreams();
if ((device_name.find("MULTI") != std::string::npos) && (device_name.find("CPU") != std::string::npos)) {
slog::warn << "Turn on GPU trottling. Multi-device execution with "
"the CPU + GPU performs best with GPU trottling hint,"
<< "which releases another CPU thread (that is otherwise "
"used by the GPU driver for active polling)"
<< slog::endl;
device_config[GPU_CONFIG_KEY(PLUGIN_THROTTLE)] = "1";
}
} else if (device == "GNA") {
if (FLAGS_qb == 8)
device_config[GNA_CONFIG_KEY(PRECISION)] = "I8";
else
device_config[GNA_CONFIG_KEY(PRECISION)] = "I16";
if (isFlagSetInCommandLine("nthreads"))
device_config[GNA_CONFIG_KEY(LIB_N_THREADS)] = std::to_string(FLAGS_nthreads);
} else {
auto supported_config_keys = ie.GetMetric(device, METRIC_KEY(SUPPORTED_CONFIG_KEYS)).as<std::vector<std::string>>();
auto supported = [&](const std::string& key) {
return std::find(std::begin(supported_config_keys), std::end(supported_config_keys), key) != std::end(supported_config_keys);
};
if (supported(CONFIG_KEY(CPU_THREADS_NUM)) && isFlagSetInCommandLine("nthreads")) {
device_config[CONFIG_KEY(CPU_THREADS_NUM)] = std::to_string(FLAGS_nthreads);
}
if (supported(CONFIG_KEY(CPU_THROUGHPUT_STREAMS)) && isFlagSetInCommandLine("nstreams")) {
device_config[CONFIG_KEY(CPU_THROUGHPUT_STREAMS)] = FLAGS_nstreams;
}
if (supported(CONFIG_KEY(CPU_BIND_THREAD)) && isFlagSetInCommandLine("pin")) {
device_config[CONFIG_KEY(CPU_BIND_THREAD)] = FLAGS_pin;
}
}
}
for (auto&& item : config) {
ie.SetConfig(item.second, item.first);
}
auto double_to_string = [](const double number) {
std::stringstream ss;
ss << std::fixed << std::setprecision(2) << number;
return ss.str();
};
auto get_total_ms_time = [](Time::time_point& startTime) {
return std::chrono::duration_cast<ns>(Time::now() - startTime).count() * 0.000001;
};
size_t batchSize = FLAGS_b;
Precision precision = Precision::UNSPECIFIED;
std::string topology_name = "";
benchmark_app::InputsInfo app_inputs_info;
std::string output_name;
// Takes priority over config from file
if (!FLAGS_cache_dir.empty()) {
ie.SetConfig({{CONFIG_KEY(CACHE_DIR), FLAGS_cache_dir}});
}
if (FLAGS_load_from_file && !isNetworkCompiled) {
next_step();
slog::info << "Skipping the step for loading network from file" << slog::endl;
next_step();
slog::info << "Skipping the step for loading network from file" << slog::endl;
next_step();
slog::info << "Skipping the step for loading network from file" << slog::endl;
auto startTime = Time::now();
exeNetwork = ie.LoadNetwork(FLAGS_m, device_name);
auto duration_ms = double_to_string(get_total_ms_time(startTime));
slog::info << "Load network took " << duration_ms << " ms" << slog::endl;
if (statistics)
statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"load network time (ms)", duration_ms}});
if (batchSize == 0) {
batchSize = 1;
}
} else if (!isNetworkCompiled) {
// ----------------- 4. Reading the Intermediate Representation network
// ----------------------------------------
next_step();
slog::info << "Loading network files" << slog::endl;
auto startTime = Time::now();
CNNNetwork cnnNetwork = ie.ReadNetwork(FLAGS_m);
auto duration_ms = double_to_string(get_total_ms_time(startTime));
slog::info << "Read network took " << duration_ms << " ms" << slog::endl;
if (statistics)
statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"read network time (ms)", duration_ms}});
const InputsDataMap inputInfo(cnnNetwork.getInputsInfo());
if (inputInfo.empty()) {
throw std::logic_error("no inputs info is provided");
}
// ----------------- 5. Resizing network to match image sizes and given
// batch ----------------------------------
next_step();
batchSize = cnnNetwork.getBatchSize();
// Parse input shapes if specified
bool reshape = false;
app_inputs_info = getInputsInfo<InputInfo::Ptr>(FLAGS_shape, FLAGS_layout, FLAGS_b, inputInfo, reshape);
if (reshape) {
InferenceEngine::ICNNNetwork::InputShapes shapes = {};
for (auto& item : app_inputs_info)
shapes[item.first] = item.second.shape;
slog::info << "Reshaping network: " << getShapesString(shapes) << slog::endl;
startTime = Time::now();
cnnNetwork.reshape(shapes);
duration_ms = double_to_string(get_total_ms_time(startTime));
slog::info << "Reshape network took " << duration_ms << " ms" << slog::endl;
if (statistics)
statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"reshape network time (ms)", duration_ms}});
}
// use batch size according to provided layout and shapes
batchSize = (!FLAGS_layout.empty()) ? getBatchSize(app_inputs_info) : cnnNetwork.getBatchSize();
topology_name = cnnNetwork.getName();
slog::info << (FLAGS_b != 0 ? "Network batch size was changed to: " : "Network batch size: ") << batchSize << slog::endl;
// ----------------- 6. Configuring inputs and outputs
// ----------------------------------------------------------------------
next_step();
processPrecision(cnnNetwork, FLAGS_ip, FLAGS_op, FLAGS_iop);
for (auto& item : cnnNetwork.getInputsInfo()) {
// if precision for input set by user, then set it to app_inputs
// if it an image, set U8
if (!FLAGS_ip.empty() || FLAGS_iop.find(item.first) != std::string::npos) {
app_inputs_info.at(item.first).precision = item.second->getPrecision();
} else if (app_inputs_info.at(item.first).isImage()) {
app_inputs_info.at(item.first).precision = Precision::U8;
item.second->setPrecision(app_inputs_info.at(item.first).precision);
}
}
printInputAndOutputsInfo(cnnNetwork);
// ----------------- 7. Loading the model to the device
// --------------------------------------------------------
next_step();
startTime = Time::now();
exeNetwork = ie.LoadNetwork(cnnNetwork, device_name);
duration_ms = double_to_string(get_total_ms_time(startTime));
slog::info << "Load network took " << duration_ms << " ms" << slog::endl;
if (statistics)
statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"load network time (ms)", duration_ms}});
} else {
next_step();
slog::info << "Skipping the step for compiled network" << slog::endl;
next_step();
slog::info << "Skipping the step for compiled network" << slog::endl;
next_step();
slog::info << "Skipping the step for compiled network" << slog::endl;
// ----------------- 7. Loading the model to the device
// --------------------------------------------------------
next_step();
auto startTime = Time::now();
exeNetwork = ie.ImportNetwork(FLAGS_m, device_name, {});
auto duration_ms = double_to_string(get_total_ms_time(startTime));
slog::info << "Import network took " << duration_ms << " ms" << slog::endl;
if (statistics)
statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"import network time (ms)", duration_ms}});
app_inputs_info = getInputsInfo<InputInfo::CPtr>(FLAGS_shape, FLAGS_layout, FLAGS_b, exeNetwork.GetInputsInfo());
if (batchSize == 0) {
batchSize = 1;
}
}
// ----------------- 8. Setting optimal runtime parameters
// -----------------------------------------------------
next_step();
// Update number of streams
for (auto&& ds : device_nstreams) {
const std::string key = ds.first + "_THROUGHPUT_STREAMS";
device_nstreams[ds.first] = ie.GetConfig(ds.first, key).as<std::string>();
}
// Number of requests
uint64_t nireq = FLAGS_nireq;
if (nireq == 0) {
if (FLAGS_api == "sync") {
nireq = 1;
} else {
std::string key = METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS);
try {
nireq = exeNetwork.GetMetric(key).as<unsigned int>();
} catch (const std::exception& ex) {
IE_THROW() << "Every device used with the benchmark_app should "
<< "support OPTIMAL_NUMBER_OF_INFER_REQUESTS "
"ExecutableNetwork metric. "
<< "Failed to query the metric for the " << device_name << " with error:" << ex.what();
}
}
}
// Iteration limit
uint64_t niter = FLAGS_niter;
if ((niter > 0) && (FLAGS_api == "async")) {
niter = ((niter + nireq - 1) / nireq) * nireq;
if (FLAGS_niter != niter) {
slog::warn << "Number of iterations was aligned by request number from " << FLAGS_niter << " to " << niter << " using number of requests "
<< nireq << slog::endl;
}
}
// Time limit
uint64_t duration_seconds = 0;
if (FLAGS_t != 0) {
// time limit
duration_seconds = FLAGS_t;
} else if (FLAGS_niter == 0) {
// default time limit
duration_seconds = deviceDefaultDeviceDurationInSeconds(device_name);
}
uint64_t duration_nanoseconds = getDurationInNanoseconds(duration_seconds);
if (statistics) {
statistics->addParameters(StatisticsReport::Category::RUNTIME_CONFIG,
{
{"topology", topology_name},
{"target device", device_name},
{"API", FLAGS_api},
{"precision", std::string(precision.name())},
{"batch size", std::to_string(batchSize)},
{"number of iterations", std::to_string(niter)},
{"number of parallel infer requests", std::to_string(nireq)},
{"duration (ms)", std::to_string(getDurationInMilliseconds(duration_seconds))},
});
for (auto& nstreams : device_nstreams) {
std::stringstream ss;
ss << "number of " << nstreams.first << " streams";
statistics->addParameters(StatisticsReport::Category::RUNTIME_CONFIG, {
{ss.str(), nstreams.second},
});
}
}
// ----------------- 9. Creating infer requests and filling input blobs
// ----------------------------------------
next_step();
InferRequestsQueue inferRequestsQueue(exeNetwork, nireq);
fillBlobs(inputFiles, batchSize, app_inputs_info, inferRequestsQueue.requests);
// ----------------- 10. Measuring performance
// ------------------------------------------------------------------
size_t progressCnt = 0;
size_t progressBarTotalCount = progressBarDefaultTotalCount;
size_t iteration = 0;
std::stringstream ss;
ss << "Start inference " << FLAGS_api << "hronously";
if (FLAGS_api == "async") {
if (!ss.str().empty()) {
ss << ", ";
}
ss << nireq << " inference requests";
std::stringstream device_ss;
for (auto& nstreams : device_nstreams) {
if (!device_ss.str().empty()) {
device_ss << ", ";
}
device_ss << nstreams.second << " streams for " << nstreams.first;
}
if (!device_ss.str().empty()) {
ss << " using " << device_ss.str();
}
}
ss << ", limits: ";
if (duration_seconds > 0) {
ss << getDurationInMilliseconds(duration_seconds) << " ms duration";
}
if (niter != 0) {
if (duration_seconds == 0) {
progressBarTotalCount = niter;
}
if (duration_seconds > 0) {
ss << ", ";
}
ss << niter << " iterations";
}
next_step(ss.str());
// warming up - out of scope
auto inferRequest = inferRequestsQueue.getIdleRequest();
if (!inferRequest) {
IE_THROW() << "No idle Infer Requests!";
}
if (FLAGS_api == "sync") {
inferRequest->infer();
} else {
inferRequest->startAsync();
}
inferRequestsQueue.waitAll();
auto duration_ms = double_to_string(inferRequestsQueue.getLatencies()[0]);
slog::info << "First inference took " << duration_ms << " ms" << slog::endl;
if (statistics)
statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"first inference time (ms)", duration_ms}});
inferRequestsQueue.resetTimes();
auto startTime = Time::now();
auto execTime = std::chrono::duration_cast<ns>(Time::now() - startTime).count();
/** Start inference & calculate performance **/
/** to align number if iterations to guarantee that last infer requests are
* executed in the same conditions **/
ProgressBar progressBar(progressBarTotalCount, FLAGS_stream_output, FLAGS_progress);
while ((niter != 0LL && iteration < niter) || (duration_nanoseconds != 0LL && (uint64_t)execTime < duration_nanoseconds) ||
(FLAGS_api == "async" && iteration % nireq != 0)) {
inferRequest = inferRequestsQueue.getIdleRequest();
if (!inferRequest) {
IE_THROW() << "No idle Infer Requests!";
}
if (FLAGS_api == "sync") {
inferRequest->infer();
} else {
// As the inference request is currently idle, the wait() adds no
// additional overhead (and should return immediately). The primary
// reason for calling the method is exception checking/re-throwing.
// Callback, that governs the actual execution can handle errors as
// well, but as it uses just error codes it has no details like ‘what()’
// method of `std::exception` So, rechecking for any exceptions here.
inferRequest->wait();
inferRequest->startAsync();
}
iteration++;
execTime = std::chrono::duration_cast<ns>(Time::now() - startTime).count();
if (niter > 0) {
progressBar.addProgress(1);
} else {
// calculate how many progress intervals are covered by current
// iteration. depends on the current iteration time and time of each
// progress interval. Previously covered progress intervals must be
// skipped.
auto progressIntervalTime = duration_nanoseconds / progressBarTotalCount;
size_t newProgress = execTime / progressIntervalTime - progressCnt;
progressBar.addProgress(newProgress);
progressCnt += newProgress;
}
}
// wait the latest inference executions
inferRequestsQueue.waitAll();
double latency = getMedianValue<double>(inferRequestsQueue.getLatencies());
double totalDuration = inferRequestsQueue.getDurationInMilliseconds();
double fps = (FLAGS_api == "sync") ? batchSize * 1000.0 / latency : batchSize * 1000.0 * iteration / totalDuration;
if (statistics) {
statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {
{"total execution time (ms)", double_to_string(totalDuration)},
{"total number of iterations", std::to_string(iteration)},
});
if (device_name.find("MULTI") == std::string::npos) {
statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {
{"latency (ms)", double_to_string(latency)},
});
}
statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"throughput", double_to_string(fps)}});
}
progressBar.finish();
// ----------------- 11. Dumping statistics report
// -------------------------------------------------------------
next_step();
#ifdef USE_OPENCV
if (!FLAGS_dump_config.empty()) {
dump_config(FLAGS_dump_config, config);
slog::info << "Inference Engine configuration settings were dumped to " << FLAGS_dump_config << slog::endl;
}
#endif
if (!FLAGS_exec_graph_path.empty()) {
try {
CNNNetwork execGraphInfo = exeNetwork.GetExecGraphInfo();
execGraphInfo.serialize(FLAGS_exec_graph_path);
slog::info << "executable graph is stored to " << FLAGS_exec_graph_path << slog::endl;
} catch (const std::exception& ex) {
slog::err << "Can't get executable graph: " << ex.what() << slog::endl;
}
}
if (perf_counts) {
std::vector<std::map<std::string, InferenceEngine::InferenceEngineProfileInfo>> perfCounts;
for (size_t ireq = 0; ireq < nireq; ireq++) {
auto reqPerfCounts = inferRequestsQueue.requests[ireq]->getPerformanceCounts();
if (FLAGS_pc) {
slog::info << "Performance counts for " << ireq << "-th infer request:" << slog::endl;
printPerformanceCounts(reqPerfCounts, std::cout, getFullDeviceName(ie, FLAGS_d), false);
}
perfCounts.push_back(reqPerfCounts);
}
if (statistics) {
statistics->dumpPerformanceCounters(perfCounts);
}
}
if (statistics)
statistics->dump();
std::cout << "Count: " << iteration << " iterations" << std::endl;
std::cout << "Duration: " << double_to_string(totalDuration) << " ms" << std::endl;
if (device_name.find("MULTI") == std::string::npos)
std::cout << "Latency: " << double_to_string(latency) << " ms" << std::endl;
std::cout << "Throughput: " << double_to_string(fps) << " FPS" << std::endl;
} catch (const std::exception& ex) {
slog::err << ex.what() << slog::endl;
if (statistics) {
statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {
{"error", ex.what()},
});
statistics->dump();
}
return 3;
}
return 0;
}

51
tools/legacy/benchmark_app/progress_bar.hpp
View File

@ -1,51 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <memory>
#include "console_progress.hpp"
/// @brief Responsible for progress bar handling within the benchmark_app
class ProgressBar {
public:
explicit ProgressBar(size_t totalNum, bool streamOutput = false, bool progressEnabled = false) {
_bar.reset(new ConsoleProgress(totalNum, streamOutput));
_streamOutput = streamOutput;
_isFinished = true;
_progressEnabled = progressEnabled;
}
void addProgress(size_t num) {
_isFinished = false;
if (_progressEnabled) {
_bar->addProgress(static_cast<int>(num));
}
}
void finish(size_t num = 0) {
if (num > 0) {
addProgress(num);
}
_isFinished = true;
_bar->finish();
if (_progressEnabled) {
std::cout << std::endl;
}
}
void newBar(size_t totalNum) {
if (_isFinished) {
_bar.reset(new ConsoleProgress(totalNum, _streamOutput));
} else {
throw std::logic_error("Cannot create a new bar. Current bar is still in progress");
}
}
private:
std::unique_ptr<ConsoleProgress> _bar;
bool _streamOutput;
bool _isFinished;
bool _progressEnabled;
};

33
tools/legacy/benchmark_app/slog.cpp
View File

@ -1,33 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "slog.hpp"
#include <iostream>
namespace slog {
LogStream info("INFO", std::cout);
LogStream warn("WARNING", std::cout);
LogStream err("ERROR", std::cerr);
LogStream::LogStream(const std::string& prefix, std::ostream& log_stream): _prefix(prefix), _new_line(true) {
_log_stream = &log_stream;
}
// Specializing for LogStreamEndLine to support slog::endl
LogStream& LogStream::operator<<(const LogStreamEndLine& /*arg*/) {
_new_line = true;
(*_log_stream) << std::endl;
return *this;
}
// Specializing for LogStreamBoolAlpha to support slog::boolalpha
LogStream& LogStream::operator<<(const LogStreamBoolAlpha& /*arg*/) {
(*_log_stream) << std::boolalpha;
return *this;
}
} // namespace slog

74
tools/legacy/benchmark_app/slog.hpp
View File

@ -1,74 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
/**
* @brief a header file with logging facility for common samples
* @file log.hpp
*/
#pragma once
#include <ostream>
#include <string>
namespace slog {
/**
* @class LogStreamEndLine
* @brief The LogStreamEndLine class implements an end line marker for a log stream
*/
class LogStreamEndLine {};
static constexpr LogStreamEndLine endl;
/**
* @class LogStreamBoolAlpha
* @brief The LogStreamBoolAlpha class implements bool printing for a log stream
*/
class LogStreamBoolAlpha {};
static constexpr LogStreamBoolAlpha boolalpha;
/**
* @class LogStream
* @brief The LogStream class implements a stream for sample logging
*/
class LogStream {
std::string _prefix;
std::ostream* _log_stream;
bool _new_line;
public:
/**
* @brief A constructor. Creates an LogStream object
* @param prefix The prefix to print
*/
LogStream(const std::string& prefix, std::ostream& log_stream);
/**
* @brief A stream output operator to be used within the logger
* @param arg Object for serialization in the logger message
*/
template <class T>
LogStream& operator<<(const T& arg) {
if (_new_line) {
(*_log_stream) << "[ " << _prefix << " ] ";
_new_line = false;
}
(*_log_stream) << arg;
return *this;
}
// Specializing for LogStreamEndLine to support slog::endl
LogStream& operator<<(const LogStreamEndLine&);
// Specializing for LogStreamBoolAlpha to support slog::boolalpha
LogStream& operator<<(const LogStreamBoolAlpha&);
};
extern LogStream info;
extern LogStream warn;
extern LogStream err;
} // namespace slog

143
tools/legacy/benchmark_app/statistics_report.cpp
View File

@ -1,143 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "statistics_report.hpp"
#include <algorithm>
#include <map>
#include <string>
#include <utility>
#include <vector>
void StatisticsReport::addParameters(const Category& category, const Parameters& parameters) {
if (_parameters.count(category) == 0)
_parameters[category] = parameters;
else
_parameters[category].insert(_parameters[category].end(), parameters.begin(), parameters.end());
}
void StatisticsReport::dump() {
CsvDumper dumper(true, _config.report_folder + _separator + "benchmark_report.csv");
auto dump_parameters = [&dumper](const Parameters& parameters) {
for (auto& parameter : parameters) {
dumper << parameter.first << parameter.second;
dumper.endLine();
}
};
if (_parameters.count(Category::COMMAND_LINE_PARAMETERS)) {
dumper << "Command line parameters";
dumper.endLine();
dump_parameters(_parameters.at(Category::COMMAND_LINE_PARAMETERS));
dumper.endLine();
}
if (_parameters.count(Category::RUNTIME_CONFIG)) {
dumper << "Configuration setup";
dumper.endLine();
dump_parameters(_parameters.at(Category::RUNTIME_CONFIG));
dumper.endLine();
}
if (_parameters.count(Category::EXECUTION_RESULTS)) {
dumper << "Execution results";
dumper.endLine();
dump_parameters(_parameters.at(Category::EXECUTION_RESULTS));
dumper.endLine();
}
slog::info << "Statistics report is stored to " << dumper.getFilename() << slog::endl;
}
void StatisticsReport::dumpPerformanceCountersRequest(CsvDumper& dumper, const PerformanceCounters& perfCounts) {
auto performanceMapSorted = perfCountersSorted(perfCounts);
long long total = 0L;
long long total_cpu = 0L;
dumper << "layerName"
<< "execStatus"
<< "layerType"
<< "execType";
dumper << "realTime (ms)"
<< "cpuTime (ms)";
dumper.endLine();
for (const auto& layer : performanceMapSorted) {
dumper << layer.first; // layer name
switch (layer.second.status) {
case InferenceEngine::InferenceEngineProfileInfo::EXECUTED:
dumper << "EXECUTED";
break;
case InferenceEngine::InferenceEngineProfileInfo::NOT_RUN:
dumper << "NOT_RUN";
break;
case InferenceEngine::InferenceEngineProfileInfo::OPTIMIZED_OUT:
dumper << "OPTIMIZED_OUT";
break;
}
dumper << layer.second.layer_type << layer.second.exec_type;
dumper << std::to_string(layer.second.realTime_uSec / 1000.0) << std::to_string(layer.second.cpu_uSec / 1000.0);
total += layer.second.realTime_uSec;
total_cpu += layer.second.cpu_uSec;
dumper.endLine();
}
dumper << "Total"
<< ""
<< ""
<< "";
dumper << total / 1000.0 << total_cpu / 1000.0;
dumper.endLine();
dumper.endLine();
}
void StatisticsReport::dumpPerformanceCounters(const std::vector<PerformanceCounters>& perfCounts) {
if ((_config.report_type.empty()) || (_config.report_type == noCntReport)) {
slog::info << "Statistics collecting for performance counters was not "
"requested. No reports are dumped."
<< slog::endl;
return;
}
if (perfCounts.empty()) {
slog::info << "Performance counters are empty. No reports are dumped." << slog::endl;
return;
}
CsvDumper dumper(true, _config.report_folder + _separator + "benchmark_" + _config.report_type + "_report.csv");
if (_config.report_type == detailedCntReport) {
for (auto& pc : perfCounts) {
dumpPerformanceCountersRequest(dumper, pc);
}
} else if (_config.report_type == averageCntReport) {
auto getAveragePerformanceCounters = [&perfCounts]() {
std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> performanceCountersAvg;
// iterate over each processed infer request and handle its PM data
for (size_t i = 0; i < perfCounts.size(); i++) {
auto performanceMapSorted = perfCountersSorted(perfCounts[i]);
// iterate over each layer from sorted vector and add required PM data
// to the per-layer maps
for (const auto& pm : performanceMapSorted) {
if (performanceCountersAvg.count(pm.first) == 0) {
performanceCountersAvg[pm.first] = perfCounts.at(i).at(pm.first);
} else {
performanceCountersAvg[pm.first].realTime_uSec += perfCounts.at(i).at(pm.first).realTime_uSec;
performanceCountersAvg[pm.first].cpu_uSec += perfCounts.at(i).at(pm.first).cpu_uSec;
}
}
}
for (auto& pm : performanceCountersAvg) {
pm.second.realTime_uSec /= perfCounts.size();
pm.second.cpu_uSec /= perfCounts.size();
}
return performanceCountersAvg;
};
dumpPerformanceCountersRequest(dumper, getAveragePerformanceCounters());
} else {
throw std::logic_error("PM data can only be collected for average or detailed report types");
}
slog::info << "Performance counters report is stored to " << dumper.getFilename() << slog::endl;
}

70
tools/legacy/benchmark_app/statistics_report.hpp
View File

@ -1,70 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <inference_engine.hpp>
#include <map>
#include "common.hpp"
#include "csv_dumper.hpp"
#include "slog.hpp"
#include <string>
#include <utility>
#include <vector>
// @brief statistics reports types
static constexpr char noCntReport[] = "no_counters";
static constexpr char averageCntReport[] = "average_counters";
static constexpr char detailedCntReport[] = "detailed_counters";
/// @brief Responsible for collecting of statistics and dumping to .csv file
class StatisticsReport {
public:
typedef std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> PerformanceCounters;
typedef std::vector<std::pair<std::string, std::string>> Parameters;
struct Config {
std::string report_type;
std::string report_folder;
};
enum class Category {
COMMAND_LINE_PARAMETERS,
RUNTIME_CONFIG,
EXECUTION_RESULTS,
};
explicit StatisticsReport(Config config): _config(std::move(config)) {
_separator =
#if defined _WIN32 || defined __CYGWIN__
#if defined UNICODE
L"\\";
#else
"\\";
#endif
#else
"/";
#endif
if (_config.report_folder.empty())
_separator = "";
}
void addParameters(const Category& category, const Parameters& parameters);
void dump();
void dumpPerformanceCounters(const std::vector<PerformanceCounters>& perfCounts);
private:
void dumpPerformanceCountersRequest(CsvDumper& dumper, const PerformanceCounters& perfCounts);
// configuration of current benchmark execution
const Config _config;
// parameters
std::map<Category, Parameters> _parameters;
// csv separator
std::string _separator;
};

194
tools/legacy/benchmark_app/utils.cpp
View File

@ -1,194 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
// clang-format off
#include <algorithm>
#include <map>
#include <regex>
#include "common.hpp"
#include "slog.hpp"
#include <string>
#include <utility>
#include <vector>
#include "utils.hpp"
// clang-format on
#ifdef USE_OPENCV
#include <opencv2/core.hpp>
#endif
namespace benchmark_app {
bool InputInfo::isImage() const {
if ((layout != "NCHW") && (layout != "NHWC") && (layout != "CHW") && (layout != "HWC"))
return false;
return (channels() == 3);
}
bool InputInfo::isImageInfo() const {
if (layout != "NC")
return false;
return (channels() >= 2);
}
size_t InputInfo::getDimentionByLayout(char character) const {
size_t pos = layout.find(character);
if (pos == std::string::npos)
throw std::runtime_error("Error: Can't get " + std::string(character, 1) + " from layout " + layout);
return shape.at(pos);
}
size_t InputInfo::width() const {
return getDimentionByLayout('W');
}
size_t InputInfo::height() const {
return getDimentionByLayout('H');
}
size_t InputInfo::channels() const {
return getDimentionByLayout('C');
}
size_t InputInfo::batch() const {
return getDimentionByLayout('N');
}
size_t InputInfo::depth() const {
return getDimentionByLayout('D');
}
} // namespace benchmark_app
uint32_t deviceDefaultDeviceDurationInSeconds(const std::string& device) {
static const std::map<std::string, uint32_t> deviceDefaultDurationInSeconds {{"CPU", 60}, {"GPU", 60}, {"VPUX", 60},
{"UNKNOWN", 120}};
uint32_t duration = 0;
for (const auto& deviceDurationInSeconds : deviceDefaultDurationInSeconds) {
if (device.find(deviceDurationInSeconds.first) != std::string::npos) {
duration = std::max(duration, deviceDurationInSeconds.second);
}
}
if (duration == 0) {
const auto unknownDeviceIt =
find_if(deviceDefaultDurationInSeconds.begin(), deviceDefaultDurationInSeconds.end(), [](std::pair<std::string, uint32_t> deviceDuration) {
return deviceDuration.first == "UNKNOWN";
});
if (unknownDeviceIt == deviceDefaultDurationInSeconds.end()) {
throw std::logic_error("UNKNOWN device was not found in the device duration list");
}
duration = unknownDeviceIt->second;
slog::warn << "Default duration " << duration << " seconds for unknown device '" << device << "' is used" << slog::endl;
}
return duration;
}
std::vector<std::string> split(const std::string& s, char delim) {
std::vector<std::string> result;
std::stringstream ss(s);
std::string item;
while (getline(ss, item, delim)) {
result.push_back(item);
}
return result;
}
std::vector<std::string> parseDevices(const std::string& device_string) {
std::string comma_separated_devices = device_string;
if (comma_separated_devices.find(":") != std::string::npos) {
comma_separated_devices = comma_separated_devices.substr(comma_separated_devices.find(":") + 1);
}
if ((comma_separated_devices == "MULTI") || (comma_separated_devices == "HETERO"))
return std::vector<std::string>();
auto devices = split(comma_separated_devices, ',');
for (auto& device : devices)
device = device.substr(0, device.find_first_of(".("));
return devices;
}
std::map<std::string, std::string> parseNStreamsValuePerDevice(const std::vector<std::string>& devices, const std::string& values_string) {
// Format: <device1>:<value1>,<device2>:<value2> or just <value>
std::map<std::string, std::string> result;
auto device_value_strings = split(values_string, ',');
for (auto& device_value_string : device_value_strings) {
auto device_value_vec = split(device_value_string, ':');
if (device_value_vec.size() == 2) {
auto device_name = device_value_vec.at(0);
auto nstreams = device_value_vec.at(1);
auto it = std::find(devices.begin(), devices.end(), device_name);
if (it != devices.end()) {
result[device_name] = nstreams;
} else {
throw std::logic_error("Can't set nstreams value " + std::string(nstreams) + " for device '" + device_name + "'! Incorrect device name!");
}
} else if (device_value_vec.size() == 1) {
auto value = device_value_vec.at(0);
for (auto& device : devices) {
result[device] = value;
}
} else if (device_value_vec.size() != 0) {
throw std::runtime_error("Unknown string format: " + values_string);
}
}
return result;
}
size_t getBatchSize(const benchmark_app::InputsInfo& inputs_info) {
size_t batch_size = 0;
for (auto& info : inputs_info) {
std::size_t batch_index = info.second.layout.find("N");
if (batch_index != std::string::npos) {
if (batch_size == 0)
batch_size = info.second.shape[batch_index];
else if (batch_size != info.second.shape[batch_index])
throw std::logic_error("Can't deterimine batch size: batch is "
"different for different inputs!");
}
}
if (batch_size == 0)
batch_size = 1;
return batch_size;
}
std::string getShapesString(const InferenceEngine::ICNNNetwork::InputShapes& shapes) {
std::stringstream ss;
for (auto& shape : shapes) {
if (!ss.str().empty())
ss << ", ";
ss << "\'" << shape.first << "': [";
for (size_t i = 0; i < shape.second.size(); i++) {
if (i > 0)
ss << ", ";
ss << shape.second.at(i);
}
ss << "]";
}
return ss.str();
}
#ifdef USE_OPENCV
void dump_config(const std::string& filename, const std::map<std::string, std::map<std::string, std::string>>& config) {
cv::FileStorage fs(filename, cv::FileStorage::WRITE);
if (!fs.isOpened())
throw std::runtime_error("Error: Can't open config file : " + filename);
for (auto device_it = config.begin(); device_it != config.end(); ++device_it) {
fs << device_it->first << "{:";
for (auto param_it = device_it->second.begin(); param_it != device_it->second.end(); ++param_it)
fs << param_it->first << param_it->second;
fs << "}";
}
fs.release();
}
void load_config(const std::string& filename, std::map<std::string, std::map<std::string, std::string>>& config) {
cv::FileStorage fs(filename, cv::FileStorage::READ);
if (!fs.isOpened())
throw std::runtime_error("Error: Can't load config file : " + filename);
cv::FileNode root = fs.root();
for (auto it = root.begin(); it != root.end(); ++it) {
auto device = *it;
if (!device.isMap()) {
throw std::runtime_error("Error: Can't parse config file : " + filename);
}
for (auto iit = device.begin(); iit != device.end(); ++iit) {
auto item = *iit;
config[device.name()][item.name()] = item.string();
}
}
}
#endif

123
tools/legacy/benchmark_app/utils.hpp
View File

@ -1,123 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <map>
#include <string>
#include <vector>
#include <inference_engine.hpp>
namespace benchmark_app {
struct InputInfo {
InferenceEngine::Precision precision;
InferenceEngine::SizeVector shape;
std::string layout;
bool isImage() const;
bool isImageInfo() const;
size_t getDimentionByLayout(char character) const;
size_t width() const;
size_t height() const;
size_t channels() const;
size_t batch() const;
size_t depth() const;
};
using InputsInfo = std::map<std::string, InputInfo>;
} // namespace benchmark_app
std::vector<std::string> parseDevices(const std::string& device_string);
uint32_t deviceDefaultDeviceDurationInSeconds(const std::string& device);
std::map<std::string, std::string> parseNStreamsValuePerDevice(const std::vector<std::string>& devices, const std::string& values_string);
std::string getShapesString(const InferenceEngine::ICNNNetwork::InputShapes& shapes);
size_t getBatchSize(const benchmark_app::InputsInfo& inputs_info);
std::vector<std::string> split(const std::string& s, char delim);
template <typename T>
std::map<std::string, std::string> parseInputParameters(const std::string parameter_string, const std::map<std::string, T>& input_info) {
// Parse parameter string like "input0[value0],input1[value1]" or "[value]" (applied to all
// inputs)
std::map<std::string, std::string> return_value;
std::string search_string = parameter_string;
auto start_pos = search_string.find_first_of('[');
while (start_pos != std::string::npos) {
auto end_pos = search_string.find_first_of(']');
if (end_pos == std::string::npos)
break;
auto input_name = search_string.substr(0, start_pos);
auto input_value = search_string.substr(start_pos + 1, end_pos - start_pos - 1);
if (!input_name.empty()) {
return_value[input_name] = input_value;
} else {
for (auto& item : input_info) {
return_value[item.first] = input_value;
}
}
search_string = search_string.substr(end_pos + 1);
if (search_string.empty() || search_string.front() != ',')
break;
search_string = search_string.substr(1);
start_pos = search_string.find_first_of('[');
}
if (!search_string.empty())
throw std::logic_error("Can't parse input parameter string: " + parameter_string);
return return_value;
}
template <typename T>
benchmark_app::InputsInfo getInputsInfo(const std::string& shape_string, const std::string& layout_string, const size_t batch_size,
const std::map<std::string, T>& input_info, bool& reshape_required) {
std::map<std::string, std::string> shape_map = parseInputParameters(shape_string, input_info);
std::map<std::string, std::string> layout_map = parseInputParameters(layout_string, input_info);
reshape_required = false;
benchmark_app::InputsInfo info_map;
for (auto& item : input_info) {
benchmark_app::InputInfo info;
auto name = item.first;
auto descriptor = item.second->getTensorDesc();
// Precision
info.precision = descriptor.getPrecision();
// Shape
if (shape_map.count(name)) {
std::vector<size_t> parsed_shape;
for (auto& dim : split(shape_map.at(name), ',')) {
parsed_shape.push_back(std::stoi(dim));
}
info.shape = parsed_shape;
reshape_required = true;
} else {
info.shape = descriptor.getDims();
}
// Layout
if (layout_map.count(name)) {
info.layout = layout_map.at(name);
std::transform(info.layout.begin(), info.layout.end(), info.layout.begin(), ::toupper);
} else {
std::stringstream ss;
ss << descriptor.getLayout();
info.layout = ss.str();
}
// Update shape with batch if needed
if (batch_size != 0) {
std::size_t batch_index = info.layout.find("N");
if ((batch_index != std::string::npos) && (info.shape.at(batch_index) != batch_size)) {
info.shape[batch_index] = batch_size;
reshape_required = true;
}
}
info_map[name] = info;
}
return info_map;
}
template <typename T>
benchmark_app::InputsInfo getInputsInfo(const std::string& shape_string, const std::string& layout_string, const size_t batch_size,
const std::map<std::string, T>& input_info) {
bool reshape_required = false;
return getInputsInfo<T>(shape_string, layout_string, batch_size, input_info, reshape_required);
}
#ifdef USE_OPENCV
void dump_config(const std::string& filename, const std::map<std::string, std::map<std::string, std::string>>& config);
void load_config(const std::string& filename, std::map<std::string, std::map<std::string, std::string>>& config);
#endif

176
tools/legacy/benchmark_app/w_dirent.h
View File

@ -1,176 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#if defined(_WIN32)
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN_UNDEF
#endif
#ifndef NOMINMAX
#define NOMINMAX
#define NOMINMAX_UNDEF
#endif
#if defined(_M_IX86) && !defined(_X86_) && !defined(_AMD64_)
#define _X86_
#endif
#if defined(_M_X64) && !defined(_X86_) && !defined(_AMD64_)
#define _AMD64_
#endif
#if defined(_M_ARM) && !defined(_ARM_) && !defined(_ARM64_)
#define _ARM_
#endif
#if defined(_M_ARM64) && !defined(_ARM_) && !defined(_ARM64_)
#define _ARM64_
#endif
// clang-format off
#include <string.h>
#include <windef.h>
#include <fileapi.h>
#include <winbase.h>
#include <sys/stat.h>
// clang-format on
// Copied from linux libc sys/stat.h:
#define S_ISREG(m) (((m)&S_IFMT) == S_IFREG)
#define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR)
/// @brief structure to store directory names
struct dirent {
char* d_name;
explicit dirent(const wchar_t* wsFilePath) {
size_t i;
auto slen = wcslen(wsFilePath);
d_name = static_cast<char*>(malloc(slen + 1));
wcstombs_s(&i, d_name, slen + 1, wsFilePath, slen);
}
~dirent() {
free(d_name);
}
};
/// @brief class to store directory data (files meta)
class DIR {
WIN32_FIND_DATAA FindFileData;
HANDLE hFind;
dirent* next;
static inline bool endsWith(const std::string& src, const char* with) {
int wl = static_cast<int>(strlen(with));
int so = static_cast<int>(src.length()) - wl;
if (so < 0)
return false;
return 0 == strncmp(with, &src[so], wl);
}
public:
DIR(const DIR& other) = delete;
DIR(DIR&& other) = delete;
DIR& operator=(const DIR& other) = delete;
DIR& operator=(DIR&& other) = delete;
explicit DIR(const char* dirPath): next(nullptr) {
std::string ws = dirPath;
if (endsWith(ws, "\\"))
ws += "*";
else
ws += "\\*";
hFind = FindFirstFileA(ws.c_str(), &FindFileData);
FindFileData.dwReserved0 = hFind != INVALID_HANDLE_VALUE;
}
~DIR() {
if (!next)
delete next;
next = nullptr;
FindClose(hFind);
}
/**
* @brief Check file handler is valid
* @return status True(success) or False(fail)
*/
bool isValid() const {
return (hFind != INVALID_HANDLE_VALUE && FindFileData.dwReserved0);
}
/**
* @brief Add directory to directory names struct
* @return pointer to directory names struct
*/
dirent* nextEnt() {
if (next != nullptr)
delete next;
next = nullptr;
if (!FindFileData.dwReserved0)
return nullptr;
wchar_t wbuf[4096];
size_t outSize;
mbstowcs_s(&outSize, wbuf, 4094, FindFileData.cFileName, 4094);
next = new dirent(wbuf);
FindFileData.dwReserved0 = FindNextFileA(hFind, &FindFileData);
return next;
}
};
/**
* @brief Create directory data struct element
* @param string directory path
* @return pointer to directory data struct element
*/
static DIR* opendir(const char* dirPath) {
auto dp = new DIR(dirPath);
if (!dp->isValid()) {
delete dp;
return nullptr;
}
return dp;
}
/**
* @brief Walk throw directory data struct
* @param pointer to directory data struct
* @return pointer to directory data struct next element
*/
static struct dirent* readdir(DIR* dp) {
return dp->nextEnt();
}
/**
* @brief Remove directory data struct
* @param pointer to struct directory data
* @return void
*/
static void closedir(DIR* dp) {
delete dp;
}
#ifdef WIN32_LEAN_AND_MEAN_UNDEF
#undef WIN32_LEAN_AND_MEAN
#undef WIN32_LEAN_AND_MEAN_UNDEF
#endif
#ifdef NOMINMAX_UNDEF
#undef NOMINMAX_UNDEF
#undef NOMINMAX
#endif
#else
#include <dirent.h>
#include <sys/types.h>
#endif