diff --git a/.ci/azure/linux_onnxruntime.yml b/.ci/azure/linux_onnxruntime.yml index 120faa65e03..480586249ab 100644 --- a/.ci/azure/linux_onnxruntime.yml +++ b/.ci/azure/linux_onnxruntime.yml @@ -131,7 +131,6 @@ jobs: -DENABLE_CPPLINT=OFF -DENABLE_PROFILING_ITT=OFF -DENABLE_SAMPLES=OFF - -DENABLE_COMPILE_TOOL=OFF -DENABLE_OV_TF_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=OFF diff --git a/CMakeLists.txt b/CMakeLists.txt index 3182d126933..bb66118126d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,7 +124,7 @@ endif() include(thirdparty/dependencies.cmake) add_subdirectory(src) -if(ENABLE_SAMPLES OR ENABLE_TESTS OR ENABLE_COMPILE_TOOL) +if(ENABLE_SAMPLES OR ENABLE_TESTS) add_subdirectory(samples) endif() diff --git a/cmake/extra_modules.cmake b/cmake/extra_modules.cmake index a5b1cd22e82..6a5df6e412c 100644 --- a/cmake/extra_modules.cmake +++ b/cmake/extra_modules.cmake @@ -15,7 +15,7 @@ function(ie_generate_dev_package_config) add_custom_target(ie_dev_targets DEPENDS ${all_dev_targets}) set(PATH_VARS "OpenVINO_SOURCE_DIR") - if(ENABLE_SAMPLES OR ENABLE_COMPILE_TOOL OR ENABLE_TESTS) + if(ENABLE_SAMPLES OR ENABLE_TESTS) list(APPEND PATH_VARS "gflags_BINARY_DIR") # if we've found system gflags if(gflags_DIR) @@ -52,7 +52,7 @@ function(ov_generate_dev_package_config) add_custom_target(ov_dev_targets DEPENDS ${all_dev_targets}) set(PATH_VARS "OpenVINO_SOURCE_DIR") - if(ENABLE_SAMPLES OR ENABLE_COMPILE_TOOL OR ENABLE_TESTS) + if(ENABLE_SAMPLES OR ENABLE_TESTS) list(APPEND PATH_VARS "gflags_BINARY_DIR") # if we've found system gflags if(gflags_DIR) diff --git a/cmake/features.cmake b/cmake/features.cmake index 3d2e926f7ac..9a405208e0d 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -12,8 +12,6 @@ ie_dependent_option (ENABLE_ARM_COMPUTE_CMAKE "Enable ARM Compute build via cmak ie_option (ENABLE_TESTS "unit, behavior and functional tests" OFF) -ie_option (ENABLE_COMPILE_TOOL "Enables compile_tool" ON) - ie_option (ENABLE_STRICT_DEPENDENCIES "Skip configuring \"convinient\" dependencies for efficient parallel builds" ON) if(X86_64) diff --git a/samples/cpp/benchmark_app/main.cpp b/samples/cpp/benchmark_app/main.cpp index d93462a81a9..0ec8267a7c8 100644 --- a/samples/cpp/benchmark_app/main.cpp +++ b/samples/cpp/benchmark_app/main.cpp @@ -95,8 +95,7 @@ bool parse_and_check_command_line(int argc, char* argv[]) { bool isPrecisionSet = !(FLAGS_ip.empty() && FLAGS_op.empty() && FLAGS_iop.empty()); if (isNetworkCompiled && isPrecisionSet) { std::string err = std::string("Cannot set precision for a compiled model. ") + - std::string("Please re-compile your model with required precision " - "using compile_tool"); + std::string("Please re-compile your model with required precision."); throw std::logic_error(err); } diff --git a/scripts/setupvars/setupvars.bat b/scripts/setupvars/setupvars.bat index 25c9fb350ec..9ef50fc88fa 100644 --- a/scripts/setupvars/setupvars.bat +++ b/scripts/setupvars/setupvars.bat @@ -60,11 +60,6 @@ if exist %INTEL_OPENVINO_DIR%\runtime\3rdparty\tbb ( ) ) -:: Compile tool -if exist %INTEL_OPENVINO_DIR%\tools\compile_tool ( - set "PATH=%INTEL_OPENVINO_DIR%\tools\compile_tool;%PATH%" -) - :: Add libs dirs to the PATH set "PATH=%OPENVINO_LIB_PATHS%;%PATH%" diff --git a/scripts/setupvars/setupvars.sh b/scripts/setupvars/setupvars.sh index e87b3538580..66a620b2d39 100755 --- a/scripts/setupvars/setupvars.sh +++ b/scripts/setupvars/setupvars.sh @@ -79,10 +79,6 @@ if [ -e "$INSTALLDIR/runtime" ]; then fi fi -if [ -e "$INSTALLDIR/tools/compile_tool" ]; then - export LD_LIBRARY_PATH=$INSTALLDIR/tools/compile_tool${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} -fi - # OpenCV environment if [ -f "$INSTALLDIR/opencv/setupvars.sh" ]; then # shellcheck source=/dev/null diff --git a/thirdparty/dependencies.cmake b/thirdparty/dependencies.cmake index c209d9ae44c..c439df4c569 100644 --- a/thirdparty/dependencies.cmake +++ b/thirdparty/dependencies.cmake @@ -362,7 +362,7 @@ endif() # Gflags # -if(ENABLE_SAMPLES OR ENABLE_COMPILE_TOOL OR ENABLE_TESTS) +if(ENABLE_SAMPLES OR ENABLE_TESTS) if(CMAKE_TOOLCHAIN_FILE MATCHES "vcpkg " OR DEFINED VCPKG_VERBOSE OR CMAKE_TOOLCHAIN_FILE MATCHES "conan_toolchain" OR DEFINED CONAN_EXPORTED) # vcpkg contains only libs compiled with threads @@ -628,7 +628,7 @@ endif() if(CPACK_GENERATOR MATCHES "^(DEB|RPM|CONDA-FORGE|BREW|CONAN|VCPKG)$") # These libraries are dependencies for openvino-samples package - if(ENABLE_SAMPLES OR ENABLE_COMPILE_TOOL OR ENABLE_TESTS) + if(ENABLE_SAMPLES OR ENABLE_TESTS) if(NOT gflags_FOUND AND CPACK_GENERATOR MATCHES "^(DEB|RPM)$") message(FATAL_ERROR "gflags must be used as a ${CPACK_GENERATOR} package. Install libgflags-dev / gflags-devel") endif() diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 21bb17cdb56..977fd50715f 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,18 +1,6 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# -# C++ tools -# - -if(ENABLE_COMPILE_TOOL) - add_subdirectory(compile_tool) -endif() - -if(ENABLE_SAMPLES) - add_subdirectory(legacy/benchmark_app) -endif() - # # Python tools # diff --git a/tools/benchmark_tool/openvino/tools/benchmark/main.py b/tools/benchmark_tool/openvino/tools/benchmark/main.py index d6692b66d28..821e81b97d0 100644 --- a/tools/benchmark_tool/openvino/tools/benchmark/main.py +++ b/tools/benchmark_tool/openvino/tools/benchmark/main.py @@ -46,8 +46,7 @@ def parse_and_check_command_line(): if is_network_compiled and is_precisiton_set: raise Exception("Cannot set precision for a compiled model. " \ - "Please re-compile your model with required precision " \ - "using compile_tool") + "Please re-compile your model with required precision.") return args, is_network_compiled diff --git a/tools/compile_tool/CMakeLists.txt b/tools/compile_tool/CMakeLists.txt deleted file mode 100644 index 3859ca11f10..00000000000 --- a/tools/compile_tool/CMakeLists.txt +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (C) 2018-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -set(TARGET_NAME compile_tool) -add_definitions(-DIN_OV_COMPONENT) - -file(GLOB SRCS - ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp -) - -add_executable(${TARGET_NAME} ${SRCS}) - -if (CMAKE_COMPILER_IS_GNUCXX) - target_compile_options(${TARGET_NAME} PRIVATE -Wall) -endif() - -# for cross-compilation with gflags -find_package(Threads REQUIRED) - -target_link_libraries(${TARGET_NAME} PRIVATE - inference_engine - inference_engine_plugin_api - gflags - ie_samples_utils - Threads::Threads -) - -set_target_properties(${TARGET_NAME} PROPERTIES - COMPILE_PDB_NAME ${TARGET_NAME} - FOLDER tools -) - -add_cpplint_target(${TARGET_NAME}_cpplint FOR_TARGETS ${TARGET_NAME}) - -# install - -ov_cpack_add_component(${OV_CPACK_COMP_CORE_TOOLS} - HIDDEN - DEPENDS ${OV_CPACK_COMP_CORE}) - -if(CPACK_GENERATOR MATCHES "^(DEB|RPM)$") - install(TARGETS compile_tool - RUNTIME DESTINATION ${OV_CPACK_TOOLSDIR} - COMPONENT ${OV_CPACK_COMP_CORE_TOOLS} - ${OV_CPACK_COMP_CORE_TOOLS_EXCLUDE_ALL}) -else() - install(TARGETS compile_tool - RUNTIME DESTINATION ${OV_CPACK_TOOLSDIR}/compile_tool - COMPONENT ${OV_CPACK_COMP_CORE_TOOLS} - ${OV_CPACK_COMP_CORE_TOOLS_EXCLUDE_ALL}) -endif() diff --git a/tools/compile_tool/main.cpp b/tools/compile_tool/main.cpp deleted file mode 100644 index ede1d97658a..00000000000 --- a/tools/compile_tool/main.cpp +++ /dev/null @@ -1,743 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "inference_engine.hpp" -#include "openvino/openvino.hpp" - -static constexpr char help_message[] = - "Optional. Print the usage message."; - -static constexpr char model_message[] = - "Required. Path to the XML model."; - -static constexpr char targetDeviceMessage[] = - "Required. Specify a target device for which executable network will be compiled.\n" -" Use \"-d HETERO:\" format to specify HETERO plugin.\n" -" Use \"-d MULTI:\" format to specify MULTI plugin.\n" -" The application looks for a suitable plugin for the specified device."; - -static constexpr char output_message[] = - "Optional. Path to the output file. Default value: \".blob\"."; - -static constexpr char log_level_message[] = - "Optional. Log level for InferenceEngine library."; - -static constexpr char config_message[] = - "Optional. Path to the configuration file."; - -static constexpr char inputs_precision_message[] = - "Optional. Specifies precision for all input layers of the network."; - -static constexpr char outputs_precision_message[] = - "Optional. Specifies precision for all output layers of the network."; - -static constexpr char iop_message[] = - "Optional. Specifies precision for input and output layers by name.\n" -" Example: -iop \"input:FP16, output:FP16\".\n" -" Notice that quotes are required.\n" -" Overwrites precision from ip and op options for specified layers."; - -static constexpr char inputs_layout_message[] = - "Optional. Specifies layout for all input layers of the network."; - -static constexpr char outputs_layout_message[] = - "Optional. Specifies layout for all output layers of the network."; - -static constexpr char iol_message[] = - "Optional. Specifies layout for input and output layers by name.\n" -" Example: -iol \"input:NCHW, output:NHWC\".\n" -" Notice that quotes are required.\n" -" Overwrites layout from il and ol options for specified layers."; - -static constexpr char inputs_model_layout_message[] = - "Optional. Specifies model layout for all input layers of the network."; - -static constexpr char outputs_model_layout_message[] = - "Optional. Specifies model layout for all output layers of the network."; - -static constexpr char ioml_message[] = - "Optional. Specifies model layout for input and output tensors by name.\n" -" Example: -ionl \"input:NCHW, output:NHWC\".\n" -" Notice that quotes are required.\n" -" Overwrites layout from il and ol options for specified layers."; - -static constexpr char api1_message[] = - "Optional. Compile model to legacy format for usage in Inference Engine API,\n" -" by default compiles to OV 2.0 API"; - -DEFINE_bool(h, false, help_message); -DEFINE_string(m, "", model_message); -DEFINE_string(d, "", targetDeviceMessage); -DEFINE_string(o, "", output_message); -DEFINE_string(log_level, "", log_level_message); -DEFINE_string(c, "", config_message); -DEFINE_string(ip, "", inputs_precision_message); -DEFINE_string(op, "", outputs_precision_message); -DEFINE_string(iop, "", iop_message); -DEFINE_string(il, "", inputs_layout_message); -DEFINE_string(ol, "", outputs_layout_message); -DEFINE_string(iol, "", iol_message); -DEFINE_string(iml, "", inputs_model_layout_message); -DEFINE_string(oml, "", outputs_model_layout_message); -DEFINE_string(ioml, "", ioml_message); -DEFINE_bool(ov_api_1_0, false, api1_message); - -namespace { -std::vector splitStringList(const std::string& str, char delim) { - if (str.empty()) - return {}; - - std::istringstream istr(str); - - std::vector result; - std::string elem; - while (std::getline(istr, elem, delim)) { - if (elem.empty()) { - continue; - } - result.emplace_back(std::move(elem)); - } - - return result; -} - -std::map parseArgMap(std::string argMap) { - argMap.erase(std::remove_if(argMap.begin(), argMap.end(), ::isspace), argMap.end()); - - const auto pairs = splitStringList(argMap, ','); - - std::map parsedMap; - for (auto&& pair : pairs) { - const auto lastDelimPos = pair.find_last_of(':'); - auto key = pair.substr(0, lastDelimPos); - auto value = pair.substr(lastDelimPos + 1); - - if (lastDelimPos == std::string::npos || key.empty() || value.empty()) { - throw std::invalid_argument("Invalid key/value pair " + pair + ". Expected :"); - } - - parsedMap[std::move(key)] = std::move(value); - } - - return parsedMap; -} - -using supported_precisions_t = std::unordered_map; - -InferenceEngine::Precision getPrecision(std::string value, const supported_precisions_t& supported_precisions) { - std::transform(value.begin(), value.end(), value.begin(), ::toupper); - - const auto precision = supported_precisions.find(value); - if (precision == supported_precisions.end()) { - throw std::logic_error("\"" + value + "\"" + " is not a valid precision"); - } - - return precision->second; -} - -InferenceEngine::Precision getPrecision(const std::string& value) { - static const supported_precisions_t supported_precisions = { - {"FP32", InferenceEngine::Precision::FP32}, {"f32", InferenceEngine::Precision::FP32}, - {"FP16", InferenceEngine::Precision::FP16}, {"f16", InferenceEngine::Precision::FP16}, - {"BF16", InferenceEngine::Precision::BF16}, {"bf16", InferenceEngine::Precision::BF16}, - {"U64", InferenceEngine::Precision::U64}, {"u64", InferenceEngine::Precision::U64}, - {"I64", InferenceEngine::Precision::I64}, {"i64", InferenceEngine::Precision::I64}, - {"U32", InferenceEngine::Precision::U32}, {"u32", InferenceEngine::Precision::U32}, - {"I32", InferenceEngine::Precision::I32}, {"i32", InferenceEngine::Precision::I32}, - {"U16", InferenceEngine::Precision::U16}, {"u16", InferenceEngine::Precision::U16}, - {"I16", InferenceEngine::Precision::I16}, {"i16", InferenceEngine::Precision::I16}, - {"U8", InferenceEngine::Precision::U8}, {"u8", InferenceEngine::Precision::U8}, - {"I8", InferenceEngine::Precision::I8}, {"i8", InferenceEngine::Precision::I8}, - {"BOOL", InferenceEngine::Precision::BOOL}, {"boolean", InferenceEngine::Precision::BOOL}, - }; - - return getPrecision(value, supported_precisions); -} - -void setPrecisions(const InferenceEngine::CNNNetwork& network, const std::string& iop) { - const auto user_precisions_map = parseArgMap(iop); - - auto inputs = network.getInputsInfo(); - auto outputs = network.getOutputsInfo(); - - for (auto&& item : user_precisions_map) { - const auto& layer_name = item.first; - const auto& user_precision = item.second; - - const auto input = inputs.find(layer_name); - const auto output = outputs.find(layer_name); - - if (input != inputs.end()) { - input->second->setPrecision(getPrecision(user_precision)); - } else if (output != outputs.end()) { - output->second->setPrecision(getPrecision(user_precision)); - } else { - throw std::logic_error(layer_name + " is not an input neither output"); - } - } -} - -} //namespace - - -void processPrecision(InferenceEngine::CNNNetwork& network, - const std::string& ip, - const std::string& op, - const std::string& iop) { - if (!ip.empty()) { - const auto user_precision = getPrecision(ip); - for (auto&& layer : network.getInputsInfo()) { - layer.second->setPrecision(user_precision); - } - } - - if (!op.empty()) { - auto user_precision = getPrecision(op); - for (auto&& layer : network.getOutputsInfo()) { - layer.second->setPrecision(user_precision); - } - } - - if (!iop.empty()) { - setPrecisions(network, iop); - } -} - -using supported_layouts_t = std::unordered_map; -using matchLayoutToDims_t = std::unordered_map; - -InferenceEngine::Layout getLayout(std::string value, const supported_layouts_t& supported_layouts) { - std::transform(value.begin(), value.end(), value.begin(), ::toupper); - - const auto layout = supported_layouts.find(value); - if (layout == supported_layouts.end()) { - throw std::logic_error("\"" + value + "\"" + " is not a valid layout"); - } - - return layout->second; -} - -InferenceEngine::Layout getLayout(const std::string& value) { - static const supported_layouts_t supported_layouts = { - {"NCDHW", InferenceEngine::Layout::NCDHW}, - {"NDHWC", InferenceEngine::Layout::NDHWC}, - {"NCHW", InferenceEngine::Layout::NCHW}, - {"NHWC", InferenceEngine::Layout::NHWC}, - {"CHW", InferenceEngine::Layout::CHW}, - {"HWC", InferenceEngine::Layout::HWC}, - {"NC", InferenceEngine::Layout::NC}, - {"C", InferenceEngine::Layout::C}, - }; - - return getLayout(value, supported_layouts); -} - -bool isMatchLayoutToDims(InferenceEngine::Layout layout, size_t dimension) { - static const matchLayoutToDims_t matchLayoutToDims = { {static_cast(InferenceEngine::Layout::NCDHW), 5}, - {static_cast(InferenceEngine::Layout::NDHWC), 5}, - {static_cast(InferenceEngine::Layout::NCHW), 4}, - {static_cast(InferenceEngine::Layout::NHWC), 4}, - {static_cast(InferenceEngine::Layout::CHW), 3}, - {static_cast(InferenceEngine::Layout::NC), 2}, - {static_cast(InferenceEngine::Layout::C), 1} }; - - const auto dims = matchLayoutToDims.find(static_cast(layout)); - if (dims == matchLayoutToDims.end()) { - throw std::logic_error("Layout is not valid."); - } - - return dimension == dims->second; -} - -void setLayouts(const InferenceEngine::CNNNetwork& network, const std::string iol) { - const auto user_layouts_map = parseArgMap(iol); - - auto inputs = network.getInputsInfo(); - auto outputs = network.getOutputsInfo(); - - for (auto&& item : user_layouts_map) { - const auto& layer_name = item.first; - const auto& user_layout = getLayout(item.second); - - const auto input = inputs.find(layer_name); - const auto output = outputs.find(layer_name); - - if (input != inputs.end()) { - if (!isMatchLayoutToDims(user_layout, input->second->getTensorDesc().getDims().size())) { - throw std::logic_error(item.second + " layout is not applicable to " + layer_name); - } - - input->second->setLayout(user_layout); - } else if (output != outputs.end()) { - if (!isMatchLayoutToDims(user_layout, output->second->getTensorDesc().getDims().size())) { - throw std::logic_error(item.second + " layout is not applicable to " + layer_name); - } - - output->second->setLayout(user_layout); - } else { - throw std::logic_error(layer_name + " is not an input neither output"); - } - } -} - -void processLayout(InferenceEngine::CNNNetwork& network, - const std::string& il, - const std::string& ol, - const std::string& iol) { - if (!il.empty()) { - const auto layout = getLayout(il); - for (auto&& layer : network.getInputsInfo()) { - if (isMatchLayoutToDims(layout, layer.second->getTensorDesc().getDims().size())) { - layer.second->setLayout(layout); - } - } - } - - if (!ol.empty()) { - const auto layout = getLayout(ol); - for (auto&& layer : network.getOutputsInfo()) { - if (isMatchLayoutToDims(layout, layer.second->getTensorDesc().getDims().size())) { - layer.second->setLayout(layout); - } - } - } - - if (!iol.empty()) { - setLayouts(network, iol); - } -} - -using supported_type_t = std::unordered_map; -ov::element::Type getType(std::string value, const supported_type_t& supported_precisions) { - std::transform(value.begin(), value.end(), value.begin(), ::toupper); - - const auto precision = supported_precisions.find(value); - if (precision == supported_precisions.end()) { - throw std::logic_error("\"" + value + "\"" + " is not a valid precision"); - } - - return precision->second; -} -ov::element::Type getType(const std::string& value) { - static const supported_type_t supported_types = { - {"FP32", ov::element::f32}, {"f32", ov::element::f32}, {"FP16", ov::element::f16}, - {"f16", ov::element::f16}, {"BF16", ov::element::bf16}, {"bf16", ov::element::bf16}, - {"U64", ov::element::u64}, {"u64", ov::element::u64}, {"I64", ov::element::i64}, - {"i64", ov::element::i64}, {"U32", ov::element::u32}, {"u32", ov::element::u32}, - {"I32", ov::element::i32}, {"i32", ov::element::i32}, {"U16", ov::element::u16}, - {"u16", ov::element::u16}, {"I16", ov::element::i16}, {"i16", ov::element::i16}, - {"U8", ov::element::u8}, {"u8", ov::element::u8}, {"I8", ov::element::i8}, - {"i8", ov::element::i8}, {"BOOL", ov::element::boolean}, {"boolean", ov::element::boolean}, - }; - - return getType(value, supported_types); -} - -bool isFP32(const ov::element::Type& type) { - return type == ov::element::f32; -} - -static void setDefaultIO(ov::preprocess::PrePostProcessor& preprocessor, - const std::vector>& inputs, - const std::vector>& outputs) { - const bool isVPUX = FLAGS_d.find("VPUX") != std::string::npos; - - if (isVPUX) { - for (size_t i = 0; i < inputs.size(); i++) { - preprocessor.input(i).tensor().set_element_type(ov::element::u8); - } - for (size_t i = 0; i < outputs.size(); i++) { - preprocessor.output(i).tensor().set_element_type(ov::element::f32); - } - } -} - -void configurePrePostProcessing(std::shared_ptr& model, - const std::string& ip, - const std::string& op, - const std::string& iop, - const std::string& il, - const std::string& ol, - const std::string& iol, - const std::string& iml, - const std::string& oml, - const std::string& ioml) { - auto preprocessor = ov::preprocess::PrePostProcessor(model); - const auto inputs = model->inputs(); - const auto outputs = model->outputs(); - setDefaultIO(preprocessor, inputs, outputs); - - if (!ip.empty()) { - auto type = getType(ip); - for (size_t i = 0; i < inputs.size(); i++) { - preprocessor.input(i).tensor().set_element_type(type); - } - } - - if (!op.empty()) { - auto type = getType(op); - for (size_t i = 0; i < outputs.size(); i++) { - preprocessor.output(i).tensor().set_element_type(type); - } - } - - if (!iop.empty()) { - const auto user_precisions_map = parseArgMap(iop); - for (auto&& item : user_precisions_map) { - const auto& tensor_name = item.first; - const auto type = getType(item.second); - - bool tensorFound = false; - for (size_t i = 0; i < inputs.size(); i++) { - if (inputs[i].get_names().count(tensor_name)) { - preprocessor.input(i).tensor().set_element_type(type); - tensorFound = true; - break; - } - } - if (!tensorFound) { - for (size_t i = 0; i < outputs.size(); i++) { - if (outputs[i].get_names().count(tensor_name)) { - preprocessor.output(i).tensor().set_element_type(type); - tensorFound = true; - break; - } - } - } - OPENVINO_ASSERT(tensorFound, "Model doesn't have input/output with tensor name: ", tensor_name); - } - } - if (!il.empty()) { - for (size_t i = 0; i < inputs.size(); i++) { - preprocessor.input(i).tensor().set_layout(ov::Layout(il)); - } - } - - if (!ol.empty()) { - for (size_t i = 0; i < outputs.size(); i++) { - preprocessor.output(i).tensor().set_layout(ov::Layout(ol)); - } - } - - if (!iol.empty()) { - const auto user_precisions_map = parseArgMap(iol); - for (auto&& item : user_precisions_map) { - const auto& tensor_name = item.first; - - bool tensorFound = false; - for (size_t i = 0; i < inputs.size(); i++) { - if (inputs[i].get_names().count(tensor_name)) { - preprocessor.input(i).tensor().set_layout(ov::Layout(item.second)); - tensorFound = true; - break; - } - } - if (!tensorFound) { - for (size_t i = 0; i < outputs.size(); i++) { - if (outputs[i].get_names().count(tensor_name)) { - preprocessor.output(i).tensor().set_layout(ov::Layout(item.second)); - tensorFound = true; - break; - } - } - } - OPENVINO_ASSERT(tensorFound, "Model doesn't have input/output with tensor name: ", tensor_name); - } - } - - if (!iml.empty()) { - for (size_t i = 0; i < inputs.size(); i++) { - preprocessor.input(i).model().set_layout(ov::Layout(iml)); - } - } - - if (!oml.empty()) { - for (size_t i = 0; i < outputs.size(); i++) { - preprocessor.output(i).model().set_layout(ov::Layout(oml)); - } - } - - if (!ioml.empty()) { - const auto user_precisions_map = parseArgMap(ioml); - for (auto&& item : user_precisions_map) { - const auto& tensor_name = item.first; - - bool tensorFound = false; - for (size_t i = 0; i < inputs.size(); i++) { - if (inputs[i].get_names().count(tensor_name)) { - preprocessor.input(i).model().set_layout(ov::Layout(item.second)); - tensorFound = true; - break; - } - } - if (!tensorFound) { - for (size_t i = 0; i < outputs.size(); i++) { - if (outputs[i].get_names().count(tensor_name)) { - preprocessor.output(i).model().set_layout(ov::Layout(item.second)); - tensorFound = true; - break; - } - } - } - OPENVINO_ASSERT(tensorFound, "Model doesn't have input/output with tensor name: ", tensor_name); - } - } - - model = preprocessor.build(); -} - -void printInputAndOutputsInfo(const InferenceEngine::CNNNetwork& network) { - std::cout << "Network inputs:" << std::endl; - for (auto&& layer : network.getInputsInfo()) { - std::cout << " " << layer.first << " : " << layer.second->getPrecision() << " / " - << layer.second->getLayout() << std::endl; - } - std::cout << "Network outputs:" << std::endl; - for (auto&& layer : network.getOutputsInfo()) { - std::cout << " " << layer.first << " : " << layer.second->getPrecision() << " / " - << layer.second->getLayout() << std::endl; - } -} - -void printInputAndOutputsInfoShort(const ov::Model& network) { - std::cout << "Network inputs:" << std::endl; - for (auto&& param : network.get_parameters()) { - auto l = param->get_layout(); - std::cout << " " << param->get_friendly_name() << " : " << param->get_element_type() << " / " - << param->get_layout().to_string() << std::endl; - } - std::cout << "Network outputs:" << std::endl; - for (auto&& result : network.get_results()) { - std::cout << " " << result->get_friendly_name() << " : " << result->get_element_type() << " / " - << result->get_layout().to_string() << std::endl; - } -} - -inline std::string fileNameNoExt(const std::string& filepath) { - auto pos = filepath.rfind('.'); - if (pos == std::string::npos) - return filepath; - return filepath.substr(0, pos); -} - - -static void showUsage() { - std::cout << "compile_tool [OPTIONS]" << std::endl; - std::cout << std::endl; - std::cout << " Common options: " << std::endl; - std::cout << " -h " << help_message << std::endl; - std::cout << " -m " << model_message << std::endl; - std::cout << " -d " << targetDeviceMessage << std::endl; - std::cout << " -o " << output_message << std::endl; - std::cout << " -c " << config_message << std::endl; - std::cout << " -ip " << inputs_precision_message << std::endl; - std::cout << " -op " << outputs_precision_message << std::endl; - std::cout << " -iop \"\" " << iop_message << std::endl; - std::cout << " -il " << inputs_layout_message << std::endl; - std::cout << " -ol " << outputs_layout_message << std::endl; - std::cout << " -iol \"\" " << iol_message << std::endl; - std::cout << " -iml " << inputs_model_layout_message << std::endl; - std::cout << " -oml " << outputs_model_layout_message << std::endl; - std::cout << " -ioml \"\" " << ioml_message << std::endl; - std::cout << " -ov_api_1_0 " << api1_message << std::endl; - std::cout << std::endl; -} - -static bool parseCommandLine(int* argc, char*** argv) { - gflags::ParseCommandLineNonHelpFlags(argc, argv, true); - - if (FLAGS_h) { - showUsage(); - return false; - } - - if (FLAGS_m.empty()) { - throw std::invalid_argument("Path to model xml file is required"); - } - - if (FLAGS_d.empty()) { - throw std::invalid_argument("Target device name is required"); - } - - if (1 < *argc) { - std::stringstream message; - message << "Unknown arguments: "; - for (auto arg = 1; arg < *argc; arg++) { - message << (*argv)[arg]; - if (arg < *argc) { - message << " "; - } - } - throw std::invalid_argument(message.str()); - } - - return true; -} - -static std::map parseConfigFile(char comment = '#') { - std::map config; - - std::ifstream file(FLAGS_c); - if (file.is_open()) { - std::string option; - while (std::getline(file, option)) { - if (option.empty() || option[0] == comment) { - continue; - } - size_t spacePos = option.find(' '); - - OPENVINO_ASSERT(spacePos != std::string::npos, - "Failed to find a space separator in " - "provided plugin config option: " + - option); - - std::string key = option.substr(0, spacePos); - std::string value = option.substr(spacePos + 1); - config[key] = value; - } - } - return config; -} - -bool isFP16(InferenceEngine::Precision precision) { - return precision == InferenceEngine::Precision::FP16; -} - -bool isFP32(InferenceEngine::Precision precision) { - return precision == InferenceEngine::Precision::FP32; -} - -bool isFloat(InferenceEngine::Precision precision) { - return isFP16(precision) || isFP32(precision); -} - -static void setDefaultIO(InferenceEngine::CNNNetwork& network) { - const bool isVPUX = FLAGS_d.find("VPUX") != std::string::npos; - - if (isVPUX) { - const InferenceEngine::Precision u8 = InferenceEngine::Precision::U8; - const InferenceEngine::Precision fp32 = InferenceEngine::Precision::FP32; - - for (auto&& layer : network.getInputsInfo()) { - layer.second->setPrecision(u8); - } - - for (auto&& layer : network.getOutputsInfo()) { - layer.second->setPrecision(fp32); - } - } -} - -std::string getFileNameFromPath(const std::string& path, -#if defined(_WIN32) - const std::string& sep = "\\") { -#else - const std::string& sep = "/") { -#endif - const auto pos = path.rfind(sep); - if (std::string::npos == pos) { - return path; - } else { - return path.substr(pos + 1); - } -} - -using TimeDiff = std::chrono::milliseconds; - -int main(int argc, char* argv[]) { - TimeDiff loadNetworkTimeElapsed {0}; - - try { - const auto& version = ov::get_openvino_version(); - std::cout << version.description << " version ......... "; - std::cout << OPENVINO_VERSION_MAJOR << "." << OPENVINO_VERSION_MINOR << "." << OPENVINO_VERSION_PATCH << std::endl; - - std::cout << "Build ........... "; - std::cout << version.buildNumber << std::endl; - - if (!parseCommandLine(&argc, &argv)) { - return EXIT_SUCCESS; - } - if (FLAGS_ov_api_1_0) { - InferenceEngine::Core ie; - if (!FLAGS_log_level.empty()) { - ie.SetConfig({{CONFIG_KEY(LOG_LEVEL), FLAGS_log_level}}, FLAGS_d); - } - - auto network = ie.ReadNetwork(FLAGS_m); - - setDefaultIO(network); - processPrecision(network, FLAGS_ip, FLAGS_op, FLAGS_iop); - processLayout(network, FLAGS_il, FLAGS_ol, FLAGS_iol); - - printInputAndOutputsInfo(network); - - auto timeBeforeLoadNetwork = std::chrono::steady_clock::now(); - auto executableNetwork = ie.LoadNetwork(network, FLAGS_d, parseConfigFile()); - loadNetworkTimeElapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - timeBeforeLoadNetwork); - - std::string outputName = FLAGS_o; - if (outputName.empty()) { - outputName = getFileNameFromPath(fileNameNoExt(FLAGS_m)) + ".blob"; - } - - std::ofstream outputFile{outputName, std::ios::out | std::ios::binary}; - if (!outputFile.is_open()) { - std::cout << "Output file " << outputName << " can't be opened for writing" << std::endl; - return EXIT_FAILURE; - } else { - executableNetwork.Export(outputFile); - } - } else { - ov::Core core; - if (!FLAGS_log_level.empty()) { - ov::log::Level level; - std::stringstream{FLAGS_log_level} >> level; - core.set_property(FLAGS_d, ov::log::level(level)); - } - - auto model = core.read_model(FLAGS_m); - - configurePrePostProcessing(model, FLAGS_ip, FLAGS_op, FLAGS_iop, FLAGS_il, FLAGS_ol, FLAGS_iol, FLAGS_iml, FLAGS_oml, FLAGS_ioml); - printInputAndOutputsInfoShort(*model); - auto timeBeforeLoadNetwork = std::chrono::steady_clock::now(); - auto configs = parseConfigFile(); - auto compiledModel = core.compile_model(model, FLAGS_d, {configs.begin(), configs.end()}); - loadNetworkTimeElapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - timeBeforeLoadNetwork); - std::string outputName = FLAGS_o; - if (outputName.empty()) { - outputName = getFileNameFromPath(fileNameNoExt(FLAGS_m)) + ".blob"; - } - - std::ofstream outputFile{outputName, std::ios::out | std::ios::binary}; - if (!outputFile.is_open()) { - std::cout << "Output file " << outputName << " can't be opened for writing" << std::endl; - return EXIT_FAILURE; - } else { - compiledModel.export_model(outputFile); - } - } - } catch (const std::exception& error) { - std::cerr << error.what() << std::endl; - return EXIT_FAILURE; - } catch (...) { - std::cerr << "Unknown/internal exception happened." << std::endl; - return EXIT_FAILURE; - } - - std::cout << "Done. LoadNetwork time elapsed: " << loadNetworkTimeElapsed.count() << " ms" << std::endl; - return EXIT_SUCCESS; -} diff --git a/tools/legacy/benchmark_app/CMakeLists.txt b/tools/legacy/benchmark_app/CMakeLists.txt deleted file mode 100644 index cd6bb4aa8bd..00000000000 --- a/tools/legacy/benchmark_app/CMakeLists.txt +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (C) 2018-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -set(TARGET_NAME benchmark_app_legacy) -add_definitions(-DIN_OV_COMPONENT) - -file (GLOB SRC ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) -file (GLOB HDR ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp) - -if(OpenVINO_SOURCE_DIR) - set(OpenVINO_DIR "${CMAKE_BINARY_DIR}") -endif() - -source_group("src" FILES ${SRC}) -source_group("include" FILES ${HDR}) - -add_executable(${TARGET_NAME} ${SRC} ${HDR}) - -find_package(OpenVINO REQUIRED COMPONENTS Runtime) - -set_target_properties(${TARGET_NAME} PROPERTIES - COMPILE_PDB_NAME ${TARGET_NAME} - FOLDER tools -) - -if (CMAKE_COMPILER_IS_GNUCXX) - target_compile_options(${TARGET_NAME} PRIVATE -Wall) -endif() - -# for cross-compilation with gflags -find_package(Threads REQUIRED) - -target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime format_reader gflags Threads::Threads) - -find_package(OpenCV QUIET COMPONENTS core) -if(NOT OpenCV_FOUND) - message(WARNING "OpenCV is disabled or not found, ${TARGET_NAME} will be built without OpenCV support. Set OpenCV_DIR") -else() - target_compile_definitions(${TARGET_NAME} PRIVATE USE_OPENCV) - target_link_libraries(${TARGET_NAME} PRIVATE opencv_core) -endif() diff --git a/tools/legacy/benchmark_app/README.md b/tools/legacy/benchmark_app/README.md deleted file mode 100644 index 6ff4143f3e0..00000000000 --- a/tools/legacy/benchmark_app/README.md +++ /dev/null @@ -1,183 +0,0 @@ -# Benchmark C++ Tool - -This topic demonstrates how to use the Benchmark C++ Tool to estimate deep learning inference performance on supported devices. Performance can be measured for two inference modes: synchronous (latency-oriented) and asynchronous (throughput-oriented). - -> **NOTE**: This topic describes usage of C++ implementation of the Benchmark Tool. For the Python* implementation, refer to [Benchmark Python* Tool](../../benchmark_tool/README.md). - - -## How It Works - -Upon start-up, the application reads command-line parameters and loads a network and images/binary files to the Inference Engine plugin, which is chosen depending on a specified device. The number of infer requests and execution approach depend on the mode defined with the `-api` command-line parameter. - -> **NOTE**: By default, Inference Engine samples, tools and demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with `--reverse_input_channels` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of [Converting a Model to Intermediate Representation (IR)](../../../docs/MO_DG/prepare_model/convert_model/Converting_Model.md). - -If you run the application in the synchronous mode, it creates one infer request and executes the `Infer` method. -If you run the application in the asynchronous mode, it creates as many infer requests as specified in the `-nireq` command-line parameter and executes the `StartAsync` method for each of them. If `-nireq` is not set, the application will use the default value for specified device. - -A number of execution steps is defined by one of the following parameters: -* Number of iterations specified with the `-niter` command-line argument -* Time duration specified with the `-t` command-line argument -* Both of them (execution will continue until both conditions are met) -* Predefined duration if `-niter` and `-t` are not specified. Predefined duration value depends on a device. - -During the execution, the application collects latency for each executed infer request. - -Reported latency value is calculated as a median value of all collected latencies. Reported throughput value is reported -in frames per second (FPS) and calculated as a derivative from: -* Reported latency in the Sync mode -* The total execution time in the Async mode - -Throughput value also depends on batch size. - -The application also collects per-layer Performance Measurement (PM) counters for each executed infer request if you -enable statistics dumping by setting the `-report_type` parameter to one of the possible values: -* `no_counters` report includes configuration options specified, resulting FPS and latency. -* `average_counters` report extends the `no_counters` report and additionally includes average PM counters values for each layer from the network. -* `detailed_counters` report extends the `average_counters` report and additionally includes per-layer PM counters and latency for each executed infer request. - -Depending on the type, the report is stored to `benchmark_no_counters_report.csv`, `benchmark_average_counters_report.csv`, -or `benchmark_detailed_counters_report.csv` file located in the path specified in `-report_folder`. - -The application also saves executable graph information serialized to an XML file if you specify a path to it with the -`-exec_graph_path` parameter. - -## Building -This tools can be built as part of OpenVINO, during standard building process. More information about building OpenVINO can be found here[Build OpenVINO Inference Engine](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md) - -## Run the Tool - -Note that the benchmark_app usually produces optimal performance for any device out of the box. - -**So in most cases you don't need to play the app options explicitly and the plain device name is enough**, for example, for CPU: -```sh -./benchmark_app -m -i -d CPU -``` - -It still may be sub-optimal for some cases, especially for very small networks. For all devices, including the [MULTI device](../../../docs/OV_Runtime_UG/multi_device.md) it is preferable to use the FP16 IR for the model. If latency of the CPU inference on the multi-socket machines is of concern. -These, as well as other topics are explained in the [Performance Optimization Guide](../../../docs/optimization_guide/dldt_deployment_optimization_guide.md). - -Running the application with the `-h` option yields the following usage message: -``` -./benchmark_app -h -InferenceEngine: - API version ............ - Build .................. -[ INFO ] Parsing input parameters - -benchmark_app [OPTION] -Options: - - -h, --help Print a usage message - -m "" Required. Path to an .xml/.onnx/.prototxt file with a trained model or to a .blob files with a trained compiled model. - -i "" Optional. Path to a folder with images and/or binaries or to specific image or binary file. - -d "" Optional. Specify a target device to infer on (the list of available devices is shown below). Default value is CPU. - Use "-d HETERO:" format to specify HETERO plugin. - Use "-d MULTI:" format to specify MULTI plugin. - The application looks for a suitable plugin for the specified device. - -l "" Required for CPU custom layers. Absolute path to a shared library with the kernels implementations. - Or - -c "" Required for GPU custom kernels. Absolute path to an .xml file with the kernels description. - -api "" Optional. Enable Sync/Async API. Default value is "async". - -niter "" Optional. Number of iterations. If not specified, the number of iterations is calculated depending on a device. - -nireq "" Optional. Number of infer requests. Default value is determined automatically for a device. - -b "" Optional. Batch size value. If not specified, the batch size value is determined from Intermediate Representation. - -stream_output Optional. Print progress as a plain text. When specified, an interactive progress bar is replaced with a multiline output. - -t Optional. Time, in seconds, to execute topology. - -progress Optional. Show progress bar (can affect performance measurement). Default values is "false". - -shape Optional. Set shape for input. For example, "input1[1,3,224,224],input2[1,4]" or "[1,3,224,224]" in case of one input size. - -layout Optional. Prompts how network layouts should be treated by application. For example, "input1[NCHW],input2[NC]" or "[NCHW]" in case of one input size. - -cache_dir "" Optional. Enables caching of loaded models to specified directory. - -load_from_file Optional. Loads model from file directly without ReadNetwork. - - CPU-specific performance options: - -nstreams "" Optional. Number of streams to use for inference on the CPU, GPU devices - (for HETERO and MULTI device cases use format :,: or just ). - Default value is determined automatically for a device. - Please note that although the automatic selection usually provides a reasonable performance, - it still may be non-optimal for some cases, especially for very small networks. - Also, using nstreams>1 is inherently throughput-oriented option, while for the best-latency - estimations the number of streams should be set to 1. - -nthreads "" Optional. Number of threads to use for inference on the CPU (including HETERO and MULTI cases). - -enforcebf16="" Optional. By default floating point operations execution in bfloat16 precision are enforced if supported by platform. - -pin "YES"/"HYBRID_AWARE"/"NUMA"/"NO" - Optional. Explicit inference threads binding options (leave empty to let the OpenVINO to make a choice): - enabling threads->cores pinning ("YES", which is already default for a conventional CPU), - letting the runtime to decide on the threads->different core types ("HYBRID_AWARE", which is default on the hybrid CPUs) - threads->(NUMA)nodes ("NUMA") or - completely disable ("NO") CPU inference threads pinning. - -ip "U8"/"FP16"/"FP32" Optional. Specifies precision for all input layers of the network. - -op "U8"/"FP16"/"FP32" Optional. Specifies precision for all output layers of the network. - -iop Optional. Specifies precision for input and output layers by name. Example: -iop "input:FP16, output:FP16". Notice that quotes are required. Overwrites precision from ip and op options for specified layers. - - Statistics dumping options: - -report_type "" Optional. Enable collecting statistics report. "no_counters" report contains configuration options specified, resulting FPS and latency. "average_counters" report extends "no_counters" report and additionally includes average PM counters values for each layer from the network. "detailed_counters" report extends "average_counters" report and additionally includes per-layer PM counters and latency for each executed infer request. - -report_folder Optional. Path to a folder where statistics report is stored. - -exec_graph_path Optional. Path to a file where to store executable graph information serialized. - -pc Optional. Report performance counters. - -dump_config Optional. Path to XML/YAML/JSON file to dump IE parameters, which were set by application. - -load_config Optional. Path to XML/YAML/JSON file to load custom IE parameters. Please note, command line parameters have higher priority then parameters from configuration file. -``` - -Running the application with the empty list of options yields the usage message given above and an error message. - -Application supports topologies with one or more inputs. If a topology is not data-sensitive, you can skip the input parameter. In this case, inputs are filled with random values. -If a model has only image input(s), please provide a folder with images or a path to an image as input. -If a model has some specific input(s) (not images), please prepare a binary file(s) that is filled with data of appropriate precision and provide a path to them as input. -If a model has mixed input types, input folder should contain all required files. Image inputs are filled with image files one by one. Binary inputs are filled with binary inputs one by one. - -To run the tool, you can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). - -> **NOTE**: Before running the tool with a trained model, make sure the model is converted to the Inference Engine format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> The sample accepts models in ONNX format (.onnx) that do not require preprocessing. - -## Examples of Running the Tool - -This section provides step-by-step instructions on how to run the Benchmark Tool with the `googlenet-v1` public model on CPU or GPU devices. As an input, the `car.png` file from the `/samples/scripts/` directory is used. - -> **NOTE**: The Internet access is required to execute the following steps successfully. If you have access to the Internet through the proxy server only, please make sure that it is configured in your OS environment. - -1. Download the model. Install openvino-dev package into Python virtual environment from PyPi and run omz_downloader: - ```sh - omz_downloader --name googlenet-v1 -o -2. Convert the model to the Inference Engine IR format. Run the Model Optimizer using the `mo` command with the path to the model, model format (which must be FP32 for CPU and FPG) and output directory to generate the IR files: - ```sh - mo --input_model /public/googlenet-v1/googlenet-v1.caffemodel - ``` -3. Run the tool with specifying the `/samples/scripts/car.png` file as an input image, the IR of the `googlenet-v1` model and a device to perform inference on. The following commands demonstrate running the Benchmark Tool in the asynchronous mode on CPU and GPU devices: - - * On CPU: - ```sh - ./benchmark_app -m /googlenet-v1.xml -i /samples/scripts/car.png -d CPU -api async --progress true - ``` - * On GPU: - ```sh - ./benchmark_app -m /googlenet-v1.xml -i /samples/scripts/car.png -d GPU -api async --progress true - ``` - -The application outputs the number of executed iterations, total duration of execution, latency, and throughput. -Additionally, if you set the `-report_type` parameter, the application outputs statistics report. If you set the `-pc` parameter, the application outputs performance counters. If you set `-exec_graph_path`, the application reports executable graph information serialized. All measurements including per-layer PM counters are reported in milliseconds. - -Below are fragments of sample output: - - ``` - [Step 10/11] Measuring performance (Start inference asynchronously, 4 inference requests using 4 streams for CPU, limits: 60000 ms duration) - [ INFO ] BENCHMARK IS IN INFERENCE ONLY MODE. - [ INFO ] Input blobs will be filled once before performance measurements. - [ INFO ] First inference took 26.26 ms - Progress: [................... ] 99% done - - [Step 11/11] Dumping statistics report - [ INFO ] Count: 6640 iterations - [ INFO ] Duration: 60039.70 ms - [ INFO ] Latency: - [ INFO ] Median: 35.36 ms - [ INFO ] Avg: 36.12 ms - [ INFO ] Min: 18.55 ms - [ INFO ] Max: 88.96 ms - [ INFO ] Throughput: 110.59 FPS - ``` - -## See Also -* [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) -* [Model Downloader](@ref omz_tools_downloader) diff --git a/tools/legacy/benchmark_app/args_helper.cpp b/tools/legacy/benchmark_app/args_helper.cpp deleted file mode 100644 index ba6e2fe9c65..00000000000 --- a/tools/legacy/benchmark_app/args_helper.cpp +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright (C) 2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "args_helper.hpp" - -#include -#include - -#include -#include "slog.hpp" - -#ifdef _WIN32 - #include "w_dirent.h" -#else - #include -#endif - -/** - * @brief Checks input file argument and add it to files vector - * @param files reference to vector to store file names - * @param arg file or folder name - * @return none - */ -void readInputFilesArguments(std::vector& files, const std::string& arg) { - struct stat sb; - if (stat(arg.c_str(), &sb) != 0) { - slog::warn << "File " << arg << " cannot be opened!" << slog::endl; - return; - } - if (S_ISDIR(sb.st_mode)) { - struct CloseDir { - void operator()(DIR* d) const noexcept { - if (d) { - closedir(d); - } - } - }; - using Dir = std::unique_ptr; - Dir dp(opendir(arg.c_str())); - if (dp == nullptr) { - slog::warn << "Directory " << arg << " cannot be opened!" << slog::endl; - return; - } - - struct dirent* ep; - while (nullptr != (ep = readdir(dp.get()))) { - std::string fileName = ep->d_name; - if (fileName == "." || fileName == "..") - continue; - files.push_back(arg + "/" + ep->d_name); - } - } else { - files.push_back(arg); - } -} - -/** - * @brief This function find -i key in input args. It's necessary to process multiple values for - * single key - * @param files reference to vector - * @return none. - */ -void parseInputFilesArguments(std::vector& files) { - std::vector args = gflags::GetArgvs(); - auto args_it = begin(args); - const auto is_image_arg = [](const std::string& s) { - return s == "-i" || s == "--images"; - }; - const auto is_arg = [](const std::string& s) { - return s.front() == '-'; - }; - - while (args_it != args.end()) { - const auto img_start = std::find_if(args_it, end(args), is_image_arg); - if (img_start == end(args)) { - break; - } - const auto img_begin = std::next(img_start); - const auto img_end = std::find_if(img_begin, end(args), is_arg); - for (auto img = img_begin; img != img_end; ++img) { - readInputFilesArguments(files, *img); - } - args_it = img_end; - } - - if (files.empty()) { - return; - } - size_t max_files = 20; - if (files.size() < max_files) { - slog::info << "Files were added: " << files.size() << slog::endl; - for (const auto& filePath : files) { - slog::info << " " << filePath << slog::endl; - } - } else { - slog::info << "Files were added: " << files.size() << ". Too many to display each of them." << slog::endl; - } -} - -namespace { -std::vector splitStringList(const std::string& str, char delim) { - if (str.empty()) - return {}; - - std::istringstream istr(str); - - std::vector result; - std::string elem; - while (std::getline(istr, elem, delim)) { - if (elem.empty()) { - continue; - } - result.emplace_back(std::move(elem)); - } - - return result; -} - -std::map parseArgMap(std::string argMap) { - argMap.erase(std::remove_if(argMap.begin(), argMap.end(), ::isspace), argMap.end()); - - const auto pairs = splitStringList(argMap, ','); - - std::map parsedMap; - for (auto&& pair : pairs) { - const auto keyValue = splitStringList(pair, ':'); - if (keyValue.size() != 2) { - throw std::invalid_argument("Invalid key/value pair " + pair + ". Expected :"); - } - - parsedMap[keyValue[0]] = keyValue[1]; - } - - return parsedMap; -} - -using supported_precisions_t = std::unordered_map; - -InferenceEngine::Precision getPrecision(std::string value, const supported_precisions_t& supported_precisions) { - std::transform(value.begin(), value.end(), value.begin(), ::toupper); - - const auto precision = supported_precisions.find(value); - if (precision == supported_precisions.end()) { - throw std::logic_error("\"" + value + "\"" + " is not a valid precision"); - } - - return precision->second; -} - -InferenceEngine::Precision getPrecision(const std::string& value) { - static const supported_precisions_t supported_precisions = { - {"FP32", InferenceEngine::Precision::FP32}, {"FP16", InferenceEngine::Precision::FP16}, {"BF16", InferenceEngine::Precision::BF16}, - {"U64", InferenceEngine::Precision::U64}, {"I64", InferenceEngine::Precision::I64}, {"U32", InferenceEngine::Precision::U32}, - {"I32", InferenceEngine::Precision::I32}, {"U16", InferenceEngine::Precision::U16}, {"I16", InferenceEngine::Precision::I16}, - {"U8", InferenceEngine::Precision::U8}, {"I8", InferenceEngine::Precision::I8}, {"BOOL", InferenceEngine::Precision::BOOL}, - }; - - return getPrecision(value, supported_precisions); -} - -void setPrecisions(const InferenceEngine::CNNNetwork& network, const std::string& iop) { - const auto user_precisions_map = parseArgMap(iop); - - auto inputs = network.getInputsInfo(); - auto outputs = network.getOutputsInfo(); - - for (auto&& item : user_precisions_map) { - const auto& layer_name = item.first; - const auto& user_precision = item.second; - - const auto input = inputs.find(layer_name); - const auto output = outputs.find(layer_name); - - if (input != inputs.end()) { - input->second->setPrecision(getPrecision(user_precision)); - } else if (output != outputs.end()) { - output->second->setPrecision(getPrecision(user_precision)); - } else { - throw std::logic_error(layer_name + " is not an input neither output"); - } - } -} - -} // namespace - -void processPrecision(InferenceEngine::CNNNetwork& network, const std::string& ip, const std::string& op, const std::string& iop) { - if (!ip.empty()) { - const auto user_precision = getPrecision(ip); - for (auto&& layer : network.getInputsInfo()) { - layer.second->setPrecision(user_precision); - } - } - - if (!op.empty()) { - auto user_precision = getPrecision(op); - for (auto&& layer : network.getOutputsInfo()) { - layer.second->setPrecision(user_precision); - } - } - - if (!iop.empty()) { - setPrecisions(network, iop); - } -} - -void printInputAndOutputsInfo(const InferenceEngine::CNNNetwork& network) { - std::cout << "Network inputs:" << std::endl; - for (auto&& layer : network.getInputsInfo()) { - std::cout << " " << layer.first << " : " << layer.second->getPrecision() << " / " << layer.second->getLayout() << std::endl; - } - std::cout << "Network outputs:" << std::endl; - for (auto&& layer : network.getOutputsInfo()) { - std::cout << " " << layer.first << " : " << layer.second->getPrecision() << " / " << layer.second->getLayout() << std::endl; - } -} diff --git a/tools/legacy/benchmark_app/args_helper.hpp b/tools/legacy/benchmark_app/args_helper.hpp deleted file mode 100644 index 2ffabc0a90a..00000000000 --- a/tools/legacy/benchmark_app/args_helper.hpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -/** - * @brief a header file with common samples functionality - * @file args_helper.hpp - */ - -#pragma once - -#include -#include -#include - -/** - * @brief This function checks input args and existence of specified files in a given folder - * @param arg path to a file to be checked for existence - * @return files updated vector of verified input files - */ -void readInputFilesArguments(std::vector& files, const std::string& arg); - -/** - * @brief This function find -i/--images key in input args - * It's necessary to process multiple values for single key - * @return files updated vector of verified input files - */ -void parseInputFilesArguments(std::vector& files); - -void processPrecision(InferenceEngine::CNNNetwork& network, const std::string& ip, const std::string& op, const std::string& iop); - -void printInputAndOutputsInfo(const InferenceEngine::CNNNetwork& network); diff --git a/tools/legacy/benchmark_app/benchmark_app.hpp b/tools/legacy/benchmark_app/benchmark_app.hpp deleted file mode 100644 index 2e161b0141c..00000000000 --- a/tools/legacy/benchmark_app/benchmark_app.hpp +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include - -#include -#include -#include - -/// @brief message for help argument -static const char help_message[] = "Print a usage message"; - -/// @brief message for images argument -static const char input_message[] = "Optional. Path to a folder with images and/or binaries or to specific image or binary file."; - -/// @brief message for model argument -static const char model_message[] = "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to a .blob files with " - "a trained compiled model."; - -/// @brief message for execution mode -static const char api_message[] = "Optional. Enable Sync/Async API. Default value is \"async\"."; - -/// @brief message for assigning cnn calculation to device -static const char target_device_message[] = "Optional. Specify a target device to infer on (the list of available devices is shown below). " - "Default value is CPU. Use \"-d HETERO:\" format to specify " - "HETERO plugin. " - "Use \"-d MULTI:\" format to specify MULTI plugin. " - "The application looks for a suitable plugin for the specified device."; - -/// @brief message for iterations count -static const char iterations_count_message[] = "Optional. Number of iterations. " - "If not specified, the number of iterations is calculated depending on a device."; - -/// @brief message for requests count -static const char infer_requests_count_message[] = "Optional. Number of infer requests. Default value is determined automatically for device."; - -/// @brief message for execution time -static const char execution_time_message[] = "Optional. Time in seconds to execute topology."; - -/// @brief message for #threads for CPU inference -static const char infer_num_threads_message[] = "Optional. Number of threads to use for inference on the CPU " - "(including HETERO and MULTI cases)."; - -/// @brief message for #streams for CPU inference -static const char infer_num_streams_message[] = "Optional. Number of streams to use for inference on the CPU or GPU devices " - "(for HETERO and MULTI device cases use format :,: or just " - "). " - "Default value is determined automatically for a device.Please note that although the " - "automatic selection " - "usually provides a reasonable performance, it still may be non - optimal for some cases, " - "especially for " - "very small networks. See sample's README for more details. " - "Also, using nstreams>1 is inherently throughput-oriented option, " - "while for the best-latency estimations the number of streams should be set to 1."; - -/// @brief message for enforcing of BF16 execution where it is possible -static const char enforce_bf16_message[] = "Optional. By default floating point operations execution in bfloat16 precision are enforced " - "if supported by platform.\n" - " 'true' - enable bfloat16 regardless of platform support\n" - " 'false' - disable bfloat16 regardless of platform support"; - -/// @brief message for user library argument -static const char custom_cpu_library_message[] = "Required for CPU custom layers. Absolute path to a shared library with the kernels " - "implementations."; - -/// @brief message for clDNN custom kernels desc -static const char custom_cldnn_message[] = "Required for GPU custom kernels. Absolute path to an .xml file with the kernels description."; - -static const char batch_size_message[] = "Optional. Batch size value. If not specified, the batch size value is determined from " - "Intermediate Representation."; - -// @brief message for CPU threads pinning option -static const char infer_threads_pinning_message[] = - "Optional. Explicit inference threads binding options (leave empty to let the OpenVINO to make a choice):\n" - "\t\t\t\tenabling threads->cores pinning(\"YES\", which is already default for any conventional CPU), \n" - "\t\t\t\tletting the runtime to decide on the threads->different core types(\"HYBRID_AWARE\", which is default on the hybrid CPUs) \n" - "\t\t\t\tthreads->(NUMA)nodes(\"NUMA\") or \n" - "\t\t\t\tcompletely disable(\"NO\") CPU inference threads pinning"; -// @brief message for stream_output option -static const char stream_output_message[] = "Optional. Print progress as a plain text. When specified, an interactive progress bar is " - "replaced with a " - "multiline output."; - -// @brief message for report_type option -static const char report_type_message[] = "Optional. Enable collecting statistics report. \"no_counters\" report contains " - "configuration options specified, resulting FPS and latency. \"average_counters\" " - "report extends \"no_counters\" report and additionally includes average PM " - "counters values for each layer from the network. \"detailed_counters\" report " - "extends \"average_counters\" report and additionally includes per-layer PM " - "counters and latency for each executed infer request."; - -// @brief message for report_folder option -static const char report_folder_message[] = "Optional. Path to a folder where statistics report is stored."; - -// @brief message for exec_graph_path option -static const char exec_graph_path_message[] = "Optional. Path to a file where to store executable graph information serialized."; - -// @brief message for progress bar option -static const char progress_message[] = "Optional. Show progress bar (can affect performance measurement). Default values is " - "\"false\"."; - -// @brief message for performance counters option -static const char pc_message[] = "Optional. Report performance counters."; - -#ifdef USE_OPENCV -// @brief message for load config option -static const char load_config_message[] = "Optional. Path to XML/YAML/JSON file to load custom IE parameters." - " Please note, command line parameters have higher priority then parameters from configuration " - "file."; - -// @brief message for dump config option -static const char dump_config_message[] = "Optional. Path to XML/YAML/JSON file to dump IE parameters, which were set by application."; -#endif - -static const char shape_message[] = "Optional. Set shape for input. For example, \"input1[1,3,224,224],input2[1,4]\" or " - "\"[1,3,224,224]\"" - " in case of one input size."; - -static const char layout_message[] = "Optional. Prompts how network layouts should be treated by application. " - "For example, \"input1[NCHW],input2[NC]\" or \"[NCHW]\" in case of one input size."; - -// @brief message for enabling caching -static const char cache_dir_message[] = "Optional. Enables caching of loaded models to specified directory. " - "List of devices which support caching is shown at the end of this message."; - -// @brief message for single load network -static const char load_from_file_message[] = "Optional. Loads model from file directly without ReadNetwork." - "All CNNNetwork options (like re-shape) will be ignored"; - -// @brief message for quantization bits -static const char gna_qb_message[] = "Optional. Weight bits for quantization: 8 or 16 (default)"; - -static constexpr char inputs_precision_message[] = "Optional. Specifies precision for all input layers of the network."; - -static constexpr char outputs_precision_message[] = "Optional. Specifies precision for all output layers of the network."; - -static constexpr char iop_message[] = "Optional. Specifies precision for input and output layers by name.\n" - " Example: -iop \"input:FP16, output:FP16\".\n" - " Notice that quotes are required.\n" - " Overwrites precision from ip and op options for " - "specified layers."; - -/// @brief Define flag for showing help message
-DEFINE_bool(h, false, help_message); - -/// @brief Declare flag for showing help message
-DECLARE_bool(help); - -/// @brief Define parameter for set image file
-/// i or mif is a required parameter -DEFINE_string(i, "", input_message); - -/// @brief Define parameter for set model file
-/// It is a required parameter -DEFINE_string(m, "", model_message); - -/// @brief Define execution mode -DEFINE_string(api, "async", api_message); - -/// @brief device the target device to infer on
-DEFINE_string(d, "CPU", target_device_message); - -/// @brief Absolute path to CPU library with user layers
-/// It is a required parameter -DEFINE_string(l, "", custom_cpu_library_message); - -/// @brief Define parameter for clDNN custom kernels path
-/// Default is ./lib -DEFINE_string(c, "", custom_cldnn_message); - -/// @brief Iterations count (default 0) -/// Sync mode: iterations count -/// Async mode: StartAsync counts -DEFINE_uint64(niter, 0, iterations_count_message); - -/// @brief Time to execute topology in seconds -DEFINE_uint64(t, 0, execution_time_message); - -/// @brief Number of infer requests in parallel -DEFINE_uint64(nireq, 0, infer_requests_count_message); - -/// @brief Number of threads to use for inference on the CPU in throughput mode (also affects Hetero -/// cases) -DEFINE_uint64(nthreads, 0, infer_num_threads_message); - -/// @brief Number of streams to use for inference on the CPU (also affects Hetero cases) -DEFINE_string(nstreams, "", infer_num_streams_message); - -/// @brief Enforces bf16 execution with bfloat16 precision on systems having this capability -DEFINE_bool(enforcebf16, false, enforce_bf16_message); - -/// @brief Define parameter for batch size
-/// Default is 0 (that means don't specify) -DEFINE_uint64(b, 0, batch_size_message); - -// @brief Enable plugin messages -DEFINE_string(pin, "", infer_threads_pinning_message); - -/// @brief Enables multiline text output instead of progress bar -DEFINE_bool(stream_output, false, stream_output_message); - -/// @brief Enables statistics report collecting -DEFINE_string(report_type, "", report_type_message); - -/// @brief Path to a folder where statistics report is stored -DEFINE_string(report_folder, "", report_folder_message); - -/// @brief Path to a file where to store executable graph information serialized -DEFINE_string(exec_graph_path, "", exec_graph_path_message); - -/// @brief Define flag for showing progress bar
-DEFINE_bool(progress, false, progress_message); - -/// @brief Define flag for showing performance counters
-DEFINE_bool(pc, false, pc_message); - -#ifdef USE_OPENCV -/// @brief Define flag for loading configuration file
-DEFINE_string(load_config, "", load_config_message); - -/// @brief Define flag for dumping configuration file
-DEFINE_string(dump_config, "", dump_config_message); -#endif - -/// @brief Define flag for input shape
-DEFINE_string(shape, "", shape_message); - -/// @brief Define flag for layout shape
-DEFINE_string(layout, "", layout_message); - -/// @brief Define flag for quantization bits (default 16) -DEFINE_int32(qb, 16, gna_qb_message); - -/// @brief Specify precision for all input layers of the network -DEFINE_string(ip, "", inputs_precision_message); - -/// @brief Specify precision for all ouput layers of the network -DEFINE_string(op, "", outputs_precision_message); - -/// @brief Specify precision for input and output layers by name.\n" -/// Example: -iop \"input:FP16, output:FP16\".\n" -/// Notice that quotes are required.\n" -/// Overwrites layout from ip and op options for specified layers."; -DEFINE_string(iop, "", iop_message); - -/// @brief Define parameter for cache model dir
-DEFINE_string(cache_dir, "", cache_dir_message); - -/// @brief Define flag for load network from model file by name without ReadNetwork
-DEFINE_bool(load_from_file, false, load_from_file_message); - -/** - * @brief This function show a help message - */ -static void showUsage() { - std::cout << std::endl; - std::cout << "benchmark_app [OPTION]" << std::endl; - std::cout << "Options:" << std::endl; - std::cout << std::endl; - std::cout << " -h, --help " << help_message << std::endl; - std::cout << " -m \"\" " << model_message << std::endl; - std::cout << " -i \"\" " << input_message << std::endl; - std::cout << " -d \"\" " << target_device_message << std::endl; - std::cout << " -l \"\" " << custom_cpu_library_message << std::endl; - std::cout << " Or" << std::endl; - std::cout << " -c \"\" " << custom_cldnn_message << std::endl; - std::cout << " -api \"\" " << api_message << std::endl; - std::cout << " -niter \"\" " << iterations_count_message << std::endl; - std::cout << " -nireq \"\" " << infer_requests_count_message << std::endl; - std::cout << " -b \"\" " << batch_size_message << std::endl; - std::cout << " -stream_output " << stream_output_message << std::endl; - std::cout << " -t " << execution_time_message << std::endl; - std::cout << " -progress " << progress_message << std::endl; - std::cout << " -shape " << shape_message << std::endl; - std::cout << " -layout " << layout_message << std::endl; - std::cout << " -cache_dir \"\" " << cache_dir_message << std::endl; - std::cout << " -load_from_file " << load_from_file_message << std::endl; - std::cout << std::endl << " device-specific performance options:" << std::endl; - std::cout << " -nstreams \"\" " << infer_num_streams_message << std::endl; - std::cout << " -nthreads \"\" " << infer_num_threads_message << std::endl; - std::cout << " -enforcebf16= " << enforce_bf16_message << std::endl; - std::cout << " -pin \"YES\"/\"HYBRID_AWARE\"/\"NO\"/\"NUMA\" " << infer_threads_pinning_message << std::endl; - std::cout << std::endl << " Statistics dumping options:" << std::endl; - std::cout << " -report_type \"\" " << report_type_message << std::endl; - std::cout << " -report_folder " << report_folder_message << std::endl; - std::cout << " -exec_graph_path " << exec_graph_path_message << std::endl; - std::cout << " -pc " << pc_message << std::endl; -#ifdef USE_OPENCV - std::cout << " -dump_config " << dump_config_message << std::endl; - std::cout << " -load_config " << load_config_message << std::endl; -#endif - std::cout << " -qb " << gna_qb_message << std::endl; - std::cout << " -ip " << inputs_precision_message << std::endl; - std::cout << " -op " << outputs_precision_message << std::endl; - std::cout << " -iop \"\" " << iop_message << std::endl; -} diff --git a/tools/legacy/benchmark_app/common.hpp b/tools/legacy/benchmark_app/common.hpp deleted file mode 100644 index 7e1aaa9ded6..00000000000 --- a/tools/legacy/benchmark_app/common.hpp +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -/** - * @brief a header file with common samples functionality - * @file common.hpp - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef UNUSED - #if defined(_MSC_VER) && !defined(__clang__) - #define UNUSED - #else - #define UNUSED __attribute__((unused)) - #endif -#endif -\ - -/** - * @brief Get extension from filename - * @param filename - name of the file which extension should be extracted - * @return string with extracted file extension - */ -inline std::string fileExt(const std::string& filename) { - auto pos = filename.rfind('.'); - if (pos == std::string::npos) - return ""; - return filename.substr(pos + 1); -} - -inline std::ostream& operator<<(std::ostream& os, const InferenceEngine::Version& version) { - os << "\t" << version.description << " version ......... "; - os << IE_VERSION_MAJOR << "." << IE_VERSION_MINOR << "." << IE_VERSION_PATCH; - - os << "\n\tBuild ........... "; - os << version.buildNumber; - - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const InferenceEngine::Version* version) { - if (nullptr != version) { - os << std::endl << *version; - } - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const std::map& versions) { - for (auto&& version : versions) { - os << "\t" << version.first << std::endl; - os << version.second << std::endl; - } - - return os; -} - -static std::vector> perfCountersSorted( - std::map perfMap) { - using perfItem = std::pair; - std::vector sorted; - for (auto& kvp : perfMap) - sorted.push_back(kvp); - - std::stable_sort(sorted.begin(), sorted.end(), [](const perfItem& l, const perfItem& r) { - return l.second.execution_index < r.second.execution_index; - }); - - return sorted; -} - -static UNUSED void printPerformanceCounts(const std::map& performanceMap, std::ostream& stream, - std::string deviceName, bool bshowHeader = true) { - std::ios::fmtflags fmt(stream.flags()); - long long totalTime = 0; - // Print performance counts - if (bshowHeader) { - stream << std::endl << "performance counts:" << std::endl << std::endl; - } - - auto performanceMapSorted = perfCountersSorted(performanceMap); - - for (const auto& it : performanceMapSorted) { - std::string toPrint(it.first); - const int maxLayerName = 30; - - if (it.first.length() >= maxLayerName) { - toPrint = it.first.substr(0, maxLayerName - 4); - toPrint += "..."; - } - - stream << std::setw(maxLayerName) << std::left << toPrint; - switch (it.second.status) { - case InferenceEngine::InferenceEngineProfileInfo::EXECUTED: - stream << std::setw(15) << std::left << "EXECUTED"; - break; - case InferenceEngine::InferenceEngineProfileInfo::NOT_RUN: - stream << std::setw(15) << std::left << "NOT_RUN"; - break; - case InferenceEngine::InferenceEngineProfileInfo::OPTIMIZED_OUT: - stream << std::setw(15) << std::left << "OPTIMIZED_OUT"; - break; - } - stream << std::setw(30) << std::left << "layerType: " + std::string(it.second.layer_type) + " "; - stream << std::setw(20) << std::left << "realTime: " + std::to_string(it.second.realTime_uSec); - stream << std::setw(20) << std::left << "cpu: " + std::to_string(it.second.cpu_uSec); - stream << " execType: " << it.second.exec_type << std::endl; - if (it.second.realTime_uSec > 0) { - totalTime += it.second.realTime_uSec; - } - } - stream << std::setw(20) << std::left << "Total time: " + std::to_string(totalTime) << " microseconds" << std::endl; - std::cout << std::endl; - std::cout << "Full device name: " << deviceName << std::endl; - std::cout << std::endl; - stream.flags(fmt); -} - -static UNUSED void printPerformanceCounts(InferenceEngine::InferRequest request, std::ostream& stream, std::string deviceName, bool bshowHeader = true) { - auto performanceMap = request.GetPerformanceCounts(); - printPerformanceCounts(performanceMap, stream, deviceName, bshowHeader); -} - -inline std::string getFullDeviceName(std::map& devicesMap, std::string device) { - std::map::iterator it = devicesMap.find(device); - if (it != devicesMap.end()) { - return it->second; - } else { - return ""; - } -} - -inline std::string getFullDeviceName(InferenceEngine::Core& ie, std::string device) { - InferenceEngine::Parameter p; - try { - p = ie.GetMetric(device, METRIC_KEY(FULL_DEVICE_NAME)); - return p.as(); - } catch (InferenceEngine::Exception&) { - return ""; - } -} - -inline void showAvailableDevices() { - InferenceEngine::Core ie; - std::vector devices = ie.GetAvailableDevices(); - - std::cout << std::endl; - std::cout << "Available target devices:"; - for (const auto& device : devices) { - std::cout << " " << device; - } - std::cout << std::endl; -} diff --git a/tools/legacy/benchmark_app/console_progress.hpp b/tools/legacy/benchmark_app/console_progress.hpp deleted file mode 100644 index 36c7233ae18..00000000000 --- a/tools/legacy/benchmark_app/console_progress.hpp +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include - -/** - * @class ConsoleProgress - * @brief A ConsoleProgress class provides functionality for printing progress dynamics - */ -class ConsoleProgress { - static const size_t DEFAULT_DETALIZATION = 20; - static const size_t DEFAULT_PERCENT_TO_UPDATE_PROGRESS = 1; - - size_t total; - size_t cur_progress = 0; - size_t prev_progress = 0; - bool stream_output; - size_t detalization; - size_t percent_to_update; - -public: - /** - * @brief A constructor of ConsoleProgress class - * @param _total - maximum value that is correspondent to 100% - * @param _detalization - number of symbols(.) to use to represent progress - */ - explicit ConsoleProgress(size_t _total, - bool _stream_output = false, - size_t _percent_to_update = DEFAULT_PERCENT_TO_UPDATE_PROGRESS, - size_t _detalization = DEFAULT_DETALIZATION) - : total(_total), - detalization(_detalization), - percent_to_update(_percent_to_update) { - stream_output = _stream_output; - if (total == 0) { - total = 1; - } - } - - /** - * @brief Shows progress with current data. Progress is shown from the beginning of the current - * line. - */ - void showProgress() const { - std::stringstream strm; - if (!stream_output) { - strm << '\r'; - } - strm << "Progress: ["; - size_t i = 0; - for (; i < detalization * cur_progress / total; i++) { - strm << "."; - } - for (; i < detalization; i++) { - strm << " "; - } - strm << "] " << std::setw(3) << 100 * cur_progress / total << "% done"; - if (stream_output) { - strm << std::endl; - } - std::fputs(strm.str().c_str(), stdout); - std::fflush(stdout); - } - - /** - * @brief Updates current value and progressbar - */ - void updateProgress() { - if (cur_progress > total) - cur_progress = total; - size_t prev_percent = 100 * prev_progress / total; - size_t cur_percent = 100 * cur_progress / total; - - if (prev_progress == 0 || cur_progress == total || prev_percent + percent_to_update <= cur_percent) { - showProgress(); - prev_progress = cur_progress; - } - } - - /** - * @brief Adds value to currently represented and redraw progressbar - * @param add - value to add - */ - void addProgress(int add) { - if (add < 0 && -add > static_cast(cur_progress)) { - add = -static_cast(cur_progress); - } - cur_progress += add; - updateProgress(); - } - - /** - * @brief Output end line. - * @return - */ - void finish() { - std::stringstream strm; - strm << std::endl; - std::fputs(strm.str().c_str(), stdout); - std::fflush(stdout); - } -}; diff --git a/tools/legacy/benchmark_app/csv_dumper.hpp b/tools/legacy/benchmark_app/csv_dumper.hpp deleted file mode 100644 index 5e0bc039eae..00000000000 --- a/tools/legacy/benchmark_app/csv_dumper.hpp +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include - -#include -#include -#include "slog.hpp" -#include -#include - -/** - * @class CsvDumper - * @brief A CsvDumper class provides functionality for dumping the values in CSV files - */ -class CsvDumper { - std::ofstream file; - std::string filename; - bool canDump = true; - char delimiter = ';'; - - std::string generateFilename() { - std::stringstream filename; - filename << "dumpfile-"; - filename << time(nullptr); - filename << ".csv"; - return filename.str(); - } - -public: - /** - * @brief A constructor. Disables dumping in case dump file cannot be created - * @param enabled - True if dumping is enabled by default. - * @param name - name of file to dump to. File won't be created if first parameter is false. - */ - explicit CsvDumper(bool enabled = true, const std::string& name = ""): canDump(enabled) { - if (!canDump) { - return; - } - filename = (name == "" ? generateFilename() : name); - file.open(filename, std::ios::out); - if (!file) { - slog::warn << "Cannot create dump file! Disabling dump." << slog::endl; - canDump = false; - } - } - - /** - * @brief Sets a delimiter to use in csv file - * @param c - Delimiter char - * @return - */ - void setDelimiter(char c) { - delimiter = c; - } - - /** - * @brief Overloads operator to organize streaming values to file. Does nothing if dumping is - * disabled Adds delimiter at the end of value provided - * @param add - value to add to dump - * @return reference to same object - */ - template - CsvDumper& operator<<(const T& add) { - if (canDump) { - file << add << delimiter; - } - return *this; - } - - /** - * @brief Finishes line in dump file. Does nothing if dumping is disabled - */ - void endLine() { - if (canDump) { - file << "\n"; - } - } - - /** - * @brief Gets information if dump is enabled. - * @return true if dump is enabled and file was successfully created - */ - bool dumpEnabled() { - return canDump; - } - - /** - * @brief Gets name of a dump file - * @return name of a dump file - */ - std::string getFilename() const { - return filename; - } -}; diff --git a/tools/legacy/benchmark_app/infer_request_wrap.hpp b/tools/legacy/benchmark_app/infer_request_wrap.hpp deleted file mode 100644 index 741b2ad7f13..00000000000 --- a/tools/legacy/benchmark_app/infer_request_wrap.hpp +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "statistics_report.hpp" - -typedef std::chrono::high_resolution_clock Time; -typedef std::chrono::nanoseconds ns; - -typedef std::function QueueCallbackFunction; - -/// @brief Wrapper class for InferenceEngine::InferRequest. Handles asynchronous callbacks and calculates execution time. -class InferReqWrap final { -public: - using Ptr = std::shared_ptr; - - ~InferReqWrap() = default; - - explicit InferReqWrap(InferenceEngine::ExecutableNetwork& net, size_t id, QueueCallbackFunction callbackQueue) - : _request(net.CreateInferRequest()), _id(id), _callbackQueue(callbackQueue) { - _request.SetCompletionCallback([&]() { - _endTime = Time::now(); - _callbackQueue(_id, getExecutionTimeInMilliseconds()); - }); - } - - void startAsync() { - _startTime = Time::now(); - _request.StartAsync(); - } - - void wait() { - _request.Wait(InferenceEngine::InferRequest::RESULT_READY); - } - - void infer() { - _startTime = Time::now(); - _request.Infer(); - _endTime = Time::now(); - _callbackQueue(_id, getExecutionTimeInMilliseconds()); - } - - std::map getPerformanceCounts() { - return _request.GetPerformanceCounts(); - } - - InferenceEngine::Blob::Ptr getBlob(const std::string& name) { - return _request.GetBlob(name); - } - - double getExecutionTimeInMilliseconds() const { - auto execTime = std::chrono::duration_cast(_endTime - _startTime); - return static_cast(execTime.count()) * 0.000001; - } - -private: - InferenceEngine::InferRequest _request; - Time::time_point _startTime; - Time::time_point _endTime; - size_t _id; - QueueCallbackFunction _callbackQueue; -}; - -class InferRequestsQueue final { -public: - InferRequestsQueue(InferenceEngine::ExecutableNetwork& net, size_t nireq) { - for (size_t id = 0; id < nireq; id++) { - requests.push_back( - std::make_shared(net, id, std::bind(&InferRequestsQueue::putIdleRequest, this, std::placeholders::_1, std::placeholders::_2))); - _idleIds.push(id); - } - resetTimes(); - } - ~InferRequestsQueue() { - // Inference Request guarantee that it will wait for all asynchronous internal tasks in destructor - // So it should be released before any context that the request can use inside internal asynchronous tasks - // For example all members of InferRequestsQueue would be destroyed before `requests` vector - // So requests can try to use this members from `putIdleRequest()` that would be called from request callback - // To avoid this we should move this vector declaration after all members declaration or just clear it manually in destructor - requests.clear(); - } - - void resetTimes() { - _startTime = Time::time_point::max(); - _endTime = Time::time_point::min(); - _latencies.clear(); - } - - double getDurationInMilliseconds() { - return std::chrono::duration_cast(_endTime - _startTime).count() * 0.000001; - } - - void putIdleRequest(size_t id, const double latency) { - std::unique_lock lock(_mutex); - _latencies.push_back(latency); - _idleIds.push(id); - _endTime = std::max(Time::now(), _endTime); - _cv.notify_one(); - } - - InferReqWrap::Ptr getIdleRequest() { - std::unique_lock lock(_mutex); - _cv.wait(lock, [this] { - return _idleIds.size() > 0; - }); - auto request = requests.at(_idleIds.front()); - _idleIds.pop(); - _startTime = std::min(Time::now(), _startTime); - return request; - } - - void waitAll() { - std::unique_lock lock(_mutex); - _cv.wait(lock, [this] { - return _idleIds.size() == requests.size(); - }); - } - - std::vector getLatencies() { - return _latencies; - } - - std::vector requests; - -private: - std::queue _idleIds; - std::mutex _mutex; - std::condition_variable _cv; - Time::time_point _startTime; - Time::time_point _endTime; - std::vector _latencies; -}; diff --git a/tools/legacy/benchmark_app/inputs_filling.cpp b/tools/legacy/benchmark_app/inputs_filling.cpp deleted file mode 100644 index 4ffa8f8ffdd..00000000000 --- a/tools/legacy/benchmark_app/inputs_filling.cpp +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "inputs_filling.hpp" - -#include - -#include -#include -#include "slog.hpp" -#include -#include -#include - -using namespace InferenceEngine; - -#ifdef USE_OPENCV -static const std::vector supported_image_extensions = {"bmp", "dib", "jpeg", "jpg", "jpe", "jp2", "png", - "pbm", "pgm", "ppm", "sr", "ras", "tiff", "tif"}; -#else -static const std::vector supported_image_extensions = {"bmp"}; -#endif -static const std::vector supported_binary_extensions = {"bin"}; - -std::vector filterFilesByExtensions(const std::vector& filePaths, const std::vector& extensions) { - std::vector filtered; - auto getExtension = [](const std::string& name) { - auto extensionPosition = name.rfind('.', name.size()); - return extensionPosition == std::string::npos ? "" : name.substr(extensionPosition + 1, name.size() - 1); - }; - for (auto& filePath : filePaths) { - auto extension = getExtension(filePath); - std::transform(extension.begin(), extension.end(), extension.begin(), ::tolower); - if (std::find(extensions.begin(), extensions.end(), extension) != extensions.end()) { - filtered.push_back(filePath); - } - } - return filtered; -} - -template -void fillBlobImage(Blob::Ptr& inputBlob, const std::vector& filePaths, const size_t& batchSize, const benchmark_app::InputInfo& app_info, - const size_t& requestId, const size_t& inputId, const size_t& inputSize) { - MemoryBlob::Ptr minput = as(inputBlob); - if (!minput) { - IE_THROW() << "We expect inputBlob to be inherited from MemoryBlob in " - "fillBlobImage, " - << "but by fact we were not able to cast inputBlob to MemoryBlob"; - } - // locked memory holder should be alive all time while access to its buffer - // happens - auto minputHolder = minput->wmap(); - auto inputBlobData = minputHolder.as(); - - /** Collect images data ptrs **/ - std::vector> vreader; - vreader.reserve(batchSize); - - for (size_t i = 0ULL, inputIndex = requestId * batchSize * inputSize + inputId; i < batchSize; i++, inputIndex += inputSize) { - inputIndex %= filePaths.size(); - - slog::info << "Prepare image " << filePaths[inputIndex] << slog::endl; - FormatReader::ReaderPtr reader(filePaths[inputIndex].c_str()); - if (reader.get() == nullptr) { - slog::warn << "Image " << filePaths[inputIndex] << " cannot be read!" << slog::endl << slog::endl; - continue; - } - - /** Getting image data **/ - std::shared_ptr imageData(reader->getData(app_info.width(), app_info.height())); - if (imageData) { - vreader.push_back(imageData); - } - } - - /** Fill input tensor with images. First b channel, then g and r channels **/ - const size_t numChannels = app_info.channels(); - const size_t width = app_info.width(); - const size_t height = app_info.height(); - /** Iterate over all input images **/ - for (size_t imageId = 0; imageId < vreader.size(); ++imageId) { - /** Iterate over all width **/ - for (size_t w = 0; w < app_info.width(); ++w) { - /** Iterate over all height **/ - for (size_t h = 0; h < app_info.height(); ++h) { - /** Iterate over all channels **/ - for (size_t ch = 0; ch < numChannels; ++ch) { - /** [images stride + channels stride + pixel id ] all in - * bytes **/ - size_t offset = imageId * numChannels * width * height + (((app_info.layout == "NCHW") || (app_info.layout == "CHW")) - ? (ch * width * height + h * width + w) - : (h * width * numChannels + w * numChannels + ch)); - inputBlobData[offset] = static_cast(vreader.at(imageId).get()[h * width * numChannels + w * numChannels + ch]); - } - } - } - } -} - -template -void fillBlobBinary(Blob::Ptr& inputBlob, const std::vector& filePaths, const size_t& batchSize, const size_t& requestId, const size_t& inputId, - const size_t& inputSize) { - MemoryBlob::Ptr minput = as(inputBlob); - if (!minput) { - IE_THROW() << "We expect inputBlob to be inherited from MemoryBlob in " - "fillBlobBinary, " - << "but by fact we were not able to cast inputBlob to MemoryBlob"; - } - // locked memory holder should be alive all time while access to its buffer - // happens - auto minputHolder = minput->wmap(); - - auto inputBlobData = minputHolder.as(); - for (size_t i = 0ULL, inputIndex = requestId * batchSize * inputSize + inputId; i < batchSize; i++, inputIndex += inputSize) { - inputIndex %= filePaths.size(); - - slog::info << "Prepare binary file " << filePaths[inputIndex] << slog::endl; - std::ifstream binaryFile(filePaths[inputIndex], std::ios_base::binary | std::ios_base::ate); - if (!binaryFile) { - IE_THROW() << "Cannot open " << filePaths[inputIndex]; - } - - auto fileSize = static_cast(binaryFile.tellg()); - binaryFile.seekg(0, std::ios_base::beg); - if (!binaryFile.good()) { - IE_THROW() << "Can not read " << filePaths[inputIndex]; - } - auto inputSize = inputBlob->size() * sizeof(T) / batchSize; - if (fileSize != inputSize) { - IE_THROW() << "File " << filePaths[inputIndex] << " contains " << std::to_string(fileSize) - << " bytes " - "but the network expects " - << std::to_string(inputSize); - } - binaryFile.read(&inputBlobData[i * inputSize], inputSize); - } -} - -template -using uniformDistribution = - typename std::conditional::value, std::uniform_real_distribution, - typename std::conditional::value, std::uniform_int_distribution, void>::type>::type; - -template -void fillBlobRandom(Blob::Ptr& inputBlob, T rand_min = std::numeric_limits::min(), T rand_max = std::numeric_limits::max()) { - MemoryBlob::Ptr minput = as(inputBlob); - if (!minput) { - IE_THROW() << "We expect inputBlob to be inherited from MemoryBlob in " - "fillBlobRandom, " - << "but by fact we were not able to cast inputBlob to MemoryBlob"; - } - // locked memory holder should be alive all time while access to its buffer - // happens - auto minputHolder = minput->wmap(); - - auto inputBlobData = minputHolder.as(); - std::mt19937 gen(0); - uniformDistribution distribution(rand_min, rand_max); - for (size_t i = 0; i < inputBlob->size(); i++) { - inputBlobData[i] = static_cast(distribution(gen)); - } -} - -template -void fillBlobImInfo(Blob::Ptr& inputBlob, const size_t& batchSize, std::pair image_size) { - MemoryBlob::Ptr minput = as(inputBlob); - if (!minput) { - IE_THROW() << "We expect inputBlob to be inherited from MemoryBlob in " - "fillBlobImInfo, " - << "but by fact we were not able to cast inputBlob to MemoryBlob"; - } - // locked memory holder should be alive all time while access to its buffer - // happens - auto minputHolder = minput->wmap(); - - auto inputBlobData = minputHolder.as(); - for (size_t b = 0; b < batchSize; b++) { - size_t iminfoSize = inputBlob->size() / batchSize; - for (size_t i = 0; i < iminfoSize; i++) { - size_t index = b * iminfoSize + i; - if (0 == i) - inputBlobData[index] = static_cast(image_size.first); - else if (1 == i) - inputBlobData[index] = static_cast(image_size.second); - else - inputBlobData[index] = 1; - } - } -} - -void fillBlobs(const std::vector& inputFiles, const size_t& batchSize, benchmark_app::InputsInfo& app_inputs_info, - std::vector requests) { - std::vector> input_image_sizes; - for (auto& item : app_inputs_info) { - if (item.second.isImage()) { - input_image_sizes.push_back(std::make_pair(item.second.width(), item.second.height())); - } - slog::info << "Network input '" << item.first << "' precision " << item.second.precision << ", dimensions (" << item.second.layout << "): "; - for (const auto& i : item.second.shape) { - slog::info << i << " "; - } - slog::info << slog::endl; - } - - size_t imageInputCount = input_image_sizes.size(); - size_t binaryInputCount = app_inputs_info.size() - imageInputCount; - - std::vector binaryFiles; - std::vector imageFiles; - - if (inputFiles.empty()) { - slog::warn << "No input files were given: all inputs will be filled with " - "random values!" - << slog::endl; - } else { - binaryFiles = filterFilesByExtensions(inputFiles, supported_binary_extensions); - std::sort(std::begin(binaryFiles), std::end(binaryFiles)); - - auto binaryToBeUsed = binaryInputCount * batchSize * requests.size(); - if (binaryToBeUsed > 0 && binaryFiles.empty()) { - std::stringstream ss; - for (auto& ext : supported_binary_extensions) { - if (!ss.str().empty()) { - ss << ", "; - } - ss << ext; - } - slog::warn << "No supported binary inputs found! Please check your file " - "extensions: " - << ss.str() << slog::endl; - } else if (binaryToBeUsed > binaryFiles.size()) { - slog::warn << "Some binary input files will be duplicated: " << binaryToBeUsed << " files are required but only " << binaryFiles.size() - << " are provided" << slog::endl; - } else if (binaryToBeUsed < binaryFiles.size()) { - slog::warn << "Some binary input files will be ignored: only " << binaryToBeUsed << " are required from " << binaryFiles.size() << slog::endl; - } - - imageFiles = filterFilesByExtensions(inputFiles, supported_image_extensions); - std::sort(std::begin(imageFiles), std::end(imageFiles)); - - auto imagesToBeUsed = imageInputCount * batchSize * requests.size(); - if (imagesToBeUsed > 0 && imageFiles.empty()) { - std::stringstream ss; - for (auto& ext : supported_image_extensions) { - if (!ss.str().empty()) { - ss << ", "; - } - ss << ext; - } - slog::warn << "No supported image inputs found! Please check your file " - "extensions: " - << ss.str() << slog::endl; - } else if (imagesToBeUsed > imageFiles.size()) { - slog::warn << "Some image input files will be duplicated: " << imagesToBeUsed << " files are required but only " << imageFiles.size() - << " are provided" << slog::endl; - } else if (imagesToBeUsed < imageFiles.size()) { - slog::warn << "Some image input files will be ignored: only " << imagesToBeUsed << " are required from " << imageFiles.size() << slog::endl; - } - } - - for (size_t requestId = 0; requestId < requests.size(); requestId++) { - slog::info << "Infer Request " << requestId << " filling" << slog::endl; - - size_t imageInputId = 0; - size_t binaryInputId = 0; - for (auto& item : app_inputs_info) { - Blob::Ptr inputBlob = requests.at(requestId)->getBlob(item.first); - auto app_info = app_inputs_info.at(item.first); - auto precision = app_info.precision; - if (app_info.isImage()) { - if (!imageFiles.empty()) { - // Fill with Images - if (precision == InferenceEngine::Precision::FP32) { - fillBlobImage(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount); - } else if (precision == InferenceEngine::Precision::FP16) { - fillBlobImage(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount); - } else if (precision == InferenceEngine::Precision::I32) { - fillBlobImage(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount); - } else if (precision == InferenceEngine::Precision::I64) { - fillBlobImage(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount); - } else if (precision == InferenceEngine::Precision::U8) { - fillBlobImage(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount); - } else { - IE_THROW() << "Input precision is not supported for " << item.first; - } - continue; - } - } else { - if (!binaryFiles.empty()) { - // Fill with binary files - if (precision == InferenceEngine::Precision::FP32) { - fillBlobBinary(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount); - } else if (precision == InferenceEngine::Precision::FP16) { - fillBlobBinary(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount); - } else if (precision == InferenceEngine::Precision::I32) { - fillBlobBinary(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount); - } else if (precision == InferenceEngine::Precision::I64) { - fillBlobBinary(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount); - } else if ((precision == InferenceEngine::Precision::U8) || (precision == InferenceEngine::Precision::BOOL)) { - fillBlobBinary(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount); - } else { - IE_THROW() << "Input precision is not supported for " << item.first; - } - continue; - } - - if (app_info.isImageInfo() && (input_image_sizes.size() == 1)) { - // Most likely it is image info: fill with image information - auto image_size = input_image_sizes.at(0); - slog::info << "Fill input '" << item.first << "' with image size " << image_size.first << "x" << image_size.second << slog::endl; - if (precision == InferenceEngine::Precision::FP32) { - fillBlobImInfo(inputBlob, batchSize, image_size); - } else if (precision == InferenceEngine::Precision::FP16) { - fillBlobImInfo(inputBlob, batchSize, image_size); - } else if (precision == InferenceEngine::Precision::I32) { - fillBlobImInfo(inputBlob, batchSize, image_size); - } else if (precision == InferenceEngine::Precision::I64) { - fillBlobImInfo(inputBlob, batchSize, image_size); - } else { - IE_THROW() << "Input precision is not supported for image info!"; - } - continue; - } - } - // Fill random - slog::info << "Fill input '" << item.first << "' with random values (" << std::string((app_info.isImage() ? "image" : "some binary data")) - << " is expected)" << slog::endl; - if (precision == InferenceEngine::Precision::FP32) { - fillBlobRandom(inputBlob); - } else if (precision == InferenceEngine::Precision::FP16) { - fillBlobRandom(inputBlob); - } else if (precision == InferenceEngine::Precision::I32) { - fillBlobRandom(inputBlob); - } else if (precision == InferenceEngine::Precision::I64) { - fillBlobRandom(inputBlob); - } else if (precision == InferenceEngine::Precision::U8) { - // uniform_int_distribution is not allowed in the C++17 - // standard and vs2017/19 - fillBlobRandom(inputBlob); - } else if (precision == InferenceEngine::Precision::I8) { - // uniform_int_distribution is not allowed in the C++17 standard - // and vs2017/19 - fillBlobRandom(inputBlob); - } else if (precision == InferenceEngine::Precision::U16) { - fillBlobRandom(inputBlob); - } else if (precision == InferenceEngine::Precision::I16) { - fillBlobRandom(inputBlob); - } else if (precision == InferenceEngine::Precision::BOOL) { - fillBlobRandom(inputBlob, 0, 1); - } else { - IE_THROW() << "Input precision is not supported for " << item.first; - } - } - } -} diff --git a/tools/legacy/benchmark_app/inputs_filling.hpp b/tools/legacy/benchmark_app/inputs_filling.hpp deleted file mode 100644 index 4410faae11e..00000000000 --- a/tools/legacy/benchmark_app/inputs_filling.hpp +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include - -#include "infer_request_wrap.hpp" -#include "utils.hpp" - -void fillBlobs(const std::vector& inputFiles, const size_t& batchSize, benchmark_app::InputsInfo& app_inputs_info, - std::vector requests); diff --git a/tools/legacy/benchmark_app/main.cpp b/tools/legacy/benchmark_app/main.cpp deleted file mode 100644 index 08cadd62db8..00000000000 --- a/tools/legacy/benchmark_app/main.cpp +++ /dev/null @@ -1,700 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include -#include -#include -#include -#include -#include -#include -#include "args_helper.hpp" -#include "common.hpp" -#include "slog.hpp" -#include -#include -#include - -#include "benchmark_app.hpp" -#include "infer_request_wrap.hpp" -#include "inputs_filling.hpp" -#include "progress_bar.hpp" -#include "statistics_report.hpp" -#include "utils.hpp" - -using namespace InferenceEngine; - -static const size_t progressBarDefaultTotalCount = 1000; - -uint64_t getDurationInMilliseconds(uint64_t duration) { - return duration * 1000LL; -} - -uint64_t getDurationInNanoseconds(uint64_t duration) { - return duration * 1000000000LL; -} - -bool ParseAndCheckCommandLine(int argc, char* argv[]) { - // ---------------------------Parsing and validating input - // arguments-------------------------------------- - slog::info << "Parsing input parameters" << slog::endl; - gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); - if (FLAGS_help || FLAGS_h) { - showUsage(); - showAvailableDevices(); - return false; - } - - if (FLAGS_m.empty()) { - showUsage(); - throw std::logic_error("Model is required but not set. Please set -m option."); - } - - if (FLAGS_api != "async" && FLAGS_api != "sync") { - throw std::logic_error("Incorrect API. Please set -api option to `sync` or `async` value."); - } - - if (!FLAGS_report_type.empty() && FLAGS_report_type != noCntReport && FLAGS_report_type != averageCntReport && FLAGS_report_type != detailedCntReport) { - std::string err = "only " + std::string(noCntReport) + "/" + std::string(averageCntReport) + "/" + std::string(detailedCntReport) + - " report types are supported (invalid -report_type option value)"; - throw std::logic_error(err); - } - - if ((FLAGS_report_type == averageCntReport) && ((FLAGS_d.find("MULTI") != std::string::npos))) { - throw std::logic_error("only " + std::string(detailedCntReport) + " report type is supported for MULTI device"); - } - - bool isNetworkCompiled = fileExt(FLAGS_m) == "blob"; - bool isPrecisionSet = !(FLAGS_ip.empty() && FLAGS_op.empty() && FLAGS_iop.empty()); - if (isNetworkCompiled && isPrecisionSet) { - std::string err = std::string("Cannot set precision for a compiled network. ") + std::string("Please re-compile your network with required precision " - "using compile_tool"); - - throw std::logic_error(err); - } - return true; -} - -static void next_step(const std::string additional_info = "") { - static size_t step_id = 0; - static const std::map step_names = {{1, "Parsing and validating input arguments"}, - {2, "Loading Inference Engine"}, - {3, "Setting device configuration"}, - {4, "Reading network files"}, - {5, "Resizing network to match image sizes and given batch"}, - {6, "Configuring input of the model"}, - {7, "Loading the model to the device"}, - {8, "Setting optimal runtime parameters"}, - {9, "Creating infer requests and filling input blobs with images"}, - {10, "Measuring performance"}, - {11, "Dumping statistics report"}}; - - step_id++; - if (step_names.count(step_id) == 0) - IE_THROW() << "Step ID " << step_id << " is out of total steps number " << step_names.size(); - - std::cout << "[Step " << step_id << "/" << step_names.size() << "] " << step_names.at(step_id) - << (additional_info.empty() ? "" : " (" + additional_info + ")") << std::endl; -} - -template -T getMedianValue(const std::vector& vec) { - std::vector sortedVec(vec); - std::sort(sortedVec.begin(), sortedVec.end()); - return (sortedVec.size() % 2 != 0) ? sortedVec[sortedVec.size() / 2ULL] - : (sortedVec[sortedVec.size() / 2ULL] + sortedVec[sortedVec.size() / 2ULL - 1ULL]) / static_cast(2.0); -} - -/** - * @brief The entry point of the benchmark application - */ -int main(int argc, char* argv[]) { - std::shared_ptr statistics; - try { - ExecutableNetwork exeNetwork; - - // ----------------- 1. Parsing and validating input arguments - // ------------------------------------------------- - next_step(); - - if (!ParseAndCheckCommandLine(argc, argv)) { - return 0; - } - - bool isNetworkCompiled = fileExt(FLAGS_m) == "blob"; - if (isNetworkCompiled) { - slog::info << "Network is compiled" << slog::endl; - } - - std::vector flags; - StatisticsReport::Parameters command_line_arguments; - gflags::GetAllFlags(&flags); - for (auto& flag : flags) { - if (!flag.is_default) { - command_line_arguments.push_back({flag.name, flag.current_value}); - } - } - if (!FLAGS_report_type.empty()) { - statistics = std::make_shared(StatisticsReport::Config {FLAGS_report_type, FLAGS_report_folder}); - statistics->addParameters(StatisticsReport::Category::COMMAND_LINE_PARAMETERS, command_line_arguments); - } - auto isFlagSetInCommandLine = [&command_line_arguments](const std::string& name) { - return (std::find_if(command_line_arguments.begin(), command_line_arguments.end(), [name](const std::pair& p) { - return p.first == name; - }) != command_line_arguments.end()); - }; - - std::string device_name = FLAGS_d; - - // Parse devices - auto devices = parseDevices(device_name); - - // Parse nstreams per device - std::map device_nstreams = parseNStreamsValuePerDevice(devices, FLAGS_nstreams); - - // Load device config file if specified - std::map> config; -#ifdef USE_OPENCV - if (!FLAGS_load_config.empty()) { - load_config(FLAGS_load_config, config); - } -#endif - /** This vector stores paths to the processed images **/ - std::vector inputFiles; - parseInputFilesArguments(inputFiles); - - // ----------------- 2. Loading the Inference Engine - // ----------------------------------------------------------- - next_step(); - - Core ie; - if (FLAGS_d.find("CPU") != std::string::npos && !FLAGS_l.empty()) { - // CPU plugin extensions are loaded as a shared library and passed as a - // pointer to base extension - const auto extension_ptr = std::make_shared(FLAGS_l); - ie.AddExtension(extension_ptr); - slog::info << "CPU plugin extensions are loaded " << FLAGS_l << slog::endl; - } - - // Load clDNN Extensions - if ((FLAGS_d.find("GPU") != std::string::npos) && !FLAGS_c.empty()) { - // Override config if command line parameter is specified - if (!config.count("GPU")) - config["GPU"] = {}; - config["GPU"][CONFIG_KEY(CONFIG_FILE)] = FLAGS_c; - } - if (config.count("GPU") && config.at("GPU").count(CONFIG_KEY(CONFIG_FILE))) { - auto ext = config.at("GPU").at(CONFIG_KEY(CONFIG_FILE)); - ie.SetConfig({{CONFIG_KEY(CONFIG_FILE), ext}}, "GPU"); - slog::info << "GPU extensions is loaded " << ext << slog::endl; - } - - slog::info << "InferenceEngine: " << GetInferenceEngineVersion() << slog::endl; - slog::info << "Device info: " << slog::endl; - std::cout << ie.GetVersions(device_name) << std::endl; - - // ----------------- 3. Setting device configuration - // ----------------------------------------------------------- - next_step(); - - bool perf_counts = false; - // Update config per device according to command line parameters - for (auto& device : devices) { - if (!config.count(device)) - config[device] = {}; - std::map& device_config = config.at(device); - - // Set performance counter - if (isFlagSetInCommandLine("pc")) { - // set to user defined value - device_config[CONFIG_KEY(PERF_COUNT)] = FLAGS_pc ? CONFIG_VALUE(YES) : CONFIG_VALUE(NO); - } else if (device_config.count(CONFIG_KEY(PERF_COUNT)) && (device_config.at(CONFIG_KEY(PERF_COUNT)) == "YES")) { - slog::warn << "Performance counters for " << device << " device is turned on. To print results use -pc option." << slog::endl; - } else if (FLAGS_report_type == detailedCntReport || FLAGS_report_type == averageCntReport) { - slog::warn << "Turn on performance counters for " << device << " device since report type is " << FLAGS_report_type << "." << slog::endl; - device_config[CONFIG_KEY(PERF_COUNT)] = CONFIG_VALUE(YES); - } else if (!FLAGS_exec_graph_path.empty()) { - slog::warn << "Turn on performance counters for " << device << " device due to execution graph dumping." << slog::endl; - device_config[CONFIG_KEY(PERF_COUNT)] = CONFIG_VALUE(YES); - } else { - // set to default value - device_config[CONFIG_KEY(PERF_COUNT)] = FLAGS_pc ? CONFIG_VALUE(YES) : CONFIG_VALUE(NO); - } - perf_counts = (device_config.at(CONFIG_KEY(PERF_COUNT)) == CONFIG_VALUE(YES)) ? true : perf_counts; - - auto setThroughputStreams = [&]() { - const std::string key = device + "_THROUGHPUT_STREAMS"; - if (device_nstreams.count(device)) { - // set to user defined value - auto supported_config_keys = ie.GetMetric(device, METRIC_KEY(SUPPORTED_CONFIG_KEYS)).as>(); - if (std::find(supported_config_keys.begin(), supported_config_keys.end(), key) == supported_config_keys.end()) { - throw std::logic_error("Device " + device + " doesn't support config key '" + key + "'! " + - "Please specify -nstreams for correct devices in format " - ":,:" + - " or via configuration file."); - } - device_config[key] = device_nstreams.at(device); - } else if (!device_config.count(key) && (FLAGS_api == "async")) { - slog::warn << "-nstreams default value is determined automatically for " << device - << " device. " - "Although the automatic selection usually provides a " - "reasonable performance, " - "but it still may be non-optimal for some cases, for more " - "information look at README." - << slog::endl; - if (std::string::npos == device.find("MYRIAD")) // MYRIAD sets the default number of - // streams implicitly (without _AUTO) - device_config[key] = std::string(device + "_THROUGHPUT_AUTO"); - } - if (device_config.count(key)) - device_nstreams[device] = device_config.at(key); - }; - - if (device == "CPU") { // CPU supports few special performance-oriented keys - // limit threading for CPU portion of inference - if (isFlagSetInCommandLine("nthreads")) - device_config[CONFIG_KEY(CPU_THREADS_NUM)] = std::to_string(FLAGS_nthreads); - - if (isFlagSetInCommandLine("enforcebf16")) - device_config[CONFIG_KEY(ENFORCE_BF16)] = FLAGS_enforcebf16 ? CONFIG_VALUE(YES) : CONFIG_VALUE(NO); - - if (isFlagSetInCommandLine("pin")) { - // set to user defined value - device_config[CONFIG_KEY(CPU_BIND_THREAD)] = FLAGS_pin; - } else if (!device_config.count(CONFIG_KEY(CPU_BIND_THREAD))) { - if ((device_name.find("MULTI") != std::string::npos) && (device_name.find("GPU") != std::string::npos)) { - slog::warn << "Turn off threads pinning for " << device << " device since multi-scenario with GPU device is used." << slog::endl; - device_config[CONFIG_KEY(CPU_BIND_THREAD)] = CONFIG_VALUE(NO); - } - } - - // for CPU execution, more throughput-oriented execution via streams - setThroughputStreams(); - } else if (device == ("GPU")) { - // for GPU execution, more throughput-oriented execution via streams - setThroughputStreams(); - - if ((device_name.find("MULTI") != std::string::npos) && (device_name.find("CPU") != std::string::npos)) { - slog::warn << "Turn on GPU trottling. Multi-device execution with " - "the CPU + GPU performs best with GPU trottling hint," - << "which releases another CPU thread (that is otherwise " - "used by the GPU driver for active polling)" - << slog::endl; - device_config[GPU_CONFIG_KEY(PLUGIN_THROTTLE)] = "1"; - } - } else if (device == "GNA") { - if (FLAGS_qb == 8) - device_config[GNA_CONFIG_KEY(PRECISION)] = "I8"; - else - device_config[GNA_CONFIG_KEY(PRECISION)] = "I16"; - - if (isFlagSetInCommandLine("nthreads")) - device_config[GNA_CONFIG_KEY(LIB_N_THREADS)] = std::to_string(FLAGS_nthreads); - } else { - auto supported_config_keys = ie.GetMetric(device, METRIC_KEY(SUPPORTED_CONFIG_KEYS)).as>(); - auto supported = [&](const std::string& key) { - return std::find(std::begin(supported_config_keys), std::end(supported_config_keys), key) != std::end(supported_config_keys); - }; - if (supported(CONFIG_KEY(CPU_THREADS_NUM)) && isFlagSetInCommandLine("nthreads")) { - device_config[CONFIG_KEY(CPU_THREADS_NUM)] = std::to_string(FLAGS_nthreads); - } - if (supported(CONFIG_KEY(CPU_THROUGHPUT_STREAMS)) && isFlagSetInCommandLine("nstreams")) { - device_config[CONFIG_KEY(CPU_THROUGHPUT_STREAMS)] = FLAGS_nstreams; - } - if (supported(CONFIG_KEY(CPU_BIND_THREAD)) && isFlagSetInCommandLine("pin")) { - device_config[CONFIG_KEY(CPU_BIND_THREAD)] = FLAGS_pin; - } - } - } - - for (auto&& item : config) { - ie.SetConfig(item.second, item.first); - } - - auto double_to_string = [](const double number) { - std::stringstream ss; - ss << std::fixed << std::setprecision(2) << number; - return ss.str(); - }; - auto get_total_ms_time = [](Time::time_point& startTime) { - return std::chrono::duration_cast(Time::now() - startTime).count() * 0.000001; - }; - - size_t batchSize = FLAGS_b; - Precision precision = Precision::UNSPECIFIED; - std::string topology_name = ""; - benchmark_app::InputsInfo app_inputs_info; - std::string output_name; - - // Takes priority over config from file - if (!FLAGS_cache_dir.empty()) { - ie.SetConfig({{CONFIG_KEY(CACHE_DIR), FLAGS_cache_dir}}); - } - - if (FLAGS_load_from_file && !isNetworkCompiled) { - next_step(); - slog::info << "Skipping the step for loading network from file" << slog::endl; - next_step(); - slog::info << "Skipping the step for loading network from file" << slog::endl; - next_step(); - slog::info << "Skipping the step for loading network from file" << slog::endl; - auto startTime = Time::now(); - exeNetwork = ie.LoadNetwork(FLAGS_m, device_name); - auto duration_ms = double_to_string(get_total_ms_time(startTime)); - slog::info << "Load network took " << duration_ms << " ms" << slog::endl; - if (statistics) - statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"load network time (ms)", duration_ms}}); - if (batchSize == 0) { - batchSize = 1; - } - } else if (!isNetworkCompiled) { - // ----------------- 4. Reading the Intermediate Representation network - // ---------------------------------------- - next_step(); - - slog::info << "Loading network files" << slog::endl; - - auto startTime = Time::now(); - CNNNetwork cnnNetwork = ie.ReadNetwork(FLAGS_m); - auto duration_ms = double_to_string(get_total_ms_time(startTime)); - slog::info << "Read network took " << duration_ms << " ms" << slog::endl; - if (statistics) - statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"read network time (ms)", duration_ms}}); - - const InputsDataMap inputInfo(cnnNetwork.getInputsInfo()); - if (inputInfo.empty()) { - throw std::logic_error("no inputs info is provided"); - } - - // ----------------- 5. Resizing network to match image sizes and given - // batch ---------------------------------- - next_step(); - batchSize = cnnNetwork.getBatchSize(); - // Parse input shapes if specified - bool reshape = false; - app_inputs_info = getInputsInfo(FLAGS_shape, FLAGS_layout, FLAGS_b, inputInfo, reshape); - if (reshape) { - InferenceEngine::ICNNNetwork::InputShapes shapes = {}; - for (auto& item : app_inputs_info) - shapes[item.first] = item.second.shape; - slog::info << "Reshaping network: " << getShapesString(shapes) << slog::endl; - startTime = Time::now(); - cnnNetwork.reshape(shapes); - duration_ms = double_to_string(get_total_ms_time(startTime)); - slog::info << "Reshape network took " << duration_ms << " ms" << slog::endl; - if (statistics) - statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"reshape network time (ms)", duration_ms}}); - } - // use batch size according to provided layout and shapes - batchSize = (!FLAGS_layout.empty()) ? getBatchSize(app_inputs_info) : cnnNetwork.getBatchSize(); - - topology_name = cnnNetwork.getName(); - slog::info << (FLAGS_b != 0 ? "Network batch size was changed to: " : "Network batch size: ") << batchSize << slog::endl; - - // ----------------- 6. Configuring inputs and outputs - // ---------------------------------------------------------------------- - next_step(); - - processPrecision(cnnNetwork, FLAGS_ip, FLAGS_op, FLAGS_iop); - for (auto& item : cnnNetwork.getInputsInfo()) { - // if precision for input set by user, then set it to app_inputs - // if it an image, set U8 - if (!FLAGS_ip.empty() || FLAGS_iop.find(item.first) != std::string::npos) { - app_inputs_info.at(item.first).precision = item.second->getPrecision(); - } else if (app_inputs_info.at(item.first).isImage()) { - app_inputs_info.at(item.first).precision = Precision::U8; - item.second->setPrecision(app_inputs_info.at(item.first).precision); - } - } - - printInputAndOutputsInfo(cnnNetwork); - // ----------------- 7. Loading the model to the device - // -------------------------------------------------------- - next_step(); - startTime = Time::now(); - exeNetwork = ie.LoadNetwork(cnnNetwork, device_name); - duration_ms = double_to_string(get_total_ms_time(startTime)); - slog::info << "Load network took " << duration_ms << " ms" << slog::endl; - if (statistics) - statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"load network time (ms)", duration_ms}}); - } else { - next_step(); - slog::info << "Skipping the step for compiled network" << slog::endl; - next_step(); - slog::info << "Skipping the step for compiled network" << slog::endl; - next_step(); - slog::info << "Skipping the step for compiled network" << slog::endl; - // ----------------- 7. Loading the model to the device - // -------------------------------------------------------- - next_step(); - auto startTime = Time::now(); - exeNetwork = ie.ImportNetwork(FLAGS_m, device_name, {}); - auto duration_ms = double_to_string(get_total_ms_time(startTime)); - slog::info << "Import network took " << duration_ms << " ms" << slog::endl; - if (statistics) - statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"import network time (ms)", duration_ms}}); - app_inputs_info = getInputsInfo(FLAGS_shape, FLAGS_layout, FLAGS_b, exeNetwork.GetInputsInfo()); - if (batchSize == 0) { - batchSize = 1; - } - } - // ----------------- 8. Setting optimal runtime parameters - // ----------------------------------------------------- - next_step(); - - // Update number of streams - for (auto&& ds : device_nstreams) { - const std::string key = ds.first + "_THROUGHPUT_STREAMS"; - device_nstreams[ds.first] = ie.GetConfig(ds.first, key).as(); - } - - // Number of requests - uint64_t nireq = FLAGS_nireq; - if (nireq == 0) { - if (FLAGS_api == "sync") { - nireq = 1; - } else { - std::string key = METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS); - try { - nireq = exeNetwork.GetMetric(key).as(); - } catch (const std::exception& ex) { - IE_THROW() << "Every device used with the benchmark_app should " - << "support OPTIMAL_NUMBER_OF_INFER_REQUESTS " - "ExecutableNetwork metric. " - << "Failed to query the metric for the " << device_name << " with error:" << ex.what(); - } - } - } - - // Iteration limit - uint64_t niter = FLAGS_niter; - if ((niter > 0) && (FLAGS_api == "async")) { - niter = ((niter + nireq - 1) / nireq) * nireq; - if (FLAGS_niter != niter) { - slog::warn << "Number of iterations was aligned by request number from " << FLAGS_niter << " to " << niter << " using number of requests " - << nireq << slog::endl; - } - } - - // Time limit - uint64_t duration_seconds = 0; - if (FLAGS_t != 0) { - // time limit - duration_seconds = FLAGS_t; - } else if (FLAGS_niter == 0) { - // default time limit - duration_seconds = deviceDefaultDeviceDurationInSeconds(device_name); - } - uint64_t duration_nanoseconds = getDurationInNanoseconds(duration_seconds); - - if (statistics) { - statistics->addParameters(StatisticsReport::Category::RUNTIME_CONFIG, - { - {"topology", topology_name}, - {"target device", device_name}, - {"API", FLAGS_api}, - {"precision", std::string(precision.name())}, - {"batch size", std::to_string(batchSize)}, - {"number of iterations", std::to_string(niter)}, - {"number of parallel infer requests", std::to_string(nireq)}, - {"duration (ms)", std::to_string(getDurationInMilliseconds(duration_seconds))}, - }); - for (auto& nstreams : device_nstreams) { - std::stringstream ss; - ss << "number of " << nstreams.first << " streams"; - statistics->addParameters(StatisticsReport::Category::RUNTIME_CONFIG, { - {ss.str(), nstreams.second}, - }); - } - } - - // ----------------- 9. Creating infer requests and filling input blobs - // ---------------------------------------- - next_step(); - - InferRequestsQueue inferRequestsQueue(exeNetwork, nireq); - fillBlobs(inputFiles, batchSize, app_inputs_info, inferRequestsQueue.requests); - - // ----------------- 10. Measuring performance - // ------------------------------------------------------------------ - size_t progressCnt = 0; - size_t progressBarTotalCount = progressBarDefaultTotalCount; - size_t iteration = 0; - - std::stringstream ss; - ss << "Start inference " << FLAGS_api << "hronously"; - if (FLAGS_api == "async") { - if (!ss.str().empty()) { - ss << ", "; - } - ss << nireq << " inference requests"; - std::stringstream device_ss; - for (auto& nstreams : device_nstreams) { - if (!device_ss.str().empty()) { - device_ss << ", "; - } - device_ss << nstreams.second << " streams for " << nstreams.first; - } - if (!device_ss.str().empty()) { - ss << " using " << device_ss.str(); - } - } - ss << ", limits: "; - if (duration_seconds > 0) { - ss << getDurationInMilliseconds(duration_seconds) << " ms duration"; - } - if (niter != 0) { - if (duration_seconds == 0) { - progressBarTotalCount = niter; - } - if (duration_seconds > 0) { - ss << ", "; - } - ss << niter << " iterations"; - } - next_step(ss.str()); - - // warming up - out of scope - auto inferRequest = inferRequestsQueue.getIdleRequest(); - if (!inferRequest) { - IE_THROW() << "No idle Infer Requests!"; - } - if (FLAGS_api == "sync") { - inferRequest->infer(); - } else { - inferRequest->startAsync(); - } - inferRequestsQueue.waitAll(); - auto duration_ms = double_to_string(inferRequestsQueue.getLatencies()[0]); - slog::info << "First inference took " << duration_ms << " ms" << slog::endl; - if (statistics) - statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"first inference time (ms)", duration_ms}}); - inferRequestsQueue.resetTimes(); - - auto startTime = Time::now(); - auto execTime = std::chrono::duration_cast(Time::now() - startTime).count(); - - /** Start inference & calculate performance **/ - /** to align number if iterations to guarantee that last infer requests are - * executed in the same conditions **/ - ProgressBar progressBar(progressBarTotalCount, FLAGS_stream_output, FLAGS_progress); - - while ((niter != 0LL && iteration < niter) || (duration_nanoseconds != 0LL && (uint64_t)execTime < duration_nanoseconds) || - (FLAGS_api == "async" && iteration % nireq != 0)) { - inferRequest = inferRequestsQueue.getIdleRequest(); - if (!inferRequest) { - IE_THROW() << "No idle Infer Requests!"; - } - - if (FLAGS_api == "sync") { - inferRequest->infer(); - } else { - // As the inference request is currently idle, the wait() adds no - // additional overhead (and should return immediately). The primary - // reason for calling the method is exception checking/re-throwing. - // Callback, that governs the actual execution can handle errors as - // well, but as it uses just error codes it has no details like ‘what()’ - // method of `std::exception` So, rechecking for any exceptions here. - inferRequest->wait(); - inferRequest->startAsync(); - } - iteration++; - - execTime = std::chrono::duration_cast(Time::now() - startTime).count(); - - if (niter > 0) { - progressBar.addProgress(1); - } else { - // calculate how many progress intervals are covered by current - // iteration. depends on the current iteration time and time of each - // progress interval. Previously covered progress intervals must be - // skipped. - auto progressIntervalTime = duration_nanoseconds / progressBarTotalCount; - size_t newProgress = execTime / progressIntervalTime - progressCnt; - progressBar.addProgress(newProgress); - progressCnt += newProgress; - } - } - - // wait the latest inference executions - inferRequestsQueue.waitAll(); - - double latency = getMedianValue(inferRequestsQueue.getLatencies()); - double totalDuration = inferRequestsQueue.getDurationInMilliseconds(); - double fps = (FLAGS_api == "sync") ? batchSize * 1000.0 / latency : batchSize * 1000.0 * iteration / totalDuration; - - if (statistics) { - statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, { - {"total execution time (ms)", double_to_string(totalDuration)}, - {"total number of iterations", std::to_string(iteration)}, - }); - if (device_name.find("MULTI") == std::string::npos) { - statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, { - {"latency (ms)", double_to_string(latency)}, - }); - } - statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"throughput", double_to_string(fps)}}); - } - - progressBar.finish(); - - // ----------------- 11. Dumping statistics report - // ------------------------------------------------------------- - next_step(); - -#ifdef USE_OPENCV - if (!FLAGS_dump_config.empty()) { - dump_config(FLAGS_dump_config, config); - slog::info << "Inference Engine configuration settings were dumped to " << FLAGS_dump_config << slog::endl; - } -#endif - - if (!FLAGS_exec_graph_path.empty()) { - try { - CNNNetwork execGraphInfo = exeNetwork.GetExecGraphInfo(); - execGraphInfo.serialize(FLAGS_exec_graph_path); - slog::info << "executable graph is stored to " << FLAGS_exec_graph_path << slog::endl; - } catch (const std::exception& ex) { - slog::err << "Can't get executable graph: " << ex.what() << slog::endl; - } - } - - if (perf_counts) { - std::vector> perfCounts; - for (size_t ireq = 0; ireq < nireq; ireq++) { - auto reqPerfCounts = inferRequestsQueue.requests[ireq]->getPerformanceCounts(); - if (FLAGS_pc) { - slog::info << "Performance counts for " << ireq << "-th infer request:" << slog::endl; - printPerformanceCounts(reqPerfCounts, std::cout, getFullDeviceName(ie, FLAGS_d), false); - } - perfCounts.push_back(reqPerfCounts); - } - if (statistics) { - statistics->dumpPerformanceCounters(perfCounts); - } - } - - if (statistics) - statistics->dump(); - - std::cout << "Count: " << iteration << " iterations" << std::endl; - std::cout << "Duration: " << double_to_string(totalDuration) << " ms" << std::endl; - if (device_name.find("MULTI") == std::string::npos) - std::cout << "Latency: " << double_to_string(latency) << " ms" << std::endl; - std::cout << "Throughput: " << double_to_string(fps) << " FPS" << std::endl; - } catch (const std::exception& ex) { - slog::err << ex.what() << slog::endl; - - if (statistics) { - statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, { - {"error", ex.what()}, - }); - statistics->dump(); - } - - return 3; - } - - return 0; -} diff --git a/tools/legacy/benchmark_app/progress_bar.hpp b/tools/legacy/benchmark_app/progress_bar.hpp deleted file mode 100644 index fa2105be827..00000000000 --- a/tools/legacy/benchmark_app/progress_bar.hpp +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include "console_progress.hpp" - -/// @brief Responsible for progress bar handling within the benchmark_app -class ProgressBar { -public: - explicit ProgressBar(size_t totalNum, bool streamOutput = false, bool progressEnabled = false) { - _bar.reset(new ConsoleProgress(totalNum, streamOutput)); - _streamOutput = streamOutput; - _isFinished = true; - _progressEnabled = progressEnabled; - } - - void addProgress(size_t num) { - _isFinished = false; - if (_progressEnabled) { - _bar->addProgress(static_cast(num)); - } - } - - void finish(size_t num = 0) { - if (num > 0) { - addProgress(num); - } - _isFinished = true; - _bar->finish(); - if (_progressEnabled) { - std::cout << std::endl; - } - } - - void newBar(size_t totalNum) { - if (_isFinished) { - _bar.reset(new ConsoleProgress(totalNum, _streamOutput)); - } else { - throw std::logic_error("Cannot create a new bar. Current bar is still in progress"); - } - } - -private: - std::unique_ptr _bar; - bool _streamOutput; - bool _isFinished; - bool _progressEnabled; -}; diff --git a/tools/legacy/benchmark_app/slog.cpp b/tools/legacy/benchmark_app/slog.cpp deleted file mode 100644 index 0fc7d391ef3..00000000000 --- a/tools/legacy/benchmark_app/slog.cpp +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "slog.hpp" - -#include - -namespace slog { - -LogStream info("INFO", std::cout); -LogStream warn("WARNING", std::cout); -LogStream err("ERROR", std::cerr); - -LogStream::LogStream(const std::string& prefix, std::ostream& log_stream): _prefix(prefix), _new_line(true) { - _log_stream = &log_stream; -} - -// Specializing for LogStreamEndLine to support slog::endl -LogStream& LogStream::operator<<(const LogStreamEndLine& /*arg*/) { - _new_line = true; - - (*_log_stream) << std::endl; - return *this; -} - -// Specializing for LogStreamBoolAlpha to support slog::boolalpha -LogStream& LogStream::operator<<(const LogStreamBoolAlpha& /*arg*/) { - (*_log_stream) << std::boolalpha; - return *this; -} - -} // namespace slog \ No newline at end of file diff --git a/tools/legacy/benchmark_app/slog.hpp b/tools/legacy/benchmark_app/slog.hpp deleted file mode 100644 index 0f5150b8ef0..00000000000 --- a/tools/legacy/benchmark_app/slog.hpp +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -/** - * @brief a header file with logging facility for common samples - * @file log.hpp - */ - -#pragma once - -#include -#include - -namespace slog { -/** - * @class LogStreamEndLine - * @brief The LogStreamEndLine class implements an end line marker for a log stream - */ -class LogStreamEndLine {}; - -static constexpr LogStreamEndLine endl; - -/** - * @class LogStreamBoolAlpha - * @brief The LogStreamBoolAlpha class implements bool printing for a log stream - */ -class LogStreamBoolAlpha {}; - -static constexpr LogStreamBoolAlpha boolalpha; - -/** - * @class LogStream - * @brief The LogStream class implements a stream for sample logging - */ -class LogStream { - std::string _prefix; - std::ostream* _log_stream; - bool _new_line; - -public: - /** - * @brief A constructor. Creates an LogStream object - * @param prefix The prefix to print - */ - LogStream(const std::string& prefix, std::ostream& log_stream); - - /** - * @brief A stream output operator to be used within the logger - * @param arg Object for serialization in the logger message - */ - template - LogStream& operator<<(const T& arg) { - if (_new_line) { - (*_log_stream) << "[ " << _prefix << " ] "; - _new_line = false; - } - - (*_log_stream) << arg; - return *this; - } - - // Specializing for LogStreamEndLine to support slog::endl - LogStream& operator<<(const LogStreamEndLine&); - - // Specializing for LogStreamBoolAlpha to support slog::boolalpha - LogStream& operator<<(const LogStreamBoolAlpha&); -}; - -extern LogStream info; -extern LogStream warn; -extern LogStream err; - -} // namespace slog diff --git a/tools/legacy/benchmark_app/statistics_report.cpp b/tools/legacy/benchmark_app/statistics_report.cpp deleted file mode 100644 index 6f2a9454ad6..00000000000 --- a/tools/legacy/benchmark_app/statistics_report.cpp +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "statistics_report.hpp" - -#include -#include -#include -#include -#include - -void StatisticsReport::addParameters(const Category& category, const Parameters& parameters) { - if (_parameters.count(category) == 0) - _parameters[category] = parameters; - else - _parameters[category].insert(_parameters[category].end(), parameters.begin(), parameters.end()); -} - -void StatisticsReport::dump() { - CsvDumper dumper(true, _config.report_folder + _separator + "benchmark_report.csv"); - - auto dump_parameters = [&dumper](const Parameters& parameters) { - for (auto& parameter : parameters) { - dumper << parameter.first << parameter.second; - dumper.endLine(); - } - }; - if (_parameters.count(Category::COMMAND_LINE_PARAMETERS)) { - dumper << "Command line parameters"; - dumper.endLine(); - - dump_parameters(_parameters.at(Category::COMMAND_LINE_PARAMETERS)); - dumper.endLine(); - } - - if (_parameters.count(Category::RUNTIME_CONFIG)) { - dumper << "Configuration setup"; - dumper.endLine(); - - dump_parameters(_parameters.at(Category::RUNTIME_CONFIG)); - dumper.endLine(); - } - - if (_parameters.count(Category::EXECUTION_RESULTS)) { - dumper << "Execution results"; - dumper.endLine(); - - dump_parameters(_parameters.at(Category::EXECUTION_RESULTS)); - dumper.endLine(); - } - - slog::info << "Statistics report is stored to " << dumper.getFilename() << slog::endl; -} - -void StatisticsReport::dumpPerformanceCountersRequest(CsvDumper& dumper, const PerformanceCounters& perfCounts) { - auto performanceMapSorted = perfCountersSorted(perfCounts); - - long long total = 0L; - long long total_cpu = 0L; - - dumper << "layerName" - << "execStatus" - << "layerType" - << "execType"; - dumper << "realTime (ms)" - << "cpuTime (ms)"; - dumper.endLine(); - - for (const auto& layer : performanceMapSorted) { - dumper << layer.first; // layer name - - switch (layer.second.status) { - case InferenceEngine::InferenceEngineProfileInfo::EXECUTED: - dumper << "EXECUTED"; - break; - case InferenceEngine::InferenceEngineProfileInfo::NOT_RUN: - dumper << "NOT_RUN"; - break; - case InferenceEngine::InferenceEngineProfileInfo::OPTIMIZED_OUT: - dumper << "OPTIMIZED_OUT"; - break; - } - dumper << layer.second.layer_type << layer.second.exec_type; - dumper << std::to_string(layer.second.realTime_uSec / 1000.0) << std::to_string(layer.second.cpu_uSec / 1000.0); - total += layer.second.realTime_uSec; - total_cpu += layer.second.cpu_uSec; - dumper.endLine(); - } - dumper << "Total" - << "" - << "" - << ""; - dumper << total / 1000.0 << total_cpu / 1000.0; - dumper.endLine(); - dumper.endLine(); -} - -void StatisticsReport::dumpPerformanceCounters(const std::vector& perfCounts) { - if ((_config.report_type.empty()) || (_config.report_type == noCntReport)) { - slog::info << "Statistics collecting for performance counters was not " - "requested. No reports are dumped." - << slog::endl; - return; - } - if (perfCounts.empty()) { - slog::info << "Performance counters are empty. No reports are dumped." << slog::endl; - return; - } - CsvDumper dumper(true, _config.report_folder + _separator + "benchmark_" + _config.report_type + "_report.csv"); - if (_config.report_type == detailedCntReport) { - for (auto& pc : perfCounts) { - dumpPerformanceCountersRequest(dumper, pc); - } - } else if (_config.report_type == averageCntReport) { - auto getAveragePerformanceCounters = [&perfCounts]() { - std::map performanceCountersAvg; - // iterate over each processed infer request and handle its PM data - for (size_t i = 0; i < perfCounts.size(); i++) { - auto performanceMapSorted = perfCountersSorted(perfCounts[i]); - // iterate over each layer from sorted vector and add required PM data - // to the per-layer maps - for (const auto& pm : performanceMapSorted) { - if (performanceCountersAvg.count(pm.first) == 0) { - performanceCountersAvg[pm.first] = perfCounts.at(i).at(pm.first); - } else { - performanceCountersAvg[pm.first].realTime_uSec += perfCounts.at(i).at(pm.first).realTime_uSec; - performanceCountersAvg[pm.first].cpu_uSec += perfCounts.at(i).at(pm.first).cpu_uSec; - } - } - } - for (auto& pm : performanceCountersAvg) { - pm.second.realTime_uSec /= perfCounts.size(); - pm.second.cpu_uSec /= perfCounts.size(); - } - return performanceCountersAvg; - }; - dumpPerformanceCountersRequest(dumper, getAveragePerformanceCounters()); - } else { - throw std::logic_error("PM data can only be collected for average or detailed report types"); - } - slog::info << "Performance counters report is stored to " << dumper.getFilename() << slog::endl; -} diff --git a/tools/legacy/benchmark_app/statistics_report.hpp b/tools/legacy/benchmark_app/statistics_report.hpp deleted file mode 100644 index 3d5abc2d34b..00000000000 --- a/tools/legacy/benchmark_app/statistics_report.hpp +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include "common.hpp" -#include "csv_dumper.hpp" -#include "slog.hpp" -#include -#include -#include - -// @brief statistics reports types -static constexpr char noCntReport[] = "no_counters"; -static constexpr char averageCntReport[] = "average_counters"; -static constexpr char detailedCntReport[] = "detailed_counters"; - -/// @brief Responsible for collecting of statistics and dumping to .csv file -class StatisticsReport { -public: - typedef std::map PerformanceCounters; - typedef std::vector> Parameters; - - struct Config { - std::string report_type; - std::string report_folder; - }; - - enum class Category { - COMMAND_LINE_PARAMETERS, - RUNTIME_CONFIG, - EXECUTION_RESULTS, - }; - - explicit StatisticsReport(Config config): _config(std::move(config)) { - _separator = -#if defined _WIN32 || defined __CYGWIN__ - #if defined UNICODE - L"\\"; - #else - "\\"; - #endif -#else - "/"; -#endif - if (_config.report_folder.empty()) - _separator = ""; - } - - void addParameters(const Category& category, const Parameters& parameters); - - void dump(); - - void dumpPerformanceCounters(const std::vector& perfCounts); - -private: - void dumpPerformanceCountersRequest(CsvDumper& dumper, const PerformanceCounters& perfCounts); - - // configuration of current benchmark execution - const Config _config; - - // parameters - std::map _parameters; - - // csv separator - std::string _separator; -}; diff --git a/tools/legacy/benchmark_app/utils.cpp b/tools/legacy/benchmark_app/utils.cpp deleted file mode 100644 index e1cd0622dd6..00000000000 --- a/tools/legacy/benchmark_app/utils.cpp +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -// clang-format off -#include -#include -#include -#include "common.hpp" -#include "slog.hpp" -#include -#include -#include - -#include "utils.hpp" -// clang-format on - -#ifdef USE_OPENCV - #include -#endif - -namespace benchmark_app { -bool InputInfo::isImage() const { - if ((layout != "NCHW") && (layout != "NHWC") && (layout != "CHW") && (layout != "HWC")) - return false; - return (channels() == 3); -} -bool InputInfo::isImageInfo() const { - if (layout != "NC") - return false; - return (channels() >= 2); -} -size_t InputInfo::getDimentionByLayout(char character) const { - size_t pos = layout.find(character); - if (pos == std::string::npos) - throw std::runtime_error("Error: Can't get " + std::string(character, 1) + " from layout " + layout); - return shape.at(pos); -} -size_t InputInfo::width() const { - return getDimentionByLayout('W'); -} -size_t InputInfo::height() const { - return getDimentionByLayout('H'); -} -size_t InputInfo::channels() const { - return getDimentionByLayout('C'); -} -size_t InputInfo::batch() const { - return getDimentionByLayout('N'); -} -size_t InputInfo::depth() const { - return getDimentionByLayout('D'); -} -} // namespace benchmark_app - -uint32_t deviceDefaultDeviceDurationInSeconds(const std::string& device) { - static const std::map deviceDefaultDurationInSeconds {{"CPU", 60}, {"GPU", 60}, {"VPUX", 60}, - {"UNKNOWN", 120}}; - uint32_t duration = 0; - for (const auto& deviceDurationInSeconds : deviceDefaultDurationInSeconds) { - if (device.find(deviceDurationInSeconds.first) != std::string::npos) { - duration = std::max(duration, deviceDurationInSeconds.second); - } - } - if (duration == 0) { - const auto unknownDeviceIt = - find_if(deviceDefaultDurationInSeconds.begin(), deviceDefaultDurationInSeconds.end(), [](std::pair deviceDuration) { - return deviceDuration.first == "UNKNOWN"; - }); - - if (unknownDeviceIt == deviceDefaultDurationInSeconds.end()) { - throw std::logic_error("UNKNOWN device was not found in the device duration list"); - } - duration = unknownDeviceIt->second; - slog::warn << "Default duration " << duration << " seconds for unknown device '" << device << "' is used" << slog::endl; - } - return duration; -} - -std::vector split(const std::string& s, char delim) { - std::vector result; - std::stringstream ss(s); - std::string item; - - while (getline(ss, item, delim)) { - result.push_back(item); - } - return result; -} - -std::vector parseDevices(const std::string& device_string) { - std::string comma_separated_devices = device_string; - if (comma_separated_devices.find(":") != std::string::npos) { - comma_separated_devices = comma_separated_devices.substr(comma_separated_devices.find(":") + 1); - } - if ((comma_separated_devices == "MULTI") || (comma_separated_devices == "HETERO")) - return std::vector(); - auto devices = split(comma_separated_devices, ','); - for (auto& device : devices) - device = device.substr(0, device.find_first_of(".(")); - return devices; -} - -std::map parseNStreamsValuePerDevice(const std::vector& devices, const std::string& values_string) { - // Format: :,: or just - std::map result; - auto device_value_strings = split(values_string, ','); - for (auto& device_value_string : device_value_strings) { - auto device_value_vec = split(device_value_string, ':'); - if (device_value_vec.size() == 2) { - auto device_name = device_value_vec.at(0); - auto nstreams = device_value_vec.at(1); - auto it = std::find(devices.begin(), devices.end(), device_name); - if (it != devices.end()) { - result[device_name] = nstreams; - } else { - throw std::logic_error("Can't set nstreams value " + std::string(nstreams) + " for device '" + device_name + "'! Incorrect device name!"); - } - } else if (device_value_vec.size() == 1) { - auto value = device_value_vec.at(0); - for (auto& device : devices) { - result[device] = value; - } - } else if (device_value_vec.size() != 0) { - throw std::runtime_error("Unknown string format: " + values_string); - } - } - return result; -} - -size_t getBatchSize(const benchmark_app::InputsInfo& inputs_info) { - size_t batch_size = 0; - for (auto& info : inputs_info) { - std::size_t batch_index = info.second.layout.find("N"); - if (batch_index != std::string::npos) { - if (batch_size == 0) - batch_size = info.second.shape[batch_index]; - else if (batch_size != info.second.shape[batch_index]) - throw std::logic_error("Can't deterimine batch size: batch is " - "different for different inputs!"); - } - } - if (batch_size == 0) - batch_size = 1; - return batch_size; -} - -std::string getShapesString(const InferenceEngine::ICNNNetwork::InputShapes& shapes) { - std::stringstream ss; - for (auto& shape : shapes) { - if (!ss.str().empty()) - ss << ", "; - ss << "\'" << shape.first << "': ["; - for (size_t i = 0; i < shape.second.size(); i++) { - if (i > 0) - ss << ", "; - ss << shape.second.at(i); - } - ss << "]"; - } - return ss.str(); -} - -#ifdef USE_OPENCV -void dump_config(const std::string& filename, const std::map>& config) { - cv::FileStorage fs(filename, cv::FileStorage::WRITE); - if (!fs.isOpened()) - throw std::runtime_error("Error: Can't open config file : " + filename); - for (auto device_it = config.begin(); device_it != config.end(); ++device_it) { - fs << device_it->first << "{:"; - for (auto param_it = device_it->second.begin(); param_it != device_it->second.end(); ++param_it) - fs << param_it->first << param_it->second; - fs << "}"; - } - fs.release(); -} - -void load_config(const std::string& filename, std::map>& config) { - cv::FileStorage fs(filename, cv::FileStorage::READ); - if (!fs.isOpened()) - throw std::runtime_error("Error: Can't load config file : " + filename); - cv::FileNode root = fs.root(); - for (auto it = root.begin(); it != root.end(); ++it) { - auto device = *it; - if (!device.isMap()) { - throw std::runtime_error("Error: Can't parse config file : " + filename); - } - for (auto iit = device.begin(); iit != device.end(); ++iit) { - auto item = *iit; - config[device.name()][item.name()] = item.string(); - } - } -} -#endif \ No newline at end of file diff --git a/tools/legacy/benchmark_app/utils.hpp b/tools/legacy/benchmark_app/utils.hpp deleted file mode 100644 index d2923002242..00000000000 --- a/tools/legacy/benchmark_app/utils.hpp +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include - -namespace benchmark_app { -struct InputInfo { - InferenceEngine::Precision precision; - InferenceEngine::SizeVector shape; - std::string layout; - bool isImage() const; - bool isImageInfo() const; - size_t getDimentionByLayout(char character) const; - size_t width() const; - size_t height() const; - size_t channels() const; - size_t batch() const; - size_t depth() const; -}; -using InputsInfo = std::map; -} // namespace benchmark_app - -std::vector parseDevices(const std::string& device_string); -uint32_t deviceDefaultDeviceDurationInSeconds(const std::string& device); -std::map parseNStreamsValuePerDevice(const std::vector& devices, const std::string& values_string); -std::string getShapesString(const InferenceEngine::ICNNNetwork::InputShapes& shapes); -size_t getBatchSize(const benchmark_app::InputsInfo& inputs_info); -std::vector split(const std::string& s, char delim); - -template -std::map parseInputParameters(const std::string parameter_string, const std::map& input_info) { - // Parse parameter string like "input0[value0],input1[value1]" or "[value]" (applied to all - // inputs) - std::map return_value; - std::string search_string = parameter_string; - auto start_pos = search_string.find_first_of('['); - while (start_pos != std::string::npos) { - auto end_pos = search_string.find_first_of(']'); - if (end_pos == std::string::npos) - break; - auto input_name = search_string.substr(0, start_pos); - auto input_value = search_string.substr(start_pos + 1, end_pos - start_pos - 1); - if (!input_name.empty()) { - return_value[input_name] = input_value; - } else { - for (auto& item : input_info) { - return_value[item.first] = input_value; - } - } - search_string = search_string.substr(end_pos + 1); - if (search_string.empty() || search_string.front() != ',') - break; - search_string = search_string.substr(1); - start_pos = search_string.find_first_of('['); - } - if (!search_string.empty()) - throw std::logic_error("Can't parse input parameter string: " + parameter_string); - return return_value; -} - -template -benchmark_app::InputsInfo getInputsInfo(const std::string& shape_string, const std::string& layout_string, const size_t batch_size, - const std::map& input_info, bool& reshape_required) { - std::map shape_map = parseInputParameters(shape_string, input_info); - std::map layout_map = parseInputParameters(layout_string, input_info); - reshape_required = false; - benchmark_app::InputsInfo info_map; - for (auto& item : input_info) { - benchmark_app::InputInfo info; - auto name = item.first; - auto descriptor = item.second->getTensorDesc(); - // Precision - info.precision = descriptor.getPrecision(); - // Shape - if (shape_map.count(name)) { - std::vector parsed_shape; - for (auto& dim : split(shape_map.at(name), ',')) { - parsed_shape.push_back(std::stoi(dim)); - } - info.shape = parsed_shape; - reshape_required = true; - } else { - info.shape = descriptor.getDims(); - } - // Layout - if (layout_map.count(name)) { - info.layout = layout_map.at(name); - std::transform(info.layout.begin(), info.layout.end(), info.layout.begin(), ::toupper); - } else { - std::stringstream ss; - ss << descriptor.getLayout(); - info.layout = ss.str(); - } - // Update shape with batch if needed - if (batch_size != 0) { - std::size_t batch_index = info.layout.find("N"); - if ((batch_index != std::string::npos) && (info.shape.at(batch_index) != batch_size)) { - info.shape[batch_index] = batch_size; - reshape_required = true; - } - } - info_map[name] = info; - } - return info_map; -} - -template -benchmark_app::InputsInfo getInputsInfo(const std::string& shape_string, const std::string& layout_string, const size_t batch_size, - const std::map& input_info) { - bool reshape_required = false; - return getInputsInfo(shape_string, layout_string, batch_size, input_info, reshape_required); -} - -#ifdef USE_OPENCV -void dump_config(const std::string& filename, const std::map>& config); -void load_config(const std::string& filename, std::map>& config); -#endif \ No newline at end of file diff --git a/tools/legacy/benchmark_app/w_dirent.h b/tools/legacy/benchmark_app/w_dirent.h deleted file mode 100644 index 88ef0bcd06a..00000000000 --- a/tools/legacy/benchmark_app/w_dirent.h +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#if defined(_WIN32) - - #ifndef WIN32_LEAN_AND_MEAN - #define WIN32_LEAN_AND_MEAN - #define WIN32_LEAN_AND_MEAN_UNDEF - #endif - - #ifndef NOMINMAX - #define NOMINMAX - #define NOMINMAX_UNDEF - #endif - - #if defined(_M_IX86) && !defined(_X86_) && !defined(_AMD64_) - #define _X86_ - #endif - - #if defined(_M_X64) && !defined(_X86_) && !defined(_AMD64_) - #define _AMD64_ - #endif - - #if defined(_M_ARM) && !defined(_ARM_) && !defined(_ARM64_) - #define _ARM_ - #endif - - #if defined(_M_ARM64) && !defined(_ARM_) && !defined(_ARM64_) - #define _ARM64_ - #endif - - // clang-format off - #include - #include - #include - #include - #include - // clang-format on - - // Copied from linux libc sys/stat.h: - #define S_ISREG(m) (((m)&S_IFMT) == S_IFREG) - #define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR) - -/// @brief structure to store directory names -struct dirent { - char* d_name; - - explicit dirent(const wchar_t* wsFilePath) { - size_t i; - auto slen = wcslen(wsFilePath); - d_name = static_cast(malloc(slen + 1)); - wcstombs_s(&i, d_name, slen + 1, wsFilePath, slen); - } - ~dirent() { - free(d_name); - } -}; - -/// @brief class to store directory data (files meta) -class DIR { - WIN32_FIND_DATAA FindFileData; - HANDLE hFind; - dirent* next; - - static inline bool endsWith(const std::string& src, const char* with) { - int wl = static_cast(strlen(with)); - int so = static_cast(src.length()) - wl; - if (so < 0) - return false; - return 0 == strncmp(with, &src[so], wl); - } - -public: - DIR(const DIR& other) = delete; - DIR(DIR&& other) = delete; - DIR& operator=(const DIR& other) = delete; - DIR& operator=(DIR&& other) = delete; - - explicit DIR(const char* dirPath): next(nullptr) { - std::string ws = dirPath; - if (endsWith(ws, "\\")) - ws += "*"; - else - ws += "\\*"; - hFind = FindFirstFileA(ws.c_str(), &FindFileData); - FindFileData.dwReserved0 = hFind != INVALID_HANDLE_VALUE; - } - - ~DIR() { - if (!next) - delete next; - next = nullptr; - FindClose(hFind); - } - - /** - * @brief Check file handler is valid - * @return status True(success) or False(fail) - */ - bool isValid() const { - return (hFind != INVALID_HANDLE_VALUE && FindFileData.dwReserved0); - } - - /** - * @brief Add directory to directory names struct - * @return pointer to directory names struct - */ - dirent* nextEnt() { - if (next != nullptr) - delete next; - next = nullptr; - - if (!FindFileData.dwReserved0) - return nullptr; - - wchar_t wbuf[4096]; - - size_t outSize; - mbstowcs_s(&outSize, wbuf, 4094, FindFileData.cFileName, 4094); - next = new dirent(wbuf); - FindFileData.dwReserved0 = FindNextFileA(hFind, &FindFileData); - return next; - } -}; - -/** - * @brief Create directory data struct element - * @param string directory path - * @return pointer to directory data struct element - */ -static DIR* opendir(const char* dirPath) { - auto dp = new DIR(dirPath); - if (!dp->isValid()) { - delete dp; - return nullptr; - } - return dp; -} - -/** - * @brief Walk throw directory data struct - * @param pointer to directory data struct - * @return pointer to directory data struct next element - */ -static struct dirent* readdir(DIR* dp) { - return dp->nextEnt(); -} - -/** - * @brief Remove directory data struct - * @param pointer to struct directory data - * @return void - */ -static void closedir(DIR* dp) { - delete dp; -} - - #ifdef WIN32_LEAN_AND_MEAN_UNDEF - #undef WIN32_LEAN_AND_MEAN - #undef WIN32_LEAN_AND_MEAN_UNDEF - #endif - - #ifdef NOMINMAX_UNDEF - #undef NOMINMAX_UNDEF - #undef NOMINMAX - #endif - -#else - - #include - #include - -#endif