natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

publish master branch snapshot, revision 49482ae3bea0cbaa07474f86f36db11943142687

This commit is contained in:
Alexey Suhov 2020-05-13 21:12:22 +03:00
parent 9d6501e9a6
commit 5b428f0655
924 changed files with 30841 additions and 8905 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
.clang-format
View File

@ -1,19 +0,0 @@
BasedOnStyle: Google
IndentWidth: 4
UseTab: Never
---
Language: Cpp
Standard: Cpp11
AccessModifierOffset: -4
AllowAllArgumentsOnNextLine: false
AllowShortFunctionsOnASingleLine: Empty
AllowShortLambdasOnASingleLine: Empty
AlwaysBreakBeforeMultilineStrings: false
ColumnLimit: 120
DerivePointerAlignment: false
FixNamespaceComments: true
IndentCaseLabels: false
SpaceBeforeCpp11BracedList: true
SpaceBeforeCtorInitializerColon: false
---

39
.coveragerc
View File

@ -1,39 +0,0 @@
# .coveragerc to control coverage.py
[run]
branch = True
source =
mo/
mo.py
omit =
# omit anything in a .local directory anywhere
*/.local/*
# omit everything in /usr
/usr/*
# omit tests
*/test_*.py
# init scripts
*/__init__.py
[report]
# Regexes for lines to exclude from consideration
exclude_lines =
# Have to re-enable the standard pragma
pragma: no cover
# Don't complain about missing debug-only code:
def __repr__
# Don't complain if tests don't hit defensive assertion code:
raise AssertionError
raise NotImplementedError
# Don't complain if non-runnable code isn't run:
if 0:
if __name__ == .__main__.:
ignore_errors = True
[html]
directory = htmlcov

36
.gitignore vendored
View File

@ -3,7 +3,7 @@ _*
# but ensure we don't skip __init__.py
!__init__.py
# developer tools
.idea
*.idea
.vscode
cmake-build-debug
cmake-build-release
@ -18,7 +18,41 @@ build/
doc/
docs/build_documentation/work_dir/
inference-engine/plugins/
inference-engine/temp
inference-engine/report
.repo/
docs/template_plugin/html/
CMakeLists.txt.user
docs/IE_PLUGIN_DG/html/
*.project
*.cproject
*.pydevproject
*.settings
*/gen/
__pycache__
*.swp
/config.xml
# Python-specific
*.env3
*.pyc
# Tests-specific
*.coverage
*htmlcov
*pylint_report.txt
*pylint_report_comments.txt
# Artifacts
/model-optimizer/*.bin
/model-optimizer/*.xml
/model-optimizer/*.json
/model-optimizer/*.so
/model-optimizer/*.txt
/model-optimizer/*.pb
/model-optimizer/*.pbtxt
/model-optimizer/!CMakeLists.txt
/model-optimizer/*.mapping
/model-optimizer/*.dat
/model-optimizer/*.svg

9
CMakeLists.txt
View File

@ -15,7 +15,6 @@ else()
cmake_minimum_required(VERSION 3.7.2 FATAL_ERROR)
endif()
project(OpenVINO)
set(OpenVINO_MAIN_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
@ -26,7 +25,7 @@ include(CTest)
include(features)
# include developer package
include(developer_package NO_POLICY_SCOPE)
include(developer_package)
# These options are shared with 3rdparty plugins
# by means of developer package
@ -37,7 +36,7 @@ include(dependencies)
message (STATUS "PROJECT ............................... " ${PROJECT_NAME})
message (STATUS "CMAKE_BINARY_DIR ...................... " ${CMAKE_BINARY_DIR})
message (STATUS "OpenVINO_MAIN_SOURCE_DIR .............. " ${OpenVINO_MAIN_SOURCE_DIR})
message (STATUS "IE_MAIN_SOURCE_DIR .............. " ${IE_MAIN_SOURCE_DIR})
message (STATUS "IE_MAIN_SOURCE_DIR .................... " ${IE_MAIN_SOURCE_DIR})
message (STATUS "CMAKE_GENERATOR ....................... " ${CMAKE_GENERATOR})
message (STATUS "CMAKE_C_COMPILER_ID ................... " ${CMAKE_C_COMPILER_ID})
message (STATUS "CMAKE_BUILD_TYPE ...................... " ${CMAKE_BUILD_TYPE})
@ -76,7 +75,7 @@ function(build_ngraph)
if (NOT ANDROID)
ngraph_set(NGRAPH_UNIT_TEST_ENABLE TRUE)
ngraph_set(NGRAPH_UNIT_TEST_OPENVINO_ENABLE TRUE)
ngraph_set(NGRAPH_IE_ENABLE TRUE)
ngraph_set(NGRAPH_ONNX_IMPORT_ENABLE TRUE)
else()
ngraph_set(NGRAPH_UNIT_TEST_ENABLE FALSE)
@ -85,7 +84,7 @@ function(build_ngraph)
ngraph_set(NGRAPH_ONNX_IMPORT_ENABLE FALSE)
endif()
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
if(CMAKE_CXX_COMPILER_ID MATCHES "^(Apple)?Clang$")
ie_add_compiler_flags(-Wno-error=uninitialized -Wno-error=literal-conversion)
elseif(UNIX)
ie_add_compiler_flags(-Wno-error=maybe-uninitialized -Wno-error=return-type -fPIC)

6
cmake/check_features.cmake
View File

@ -20,14 +20,14 @@ if (NOT ENABLE_MKL_DNN)
endif()
if(ENABLE_AVX512F)
if ((CMAKE_CXX_COMPILER_ID MATCHES MSVC) AND (MSVC_VERSION VERSION_LESS 1920))
if ((CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") AND (MSVC_VERSION VERSION_LESS 1920))
# 1920 version of MSVC 2019. In MSVC 2017 AVX512F not work
set(ENABLE_AVX512F OFF CACHE BOOL "" FORCE)
endif()
if (CMAKE_CXX_COMPILER_ID MATCHES Clang)
if (CMAKE_CXX_COMPILER_ID MATCHES "^(Apple)?Clang$")
set(ENABLE_AVX512F OFF CACHE BOOL "" FORCE)
endif()
if ((CMAKE_CXX_COMPILER_ID STREQUAL GNU) AND (NOT (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.9)))
if ((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (NOT (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.9)))
set(ENABLE_AVX512F OFF CACHE BOOL "" FORCE)
endif()
endif()

9
cmake/developer_package.cmake
View File

@ -143,7 +143,10 @@ if("${CMAKE_BUILD_TYPE}" STREQUAL "")
set(CMAKE_BUILD_TYPE "Release")
endif()
set(OUTPUT_ROOT ${OpenVINO_MAIN_SOURCE_DIR})
# allow to override default OUTPUT_ROOT root
if(NOT DEFINED OUTPUT_ROOT)
set(OUTPUT_ROOT ${OpenVINO_MAIN_SOURCE_DIR})
endif()
# Enable postfixes for Debug/Release builds
set(IE_DEBUG_POSTFIX_WIN "d")
@ -206,8 +209,10 @@ endif()
# Use solution folders
set_property(GLOBAL PROPERTY USE_FOLDERS ON)
set(CMAKE_POLICY_DEFAULT_CMP0054 NEW)
include(sdl)
include(os_flags NO_POLICY_SCOPE)
include(os_flags)
include(sanitizer)
function(set_ci_build_number)

2
cmake/fuzzing.cmake
View File

@ -4,7 +4,7 @@
function(enable_fuzzing)
# Enable (libFuzzer)[https://llvm.org/docs/LibFuzzer.html] if supported.
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND NOT WIN32)
if(CMAKE_CXX_COMPILER_ID MATCHES "^(Apple)?Clang$" AND NOT WIN32)
# Communicate libfuzzer is enabled
set(WITH_LIBFUZZER ON PARENT_SCOPE)
add_compile_definitions(WITH_LIBFUZZER)

70
cmake/os_flags.cmake
View File

@ -8,13 +8,13 @@
#
macro(disable_deprecated_warnings)
if(WIN32)
if(CMAKE_CXX_COMPILER_ID MATCHES Intel)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
set(ie_c_cxx_deprecated "/Qdiag-disable:1478,1786")
elseif(CMAKE_CXX_COMPILER_ID MATCHES MSVC)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set(ie_c_cxx_deprecated "/wd4996")
endif()
else()
if(CMAKE_CXX_COMPILER_ID STREQUAL Intel)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
set(ie_c_cxx_deprecated "-diag-disable=1478,1786")
else()
set(ie_c_cxx_deprecated "-Wno-deprecated-declarations")
@ -35,13 +35,13 @@ endmacro()
#
macro(ie_deprecated_no_errors)
if(WIN32)
if(CMAKE_CXX_COMPILER_ID MATCHES Intel)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
set(ie_c_cxx_deprecated "/Qdiag-warning:1478,1786")
elseif(CMAKE_CXX_COMPILER_ID MATCHES MSVC)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set(ie_c_cxx_deprecated "/wd4996")
endif()
else()
if(CMAKE_CXX_COMPILER_ID MATCHES Intel)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
set(ie_c_cxx_deprecated_no_errors "-diag-warning=1478,1786")
else()
set(ie_c_cxx_deprecated_no_errors "-Wno-error=deprecated-declarations")
@ -61,15 +61,15 @@ endmacro()
#
function(ie_sse42_optimization_flags flags)
if(WIN32)
if(CMAKE_CXX_COMPILER_ID MATCHES MSVC)
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# No such option for MSVC 2019
elseif(CMAKE_CXX_COMPILER_ID STREQUAL Intel)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
set(${flags} "/arch:SSE4.2 /QxSSE4.2" PARENT_SCOPE)
else()
message(WARNING "Unsupported CXX compiler ${CMAKE_CXX_COMPILER_ID}")
endif()
else()
if(CMAKE_CXX_COMPILER_ID STREQUAL Intel)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
set(${flags} "-msse4.2 -xSSE4.2" PARENT_SCOPE)
else()
set(${flags} "-msse4.2" PARENT_SCOPE)
@ -82,15 +82,15 @@ endfunction()
#
function(ie_avx2_optimization_flags flags)
if(WIN32)
if(CMAKE_CXX_COMPILER_ID STREQUAL Intel)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
set(${flags} "/QxCORE-AVX2" PARENT_SCOPE)
elseif(CMAKE_CXX_COMPILER_ID MATCHES MSVC)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set(${flags} "/arch:AVX2" PARENT_SCOPE)
else()
message(WARNING "Unsupported CXX compiler ${CMAKE_CXX_COMPILER_ID}")
endif()
else()
if(CMAKE_CXX_COMPILER_ID STREQUAL Intel)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
set(${flags} "-march=core-avx2 -xCORE-AVX2 -mtune=core-avx2" PARENT_SCOPE)
else()
set(${flags} "-mavx2 -mfma" PARENT_SCOPE)
@ -104,18 +104,18 @@ endfunction()
#
function(ie_avx512_optimization_flags flags)
if(WIN32)
if(CMAKE_CXX_COMPILER_ID STREQUAL Intel)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
set(${flags} "/QxCOMMON-AVX512" PARENT_SCOPE)
elseif(CMAKE_CXX_COMPILER_ID MATCHES MSVC)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set(${flags} "/arch:AVX512" PARENT_SCOPE)
else()
message(WARNING "Unsupported CXX compiler ${CMAKE_CXX_COMPILER_ID}")
endif()
else()
if(CMAKE_CXX_COMPILER_ID STREQUAL Intel)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
set(${flags} "-xCOMMON-AVX512" PARENT_SCOPE)
endif()
if(CMAKE_CXX_COMPILER_ID STREQUAL GNU)
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(${flags} "-mavx512f -mfma" PARENT_SCOPE)
endif()
endif()
@ -138,12 +138,12 @@ macro(ie_enable_lto)
set(CMAKE_RANLIB "gcc-ranlib")
endif()
elseif(WIN32)
if(CMAKE_BUILD_TYPE STREQUAL Release)
# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GL")
# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /GL")
# set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /LTCG:STATUS")
# set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /LTCG:STATUS")
# set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} /LTCG:STATUS")
if(CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GL")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /GL")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /LTCG:STATUS")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /LTCG:STATUS")
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} /LTCG:STATUS")
endif()
endif()
endmacro()
@ -167,7 +167,7 @@ set(THREADS_PREFER_PTHREAD_FLAG ON)
# to allows to override CMAKE_CXX_STANDARD from command line
if(NOT DEFINED CMAKE_CXX_STANDARD)
if(CMAKE_CXX_COMPILER_ID MATCHES MSVC)
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set(CMAKE_CXX_STANDARD 14)
else()
set(CMAKE_CXX_STANDARD 11)
@ -176,7 +176,7 @@ if(NOT DEFINED CMAKE_CXX_STANDARD)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
endif()
if(COVERAGE)
if(ENABLE_COVERAGE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage")
@ -198,10 +198,10 @@ if(WIN32)
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /LARGEADDRESSAWARE")
if (TREAT_WARNING_AS_ERROR)
if(CMAKE_CXX_COMPILER_ID MATCHES Intel)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
ie_add_compiler_flags(/WX)
ie_add_compiler_flags(/Qdiag-warning:47,1740,1786)
elseif (CMAKE_CXX_COMPILER_ID MATCHES MSVC)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# ie_add_compiler_flags(/WX) # Too many warnings
endif()
endif()
@ -212,14 +212,14 @@ if(WIN32)
# Disable noisy warnings
if(CMAKE_CXX_COMPILER_ID MATCHES MSVC)
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# C4251 needs to have dll-interface to be used by clients of class
ie_add_compiler_flags(/wd4251)
# C4275 non dll-interface class used as base for dll-interface class
ie_add_compiler_flags(/wd4275)
endif()
if(CMAKE_CXX_COMPILER_ID MATCHES Intel)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
# 161 unrecognized pragma
# 177 variable was declared but never referenced
# 556 not matched type of assigned function pointer
@ -236,20 +236,6 @@ if(WIN32)
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /Z7")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Z7")
if(ENABLE_DEBUG_SYMBOLS)
ie_add_compiler_flags(/Z7)
set(DEBUG_SYMBOLS_LINKER_FLAGS "/DEBUG")
if (CMAKE_BUILD_TYPE STREQUAL "Release")
# Keep default /OPT values. See /DEBUG reference for details.
set(DEBUG_SYMBOLS_LINKER_FLAGS "${DEBUG_SYMBOLS_LINKER_FLAGS} /OPT:REF /OPT:ICF")
endif()
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${DEBUG_SYMBOLS_LINKER_FLAGS}")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${DEBUG_SYMBOLS_LINKER_FLAGS}")
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${DEBUG_SYMBOLS_LINKER_FLAGS}")
endif()
else()
# TODO: enable for C sources as well
# ie_add_compiler_flags(-Werror)

7
cmake/sanitizer.cmake
View File

@ -14,7 +14,7 @@ if (ENABLE_SANITIZER)
set(SANITIZER_LINKER_FLAGS "-fsanitize=address")
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fuse-ld=gold")
elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND NOT WIN32)
elseif(CMAKE_CXX_COMPILER_ID MATCHES "^(Apple)?Clang$" AND NOT WIN32)
set(SANITIZER_LINKER_FLAGS "${SANITIZER_LINKER_FLAGS} -fuse-ld=lld")
endif()
@ -26,9 +26,8 @@ if (ENABLE_SANITIZER)
endif()
if (ENABLE_THREAD_SANITIZER)
set(SANITIZER_COMPILER_FLAGS "-g -fsanitize=thread")
set(SANITIZER_LINKER_FLAGS "-fsanitize=thread")
set(SANITIZER_COMPILER_FLAGS "-g -fsanitize=thread -fno-omit-frame-pointer")
set(SANITIZER_LINKER_FLAGS "-fsanitize=thread -static-libsan")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SANITIZER_COMPILER_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SANITIZER_COMPILER_FLAGS}")

4
cmake/sdl.cmake
View File

@ -25,7 +25,7 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release")
if (NOT ENABLE_SANITIZER)
set(IE_C_CXX_FLAGS "${IE_C_CXX_FLAGS} -s")
endif()
elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
elseif(CMAKE_CXX_COMPILER_ID MATCHES "^(Apple)?Clang$")
set(IE_C_CXX_FLAGS "${IE_C_CXX_FLAGS} -fstack-protector-all")
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
if (NOT ENABLE_SANITIZER)
@ -36,7 +36,7 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_MODULE_LINKER_FLAGS_RELEASE} -z noexecstack -z relro -z now")
set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} -z noexecstack -z relro -z now")
endif()
elseif(CMAKE_CXX_COMPILER_ID MATCHES MSVC)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set(IE_C_CXX_FLAGS "${IE_C_CXX_FLAGS} /sdl")
endif()

2
cmake/target_flags.cmake
View File

@ -7,7 +7,7 @@ if(CMAKE_CL_64)
set(MSVC64 ON)
endif()
if(WIN32 AND CMAKE_CXX_COMPILER_ID MATCHES "GNU")
if(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
execute_process(COMMAND ${CMAKE_CXX_COMPILER} -dumpmachine
OUTPUT_VARIABLE OPENVINO_GCC_TARGET_MACHINE
OUTPUT_STRIP_TRAILING_WHITESPACE)

0
.editorconfig → inference-engine/.editorconfig
View File

2
inference-engine/CMakeLists.txt
View File

@ -188,7 +188,7 @@ configure_file(
# Coverage
#
if(COVERAGE)
if(ENABLE_COVERAGE)
include(coverage_ie)
endif()

9
inference-engine/cmake/coverage_ie.cmake
View File

@ -13,14 +13,17 @@ ie_coverage_capture(INFO_FILE "dldt"
# Generate reports
ie_coverage_extract(INPUT "dldt" OUTPUT "inference_engine_with_builders"
ie_coverage_extract(INPUT "dldt" OUTPUT "inference_engine"
PATTERNS "${DLDT_COVERAGE_BASE_DIRECTORY}/inference_engine/*"
"${DLDT_COVERAGE_BASE_DIRECTORY}/plugin_api/*")
ie_coverage_remove(INPUT "inference_engine_with_builders" OUTPUT "inference_engine"
PATTERNS "${DLDT_COVERAGE_BASE_DIRECTORY}/inference_engine/builders/*")
ie_coverage_genhtml(INFO_FILE "inference_engine"
PREFIX "${DLDT_COVERAGE_BASE_DIRECTORY}")
ie_coverage_extract(INPUT "dldt" OUTPUT "inference_engine_ir_reader"
PATTERNS "${DLDT_COVERAGE_BASE_DIRECTORY}/ir_readers/*")
ie_coverage_genhtml(INFO_FILE "inference_engine_ir_reader"
PREFIX "${DLDT_COVERAGE_BASE_DIRECTORY}")
ie_coverage_extract(INPUT "dldt" OUTPUT "inference_engine_legacy"
PATTERNS "${DLDT_COVERAGE_BASE_DIRECTORY}/legacy_api/*")
ie_coverage_genhtml(INFO_FILE "inference_engine_legacy"

6
inference-engine/cmake/cpplint.cmake
View File

@ -3,10 +3,10 @@
#
if(ENABLE_CPPLINT)
find_host_package(PythonInterp)
find_package(Python3 COMPONENTS Interpreter)
if(NOT PYTHONINTERP_FOUND)
message(WARNING "Python interpreter was not found (required for cpplint check)")
if(NOT Python3_Interpreter_FOUND)
message(WARNING "Python3 interpreter was not found (required for cpplint check)")
set(ENABLE_CPPLINT OFF)
endif()
endif()

4
inference-engine/cmake/developer_package_config.cmake.in
View File

@ -25,6 +25,8 @@ endforeach()
message("")
set(gflags_DIR "@gflags_BINARY_DIR@")
# GNA lib dir
set(GNA "@GNA@")
# Targets
@ -43,7 +45,7 @@ list(APPEND CMAKE_MODULE_PATH "${OpenVINO_MAIN_SOURCE_DIR}/cmake")
list(APPEND CMAKE_MODULE_PATH "${IE_MAIN_SOURCE_DIR}/cmake")
# generic stuff from developer package
include(developer_package NO_POLICY_SCOPE)
include(developer_package)
include(developer_package_ie)
# Don't threat deprecated API warnings as errors in 3rd party apps

15
inference-engine/cmake/features_ie.cmake
View File

@ -62,8 +62,6 @@ if (ENABLE_GNA)
endif()
endif()
ie_option (ENABLE_IR_READER "Compile with IR readers / parsers" ON)
ie_option (ENABLE_VPU "vpu targeted plugins for inference engine" ON)
ie_dependent_option (ENABLE_MYRIAD "myriad targeted plugin for inference engine" ON "ENABLE_VPU" OFF)
@ -72,7 +70,7 @@ ie_dependent_option (ENABLE_MYRIAD_NO_BOOT "myriad plugin will skip device boot"
ie_option (ENABLE_TESTS "unit, behavior and functional tests" OFF)
ie_dependent_option (ENABLE_GAPI_TESTS "tests for GAPI kernels" OFF "ENABLE_TESTS" OFF)
ie_dependent_option (ENABLE_GAPI_TESTS "tests for GAPI kernels" ON "ENABLE_TESTS" OFF)
ie_dependent_option (GAPI_TEST_PERF "if GAPI unit tests should examine performance" OFF "ENABLE_GAPI_TESTS" OFF)
@ -84,7 +82,7 @@ ie_dependent_option (ENABLE_SAME_BRANCH_FOR_MODELS "uses same branch for models
ie_dependent_option (ENABLE_BEH_TESTS "tests oriented to check inference engine API corecteness" ON "ENABLE_TESTS" OFF)
ie_dependent_option (ENABLE_FUNCTIONAL_TESTS "functional tests" ON "ENABLE_TESTS;ENABLE_IR_READER" OFF)
ie_dependent_option (ENABLE_FUNCTIONAL_TESTS "functional tests" ON "ENABLE_TESTS" OFF)
ie_dependent_option (ENABLE_SAMPLES "console samples are part of inference engine package" ON "NOT MINGW" OFF)
@ -98,18 +96,15 @@ ie_option (ENABLE_ALTERNATIVE_TEMP "in case of dependency conflict, to avoid mod
ie_option (ENABLE_OPENCV "enables OpenCV" ON)
ie_option (ENABLE_DEBUG_SYMBOLS "generates symbols for debugging" OFF)
ie_option (ENABLE_PYTHON "enables ie python bridge build" OFF)
ie_option (ENABLE_CPP_CCT "enables C++ version of Cross Check Tool" OFF)
ie_option (ENABLE_C "enables ie c bridge build" ON)
ie_dependent_option(ENABLE_CPPLINT "Enable cpplint checks during the build" OFF "UNIX;NOT APPLE;NOT ANDROID" OFF)
ie_dependent_option(ENABLE_CPPLINT "Enable cpplint checks during the build" ON "UNIX;NOT APPLE;NOT ANDROID" OFF)
ie_dependent_option(ENABLE_CPPLINT_REPORT "Build cpplint report instead of failing the build" OFF "ENABLE_CPPLINT" OFF)
ie_option(ENABLE_CLANG_FORMAT "Enable clang-format checks during the build" OFF)
ie_option(ENABLE_CLANG_FORMAT "Enable clang-format checks during the build" ON)
set(IE_EXTRA_PLUGINS "" CACHE STRING "Extra paths for plugins to include into DLDT build tree")

4
inference-engine/cmake/models.cmake
View File

@ -34,6 +34,10 @@ function (add_models_repo add_to_fetcher model_name)
endfunction()
function(add_lfs_repo name prefix url tag)
if(TARGET ${name})
return()
endif()
ExternalProject_Add(${name}
PREFIX ${prefix}
GIT_REPOSITORY ${url}

2
inference-engine/cmake/share/InferenceEngineConfig.cmake.in
View File

@ -150,7 +150,7 @@ else()
set_target_properties(IE::inference_engine${ie_library_suffix} PROPERTIES
IMPORTED_LOCATION "${IE${ie_library_usuffix}_RELEASE_LIBRARY}"
INTERFACE_INCLUDE_DIRECTORIES "${IE_INCLUDE_DIR}")
if(CMAKE_CXX_COMPILER_ID MATCHES Intel)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
set_target_properties(IE::inference_engine${ie_library_suffix} PROPERTIES
INTERFACE_COMPILE_OPTIONS "-diag-warning=1786")
else()

26
inference-engine/ie_bridges/c/samples/object_detection_sample_ssd/main.c
View File

@ -105,9 +105,9 @@ void readInputFilesArgument(const char *arg) {
const char *fileName = ep->d_name;
if (strcmp(fileName, ".") == 0 || strcmp(fileName, "..") == 0) continue;
char *file_path = (char *)calloc(strlen(arg) + strlen(ep->d_name) + 2, sizeof(char));
strcpy(file_path, arg);
strcat(file_path, "/");
strcat(file_path, ep->d_name);
memcpy(file_path, arg, strlen(arg));
memcpy(file_path + strlen(arg), "/", strlen("/"));
memcpy(file_path + strlen(arg) + strlen("/"), ep->d_name, strlen(ep->d_name) + 1);
if (file_num == 0) {
file_paths = (char **)calloc(1, sizeof(char *));
@ -131,7 +131,7 @@ void readInputFilesArgument(const char *arg) {
dp = NULL;
} else {
char *file_path = (char *)calloc(strlen(arg) + 1, sizeof(char));
strcpy(file_path, arg);
memcpy(file_path, arg, strlen(arg) + 1);
if (file_num == 0) {
file_paths = (char **)calloc(1, sizeof(char *));
}
@ -187,8 +187,8 @@ ie_config_t *parseConfig(const char *config_file, char comment) {
if (fscanf(file, "%s", key)!= EOF && fscanf(file, "%s", value) != EOF) {
char *cfg_name = (char *)calloc(strlen(key) + 1, sizeof(char));
char *cfg_value = (char *)calloc(strlen(value) + 1, sizeof(char));
strcpy(cfg_name, key);
strcpy(cfg_value, value);
memcpy(cfg_name, key, strlen(key) + 1);
memcpy(cfg_value, value, strlen(value) + 1);
ie_config_t *cfg_t = (ie_config_t *)calloc(1, sizeof(ie_config_t));
cfg_t->name = cfg_name;
cfg_t->value = cfg_value;
@ -203,8 +203,8 @@ ie_config_t *parseConfig(const char *config_file, char comment) {
}
char *cfg_name = (char *)calloc(strlen(key) + 1, sizeof(char));
char *cfg_value = (char *)calloc(strlen(value) + 1, sizeof(char));
strcpy(cfg_name, key);
strcpy(cfg_value, value);
memcpy(cfg_name, key, strlen(key) + 1);
memcpy(cfg_value, value, strlen(value) + 1);
ie_config_t *cfg_t = (ie_config_t *)calloc(1, sizeof(ie_config_t));
cfg_t->name = cfg_name;
cfg_t->value = cfg_value;
@ -345,8 +345,8 @@ int main(int argc, char **argv) {
// --------------------------- 4. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------
input_weight = (char *)calloc(strlen(input_model) + 1, sizeof(char));
strncpy(input_weight, input_model, strlen(input_model)-4);
strcat(input_weight, ".bin");
memcpy(input_weight, input_model, strlen(input_model) - 4);
memcpy(input_weight + strlen(input_model) - 4, ".bin", strlen(".bin") + 1);
printf("%sLoading network files:\n", info);
printf("\t%s\n", input_model);
printf("\t%s\n", input_weight);
@ -680,9 +680,9 @@ int main(int argc, char **argv) {
char str_num[16] = {0};
int2str(str_num, batch_id);
char *img_path = (char *)calloc(strlen(out) + strlen(str_num) + strlen(".bmp") + 1, sizeof(char));
strcpy(img_path, out);
strcat(img_path, str_num);
strcat(img_path, ".bmp");
memcpy(img_path, out, strlen(out));
memcpy(img_path + strlen(out), str_num, strlen(str_num));
memcpy(img_path + strlen(out) + strlen(str_num), ".bmp", strlen(".bmp") + 1);
image_save(img_path, &originalImages[batch_id]);
printf("%sImage %s created!\n", info, img_path);
free(img_path);

2
inference-engine/ie_bridges/c/tests/CMakeLists.txt
View File

@ -27,6 +27,8 @@ target_compile_definitions(${TARGET_NAME}
DATA_PATH=\"${DATA_PATH}\"
MODELS_PATH=\"${MODELS_PATH}\" )
add_dependencies(${TARGET_NAME} MultiDevicePlugin)
if(ENABLE_MKL_DNN)
add_dependencies(${TARGET_NAME} MKLDNNPlugin)
endif()

7
inference-engine/ie_bridges/c/tests/ie_c_api_test.cpp
View File

@ -816,10 +816,8 @@ TEST(ie_exec_network_set_config, setConfig) {
IE_ASSERT_OK(ie_core_create("", &core));
ASSERT_NE(nullptr, core);
ie_core_versions_t ie_core_versions_multi;
ie_param_t param;
if (ie_core_get_versions(core, "MULTI", &ie_core_versions_multi) != IEStatusCode::OK ||
ie_core_get_metric(core, "GPU", "AVAILABLE_DEVICES", &param) != IEStatusCode::OK) {
if (ie_core_get_metric(core, "GPU", "AVAILABLE_DEVICES", &param) != IEStatusCode::OK) {
ie_core_free(&core);
GTEST_SKIP();
}
@ -837,11 +835,10 @@ TEST(ie_exec_network_set_config, setConfig) {
ie_config_t config_param = {"MULTI_DEVICE_PRIORITIES", "GPU,CPU", nullptr};
IE_EXPECT_OK(ie_exec_network_set_config(exe_network, &config_param));
ie_core_versions_free(&ie_core_versions_multi);
ie_param_free(&param);
ie_exec_network_free(&exe_network);
ie_network_free(&network);
ie_core_free(&core);
ie_param_free(&param);
}
TEST(ie_exec_network_get_metric, getMetric) {

7
inference-engine/ie_bridges/python/CMakeLists.txt
View File

@ -55,6 +55,13 @@ endif()
set (PYTHON_BRIDGE_SRC_ROOT ${CMAKE_CURRENT_SOURCE_DIR})
add_subdirectory (src/openvino/inference_engine)
# Check Cython version
if("${CYTHON_VERSION}" VERSION_LESS "0.29")
message(FATAL_ERROR "OpenVINO Python API needs at least Cython version 0.29, found verson ${CYTHON_VERSION}")
else()
message(STATUS "Found Cython version ${CYTHON_VERSION}")
endif()
# install
ie_cpack_add_component(${PYTHON_VERSION} REQUIRED)

6
inference-engine/ie_bridges/python/cmake/FindCython.cmake
View File

@ -56,4 +56,8 @@ endif()
include( FindPackageHandleStandardArgs )
FIND_PACKAGE_HANDLE_STANDARD_ARGS( Cython REQUIRED_VARS CYTHON_EXECUTABLE )
mark_as_advanced( CYTHON_EXECUTABLE )
# Find Cython version
execute_process(COMMAND ${CYTHON_EXECUTABLE} -V ERROR_VARIABLE CYTHON_OUTPUT OUTPUT_QUIET)
string(REGEX REPLACE "^Cython version ([0-9]+\\.[0-9]+\\.[0-9]+).*" "\\1" CYTHON_VERSION "${CYTHON_OUTPUT}")
mark_as_advanced( CYTHON_EXECUTABLE CYTHON_VERSION )

2
inference-engine/ie_bridges/python/src/openvino/inference_engine/ie_api.pyx
View File

@ -1439,7 +1439,7 @@ cdef class IENetwork:
# net = ie.read_network(model=path_to_xml_file, weights=path_to_bin_file)
# input_layer = next(iter(net.inputs))
# n, c, h, w = net.inputs[input_layer]
# net.reshape({input_layer: (n, c, h*2, w*2)}]
# net.reshape({input_layer: (n, c, h*2, w*2)})
# ```
def reshape(self, input_shapes: dict):
cdef map[string, vector[size_t]] c_input_shapes;

8
inference-engine/ie_bridges/python/tests/test_InferRequest.py
View File

@ -2,6 +2,7 @@ import numpy as np
import os
import pytest
import warnings
import threading
from openvino.inference_engine import ie_api as ie
from conftest import model_path, image_path
@ -324,15 +325,22 @@ def test_async_infer_callback_wait_in_callback(device):
class InferReqWrap:
def __init__(self, request):
self.request = request
self.cv = threading.Condition()
self.request.set_completion_callback(self.callback)
self.status_code = self.request.wait(ie.WaitMode.STATUS_ONLY)
assert self.status_code == ie.StatusCode.INFER_NOT_STARTED
def callback(self, statusCode, userdata):
self.status_code = self.request.wait(ie.WaitMode.STATUS_ONLY)
self.cv.acquire()
self.cv.notify()
self.cv.release()
def execute(self, input_data):
self.request.async_infer(input_data)
self.cv.acquire()
self.cv.wait()
self.cv.release()
status = self.request.wait(ie.WaitMode.RESULT_READY)
assert status == ie.StatusCode.OK
assert self.status_code == ie.StatusCode.OK

22
inference-engine/include/cpp/ie_cnn_network.h
View File

@ -49,9 +49,7 @@ public:
*/
explicit CNNNetwork(std::shared_ptr<ICNNNetwork> network): network(network) {
actual = network.get();
if (actual == nullptr) {
THROW_IE_EXCEPTION << "CNNNetwork was not initialized.";
}
if (actual == nullptr) THROW_IE_EXCEPTION << "CNNNetwork was not initialized.";
}
/**
@ -100,6 +98,7 @@ public:
* @return outputs Reference to the OutputsDataMap object
*/
virtual OutputsDataMap getOutputsInfo() const {
if (actual == nullptr) THROW_IE_EXCEPTION << "CNNNetwork was not initialized.";
OutputsDataMap outputs;
actual->getOutputsInfo(outputs);
return outputs;
@ -113,6 +112,7 @@ public:
* @return inputs Reference to InputsDataMap object
*/
virtual InputsDataMap getInputsInfo() const {
if (actual == nullptr) THROW_IE_EXCEPTION << "CNNNetwork was not initialized.";
InputsDataMap inputs;
actual->getInputsInfo(inputs);
return inputs;
@ -126,6 +126,7 @@ public:
* @return The number of layers as an integer value
*/
size_t layerCount() const {
if (actual == nullptr) THROW_IE_EXCEPTION << "CNNNetwork was not initialized.";
return actual->layerCount();
}
@ -136,7 +137,8 @@ public:
*
* @return Network name
*/
const std::string& getName() const noexcept {
const std::string& getName() const {
if (actual == nullptr) THROW_IE_EXCEPTION << "CNNNetwork was not initialized.";
return actual->getName();
}
@ -160,6 +162,7 @@ public:
* @return The size of batch as a size_t value
*/
virtual size_t getBatchSize() const {
if (actual == nullptr) THROW_IE_EXCEPTION << "CNNNetwork was not initialized.";
return actual->getBatchSize();
}
@ -168,7 +171,7 @@ public:
*
* @return A shared pointer of the current network
*/
operator std::shared_ptr<ICNNNetwork>() {
operator ICNNNetwork::Ptr() {
return network;
}
@ -178,6 +181,7 @@ public:
* @return An instance of the current network
*/
operator ICNNNetwork&() {
if (actual == nullptr) THROW_IE_EXCEPTION << "CNNNetwork was not initialized.";
return *actual;
}
@ -187,6 +191,7 @@ public:
* @return A const reference of the current network
*/
operator const ICNNNetwork&() const {
if (actual == nullptr) THROW_IE_EXCEPTION << "CNNNetwork was not initialized.";
return *actual;
}
@ -195,7 +200,8 @@ public:
*
* @return constant nGraph function
*/
std::shared_ptr<ngraph::Function> getFunction() noexcept {
std::shared_ptr<ngraph::Function> getFunction() {
if (actual == nullptr) THROW_IE_EXCEPTION << "CNNNetwork was not initialized.";
return actual->getFunction();
}
@ -204,7 +210,8 @@ public:
*
* @return constant nGraph function
*/
std::shared_ptr<const ngraph::Function> getFunction() const noexcept {
std::shared_ptr<const ngraph::Function> getFunction() const {
if (actual == nullptr) THROW_IE_EXCEPTION << "CNNNetwork was not initialized.";
return actual->getFunction();
}
@ -278,6 +285,7 @@ public:
* @return Map of pairs: input name and its dimension.
*/
virtual ICNNNetwork::InputShapes getInputShapes() const {
if (actual == nullptr) THROW_IE_EXCEPTION << "CNNNetwork was not initialized.";
ICNNNetwork::InputShapes shapes;
InputsDataMap inputs;
actual->getInputsInfo(inputs);

4
inference-engine/include/cpp/ie_executable_network.hpp
View File

@ -190,9 +190,7 @@ public:
* @return A vector of Memory State objects
*/
std::vector<MemoryState> QueryState() {
if (actual == nullptr) {
THROW_IE_EXCEPTION << "ExecutableNetwork wrapper was not initialized.";
}
if (actual == nullptr) THROW_IE_EXCEPTION << "ExecutableNetwork was not initialized.";
IMemoryState::Ptr pState = nullptr;
auto res = OK;
std::vector<MemoryState> controller;

9
inference-engine/include/cpp/ie_infer_request.hpp
View File

@ -88,9 +88,7 @@ public:
InferenceEngine::details::SharedObjectLoader::Ptr splg = {}):
actual(request), plg(splg) {
// plg can be null, but not the actual
if (actual == nullptr) {
THROW_IE_EXCEPTION << "InferRequest wrapper was not initialized.";
}
if (actual == nullptr) THROW_IE_EXCEPTION << "InferRequest was not initialized.";
}
/**
@ -230,9 +228,7 @@ public:
*/
StatusCode Wait(int64_t millis_timeout) {
ResponseDesc resp;
if (actual == nullptr) {
THROW_IE_EXCEPTION << "InferRequest wrapper was not initialized.";
}
if (actual == nullptr) THROW_IE_EXCEPTION << "InferRequest was not initialized.";
auto res = actual->Wait(millis_timeout, &resp);
if (res != OK && res != RESULT_NOT_READY && res != INFER_NOT_STARTED) {
InferenceEngine::details::extract_exception(res, resp.msg);
@ -259,6 +255,7 @@ public:
* @return A shared pointer to underlying IInferRequest interface
*/
operator IInferRequest::Ptr&() {
if (actual == nullptr) THROW_IE_EXCEPTION << "InferRequest was not initialized.";
return actual;
}

2
inference-engine/include/cpp/ie_plugin_cpp.hpp
View File

@ -57,6 +57,7 @@ public:
const Version* GetVersion() {
const Version* versionInfo = nullptr;
IE_SUPPRESS_DEPRECATED_START
if (actual == nullptr) THROW_IE_EXCEPTION << "InferencePlugin wrapper was not initialized";
actual->GetVersion(versionInfo);
IE_SUPPRESS_DEPRECATED_END
if (versionInfo == nullptr) {
@ -153,6 +154,7 @@ public:
void QueryNetwork(const ICNNNetwork& network, const std::map<std::string, std::string>& config,
QueryNetworkResult& res) const {
IE_SUPPRESS_DEPRECATED_START
if (actual == nullptr) THROW_IE_EXCEPTION << "InferencePlugin wrapper was not initialized";
actual->QueryNetwork(network, config, res);
IE_SUPPRESS_DEPRECATED_END
if (res.rc != OK) THROW_IE_EXCEPTION << res.resp.msg;

1
inference-engine/include/details/ie_cnn_network_iterator.hpp
View File

@ -53,6 +53,7 @@ public:
* scope.
*/
explicit CNNNetworkIterator(const ICNNNetwork* network) {
if (network == nullptr) THROW_IE_EXCEPTION << "ICNNNetwork object is nullptr";
InputsDataMap inputs;
network->getInputsInfo(inputs);
if (!inputs.empty()) {

6
inference-engine/include/details/ie_so_pointer.hpp
View File

@ -38,7 +38,11 @@ public:
* @brief The main constructor
* @param loader Library to load from
*/
explicit SymbolLoader(std::shared_ptr<Loader> loader): _so_loader(loader) {}
explicit SymbolLoader(std::shared_ptr<Loader> loader): _so_loader(loader) {
if (_so_loader == nullptr) {
THROW_IE_EXCEPTION << "SymbolLoader cannot be created with nullptr";
}
}
/**
* @brief Calls a function from the library that creates an object and returns StatusCode

26
inference-engine/include/ie_core.hpp
View File

@ -38,7 +38,7 @@ struct QueryNetworkResult {
StatusCode rc = OK;
/**
* @brief Response mssage
* @brief Response message
*/
ResponseDesc resp;
};
@ -61,7 +61,7 @@ public:
* @param xmlConfigFile A path to .xml file with plugins to load from. If XML configuration file is not specified,
* then default Inference Engine plugins are loaded from the default plugin.xml file.
*/
explicit Core(const std::string& xmlConfigFile = std::string());
explicit Core(const std::string& xmlConfigFile = {});
/**
* @brief Returns plugins version information
@ -92,7 +92,7 @@ public:
* if bin file with the same name was not found, will load IR without weights.
* @return CNNNetwork
*/
CNNNetwork ReadNetwork(const std::wstring& modelPath, const std::wstring& binPath = std::wstring()) const {
CNNNetwork ReadNetwork(const std::wstring& modelPath, const std::wstring& binPath = {}) const {
return ReadNetwork(details::wStringtoMBCSstringChar(modelPath), details::wStringtoMBCSstringChar(binPath));
}
#endif
@ -104,7 +104,7 @@ public:
* if bin file with the same name was not found, will load IR without weights.
* @return CNNNetwork
*/
CNNNetwork ReadNetwork(const std::string& modelPath, const std::string& binPath = "") const;
CNNNetwork ReadNetwork(const std::string& modelPath, const std::string& binPath = {}) const;
/**
* @brief Reads IR xml and bin (with the same name) files
* @param model string with IR
@ -126,8 +126,8 @@ public:
* @return An executable network reference
*/
ExecutableNetwork LoadNetwork(
const CNNNetwork network, const std::string& deviceName,
const std::map<std::string, std::string>& config = std::map<std::string, std::string>());
const CNNNetwork& network, const std::string& deviceName,
const std::map<std::string, std::string>& config = {});
/**
* @brief Registers extension
@ -141,11 +141,11 @@ public:
* @param context Pointer to RemoteContext object
* @param config Optional map of pairs: (config parameter name, config parameter value) relevant only for this load
* operation
* @return An executable network reference
* @return An executable network object
*/
ExecutableNetwork LoadNetwork(
const CNNNetwork network, RemoteContext::Ptr context,
const std::map<std::string, std::string>& config = std::map<std::string, std::string>());
const CNNNetwork& network, RemoteContext::Ptr context,
const std::map<std::string, std::string>& config = {});
/**
* @brief Registers extension for the specified plugin
@ -166,7 +166,7 @@ public:
*/
ExecutableNetwork ImportNetwork(
const std::string& modelFileName, const std::string& deviceName,
const std::map<std::string, std::string>& config = std::map<std::string, std::string>());
const std::map<std::string, std::string>& config = {});
/**
* @brief Creates an executable network from a previously exported network
@ -199,11 +199,11 @@ public:
* @param deviceName A name of a device to query
* @param network Network object to query
* @param config Optional map of pairs: (config parameter name, config parameter value)
* @return Pointer to the response message that holds a description of an error if any occurred
* @return An object containing a map of pairs a layer name -> a device name supporting this layer.
*/
QueryNetworkResult QueryNetwork(
const ICNNNetwork& network, const std::string& deviceName,
const std::map<std::string, std::string>& config = std::map<std::string, std::string>()) const;
const std::map<std::string, std::string>& config = {}) const;
/**
* @brief Sets configuration for device, acceptable keys can be found in ie_plugin_config.hpp
@ -213,7 +213,7 @@ public:
*
* @param config Map of pairs: (config parameter name, config parameter value)
*/
void SetConfig(const std::map<std::string, std::string>& config, const std::string& deviceName = std::string());
void SetConfig(const std::map<std::string, std::string>& config, const std::string& deviceName = {});
/**
* @brief Gets configuration dedicated to device behaviour.

14
inference-engine/samples/CMakeLists.txt
View File

@ -6,6 +6,10 @@ cmake_minimum_required (VERSION 2.8.12)
project(Samples)
if(POLICY CMP0054)
cmake_policy(SET CMP0054 NEW)
endif()
if(POLICY CMP0063)
cmake_policy(SET CMP0063 NEW)
endif()
@ -66,7 +70,7 @@ if (WIN32)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX") #treating warnings as errors
endif ()
if (CMAKE_CXX_COMPILER_ID MATCHES MSVC)
if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4251 /wd4275 /wd4267") #disable some warnings
endif()
else()
@ -78,7 +82,7 @@ else()
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=unused-command-line-argument")
elseif(UNIX)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wuninitialized -Winit-self")
if(NOT CMAKE_CXX_COMPILER_ID STREQUAL Clang)
if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wmaybe-uninitialized")
endif()
endif()
@ -100,7 +104,7 @@ if(NOT DEFINED CMAKE_CXX_STANDARD)
set (CMAKE_CXX_STANDARD 11)
set (CMAKE_CXX_EXTENSIONS OFF)
set (CMAKE_CXX_STANDARD_REQUIRED ON)
if (CMAKE_CXX_COMPILER_ID STREQUAL GNU)
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set (CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS}")
endif()
endif()
@ -117,7 +121,7 @@ if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/gflags")
set_target_properties(gflags_nothreads_static PROPERTIES FOLDER thirdparty)
endif()
if (CMAKE_CXX_COMPILER_ID STREQUAL GNU)
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
endif()
@ -228,7 +232,7 @@ macro(ie_add_sample)
if(COMMAND add_cpplint_target AND NOT IE_SAMPLE_EXCLUDE_CPPLINT)
if(folder_name STREQUAL "c_samples")
set(custom_filters "-readability/casting,-runtime/printf")
set(custom_filters "-readability/casting")
endif()
add_cpplint_target(${IE_SAMPLE_NAME}_cpplint FOR_TARGETS ${IE_SAMPLE_NAME}
CUSTOM_FILTERS ${custom_filters})

1
inference-engine/samples/benchmark_app/README.md
View File

@ -84,6 +84,7 @@ Options:
-stream_output Optional. Print progress as a plain text. When specified, an interactive progress bar is replaced with a multiline output.
-t Optional. Time in seconds to execute topology.
-progress Optional. Show progress bar (can affect performance measurement). Default values is "false".
-shape Optional. Set shape for input. For example, "input1[1,3,224,224],input2[1,4]" or "[1,3,224,224]" in case of one input size.
CPU-specific performance options:
-nstreams "<integer>" Optional. Number of streams to use for inference on the CPU or/and GPU in throughput mode

7
inference-engine/samples/benchmark_app/benchmark_app.hpp
View File

@ -97,6 +97,9 @@ static const char load_config_message[] = "Optional. Path to XML/YAML/JSON file
static const char dump_config_message[] = "Optional. Path to XML/YAML/JSON file to dump IE parameters, which were set by application.";
#endif
static const char shape_message[] = "Optional. Set shape for input. For example, \"input1[1,3,224,224],input2[1,4]\" or \"[1,3,224,224]\""
" in case of one input size.";
/// @brief Define flag for showing help message <br>
DEFINE_bool(h, false, help_message);
@ -178,6 +181,9 @@ DEFINE_string(load_config, "", load_config_message);
DEFINE_string(dump_config, "", dump_config_message);
#endif
/// @brief Define flag for input shape <br>
DEFINE_string(shape, "", shape_message);
/**
* @brief This function show a help message
*/
@ -200,6 +206,7 @@ static void showUsage() {
std::cout << " -stream_output " << stream_output_message << std::endl;
std::cout << " -t " << execution_time_message << std::endl;
std::cout << " -progress " << progress_message << std::endl;
std::cout << " -shape " << shape_message << std::endl;
std::cout << std::endl << " device-specific performance options:" << std::endl;
std::cout << " -nstreams \"<integer>\" " << infer_num_streams_message << std::endl;
std::cout << " -nthreads \"<integer>\" " << infer_num_threads_message << std::endl;

45
inference-engine/samples/benchmark_app/main.cpp
View File

@ -316,33 +316,28 @@ int main(int argc, char *argv[]) {
// ----------------- 5. Resizing network to match image sizes and given batch ----------------------------------
next_step();
batchSize = cnnNetwork.getBatchSize();
if ((FLAGS_b != 0) && (batchSize != FLAGS_b)) {
ICNNNetwork::InputShapes shapes = cnnNetwork.getInputShapes();
bool reshape = false;
for (const InputsDataMap::value_type& item : inputInfo) {
auto layout = item.second->getTensorDesc().getLayout();
int batchIndex = -1;
if ((layout == Layout::NCHW) || (layout == Layout::NCDHW) ||
(layout == Layout::NHWC) || (layout == Layout::NDHWC) ||
(layout == Layout::NC)) {
batchIndex = 0;
} else if (layout == CN) {
batchIndex = 1;
}
if ((batchIndex != -1) && (shapes[item.first][batchIndex] != FLAGS_b)) {
shapes[item.first][batchIndex] = FLAGS_b;
reshape = true;
}
}
if (reshape) {
slog::info << "Resizing network to batch = " << FLAGS_b << slog::endl;
cnnNetwork.reshape(shapes);
}
// Parse input shapes if specified
InferenceEngine::ICNNNetwork::InputShapes shapes = cnnNetwork.getInputShapes();
bool reshape = false;
if (!FLAGS_shape.empty()) {
reshape |= updateShapes(shapes, FLAGS_shape, inputInfo);
}
if ((FLAGS_b != 0) && (batchSize != FLAGS_b)) {
reshape |= adjustShapesBatch(shapes, FLAGS_b, inputInfo);
}
if (reshape) {
slog::info << "Reshaping network: " << getShapesString(shapes) << slog::endl;
startTime = Time::now();
cnnNetwork.reshape(shapes);
auto duration_ms = double_to_string(get_total_ms_time(startTime));
slog::info << "Reshape network took " << duration_ms << " ms" << slog::endl;
if (statistics)
statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS,
{
{"reshape network time (ms)", duration_ms}
});
}
batchSize = cnnNetwork.getBatchSize();
topology_name = cnnNetwork.getName();
slog::info << (FLAGS_b != 0 ? "Network batch size was changed to: " : "Network batch size: ") << batchSize << slog::endl;

72
inference-engine/samples/benchmark_app/utils.cpp
View File

@ -7,6 +7,7 @@
#include <utility>
#include <vector>
#include <map>
#include <regex>
#include <samples/common.hpp>
#include <samples/slog.hpp>
@ -101,6 +102,77 @@ std::map<std::string, std::string> parseNStreamsValuePerDevice(const std::vector
return result;
}
bool adjustShapesBatch(InferenceEngine::ICNNNetwork::InputShapes& shapes,
const size_t batch_size, const InferenceEngine::InputsDataMap& input_info) {
bool updated = false;
for (auto& item : input_info) {
auto layout = item.second->getTensorDesc().getLayout();
int batch_index = -1;
if ((layout == InferenceEngine::Layout::NCHW) || (layout == InferenceEngine::Layout::NCDHW) ||
(layout == InferenceEngine::Layout::NHWC) || (layout == InferenceEngine::Layout::NDHWC) ||
(layout == InferenceEngine::Layout::NC)) {
batch_index = 0;
} else if (layout == InferenceEngine::Layout::CN) {
batch_index = 1;
}
if ((batch_index != -1) && (shapes.at(item.first).at(batch_index) != batch_size)) {
shapes[item.first][batch_index] = batch_size;
updated = true;
}
}
return updated;
}
bool updateShapes(InferenceEngine::ICNNNetwork::InputShapes& shapes,
const std::string shapes_string, const InferenceEngine::InputsDataMap& input_info) {
bool updated = false;
std::string search_string = shapes_string;
auto start_pos = search_string.find_first_of('[');
while (start_pos != std::string::npos) {
auto end_pos = search_string.find_first_of(']');
if (end_pos == std::string::npos)
break;
auto input_name = search_string.substr(0, start_pos);
auto input_shape = search_string.substr(start_pos + 1, end_pos - start_pos - 1);
std::vector<size_t> parsed_shape;
for (auto& dim : split(input_shape, ',')) {
parsed_shape.push_back(std::stoi(dim));
}
if (!input_name.empty()) {
shapes[input_name] = parsed_shape;
updated = true;
} else {
for (auto& item : input_info) {
shapes[item.first] = parsed_shape;
}
updated = true;
}
search_string = search_string.substr(end_pos + 1);
if (search_string.empty() || search_string.front() != ',')
break;
search_string = search_string.substr(1);
start_pos = search_string.find_first_of('[');
}
if (!search_string.empty())
throw std::logic_error("Can't parse `shape` parameter: " + shapes_string);
return updated;
}
std::string getShapesString(const InferenceEngine::ICNNNetwork::InputShapes& shapes) {
std::stringstream ss;
for (auto& shape : shapes) {
if (!ss.str().empty()) ss << ", ";
ss << "\'" << shape.first << "': [";
for (size_t i = 0; i < shape.second.size(); i++) {
if (i > 0) ss << ", ";
ss << shape.second.at(i);
}
ss << "]";
}
return ss.str();
}
#ifdef USE_OPENCV
void dump_config(const std::string& filename,
const std::map<std::string, std::map<std::string, std::string>>& config) {

6
inference-engine/samples/benchmark_app/utils.hpp
View File

@ -12,6 +12,12 @@ std::vector<std::string> parseDevices(const std::string& device_string);
uint32_t deviceDefaultDeviceDurationInSeconds(const std::string& device);
std::map<std::string, std::string> parseNStreamsValuePerDevice(const std::vector<std::string>& devices,
const std::string& values_string);
bool updateShapes(InferenceEngine::ICNNNetwork::InputShapes& shapes,
const std::string shapes_string, const InferenceEngine::InputsDataMap& input_info);
bool adjustShapesBatch(InferenceEngine::ICNNNetwork::InputShapes& shapes,
const size_t batch_size, const InferenceEngine::InputsDataMap& input_info);
std::string getShapesString(const InferenceEngine::ICNNNetwork::InputShapes& shapes);
#ifdef USE_OPENCV
void dump_config(const std::string& filename,
const std::map<std::string, std::map<std::string, std::string>>& config);

31
inference-engine/scripts/create_vs2017_proj_x64.cmd Normal file
View File

@ -0,0 +1,31 @@
@echo off
:: Copyright (C) 2018-2020 Intel Corporation
:: SPDX-License-Identifier: Apache-2.0
pushd ..\..
if not exist "vs2017x64" (
mkdir "vs2017x64"
)
cmake -E chdir "vs2017x64" cmake -G "Visual Studio 15 2017 Win64" -T "Intel C++ Compiler 18.0" -DOS_FOLDER=ON ^
-DENABLE_MYRIAD=OFF -DENABLE_VPU=OFF -DENABLE_GNA=ON -DENABLE_CLDNN=OFF ^
-DENABLE_OPENCV=ON -DENABLE_MKL_DNN=ON ^
-DVERBOSE_BUILD=ON -DENABLE_TESTS=ON -DTHREADING=TBB ..
chdir
cd "vs2017x64\thirdparty\"
"C:\Program Files (x86)\Common Files\Intel\shared files\ia32\Bin\ICProjConvert180.exe" mkldnn.vcxproj /IC
chdir
cd "..\src\mkldnn_plugin"
"C:\Program Files (x86)\Common Files\Intel\shared files\ia32\Bin\ICProjConvert180.exe" MKLDNNPlugin.vcxproj /IC
"C:\Program Files (x86)\Common Files\Intel\shared files\ia32\Bin\ICProjConvert180.exe" test_MKLDNNPlugin.vcxproj /IC
chdir
cd "..\..\tests\unit"
"C:\Program Files (x86)\Common Files\Intel\shared files\ia32\Bin\ICProjConvert180.exe" InferenceEngineUnitTests.vcxproj /IC
popd
pause

2
inference-engine/scripts/run_code_checks.sh
View File

@ -1,4 +1,6 @@
#!/bin/bash
# Copyright (C) 2018-2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
command -v realpath >/dev/null 2>&1 || { echo >&2 "cpplint require realpath executable but it's not installed. Aborting."; exit 1; }

87
inference-engine/scripts/run_tests_myriad_multistick.sh Normal file
View File

@ -0,0 +1,87 @@
#!/bin/bash
# Copyright (C) 2018-2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
APP_NAME="MyriadFunctionalTests"
APPS_TO_RUN=$1
APPS_TO_RUN=${APPS_TO_RUN:=4}
echo "Run in parallel ${APPS_TO_RUN} applications"
TEST_DIR=../../bin/intel64
# Path to test dir is provided
if [[ -n "$2" ]]; then
TEST_DIR=$2
# Search for test dir with binaries
else
# Windows default
if [[ -f "${TEST_DIR}/${APP_NAME}" ]]; then
TEST_DIR=${TEST_DIR}
# Search for Release or Debug config
elif [[ -f "${TEST_DIR}/Release/${APP_NAME}" ]]; then
TEST_DIR="$TEST_DIR/Release/"
elif [[ -f "${TEST_DIR}/Debug/${APP_NAME}" ]]; then
TEST_DIR="$TEST_DIR/Debug/"
else
echo "Directory with binaries not found!"
exit -1
fi
fi
echo "Test directory: ${TEST_DIR}"
cd ${TEST_DIR}
export IE_VPU_MYRIADX=1
pids=""
if [[ "${APPS_TO_RUN}" -ge 1 ]] ; then
./${APP_NAME} --gtest_filter=*VPURegTest*SSD*myriad* &
pids+=" $!"
fi
if [[ "${APPS_TO_RUN}" -ge 2 ]] ; then
./${APP_NAME} --gtest_filter=*VPURegTest*VGG*myriad* &
pids+=" $!"
fi
if [[ "${APPS_TO_RUN}" -ge 3 ]] ; then
./${APP_NAME} --gtest_filter=*VPURegTest*VGG*myriad* &
pids+=" $!"
fi
if [[ "${APPS_TO_RUN}" -ge 4 ]] ; then
# For more then 4 multidevice testing
for (( VAR = 4; VAR <= ${APPS_TO_RUN}; ++VAR )); do
./${APP_NAME} --gtest_filter=*VPURegTest*YOLO*myriad* &
pids+=" $!"
done
fi
# Wait for all processes to finish
sts=""
for p in ${pids}; do
if wait ${p}; then
sts+=" 1"
else
sts+=" 0"
fi
echo "--- Process $p finished"
done
idx=0
exit_code=0
for s in ${sts}; do
if [[ ${s} -eq 1 ]]; then
echo "Task $idx PASSED"
else
echo "Task $idx FAILED"
exit_code=1
fi
((idx+=1))
done
exit ${exit_code}

2
inference-engine/src/CMakeLists.txt
View File

@ -26,6 +26,8 @@ endif()
add_subdirectory(hetero_plugin)
add_subdirectory(multi_device)
add_subdirectory(transformations)
add_subdirectory(inference_engine)

12
inference-engine/src/cldnn_engine/cldnn_engine.cpp
View File

@ -26,8 +26,10 @@
#include <ngraph/opsets/opset2.hpp>
#include <ngraph/op/fused/gelu.hpp>
#include <generic_ie.hpp>
#include <transformations/common_optimizations/common_optimizations.hpp>
#include <transformations/convert_opset1_to_legacy/convert_opset1_to_legacy.hpp>
#include <transformations/convert_opset2_to_opset1/convert_opset2_to_opset1.hpp>
#include <transformations/convert_opset3_to_opset2/convert_opset3_to_opset2.hpp>
#include "convert_function_to_cnn_network.hpp"
#undef min
@ -79,6 +81,8 @@ InferenceEngine::ICNNNetwork::Ptr clDNNEngine::CloneNetwork(const InferenceEngin
::ngraph::op::GenericIE::DisableReshape noReshape(nGraphFunc);
// Note: instead of running all Conversion Transformations you can make up your own transformation pipeline
ngraph::pass::CommonOptimizations().run_on_function(nGraphFunc);
ngraph::pass::ConvertOpSet3ToOpSet2(transformations_callback).run_on_function(nGraphFunc);
ngraph::pass::ConvertOpSet2ToOpSet1(transformations_callback).run_on_function(nGraphFunc);
ngraph::pass::ConvertOpSet1ToLegacy(transformations_callback).run_on_function(nGraphFunc);
clonedNetwork = InferenceEngine::details::convertFunctionToICNNNetwork(nGraphFunc, network);
@ -143,7 +147,7 @@ auto check_inputs = [](InferenceEngine::InputsDataMap _networkInputs) {
}
};
ExecutableNetworkInternal::Ptr clDNNEngine::LoadExeNetworkImpl(const InferenceEngine::ICore * /*core*/, const InferenceEngine::ICNNNetwork &network,
ExecutableNetworkInternal::Ptr clDNNEngine::LoadExeNetworkImpl(const InferenceEngine::ICNNNetwork &network,
const std::map<std::string, std::string> &config) {
// verification of supported input
InferenceEngine::InputsDataMap _networkInputs;
@ -189,9 +193,9 @@ ExecutableNetworkInternal::Ptr clDNNEngine::LoadExeNetworkImpl(const InferenceEn
return std::make_shared<CLDNNExecNetwork>(*CloneNetwork(network), context, conf);
}
ExecutableNetworkInternal::Ptr clDNNEngine::LoadExeNetworkImpl(const InferenceEngine::ICore * /*core*/, const InferenceEngine::ICNNNetwork &network,
RemoteContext::Ptr context,
const std::map<std::string, std::string> &config) {
ExecutableNetworkInternal::Ptr clDNNEngine::LoadExeNetworkImpl(const InferenceEngine::ICNNNetwork &network,
RemoteContext::Ptr context,
const std::map<std::string, std::string> &config) {
InferenceEngine::InputsDataMap _networkInputs;
network.getInputsInfo(_networkInputs);
check_inputs(_networkInputs);

8
inference-engine/src/cldnn_engine/cldnn_engine.h
View File

@ -30,12 +30,12 @@ class clDNNEngine : public InferenceEngine::InferencePluginInternal,
public:
clDNNEngine();
InferenceEngine::ExecutableNetworkInternal::Ptr LoadExeNetworkImpl(const InferenceEngine::ICore * core, const InferenceEngine::ICNNNetwork &network,
InferenceEngine::ExecutableNetworkInternal::Ptr LoadExeNetworkImpl(const InferenceEngine::ICNNNetwork &network,
const std::map<std::string, std::string> &config) override;
InferenceEngine::ExecutableNetworkInternal::Ptr LoadExeNetworkImpl(const InferenceEngine::ICore * core, const InferenceEngine::ICNNNetwork &network,
InferenceEngine::RemoteContext::Ptr context,
const std::map<std::string, std::string> &config) override;
InferenceEngine::ExecutableNetworkInternal::Ptr LoadExeNetworkImpl(const InferenceEngine::ICNNNetwork &network,
InferenceEngine::RemoteContext::Ptr context,
const std::map<std::string, std::string> &config) override;
void SetConfig(const std::map<std::string, std::string> &config) override;
InferenceEngine::Parameter GetConfig(const std::string& name, const std::map<std::string, InferenceEngine::Parameter>& options) const override;

76
inference-engine/src/gna_plugin/backend/am_intel_dnn.cpp
View File

@ -42,11 +42,11 @@ void GNAPluginNS::backend::AMIntelDNN::BeginNewWrite(uint32_t index) {
void GNAPluginNS::backend::AMIntelDNN::Init(void *ptr_memory,
uint32_t num_memory_bytes,
intel_dnn_number_type_t number_type,
intel_dnn_number_type_t compute_precision,
float scale_factor) {
ptr_dnn_memory_ = ptr_memory;
num_bytes_dnn_memory_ = num_memory_bytes;
number_type_ = number_type;
compute_precision_ = compute_precision;
input_scale_factor_ = scale_factor;
ptr_active_outputs_ = nullptr;
@ -342,7 +342,7 @@ void GNAPluginNS::backend::AMIntelDNN::Propagate() {
reinterpret_cast<void *>(reinterpret_cast<int32_t *>(comp->op.recurrent.ptr_feedbacks) + j * comp_pwl->num_columns_out);
ApplyRecurrentTransform(comp, j, ptr_feedbacks);
// PrintOutputs(i);
ApplyPiecewiseLinearTransform(comp_pwl, number_type_, num_active_outputs, j);
ApplyPiecewiseLinearTransform(comp_pwl, compute_precision_, num_active_outputs, j);
}
i++; // skip next component
} else {
@ -352,9 +352,9 @@ void GNAPluginNS::backend::AMIntelDNN::Propagate() {
break;
case kDnnConvolutional1dOp:ApplyConvolutional1DTransform(comp);
break;
case kDnnPiecewiselinearOp:ApplyPiecewiseLinearTransform(comp, number_type_, num_active_outputs);
case kDnnPiecewiselinearOp:ApplyPiecewiseLinearTransform(comp, compute_precision_, num_active_outputs);
break;
case kDnnMaxPoolOp:ApplyMaxPoolTransform(comp, number_type_);
case kDnnMaxPoolOp:ApplyMaxPoolTransform(comp, compute_precision_);
break;
case kDnnInterleaveOp:ApplyTranspose(comp);
break;
@ -633,8 +633,8 @@ void GNAPluginNS::backend::AMIntelDNN::WriteGraphWizModel(const char *filename)
graph << "}";
}
void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_dnn_number_type_t number_type) {
if ((number_type_ == kDnnFloat) && (number_type == kDnnInt)) {
void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_dnn_number_type_t logging_precision) {
if ((compute_precision_ == kDnnFloat) && (logging_precision == kDnnInt)) {
fprintf(stderr, "Error trying to write floating point DNN as integer in GNAPluginNS::backend::AMIntelDNN::WriteDnnText().\n");
fprintf(stderr, " Please convert to integer first.\n");
throw -1;
@ -653,7 +653,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
uint32_t layer = 0;
out_file << "<intel_dnn_file>\n";
out_file << "<number_type> " << intel_dnn_number_type_name[number_type] << "\n";
out_file << "<number_type> " << intel_dnn_number_type_name[logging_precision] << "\n";
out_file << "<softmax_type> " << intel_dnn_softmax_name[softmax_type] << "\n";
out_file << "<num_memory_bytes> " << std::dec << num_bytes_dnn_memory_ << "\n";
out_file << "<num_group> " << std::dec << num_group << "\n";
@ -701,7 +701,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
out_file << "<orientation_out> " << std::dec << (component[i].orientation_out == kDnnInterleavedOrientation ?
"interleaved" : "deinterleaved") << "\n";
if ((number_type_ == kDnnInt) && (number_type == kDnnFloat)) {
if ((compute_precision_ == kDnnInt) && (logging_precision == kDnnFloat)) {
out_file << "<num_bytes_per_input> " << std::dec << sizeof(float) << "\n";
out_file << "<num_bytes_per_output> " << std::dec << sizeof(float) << "\n";
} else {
@ -721,14 +721,14 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
float output_scale_factor = component[i].output_scale_factor;
uint32_t num_weight_rows = (component[i].operation == kDnnDiagonalOp) ? 1 : num_rows_out;
uint32_t num_weight_columns = num_rows_in;
if ((number_type_ == kDnnInt) && (number_type == kDnnFloat)) {
if ((compute_precision_ == kDnnInt) && (logging_precision == kDnnFloat)) {
out_file << "<num_bytes_per_weight> " << std::dec << 4 << "\n";
out_file << "<num_bytes_per_bias> " << std::dec << 4 << "\n";
} else {
out_file << "<num_bytes_per_weight> " << std::dec << num_bytes_per_weight << "\n";
out_file << "<num_bytes_per_bias> " << std::dec << num_bytes_per_bias << "\n";
}
if ((number_type_ == kDnnInt) && (number_type == kDnnFloat)) {
if ((compute_precision_ == kDnnInt) && (logging_precision == kDnnFloat)) {
out_file << std::setprecision(12) << std::scientific << "<weight_scale_factor> " << 1.0 << "\n";
out_file << std::setprecision(12) << std::scientific << "<output_scale_factor> " << 1.0 << "\n";
} else {
@ -751,7 +751,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
#ifdef DUMP_WB
for (uint32_t row = 0; row < num_weight_rows; row++) {
for (uint32_t col = 0; col < num_weight_columns; col++) {
if (number_type == kDnnFloat) {
if (logging_precision == kDnnFloat) {
float val =
static_cast<float>(ptr_weight[row * num_weight_columns + col]) * ptr_bias[row].multiplier
/ weight_scale_factor;
@ -768,7 +768,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
#ifdef DUMP_WB
for (uint32_t row = 0; row < num_weight_rows; row++) {
for (uint32_t col = 0; col < num_weight_columns; col++) {
if (number_type == kDnnFloat) {
if (logging_precision == kDnnFloat) {
out_wfile << std::setprecision(12)
<< ptr_weight[row * num_weight_columns + col] / weight_scale_factor << " ";
} else {
@ -778,7 +778,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
}
}
#endif
} else if (number_type_ == kDnnFloat) {
} else if (compute_precision_ == kDnnFloat) {
float *ptr_weight = reinterpret_cast<float *>(component[i].op.affine.ptr_weights);
#ifdef DUMP_WB
for (uint32_t row = 0; row < num_weight_rows; row++) {
@ -793,21 +793,25 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
fprintf(stderr, "Unsupported weight type in WriteDnnText!\n");
throw -1;
}
if (number_type_ == kDnnInt) {
if (compute_precision_ == kDnnInt) {
if (num_bytes_per_weight == 1) {
intel_compound_bias_t
*ptr_biases = reinterpret_cast<intel_compound_bias_t *>(component[i].op.affine.ptr_biases);
#ifdef DUMP_WB
for (uint32_t row = 0; row < num_rows_out; row++) {
out_bfile << std::setw(8) << ptr_biases[row].bias << ", ";
out_bfile << std::setw(8) << int(ptr_biases[row].multiplier) << "\n";
if (logging_precision == kDnnInt) {
out_bfile << std::setw(8) << ptr_biases[row].bias << ", ";
out_bfile << std::setw(8) << int(ptr_biases[row].multiplier) << "\n";
} else {
out_bfile << std::setw(8) << ptr_biases[row].bias / output_scale_factor << "\n";
}
}
#endif
} else {
int32_t *ptr_biases = reinterpret_cast<int32_t *>(component[i].op.affine.ptr_biases);
#ifdef DUMP_WB
for (uint32_t row = 0; row < num_rows_out; row++) {
if (number_type == kDnnInt) {
if (logging_precision == kDnnInt) {
out_bfile << std::setw(8) << ptr_biases[row] << "\n";
} else {
out_bfile << std::setw(8) << ptr_biases[row] / output_scale_factor << "\n";
@ -844,14 +848,14 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
out_file << "<num_feature_maps> " << std::dec << num_feature_maps << "\n";
out_file << "<num_feature_map_rows> " << std::dec << num_feature_map_rows << "\n";
out_file << "<num_feature_map_columns> " << std::dec << num_feature_map_columns << "\n";
if ((number_type_ == kDnnInt) && (number_type == kDnnFloat)) {
if ((compute_precision_ == kDnnInt) && (logging_precision == kDnnFloat)) {
out_file << "<num_bytes_per_weight> " << std::dec << 4 << "\n";
out_file << "<num_bytes_per_bias> " << std::dec << 4 << "\n";
} else {
out_file << "<num_bytes_per_weight> " << std::dec << num_bytes_per_weight << "\n";
out_file << "<num_bytes_per_bias> " << std::dec << num_bytes_per_bias << "\n";
}
if ((number_type_ == kDnnInt) && (number_type == kDnnFloat)) {
if ((compute_precision_ == kDnnInt) && (logging_precision == kDnnFloat)) {
out_file << std::setprecision(12) << std::scientific << "<weight_scale_factor> " << 1.0 << "\n";
out_file << std::setprecision(12) << std::scientific << "<output_scale_factor> " << 1.0 << "\n";
} else {
@ -876,7 +880,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
#ifdef DUMP_WB
for (uint32_t row = 0; row < num_filters; row++) {
for (uint32_t col = 0; col < num_filter_coefficients; col++) {
if (number_type == kDnnFloat) {
if (logging_precision == kDnnFloat) {
float val = static_cast<float>(ptr_weight[row * num_filter_coefficients + col])
* ptr_bias[row].multiplier / weight_scale_factor;
out_wfile << std::setprecision(12) <<val << "\n";
@ -892,7 +896,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
#ifdef DUMP_WB
for (uint32_t row = 0; row < num_filters; row++) {
for (uint32_t col = 0; col < num_filter_coefficients; col++) {
if (number_type == kDnnFloat) {
if (logging_precision == kDnnFloat) {
out_wfile << std::setprecision(12)
<< ptr_weight[row * num_filter_coefficients + col] / weight_scale_factor
<< "\n";
@ -903,7 +907,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
}
}
#endif
} else if (number_type_ == kDnnFloat) {
} else if (compute_precision_ == kDnnFloat) {
float *ptr_weight = reinterpret_cast<float *>(component[i].op.conv1D.ptr_filters);
#ifdef DUMP_WB
for (uint32_t row = 0; row < num_filters; row++) {
@ -918,8 +922,8 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
throw -1;
}
if (number_type_ == kDnnInt) {
if (number_type == kDnnInt) {
if (compute_precision_ == kDnnInt) {
if (logging_precision == kDnnInt) {
if (num_bytes_per_weight == 1) {
intel_compound_bias_t
*ptr_biases = reinterpret_cast<intel_compound_bias_t *>(component[i].op.conv1D.ptr_biases);
@ -969,14 +973,14 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
uint32_t num_weight_rows = num_columns_out;
uint32_t num_weight_columns = num_columns_in + num_columns_out;
out_file << "<num_vector_delay> " << std::dec << num_vector_delay << "\n";
if ((number_type_ == kDnnInt) && (number_type == kDnnFloat)) {
if ((compute_precision_ == kDnnInt) && (logging_precision == kDnnFloat)) {
out_file << "<num_bytes_per_weight> " << std::dec << 4 << "\n";
out_file << "<num_bytes_per_bias> " << std::dec << 4 << "\n";
} else {
out_file << "<num_bytes_per_weight> " << std::dec << num_bytes_per_weight << "\n";
out_file << "<num_bytes_per_bias> " << std::dec << num_bytes_per_bias << "\n";
}
if ((number_type_ == kDnnInt) && (number_type == kDnnFloat)) {
if ((compute_precision_ == kDnnInt) && (logging_precision == kDnnFloat)) {
out_file << std::setprecision(12) << std::scientific << "<weight_scale_factor> " << 1.0 << "\n";
out_file << std::setprecision(12) << std::scientific << "<output_scale_factor> " << 1.0 << "\n";
} else {
@ -999,7 +1003,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
for (uint32_t row = 0; row < num_weight_rows; row++) {
out_file << "<weight_row> ";
for (uint32_t col = 0; col < num_weight_columns; col++) {
if (number_type == kDnnFloat) {
if (logging_precision == kDnnFloat) {
float val =
static_cast<float>(ptr_weight[row * num_weight_columns + col]) * ptr_bias[col].multiplier
/ weight_scale_factor;
@ -1018,7 +1022,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
for (uint32_t row = 0; row < num_weight_rows; row++) {
out_file << "<weight_row> ";
for (uint32_t col = 0; col < num_weight_columns; col++) {
if (number_type == kDnnFloat) {
if (logging_precision == kDnnFloat) {
out_file << std::setprecision(12) << std::scientific
<< ptr_weight[row * num_weight_columns + col] / weight_scale_factor << " ";
} else {
@ -1029,7 +1033,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
out_file << "\n";
}
#endif
} else if (number_type_ == kDnnFloat) {
} else if (compute_precision_ == kDnnFloat) {
float *ptr_weight = reinterpret_cast<float *>(component[i].op.recurrent.ptr_weights);
#ifdef DUMP_WB
for (uint32_t row = 0; row < num_weight_rows; row++) {
@ -1045,8 +1049,8 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
fprintf(stderr, "Unsupported weight type in WriteDnnText!\n");
throw -1;
}
if (number_type_ == kDnnInt) {
if (number_type == kDnnInt) {
if (compute_precision_ == kDnnInt) {
if (logging_precision == kDnnInt) {
if (num_bytes_per_weight == 1) {
intel_compound_bias_t
*ptr_biases = reinterpret_cast<intel_compound_bias_t *>(component[i].op.recurrent.ptr_biases);
@ -1110,7 +1114,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
out_file << "<num_bytes_per_slope> " << std::dec << sizeof(int16_t) << "\n";
out_file << "<num_bytes_per_intercept> " << std::dec << sizeof(int16_t) << "\n";
out_file << "<num_bytes_per_offset> " << std::dec << sizeof(int32_t) << "\n";
if (number_type == kDnnFloat) {
if (logging_precision == kDnnFloat) {
out_file << std::setprecision(12) << std::scientific << "<output_scale_factor> " << 1.0 << "\n";
out_file << "<num_segments> " << std::dec << 0 << "\n";
out_file << "<segment_address> " << "0x" << std::setfill('0') << std::setw(8) << std::hex
@ -1121,7 +1125,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteDnnText(const char *filename, intel_
out_file << "<num_segments> " << std::dec << num_segments << "\n";
out_file << "<segment_address> " << "0x" << std::setfill('0') << std::setw(8) << std::hex
<< GNAPluginNS::memory::MemoryOffset(component[i].op.pwl.ptr_segments, ptr_dnn_memory_) << "\n";
if (number_type_ == kDnnInt) {
if (compute_precision_ == kDnnInt) {
out_file << "<slope> ";
for (int segment = 0; segment < num_segments; segment++) {
out_file << "0x" << std::setfill('0') << std::setw(4) << std::hex
@ -1862,7 +1866,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteInputAndOutputText() {
for (int j = 0; j < component[i].num_columns_out; j++) {
float floatValue = 0.f;
if (component[i].num_bytes_per_output == 4) {
if (number_type_ == kDnnInt) {
if (compute_precision_ == kDnnInt) {
auto value = reinterpret_cast<int32_t *>(component[i].ptr_outputs)[k * component[i].num_columns_out+ j];
floatValue = static_cast<float>(value);
@ -1903,7 +1907,7 @@ void GNAPluginNS::backend::AMIntelDNN::WriteInputAndOutputText() {
for (int j = 0; j < component[i].num_columns_in; j++) {
float floatValue = 0.f;
if (component[i].num_bytes_per_input == 4) {
if (number_type_ == kDnnInt) {
if (compute_precision_ == kDnnInt) {
auto value = reinterpret_cast<int32_t *>(component[i].ptr_inputs)[k * component[i].num_columns_in + j];
floatValue = static_cast<float>(value);
} else {

8
inference-engine/src/gna_plugin/backend/am_intel_dnn.hpp
View File

@ -36,14 +36,14 @@ public:
ptr_priors(NULL),
ptr_dnn_memory_(NULL),
num_bytes_dnn_memory_(0),
number_type_(kDnnNumNumberType) {
compute_precision_(kDnnNumNumberType) {
}
~AMIntelDNN();
void Init(void *ptr_memory,
uint32_t num_memory_bytes,
intel_dnn_number_type_t number_type,
intel_dnn_number_type_t compute_precision,
float scale_factor);
void InitActiveList(uint32_t *ptr_active_list);
@ -235,7 +235,7 @@ public:
void WriteGraphWizModel(const char *filename);
void WriteDnnText(const char *filename, intel_dnn_number_type_t number_type);
void WriteDnnText(const char *filename, intel_dnn_number_type_t logging_precision);
#if GNA_LIB_VER == 2
@ -291,7 +291,7 @@ private:
uint32_t num_bytes_dnn_memory_;
uint32_t *ptr_active_outputs_;
uint32_t num_active_outputs_;
intel_dnn_number_type_t number_type_;
intel_dnn_number_type_t compute_precision_;
float input_scale_factor_;
uint32_t dump_write_index = 0;

2
inference-engine/src/gna_plugin/gna_plugin.hpp
View File

@ -109,7 +109,7 @@ class GNAPlugin : public InferenceEngine::IInferencePluginInternal, public std::
InferenceEngine::RemoteContext::Ptr context) override { THROW_GNA_EXCEPTION << "Not implemented"; }
void Infer(const InferenceEngine::Blob &input, InferenceEngine::Blob &result);
void SetCore(InferenceEngine::ICore*) noexcept override {}
const InferenceEngine::ICore* GetCore() const noexcept override {return nullptr;}
InferenceEngine::ICore* GetCore() const noexcept override {return nullptr;}
void Reset();
void QueryNetwork(const InferenceEngine::ICNNNetwork &network,
const std::map<std::string, std::string>& config,

2
inference-engine/src/gna_plugin/gna_plugin_internal.hpp
View File

@ -28,7 +28,7 @@ private:
}
public:
InferenceEngine::ExecutableNetworkInternal::Ptr LoadExeNetworkImpl(const InferenceEngine::ICore * core,
InferenceEngine::ExecutableNetworkInternal::Ptr LoadExeNetworkImpl(
const InferenceEngine::ICNNNetwork &network,
const std::map<std::string, std::string> &config) override {
Config updated_config(defaultConfig);

4
inference-engine/src/gna_plugin/gna_plugin_query_api.cpp
View File

@ -24,6 +24,10 @@ Parameter GNAPlugin::GetMetric(const std::string& name, const std::map<std::stri
const std::unordered_map<std::string, std::function<Parameter()>> queryApiSupported = {
{METRIC_KEY(AVAILABLE_DEVICES), [this]() {return GetAvailableDevices();}},
{METRIC_KEY(SUPPORTED_CONFIG_KEYS), [this]() {return config.GetSupportedKeys();}},
{METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS), [this]() {
uint32_t nireq = 1;
return nireq;
}},
{METRIC_KEY(FULL_DEVICE_NAME), [&options, this]() {
auto availableDevices = GetAvailableDevices().as<std::vector<std::string>>();

1
inference-engine/src/hetero_plugin/hetero_async_infer_request.cpp
View File

@ -5,7 +5,6 @@
#include <utility>
#include <memory>
#include "hetero_async_infer_request.hpp"
#include <ie_util_internal.hpp>
#include <ie_profiling.hpp>
using namespace HeteroPlugin;

100
inference-engine/src/hetero_plugin/hetero_executable_network.cpp
View File

@ -7,7 +7,6 @@
#include "hetero_async_infer_request.hpp"
#include "ie_util_internal.hpp"
#include "hetero_graph_splitter.hpp"
#include "file_utils.h"
#include "xml_parse_utils.h"
#include <vector>
@ -22,12 +21,9 @@
#include <array>
#include <cstdint>
#include "details/caseless.hpp"
#include "ie_plugin_config.hpp"
#include "cpp_interfaces/interface/ie_internal_plugin_config.hpp"
#include "cpp_interfaces/base/ie_inference_plugin_api.hpp"
#include "hetero/hetero_plugin_config.hpp"
#include "precision_utils.h"
#include "hetero_plugin.hpp"
#include "network_serializer.h"
@ -141,10 +137,10 @@ void dumpGraph(InferenceEngine::ICNNNetwork &network,
HeteroExecutableNetwork::HeteroExecutableNetwork(const InferenceEngine::ICNNNetwork& network_,
const Engine::Configs& config,
Engine* plugin):
Engine* heteroPlugin):
InferenceEngine::ExecutableNetworkThreadSafeDefault(
nullptr, std::make_shared<InferenceEngine::ImmediateExecutor>()),
_plugin{plugin},
_heteroPlugin(heteroPlugin),
_name{network_.getName()},
_config{config} {
auto networkPtr = cloneNet(network_);
@ -171,7 +167,7 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(const InferenceEngine::ICNNNetw
if (allEmpty) {
auto it = _config.find("TARGET_FALLBACK");
if (it != _config.end()) {
plugin->SetAffinity(network, _config);
_heteroPlugin->SetAffinity(network, _config);
} else {
THROW_IE_EXCEPTION << "The 'TARGET_FALLBACK' option was not defined for heterogeneous plugin";
}
@ -230,25 +226,8 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(const InferenceEngine::ICNNNetw
network.getOutputsInfo(externalOutputsData);
auto subgraphs = splitGraph(network, getAffinities(network));
sortSubgraphs(subgraphs);
std::vector<NetworkDesc> descs;
std::vector<CNNLayerPtr> tempLayers;
for (auto &&subgraph : subgraphs) {
assert(!subgraph.empty());
auto affinity = (*subgraph.begin())->affinity;
assert(!affinity.empty());
_affinities.push_back(affinity);
if (_plugin->_plugins.end() == _plugin->_plugins.find(affinity)) {
IE_SUPPRESS_DEPRECATED_START
_plugin->_plugins[affinity] = _plugin->GetDevicePlugin(affinity);
IE_SUPPRESS_DEPRECATED_END
}
}
if (dumpDotFile) {
std::stringstream stream(std::stringstream::out);
stream << "hetero_subgraphs_" << network.getName() << ".dot";
@ -262,6 +241,8 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(const InferenceEngine::ICNNNetw
networkStats = nullptr;
}
std::vector<NetworkDesc> descs;
std::vector<CNNLayerPtr> tempLayers;
for (auto &&subgraph : subgraphs) {
auto affinity = (*subgraph.begin())->affinity;
tempLayers.assign(subgraph.begin(), subgraph.end());
@ -289,11 +270,9 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(const InferenceEngine::ICNNNetw
inp->second->getPreProcess() = it.second->getPreProcess();
}
}
// go over all inputs/outputs and right now
// set precision for intermediate data (not for external) to FP32
// later on we have to add Plugin::getPreferableInputPrecision(network) and
// Plugin::getPreferableOutputPrecision(network) and set precision based on this info
// TODO(amalyshe) add clever selectino of precision for intermediate blobs
for (auto &&it : clonedInputs) {
if (externalInputsData.find(it.first) == externalInputsData.end()) {
it.second->setPrecision(Precision::FP32);
@ -327,36 +306,24 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(const InferenceEngine::ICNNNetw
}));
auto cfg = _config;
cfg[PluginConfigInternalParams::KEY_SUBNETWORK_WITH_NETWORK_INPUTS] = isInputSubnetwork
? CONFIG_VALUE(YES)
: CONFIG_VALUE(NO);
IE_SUPPRESS_DEPRECATED_START
auto plugin = _plugin->_plugins[d._device];
d._network = plugin._ref.LoadNetwork(d._clonedNetwork, Engine::GetSupportedConfig(plugin._config, cfg, plugin._ref));
IE_SUPPRESS_DEPRECATED_END
cfg[PluginConfigInternalParams::KEY_SUBNETWORK_WITH_NETWORK_INPUTS] =
isInputSubnetwork ? CONFIG_VALUE(YES) : CONFIG_VALUE(NO);
auto deviceName = d._device;
auto metaDevices = _heteroPlugin->GetDevicePlugins(deviceName, cfg);
assert(metaDevices.size() == 1);
auto loadConfig = metaDevices[deviceName];
d._network = _heteroPlugin->GetCore()->LoadNetwork(d._clonedNetwork, deviceName, loadConfig);
}
networks = std::move(descs);
}
namespace {
IE_SUPPRESS_DEPRECATED_START
IInferencePluginAPI * getInferencePluginAPIInterface(IInferencePlugin * iplugin) {
return dynamic_cast<IInferencePluginAPI *>(iplugin);
}
IInferencePluginAPI * getInferencePluginAPIInterface(InferenceEnginePluginPtr iplugin) {
return getInferencePluginAPIInterface(static_cast<IInferencePlugin *>(iplugin.operator->()));
}
IE_SUPPRESS_DEPRECATED_END
} // namespace
HeteroExecutableNetwork::HeteroExecutableNetwork(std::istream& heteroModel,
const std::map<std::string, std::string>& configs,
Engine* plugin) :
_plugin(plugin) {
Engine* heteroPlugin) :
_heteroPlugin(heteroPlugin) {
std::string heteroXmlStr;
std::getline(heteroModel, heteroXmlStr);
@ -400,26 +367,17 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(std::istream&
pugi::xml_node subnetworksNode = heteroNode.child("subnetworks");
for (auto subnetworkNode = subnetworksNode.child("subnetwork"); !subnetworkNode.empty();
subnetworkNode = subnetworkNode.next_sibling("subnetwork")) {
auto device = GetStrAttr(subnetworkNode, "device");
_affinities.push_back(device);
auto deviceName = GetStrAttr(subnetworkNode, "device");
if (_plugin->_plugins.end() == _plugin->_plugins.find(device)) {
IE_SUPPRESS_DEPRECATED_START
_plugin->_plugins[device] = _plugin->GetDevicePlugin(device);
IE_SUPPRESS_DEPRECATED_END
}
auto& plugin = _plugin->_plugins[device];
auto supportedConfig = Engine::GetSupportedConfig(plugin._config, importedConfigs, plugin._ref);
IE_SUPPRESS_DEPRECATED_START
auto pluginAPI = getInferencePluginAPIInterface(plugin._ref);
IE_SUPPRESS_DEPRECATED_END
auto metaDevices = _heteroPlugin->GetDevicePlugins(deviceName, importedConfigs);
assert(metaDevices.size() == 1);
auto& loadConfig = metaDevices[deviceName];
InferenceEngine::ExecutableNetwork executableNetwork;
CNNNetwork cnnnetwork;
bool loaded = false;
try {
executableNetwork = pluginAPI->ImportNetwork(heteroModel, supportedConfig);
executableNetwork = _heteroPlugin->GetCore()->ImportNetwork(heteroModel, deviceName, loadConfig);
} catch(InferenceEngine::details::InferenceEngineException& ie_ex) {
if (std::string::npos != std::string{ie_ex.what()}.find(NOT_IMPLEMENTED_str)) {
// read XML content
@ -439,7 +397,7 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(std::istream&
heteroModel.read(dataBlob->buffer(), dataSize);
}
cnnnetwork = _plugin->GetCore()->ReadNetwork(xmlString, std::move(dataBlob));
cnnnetwork = _heteroPlugin->GetCore()->ReadNetwork(xmlString, std::move(dataBlob));
auto inputs = cnnnetwork.getInputsInfo();
auto inputsNode = subnetworkNode.child("inputs");
for (auto inputNode = inputsNode.child("input"); !inputNode.empty(); inputNode = inputNode.next_sibling("input")) {
@ -455,9 +413,7 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(std::istream&
for (auto outputNode = outputsNode.child("output"); !outputNode.empty(); outputNode = outputNode.next_sibling("output")) {
outputs[GetStrAttr(outputNode, "name")]->setPrecision(Precision::FromStr(GetStrAttr(outputNode, "precision")));
}
IE_SUPPRESS_DEPRECATED_START
executableNetwork = plugin._ref.LoadNetwork(cnnnetwork, supportedConfig);
IE_SUPPRESS_DEPRECATED_END
executableNetwork = _heteroPlugin->GetCore()->LoadNetwork(cnnnetwork, deviceName, loadConfig);
loaded = true;
} else {
throw;
@ -477,7 +433,7 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(std::istream&
}
descs.emplace_back(NetworkDesc{
device,
deviceName,
loaded ? CNNNetwork{cloneNet(static_cast<InferenceEngine::ICNNNetwork&>(cnnnetwork))} : CNNNetwork{},
executableNetwork,
});
@ -578,10 +534,10 @@ void HeteroExecutableNetwork::CreateInferRequest(IInferRequest::Ptr &asyncReques
auto heteroInferRequest = std::dynamic_pointer_cast<HeteroInferRequest>(
CreateInferRequestImpl(_networkInputs, _networkOutputs));
heteroInferRequest->setPointerToExecutableNetworkInternal(shared_from_this());
auto asyncTreadSafeImpl = std::make_shared<HeteroAsyncInferRequest>(heteroInferRequest, _taskExecutor, _callbackExecutor);
asyncRequest.reset(new InferRequestBase<HeteroAsyncInferRequest>(asyncTreadSafeImpl),
auto asyncThreadSafeImpl = std::make_shared<HeteroAsyncInferRequest>(heteroInferRequest, _taskExecutor, _callbackExecutor);
asyncRequest.reset(new InferRequestBase<HeteroAsyncInferRequest>(asyncThreadSafeImpl),
[](IInferRequest *p) { p->Release(); });
asyncTreadSafeImpl->SetPointerToPublicInterface(asyncRequest);
asyncThreadSafeImpl->SetPointerToPublicInterface(asyncRequest);
}
void HeteroExecutableNetwork::GetConfig(const std::string &name, InferenceEngine::Parameter &result, InferenceEngine::ResponseDesc *) const {

5
inference-engine/src/hetero_plugin/hetero_executable_network.hpp
View File

@ -39,7 +39,7 @@ public:
/**
* @brief constructor
*/
HeteroExecutableNetwork(const InferenceEngine::ICNNNetwork& network,
HeteroExecutableNetwork(const InferenceEngine::ICNNNetwork& network,
const std::map<std::string, std::string>& config,
Engine* plugin);
@ -71,9 +71,8 @@ private:
};
std::vector<NetworkDesc> networks;
Engine* _plugin;
Engine* _heteroPlugin;
std::string _name;
std::vector<std::string> _affinities;
std::map<std::string, std::string> _config;
};

232
inference-engine/src/hetero_plugin/hetero_plugin.cpp
View File

@ -3,9 +3,7 @@
//
#include "ie_metric_helpers.hpp"
#include "ie_plugin_dispatcher.hpp"
#include "hetero_plugin.hpp"
#include "ie_util_internal.hpp"
#include <memory>
#include <vector>
#include <map>
@ -17,7 +15,6 @@
#include "hetero/hetero_plugin_config.hpp"
#include <cpp_interfaces/base/ie_plugin_base.hpp>
#include "hetero_executable_network.hpp"
#include "cpp_interfaces/base/ie_inference_plugin_api.hpp"
using namespace InferenceEngine;
using namespace InferenceEngine::PluginConfigParams;
@ -31,163 +28,88 @@ static Version heteroPluginDescription = {
"heteroPlugin" // plugin description message
};
void Engine::GetVersion(const Version *&versionInfo)noexcept {
versionInfo = &heteroPluginDescription;
}
Engine::Engine() {
_pluginName = "HETERO";
_config[InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS] = "YES";
_config[KEY_EXCLUSIVE_ASYNC_REQUESTS] = YES;
_config[HETERO_CONFIG_KEY(DUMP_GRAPH_DOT)] = NO;
}
InferenceEngine::ExecutableNetworkInternal::Ptr Engine::LoadExeNetworkImpl(const ICore* /*core*/,
const InferenceEngine::ICNNNetwork& network,
const Configs& config) {
// TODO(amalyshe) do we need here verification of input precisions?
Configs tconfig;
tconfig = config;
namespace {
// we must not override the parameter, but need to copy everything from plugin config
for (auto && c : _config) {
if (tconfig.find(c.first) == tconfig.end()) {
tconfig[c.first] = c.second;
}
Engine::Configs mergeConfigs(Engine::Configs config, const Engine::Configs & local) {
for (auto && kvp : local) {
config[kvp.first] = kvp.second;
}
return config;
}
} // namespace
InferenceEngine::ExecutableNetworkInternal::Ptr Engine::LoadExeNetworkImpl(const InferenceEngine::ICNNNetwork& network,
const Configs& config) {
if (GetCore() == nullptr) {
THROW_IE_EXCEPTION << "Please, work with HETERO device via InferencEngine::Core object";
}
return std::make_shared<HeteroExecutableNetwork>(*cloneNet(network), tconfig, this);
return std::make_shared<HeteroExecutableNetwork>(*cloneNet(network), mergeConfigs(_config, config), this);
}
ExecutableNetwork Engine::ImportNetworkImpl(std::istream& heteroModel, const Configs& config) {
Configs tconfig;
tconfig = config;
// we must not override the parameter, but need to copy everything from plugin config
for (auto && c : _config) {
if (tconfig.find(c.first) == tconfig.end()) {
tconfig[c.first] = c.second;
}
if (GetCore() == nullptr) {
THROW_IE_EXCEPTION << "Please, work with HETERO device via InferencEngine::Core object";
}
IExecutableNetwork::Ptr executableNetwork;
// Use config provided by an user ignoring default config
executableNetwork.reset(new ExecutableNetworkBase<ExecutableNetworkInternal>(
std::make_shared<HeteroExecutableNetwork>(heteroModel, tconfig, this)),
std::make_shared<HeteroExecutableNetwork>(heteroModel, mergeConfigs(_config, config), this)),
[](InferenceEngine::details::IRelease *p) {p->Release();});
return ExecutableNetwork{executableNetwork};
}
namespace {
IE_SUPPRESS_DEPRECATED_START
IInferencePluginAPI * getInferencePluginAPIInterface(IInferencePlugin * iplugin) {
return dynamic_cast<IInferencePluginAPI *>(iplugin);
}
IInferencePluginAPI * getInferencePluginAPIInterface(InferenceEnginePluginPtr iplugin) {
return getInferencePluginAPIInterface(static_cast<IInferencePlugin *>(iplugin.operator->()));
}
IInferencePluginAPI * getInferencePluginAPIInterface(InferencePlugin plugin) {
return getInferencePluginAPIInterface(static_cast<InferenceEnginePluginPtr>(plugin));
}
} // namespace
Engine::Configs Engine::GetSupportedConfig(const Engine::Configs& globalConfig,
const Engine::Configs& localConfig,
const InferenceEngine::InferencePlugin& plugin) {
auto pluginApi = getInferencePluginAPIInterface(plugin);
std::vector<std::string> supportedConfigKeys = pluginApi->GetMetric(METRIC_KEY(SUPPORTED_CONFIG_KEYS), {});
Engine::Configs Engine::GetSupportedConfig(const Engine::Configs& config, const std::string & deviceName) const {
std::vector<std::string> supportedConfigKeys = GetCore()->GetMetric(deviceName, METRIC_KEY(SUPPORTED_CONFIG_KEYS));
Engine::Configs supportedConfig;
for (auto&& key : supportedConfigKeys) {
auto itKey = localConfig.find(key);
if (localConfig.end() != itKey) {
auto itKey = config.find(key);
if (config.end() != itKey) {
supportedConfig[key] = itKey->second;
} else {
itKey = globalConfig.find(key);
if (globalConfig.end() != itKey) {
supportedConfig[key] = itKey->second;
}
}
}
return supportedConfig;
}
Engine::PluginEntry Engine::GetDevicePlugin(const std::string& deviceWithID) const {
InferenceEngine::InferencePlugin plugin;
DeviceIDParser deviceParser(deviceWithID);
std::string deviceName = deviceParser.getDeviceName();
Engine::DeviceMetaInformationMap Engine::GetDevicePlugins(const std::string& targetFallback,
const Configs & localConfig) const {
auto getDeviceConfig = [&](const std::string & deviceWithID) {
DeviceIDParser deviceParser(deviceWithID);
std::string deviceName = deviceParser.getDeviceName();
Configs tconfig = mergeConfigs(_config, localConfig);
if (nullptr == _core) {
IE_SUPPRESS_DEPRECATED_START
// try to create plugin
PluginDispatcher dispatcher({file_name_t()});
plugin = dispatcher.getPluginByDevice(deviceName);
IE_SUPPRESS_DEPRECATED_END
} else {
plugin = InferencePlugin{_core->GetPluginByName(deviceName)};
}
try {
for (auto&& ext : _extensions) {
plugin.AddExtension(ext);
// set device ID if any
std::string deviceIDLocal = deviceParser.getDeviceID();
if (!deviceIDLocal.empty()) {
tconfig[KEY_DEVICE_ID] = deviceIDLocal;
}
} catch (InferenceEngine::details::InferenceEngineException &) {}
Configs pluginConfig = GetSupportedConfig(_config, {}, plugin);
return GetSupportedConfig(tconfig, deviceName);
};
// set device ID if any
std::string deviceIDLocal = deviceParser.getDeviceID();
if (!deviceIDLocal.empty()) {
pluginConfig = GetSupportedConfig(pluginConfig, { { KEY_DEVICE_ID, deviceIDLocal } }, plugin);
}
return { plugin, pluginConfig };
}
IE_SUPPRESS_DEPRECATED_END
Engine::Plugins Engine::GetDevicePlugins(const std::string& targetFallback) const {
auto devices = InferenceEngine::DeviceIDParser::getHeteroDevices(targetFallback);
Engine::Plugins plugins = _plugins;
for (auto&& device : devices) {
auto itPlugin = plugins.find(device);
if (plugins.end() == itPlugin) {
IE_SUPPRESS_DEPRECATED_START
plugins[device] = GetDevicePlugin(device);
IE_SUPPRESS_DEPRECATED_END
auto fallbackDevices = InferenceEngine::DeviceIDParser::getHeteroDevices(targetFallback);
Engine::DeviceMetaInformationMap metaDevices;
for (auto&& deviceName : fallbackDevices) {
auto itPlugin = metaDevices.find(deviceName);
if (metaDevices.end() == itPlugin) {
metaDevices[deviceName] = getDeviceConfig(deviceName);
}
}
return plugins;
}
Engine::Plugins Engine::GetDevicePlugins(const std::string& targetFallback) {
_plugins = const_cast<const Engine*>(this)->GetDevicePlugins(targetFallback);
return _plugins;
return metaDevices;
}
void Engine::SetConfig(const Configs &configs) {
for (auto&& config : configs) {
_config[config.first] = config.second;
}
for (auto&& plugin : _plugins) {
plugin.second._config = GetSupportedConfig(plugin.second._config, configs, plugin.second._ref);
}
}
void Engine::AddExtension(InferenceEngine::IExtensionPtr extension) {
_extensions.emplace_back(extension);
try {
for (auto&& plugin : _plugins) {
IE_SUPPRESS_DEPRECATED_START
plugin.second._ref.AddExtension(extension);
IE_SUPPRESS_DEPRECATED_END
}
} catch (InferenceEngine::details::InferenceEngineException &) {}
}
HeteroLayerColorer::HeteroLayerColorer(const std::vector<std::string>& devices) {
@ -206,19 +128,8 @@ void HeteroLayerColorer::operator()(const CNNLayerPtr layer,
}
void Engine::SetAffinity(InferenceEngine::ICNNNetwork &network, const Configs &config) {
Configs tconfig = _config;
for (auto && value : config) {
tconfig[value.first] = value.second;
}
auto it = tconfig.find("TARGET_FALLBACK");
if (it == tconfig.end()) {
THROW_IE_EXCEPTION << "The 'TARGET_FALLBACK' option was not defined for heterogeneous plugin";
}
GetDevicePlugins(it->second);
QueryNetworkResult qr;
QueryNetwork(network, tconfig, qr);
QueryNetwork(network, config, qr);
details::CNNNetworkIterator i(&network);
while (i != details::CNNNetworkIterator()) {
@ -230,7 +141,12 @@ void Engine::SetAffinity(InferenceEngine::ICNNNetwork &network, const Configs &c
i++;
}
if (YES == tconfig[HETERO_CONFIG_KEY(DUMP_GRAPH_DOT)]) {
auto dumpDot = [](const Configs & config) {
auto it = config.find(HETERO_CONFIG_KEY(DUMP_GRAPH_DOT));
return it != config.end() ? it->second == YES : false;
};
if (dumpDot(config) || dumpDot(_config)) {
std::unordered_set<std::string> devicesSet;
details::CNNNetworkIterator i(&network);
while (i != details::CNNNetworkIterator()) {
@ -245,52 +161,49 @@ void Engine::SetAffinity(InferenceEngine::ICNNNetwork &network, const Configs &c
stream << "hetero_affinity_" << network.getName() << ".dot";
std::ofstream file(stream.str());
saveGraphToDot(network, file, HeteroLayerColorer{devices});
}
}
void Engine::QueryNetwork(const ICNNNetwork &network, const Configs& config, QueryNetworkResult &qr) const {
auto it = config.find("TARGET_FALLBACK");
if (it == config.end()) {
it = _config.find("TARGET_FALLBACK");
if (it == _config.end()) {
THROW_IE_EXCEPTION << "The 'TARGET_FALLBACK' option was not defined for heterogeneous plugin";
}
if (GetCore() == nullptr) {
THROW_IE_EXCEPTION << "Please, work with HETERO device via InferencEngine::Core object";
}
Plugins plugins = GetDevicePlugins(it->second);
auto tconfig = mergeConfigs(_config, config);
auto it = tconfig.find("TARGET_FALLBACK");
if (it == tconfig.end()) {
THROW_IE_EXCEPTION << "The 'TARGET_FALLBACK' option was not defined for heterogeneous plugin";
}
qr.rc = StatusCode::OK;
std::string fallbackDevicesStr = it->second;
DeviceMetaInformationMap metaDevices = GetDevicePlugins(fallbackDevicesStr, tconfig);
std::map<std::string, QueryNetworkResult> queryResults;
// go over devices, create appropriate plugins and
for (auto&& value : plugins) {
auto& device = value.first;
auto& plugin = value.second;
QueryNetworkResult r;
IE_SUPPRESS_DEPRECATED_START
plugin._ref.QueryNetwork(network, GetSupportedConfig(plugin._config, config, plugin._ref), r);
IE_SUPPRESS_DEPRECATED_END
queryResults[device] = r;
// go over devices and call query network
for (auto&& metaDevice : metaDevices) {
auto& deviceName = metaDevice.first;
queryResults[deviceName] = GetCore()->QueryNetwork(network, deviceName, metaDevice.second);
}
// WARNING: Here is devices with user set priority
auto falbackDevices = InferenceEngine::DeviceIDParser::getHeteroDevices(it->second);
auto fallbackDevices = InferenceEngine::DeviceIDParser::getHeteroDevices(fallbackDevicesStr);
details::CNNNetworkIterator i(&network);
while (i != details::CNNNetworkIterator()) {
CNNLayer::Ptr layer = *i;
for (auto&& device : falbackDevices) {
auto& deviceQueryResult = queryResults[device];
for (auto&& deviceName : fallbackDevices) {
auto& deviceQueryResult = queryResults[deviceName];
if (deviceQueryResult.supportedLayersMap.find(layer->name) != deviceQueryResult.supportedLayersMap.end()) {
qr.supportedLayersMap[layer->name] = device;
qr.supportedLayersMap[layer->name] = deviceName;
break;
}
}
i++;
}
// set OK status
qr.rc = StatusCode::OK;
}
Parameter Engine::GetMetric(const std::string& name, const std::map<std::string, Parameter> & /*options*/) const {
@ -317,6 +230,13 @@ Parameter Engine::GetConfig(const std::string& name, const std::map<std::string,
IE_ASSERT(it != _config.end());
bool dump = it->second == YES;
return { dump };
} else if (name == "TARGET_FALLBACK") {
auto it = _config.find("TARGET_FALLBACK");
if (it == _config.end()) {
THROW_IE_EXCEPTION << "Value for TARGET_FALLBACK is not set";
} else {
return { it->second };
}
} else {
THROW_IE_EXCEPTION << "Unsupported config key: " << name;
}

51
inference-engine/src/hetero_plugin/hetero_plugin.hpp
View File

@ -21,55 +21,34 @@ namespace HeteroPlugin {
class Engine : public InferenceEngine::InferencePluginInternal {
public:
using Configs = std::map<std::string, std::string>;
struct PluginEntry {
IE_SUPPRESS_DEPRECATED_START
InferenceEngine::InferencePlugin _ref;
IE_SUPPRESS_DEPRECATED_END
Configs _config;
};
using Plugins = std::unordered_map<std::string, PluginEntry >;
using Devices = std::vector<std::string>;
using DeviceMetaInformationMap = std::unordered_map<std::string, Configs>;
Engine();
void GetVersion(const InferenceEngine::Version *&versionInfo) noexcept;
InferenceEngine::ExecutableNetworkInternal::Ptr
LoadExeNetworkImpl(const InferenceEngine::ICore * core, const InferenceEngine::ICNNNetwork &network, const Configs &config) override;
LoadExeNetworkImpl(const InferenceEngine::ICNNNetwork &network, const Configs &config) override;
void SetConfig(const Configs &config) override;
void SetAffinity(InferenceEngine::ICNNNetwork& network, const Configs &config);
void AddExtension(InferenceEngine::IExtensionPtr extension)override;
void QueryNetwork(const InferenceEngine::ICNNNetwork &network,
const Configs& config, InferenceEngine::QueryNetworkResult &res) const override;
InferenceEngine::Parameter GetMetric(const std::string& name,
const std::map<std::string, InferenceEngine::Parameter> & options) const override;
InferenceEngine::Parameter GetMetric(const std::string& name, const std::map<std::string,
InferenceEngine::Parameter> & options) const override;
InferenceEngine::Parameter GetConfig(const std::string& name,
const std::map<std::string, InferenceEngine::Parameter> & options) const override;
IE_SUPPRESS_DEPRECATED_START
PluginEntry GetDevicePlugin(const std::string& device) const;
static Configs GetSupportedConfig(const Configs& globalConfig, const Configs& localConfig, const InferenceEngine::InferencePlugin& plugin);
IE_SUPPRESS_DEPRECATED_END
Plugins GetDevicePlugins(const std::string& targetFallback);
Plugins GetDevicePlugins(const std::string& targetFallback) const;
InferenceEngine::Parameter GetConfig(const std::string& name, const std::map<std::string,
InferenceEngine::Parameter> & options) const override;
ExecutableNetwork ImportNetworkImpl(std::istream& heteroModel, const Configs& config) override;
Plugins _plugins;
std::vector<InferenceEngine::IExtensionPtr> _extensions;
void SetAffinity(InferenceEngine::ICNNNetwork& network, const Configs &config);
DeviceMetaInformationMap GetDevicePlugins(const std::string& targetFallback,
const Configs & localConfig) const;
private:
Configs GetSupportedConfig(const Configs& config, const std::string & deviceName) const;
};
struct HeteroLayerColorer {

6
inference-engine/src/inference_engine/CMakeLists.txt
View File

@ -131,10 +131,6 @@ if(ENABLE_PROFILING_ITT AND INTEL_ITT_LIBS)
target_compile_definitions(${TARGET_NAME}_obj PRIVATE $<TARGET_PROPERTY:ittnotify,INTERFACE_COMPILE_DEFINITIONS>)
endif()
if(ENABLE_IR_READER)
target_compile_definitions(${TARGET_NAME}_obj PRIVATE ENABLE_IR_READER)
endif()
target_include_directories(${TARGET_NAME}_obj PRIVATE $<TARGET_PROPERTY:inference_engine_transformations,INTERFACE_INCLUDE_DIRECTORIES>)
if(ENABLE_MKL_DNN)
@ -172,7 +168,7 @@ endif()
target_compile_definitions(${TARGET_NAME} PRIVATE IMPLEMENT_INFERENCE_ENGINE_API)
ie_register_plugins(MAIN_TARGET ${TARGET_NAME}
POSSIBLE_PLUGINS HeteroPlugin clDNNPlugin GNAPlugin MKLDNNPlugin myriadPlugin)
POSSIBLE_PLUGINS MultiDevicePlugin HeteroPlugin clDNNPlugin GNAPlugin MKLDNNPlugin myriadPlugin)
# Static library used for unit tests which are always built

13
inference-engine/src/inference_engine/cnn_network_ngraph_impl.cpp
View File

@ -23,6 +23,7 @@
#include <transformations/common_optimizations/common_optimizations.hpp>
#include <transformations/convert_opset1_to_legacy/convert_opset1_to_legacy.hpp>
#include <transformations/convert_opset2_to_opset1/convert_opset2_to_opset1.hpp>
#include <transformations/convert_opset3_to_opset2/convert_opset3_to_opset2.hpp>
#include <transformations/convert_opset1_to_legacy/convert_one_hot_to_one_hot_ie.hpp>
#include "ngraph_ops/eltwise.hpp"
@ -76,6 +77,10 @@ static std::shared_ptr<ngraph::Function> copyFunction(const std::shared_ptr<cons
// WA: for cnnNetwork ngraph constructor
CNNNetwork::CNNNetwork(const std::shared_ptr<const ngraph::Function>& graph) {
if (graph == nullptr) {
THROW_IE_EXCEPTION << "CNNNetwork was not initialized: 'graph' object is empty";
}
// Copy nGraph function
network = std::make_shared<CNNNetworkNGraphImpl>(copyFunction(graph, false, {}));
actual = network.get();
@ -141,8 +146,10 @@ CNNNetworkNGraphImpl::CNNNetworkNGraphImpl(const std::shared_ptr<Function>& nGra
keep_input_info(*this, ptr);
}
for (auto& output : _outputData) {
// Convert precision into native format. Be consistent with possible convertation to CNNNetwork later.
if (output.second->getPrecision() != Precision::FP32 &&
// Convert precision into native format. Be consistent with possible conversion to CNNNetwork later.
if (output.second->getPrecision() == Precision::I64) {
output.second->setPrecision(Precision::I32);
} else if (output.second->getPrecision() != Precision::FP32 &&
output.second->getPrecision() != Precision::I32) {
output.second->setPrecision(Precision::FP32);
}
@ -456,6 +463,7 @@ StatusCode CNNNetworkNGraphImpl::serialize(const std::string& xmlPath, const std
::ngraph::op::GenericIE::DisableReshape noReshape(graph);
::ngraph::pass::CommonOptimizations().run_on_function(graph);
::ngraph::pass::ConvertOpSet3ToOpSet2().run_on_function(graph);
::ngraph::pass::ConvertOpSet2ToOpSet1().run_on_function(graph);
::ngraph::pass::ConvertOpSet1ToLegacy().run_on_function(graph);
network = InferenceEngine::details::convertFunctionToICNNNetwork(graph, *this);
@ -519,6 +527,7 @@ void CNNNetworkNGraphImpl::convertToCNNNetworkImpl() {
::ngraph::op::GenericIE::DisableReshape noReshape(graph);
::ngraph::pass::CommonOptimizations().run_on_function(graph);
::ngraph::pass::ConvertOpSet3ToOpSet2().run_on_function(graph);
::ngraph::pass::ConvertOpSet2ToOpSet1().run_on_function(graph);
::ngraph::pass::ConvertOpSet1ToLegacy().run_on_function(graph);
cnnNetwork = InferenceEngine::details::convertFunctionToICNNNetwork(graph, *this);

22
inference-engine/src/inference_engine/dll_main.hpp
View File

@ -1,22 +0,0 @@
// Copyright (C) 2018-2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#ifdef _WIN32
#define _WINSOCKAPI_
#include <windows.h>
BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) {
switch (ul_reason_for_call) {
case DLL_PROCESS_ATTACH:
case DLL_THREAD_ATTACH:
case DLL_THREAD_DETACH:
case DLL_PROCESS_DETACH:
break;
}
return TRUE;
}
#endif

338
inference-engine/src/inference_engine/ie_core.cpp
View File

@ -14,6 +14,7 @@
#include <string>
#include <utility>
#include <vector>
#include <mutex>
#include <ngraph/opsets/opset.hpp>
#include "cpp/ie_cnn_net_reader.h"
@ -70,6 +71,62 @@ IInferencePluginAPI* getInferencePluginAPIInterface(InferencePlugin plugin) {
return getInferencePluginAPIInterface(static_cast<InferenceEnginePluginPtr>(plugin));
}
template <typename T>
struct Parsed {
std::string _deviceName;
std::map<std::string, T> _config;
};
template <typename T = Parameter>
Parsed<T> parseDeviceNameIntoConfig(const std::string& deviceName, const std::map<std::string, T>& config = {}) {
auto config_ = config;
auto deviceName_ = deviceName;
if (deviceName_.find("HETERO:") == 0) {
deviceName_ = "HETERO";
config_["TARGET_FALLBACK"] = deviceName.substr(7);
} else if (deviceName_.find("MULTI:") == 0) {
deviceName_ = "MULTI";
config_[InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES] = deviceName.substr(6);
} else {
DeviceIDParser parser(deviceName_);
deviceName_ = parser.getDeviceName();
std::string deviceIDLocal = parser.getDeviceID();
if (!deviceIDLocal.empty()) {
config_[KEY_DEVICE_ID] = deviceIDLocal;
}
}
return {deviceName_, config_};
}
Parameter copyParameterValue(const Parameter & value) {
if (value.is<bool>()) {
return { value.as<bool>() };
} else if (value.is<int>()) {
return { value.as<int>() };
} else if (value.is<unsigned int>()) {
return { value.as<unsigned int>() };
} else if (value.is<float>()) {
return { value.as<float>() };
} else if (value.is<std::string>()) {
return { value.as<std::string>() };
} else if (value.is<std::vector<std::string> >()) {
return { value.as<std::vector<std::string> >() };
} else if (value.is<std::vector<int> >()) {
return { value.as<std::vector<int> >() };
} else if (value.is<std::vector<float> >()) {
return { value.as<std::vector<float> >() };
} else if (value.is<std::vector<unsigned int> >()) {
return { value.as<std::vector<unsigned int> >() };
} else if (value.is<std::tuple<unsigned int, unsigned int, unsigned int> >()) {
return { value.as<std::tuple<unsigned int, unsigned int, unsigned int> >() };
} else if (value.is<std::tuple<unsigned int, unsigned int> >()) {
return { value.as<std::tuple<unsigned int, unsigned int> >() };
}
return std::move(value);
}
} // namespace
CNNNetReaderPtr CreateCNNNetReaderPtr() noexcept {
@ -151,7 +208,8 @@ class Core::Impl : public ICore {
};
/**
* Hold original blob in order to avoid situations when original blob is allocated on stack
* @brief Holds original blob in order to avoid situations
* when original blob is allocated on stack
*/
class WeightsHolderBlob : public TBlob<uint8_t> {
Blob::CPtr originBlob;
@ -167,6 +225,7 @@ class Core::Impl : public ICore {
std::vector<IExtensionPtr> extensions;
std::map<std::string, PluginDescriptor> pluginRegistry;
mutable std::mutex pluginsMutex; // to lock parallel access to pluginRegistry and plugins
public:
Impl();
@ -174,9 +233,11 @@ public:
/**
* @brief Register plugins for devices which are located in .xml configuration file. The function supports UNICODE path
* @param xmlConfigFile - an .xml configuraion with device / plugin information
* @param xmlConfigFile An .xml configuraion with device / plugin information
*/
void RegisterPluginsInRegistry(const std::string& xmlConfigFile) {
std::lock_guard<std::mutex> lock(pluginsMutex);
auto parse_result = ParseXml(xmlConfigFile.c_str());
if (!parse_result.error_msg.empty()) {
THROW_IE_EXCEPTION << parse_result.error_msg;
@ -256,7 +317,8 @@ public:
StatusCode rt = cnnReader->ReadNetwork(modelPath.c_str(), &desc);
if (rt != OK) THROW_IE_EXCEPTION << desc.msg;
if (cnnReader->getVersion(&desc) >= 10) {
cnnReader->addExtensions(getExtensions());
std::lock_guard<std::mutex> lock(pluginsMutex);
cnnReader->addExtensions(GetExtensions());
}
std::string bPath = binPath;
if (bPath.empty()) {
@ -289,7 +351,8 @@ public:
StatusCode rt = cnnReader->ReadNetwork(model.data(), model.length(), &desc);
if (rt != OK) THROW_IE_EXCEPTION << desc.msg;
if (cnnReader->getVersion(&desc) >= 10) {
cnnReader->addExtensions(getExtensions());
std::lock_guard<std::mutex> lock(pluginsMutex);
cnnReader->addExtensions(GetExtensions());
}
TBlob<uint8_t>::Ptr weights_ptr;
if (weights) {
@ -302,11 +365,90 @@ public:
return CNNNetwork(cnnReader);
}
ExecutableNetwork LoadNetwork(const CNNNetwork& network, const std::string& deviceName,
const std::map<std::string, std::string>& config) override {
IE_PROFILING_AUTO_SCOPE(Core::LoadNetwork)
auto parsed = parseDeviceNameIntoConfig(deviceName, config);
IE_SUPPRESS_DEPRECATED_START
return GetCPPPluginByName(parsed._deviceName).LoadNetwork(network, parsed._config);
IE_SUPPRESS_DEPRECATED_END
}
IE_SUPPRESS_DEPRECATED_START
ExecutableNetwork ImportNetwork(std::istream& networkModel, const std::string& deviceName,
const std::map<std::string, std::string>& config) override {
auto parsed = parseDeviceNameIntoConfig(deviceName, config);
if (parsed._deviceName.empty()) {
ExportMagic magic = {};
auto currentPos = networkModel.tellg();
networkModel.read(magic.data(), magic.size());
auto exportedWithName = (exportMagic == magic);
if (exportedWithName) {
std::getline(networkModel, parsed._deviceName);
}
networkModel.seekg(currentPos, networkModel.beg);
}
auto cppPlugin = GetCPPPluginByName(parsed._deviceName);
auto pluginAPIInterface = getInferencePluginAPIInterface(cppPlugin);
if (pluginAPIInterface == nullptr) {
THROW_IE_EXCEPTION << parsed._deviceName << " does not implement the ImportNetwork method";
}
return pluginAPIInterface->ImportNetwork(networkModel, parsed._config);
}
QueryNetworkResult QueryNetwork(const ICNNNetwork& network, const std::string& deviceName,
const std::map<std::string, std::string>& config) const override {
QueryNetworkResult res;
auto parsed = parseDeviceNameIntoConfig(deviceName, config);
IE_SUPPRESS_DEPRECATED_START
GetCPPPluginByName(parsed._deviceName).QueryNetwork(network, parsed._config, res);
IE_SUPPRESS_DEPRECATED_END
return res;
}
Parameter GetMetric(const std::string& deviceName, const std::string& name) const override {
// HETERO case
{
if (deviceName.find("HETERO:") == 0) {
THROW_IE_EXCEPTION
<< "You can get specific metrics with the GetMetric only for the HETERO itself (without devices). "
"To get individual devices's metrics call GetMetric for each device separately";
}
}
// MULTI case
{
if (deviceName.find("MULTI:") == 0) {
THROW_IE_EXCEPTION
<< "You can get specific metrics with the GetMetric only for the MULTI itself (without devices). "
"To get individual devices's metrics call GetMetric for each device separately";
}
}
auto parsed = parseDeviceNameIntoConfig(deviceName);
IE_SUPPRESS_DEPRECATED_START
InferencePlugin cppPlugin = GetCPPPluginByName(parsed._deviceName);
auto pluginAPIInterface = getInferencePluginAPIInterface(cppPlugin);
IE_SUPPRESS_DEPRECATED_END
if (pluginAPIInterface == nullptr) {
THROW_IE_EXCEPTION << parsed._deviceName << " does not implement the GetMetric method";
}
// we need to return a copy of Parameter object which is created on Core side,
// not in InferenceEngine plugin side, which can be unloaded from Core in a parallel thread
// TODO: remove this WA after *-31417 is resolved
return copyParameterValue(pluginAPIInterface->GetMetric(name, parsed._config));
}
/**
* @deprecated Use ICore::LoadNetwork, ICore::QueryNetwork, ICore::GetMetric instead
* @brief Returns reference to plugin by a device name
* @param deviceName - a name of device
* @param deviceName A name of device
* @return Reference to a plugin
*/
InferenceEnginePluginPtr GetPluginByName(const std::string& deviceName) const override {
@ -314,11 +456,14 @@ public:
}
/**
* @deprecated
* @brief Returns reference to CPP plugin wrapper by a device name
* @param deviceName - a name of device
* @param deviceName A name of device
* @return Reference to a CPP plugin wrapper
*/
InferencePlugin GetCPPPluginByName(const std::string& deviceName) const {
std::lock_guard<std::mutex> lock(pluginsMutex);
IE_SUPPRESS_DEPRECATED_START
auto it = pluginRegistry.find(deviceName);
@ -379,10 +524,11 @@ public:
IE_SUPPRESS_DEPRECATED_END
/**
* @brief Unregisters plugin for specified device
* @param deviceName - a name of device
* @brief Unload plugin for specified device, but plugin meta-data is still in plugin registry
* @param deviceName A name of device
*/
void UnregisterPluginByName(const std::string& deviceName) {
void UnloadPluginByName(const std::string& deviceName) {
std::lock_guard<std::mutex> lock(pluginsMutex);
auto it = plugins.find(deviceName);
if (it == plugins.end()) {
THROW_IE_EXCEPTION << "Device with \"" << deviceName << "\" name is not registered in the InferenceEngine";
@ -392,10 +538,12 @@ public:
}
/**
* @brief Registers plugin in registry for specified device
* @param deviceName - a name of device
* @brief Registers plugin meta-data in registry for specified device
* @param deviceName A name of device
*/
void RegisterPluginByName(const std::string& pluginName, const std::string& deviceName) {
std::lock_guard<std::mutex> lock(pluginsMutex);
auto it = pluginRegistry.find(deviceName);
if (it != pluginRegistry.end()) {
THROW_IE_EXCEPTION << "Device with \"" << deviceName << "\" is already registered in the InferenceEngine";
@ -418,7 +566,13 @@ public:
pluginRegistry[deviceName] = desc;
}
/**
* @brief Porvides a list of plugin names in registry; physically such plugins may not be created
* @return A list of plugin names
*/
std::vector<std::string> GetListOfDevicesInRegistry() const {
std::lock_guard<std::mutex> lock(pluginsMutex);
std::vector<std::string> listOfDevices;
for (auto&& pluginDesc : pluginRegistry) {
listOfDevices.push_back(pluginDesc.first);
@ -427,7 +581,14 @@ public:
return listOfDevices;
}
/**
* @brief Sets config values for a plugin or set of plugins
* @param deviceName A device name to set config to
* If empty, config is set for all the plugins / plugin's meta-data
*/
void SetConfigForPlugins(const std::map<std::string, std::string>& config, const std::string& deviceName) {
std::lock_guard<std::mutex> lock(pluginsMutex);
// set config for plugins in registry
bool configIsSet = false;
for (auto& desc : pluginRegistry) {
@ -453,7 +614,13 @@ public:
}
}
void addExtension(const IExtensionPtr& extension) {
/**
* @brief Registers the extension in a Core object
* Such extensions can be used for both CNNNetwork readers and device plugins
*/
void AddExtension(const IExtensionPtr& extension) {
std::lock_guard<std::mutex> lock(pluginsMutex);
std::map<std::string, ngraph::OpSet> opsets = extension->getOpSets();
for (const auto& it : opsets) {
if (opsetNames.find(it.first) != opsetNames.end())
@ -461,6 +628,7 @@ public:
opsetNames.insert(it.first);
}
// add extensions for already created plugins
for (auto& plugin : plugins) {
IE_SUPPRESS_DEPRECATED_START
try {
@ -471,7 +639,11 @@ public:
extensions.emplace_back(extension);
}
const std::vector<IExtensionPtr>& getExtensions() const {
/**
* @brief Provides a list of extensions
* @return A list of registered extensions
*/
const std::vector<IExtensionPtr>& GetExtensions() const {
return extensions;
}
};
@ -525,7 +697,8 @@ std::map<std::string, Version> Core::GetVersions(const std::string& deviceName)
std::string deviceNameLocal = parser.getDeviceName();
IE_SUPPRESS_DEPRECATED_START
const Version* version = _impl->GetCPPPluginByName(deviceNameLocal).GetVersion();
InferenceEngine::InferencePlugin cppPlugin = _impl->GetCPPPluginByName(deviceNameLocal);
const Version * version = cppPlugin.GetVersion();
IE_SUPPRESS_DEPRECATED_END
versions[deviceNameLocal] = *version;
}
@ -538,36 +711,6 @@ void Core::SetLogCallback(IErrorListener&) const {
}
IE_SUPPRESS_DEPRECATED_END
namespace {
template <typename T>
struct Parsed {
std::string _deviceName;
std::map<std::string, T> _config;
};
template <typename T = Parameter>
Parsed<T> parseDeviceNameIntoConfig(const std::string& deviceName, const std::map<std::string, T>& config = {}) {
auto config_ = config;
auto deviceName_ = deviceName;
if (deviceName_.find("HETERO:") == 0) {
deviceName_ = "HETERO";
config_["TARGET_FALLBACK"] = deviceName.substr(7);
} else if (deviceName_.find("MULTI:") == 0) {
deviceName_ = "MULTI";
config_[InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES] = deviceName.substr(6);
} else {
DeviceIDParser parser(deviceName_);
deviceName_ = parser.getDeviceName();
std::string deviceIDLocal = parser.getDeviceID();
if (!deviceIDLocal.empty()) {
config_[KEY_DEVICE_ID] = deviceIDLocal;
}
}
return {deviceName_, config_};
}
} // namespace
CNNNetwork Core::ReadNetwork(const std::string& modelPath, const std::string& binPath) const {
return _impl->ReadNetwork(modelPath, binPath);
}
@ -576,20 +719,16 @@ CNNNetwork Core::ReadNetwork(const std::string& model, const Blob::CPtr& weights
return _impl->ReadNetwork(model, weights);
}
ExecutableNetwork Core::LoadNetwork(const CNNNetwork network, const std::string& deviceName,
ExecutableNetwork Core::LoadNetwork(const CNNNetwork& network, const std::string& deviceName,
const std::map<std::string, std::string>& config) {
IE_PROFILING_AUTO_SCOPE(Core::LoadNetwork)
auto parsed = parseDeviceNameIntoConfig(deviceName, config);
IE_SUPPRESS_DEPRECATED_START
return _impl->GetCPPPluginByName(parsed._deviceName).LoadNetwork(network, parsed._config);
IE_SUPPRESS_DEPRECATED_END
return _impl->LoadNetwork(network, deviceName, config);
}
void Core::AddExtension(const IExtensionPtr& extension) {
_impl->addExtension(extension);
_impl->AddExtension(extension);
}
ExecutableNetwork Core::LoadNetwork(const CNNNetwork network, RemoteContext::Ptr context,
ExecutableNetwork Core::LoadNetwork(const CNNNetwork& network, RemoteContext::Ptr context,
const std::map<std::string, std::string>& config) {
IE_PROFILING_AUTO_SCOPE(Core::LoadNetwork)
std::map<std::string, std::string> config_ = config;
@ -603,7 +742,8 @@ ExecutableNetwork Core::LoadNetwork(const CNNNetwork network, RemoteContext::Ptr
std::string deviceName = device.getDeviceName();
IE_SUPPRESS_DEPRECATED_START
auto pluginAPIInterface = getInferencePluginAPIInterface(_impl->GetCPPPluginByName(deviceName));
auto cppPlugin = _impl->GetCPPPluginByName(deviceName);
auto pluginAPIInterface = getInferencePluginAPIInterface(cppPlugin);
if (pluginAPIInterface == nullptr) {
THROW_IE_EXCEPTION << deviceName << " does not implement the LoadNetwork method";
@ -625,7 +765,8 @@ RemoteContext::Ptr Core::CreateContext(const std::string& deviceName_, const Par
std::string deviceName = device.getDeviceName();
IE_SUPPRESS_DEPRECATED_START
auto pluginAPIInterface = getInferencePluginAPIInterface(_impl->GetCPPPluginByName(deviceName));
auto cppPlugin = _impl->GetCPPPluginByName(deviceName);
auto pluginAPIInterface = getInferencePluginAPIInterface(cppPlugin);
if (pluginAPIInterface == nullptr) {
THROW_IE_EXCEPTION << deviceName << " does not implement the CreateContext method";
@ -647,7 +788,8 @@ RemoteContext::Ptr Core::GetDefaultContext(const std::string& deviceName_) {
std::string deviceName = device.getDeviceName();
IE_SUPPRESS_DEPRECATED_START
auto pluginAPIInterface = getInferencePluginAPIInterface(_impl->GetCPPPluginByName(deviceName));
auto cppPlugin = _impl->GetCPPPluginByName(deviceName);
auto pluginAPIInterface = getInferencePluginAPIInterface(cppPlugin);
if (pluginAPIInterface == nullptr) {
THROW_IE_EXCEPTION << deviceName << " does not implement the CreateContext method";
@ -667,13 +809,7 @@ void Core::AddExtension(IExtensionPtr extension, const std::string& deviceName_)
<< "MULTI device does not support extensions. Please, set extensions directly to fallback devices";
}
DeviceIDParser parser(deviceName_);
std::string deviceName = parser.getDeviceName();
IE_SUPPRESS_DEPRECATED_START
_impl->GetCPPPluginByName(deviceName).AddExtension(extension);
_impl->addExtension(extension);
IE_SUPPRESS_DEPRECATED_END
_impl->AddExtension(extension);
}
ExecutableNetwork Core::ImportNetwork(const std::string& modelFileName, const std::string& deviceName,
@ -692,32 +828,11 @@ ExecutableNetwork Core::ImportNetwork(const std::string& modelFileName, const st
IE_SUPPRESS_DEPRECATED_END
}
IE_SUPPRESS_DEPRECATED_START
ExecutableNetwork Core::ImportNetwork(std::istream& networkModel, const std::string& deviceName,
const std::map<std::string, std::string>& config) {
auto parsed = parseDeviceNameIntoConfig(deviceName, config);
if (parsed._deviceName.empty()) {
ExportMagic magic = {};
networkModel.read(magic.data(), magic.size());
auto exportedWithName = (exportMagic == magic);
if (exportedWithName) {
std::getline(networkModel, parsed._deviceName);
}
networkModel.seekg(0, networkModel.beg);
}
auto pluginAPIInterface = getInferencePluginAPIInterface(_impl->GetCPPPluginByName(parsed._deviceName));
if (pluginAPIInterface == nullptr) {
THROW_IE_EXCEPTION << parsed._deviceName << " does not implement the ImportNetwork method";
}
return pluginAPIInterface->ImportNetwork(networkModel, parsed._config);
return _impl->ImportNetwork(networkModel, deviceName, config);
}
IE_SUPPRESS_DEPRECATED_END
ExecutableNetwork Core::ImportNetwork(std::istream& networkModel,
const RemoteContext::Ptr& context,
const std::map<std::string, std::string>& config) {
@ -734,7 +849,8 @@ ExecutableNetwork Core::ImportNetwork(std::istream& networkModel,
auto parsed = parseDeviceNameIntoConfig(deviceName, config);
IE_SUPPRESS_DEPRECATED_START
auto pluginAPIInterface = getInferencePluginAPIInterface(_impl->GetCPPPluginByName(parsed._deviceName));
auto cppPlugin = _impl->GetCPPPluginByName(deviceName);
auto pluginAPIInterface = getInferencePluginAPIInterface(cppPlugin);
if (pluginAPIInterface == nullptr) {
THROW_IE_EXCEPTION << deviceName << " does not implement the ImportNetwork method";
@ -745,12 +861,7 @@ ExecutableNetwork Core::ImportNetwork(std::istream& networkModel,
QueryNetworkResult Core::QueryNetwork(const ICNNNetwork& network, const std::string& deviceName,
const std::map<std::string, std::string>& config) const {
QueryNetworkResult res;
auto parsed = parseDeviceNameIntoConfig(deviceName, config);
IE_SUPPRESS_DEPRECATED_START
_impl->GetCPPPluginByName(parsed._deviceName).QueryNetwork(network, parsed._config, res);
IE_SUPPRESS_DEPRECATED_END
return res;
return _impl->QueryNetwork(network, deviceName, config);
}
void Core::SetConfig(const std::map<std::string, std::string>& config, const std::string& deviceName) {
@ -798,42 +909,22 @@ Parameter Core::GetConfig(const std::string& deviceName, const std::string& name
auto parsed = parseDeviceNameIntoConfig(deviceName);
IE_SUPPRESS_DEPRECATED_START
auto pluginAPIInterface = getInferencePluginAPIInterface(_impl->GetCPPPluginByName(parsed._deviceName));
auto cppPlugin = _impl->GetCPPPluginByName(parsed._deviceName);
auto pluginAPIInterface = getInferencePluginAPIInterface(cppPlugin);
IE_SUPPRESS_DEPRECATED_END
if (pluginAPIInterface == nullptr) {
THROW_IE_EXCEPTION << parsed._deviceName << " does not implement the GetConfig method";
}
return pluginAPIInterface->GetConfig(name, parsed._config);
// we need to return a copy of Parameter object which is created on Core side,
// not in InferenceEngine plugin side, which can be unloaded from Core in a parallel thread
// TODO: remove this WA after *-31417 is resolved
return copyParameterValue(pluginAPIInterface->GetConfig(name, parsed._config));
}
Parameter Core::GetMetric(const std::string& deviceName, const std::string& name) const {
// HETERO case
{
if (deviceName.find("HETERO:") == 0) {
THROW_IE_EXCEPTION
<< "You can get specific metrics with the GetMetric only for the HETERO itself (without devices). "
"To get individual devices's metrics call GetMetric for each device separately";
}
}
// MULTI case
{
if (deviceName.find("MULTI:") == 0) {
THROW_IE_EXCEPTION
<< "You can get specific metrics with the GetMetric only for the MULTI itself (without devices). "
"To get individual devices's metrics call GetMetric for each device separately";
}
}
auto parsed = parseDeviceNameIntoConfig(deviceName);
IE_SUPPRESS_DEPRECATED_START
auto pluginAPIInterface = getInferencePluginAPIInterface(_impl->GetCPPPluginByName(parsed._deviceName));
IE_SUPPRESS_DEPRECATED_END
if (pluginAPIInterface == nullptr) {
THROW_IE_EXCEPTION << parsed._deviceName << " does not implement the GetMetric method";
}
return pluginAPIInterface->GetMetric(name, parsed._config);
return _impl->GetMetric(deviceName, name);
}
std::vector<std::string> Core::GetAvailableDevices() const {
@ -842,11 +933,11 @@ std::vector<std::string> Core::GetAvailableDevices() const {
std::string propertyName = METRIC_KEY(AVAILABLE_DEVICES);
for (auto&& deviceName : _impl->GetListOfDevicesInRegistry()) {
Parameter p;
std::vector<std::string> devicesIDs;
IE_SUPPRESS_DEPRECATED_START
try {
p = GetMetric(deviceName, propertyName);
Parameter p = GetMetric(deviceName, propertyName);
devicesIDs = p.as<std::vector<std::string>>();
} catch (details::InferenceEngineException&) {
// plugin is not created by e.g. invalid env
@ -857,6 +948,7 @@ std::vector<std::string> Core::GetAvailableDevices() const {
THROW_IE_EXCEPTION << "Unknown exception is thrown while trying to create the " << deviceName
<< " device and call GetMetric";
}
IE_SUPPRESS_DEPRECATED_END
if (devicesIDs.size() > 1) {
for (auto&& deviceID : devicesIDs) {
@ -882,7 +974,7 @@ void Core::UnregisterPlugin(const std::string& deviceName_) {
DeviceIDParser parser(deviceName_);
std::string deviceName = parser.getDeviceName();
_impl->UnregisterPluginByName(deviceName);
_impl->UnloadPluginByName(deviceName);
}
} // namespace InferenceEngine

344
inference-engine/src/inference_engine/threading/ie_cpu_streams_executor.cpp
View File

@ -23,159 +23,215 @@
#include "threading/ie_cpu_streams_executor.hpp"
namespace InferenceEngine {
#if IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO
struct PinningObserver: public tbb::task_scheduler_observer {
CpuSet& _mask;
int _ncpus = 0;
int _streamId = 0;
int _threadsPerStream = 0;
int _threadBindingStep = 0;
int _threadBindingOffset = 0;
PinningObserver(tbb::task_arena& arena,
CpuSet& mask,
int ncpus,
const int streamId,
const int threadsPerStream,
const int threadBindingStep,
const int threadBindingOffset) :
tbb::task_scheduler_observer(arena),
_mask(mask),
_ncpus(ncpus),
_streamId(streamId),
_threadsPerStream(threadsPerStream),
_threadBindingStep(threadBindingStep),
_threadBindingOffset(threadBindingOffset) {
observe(true);
}
void on_scheduler_entry(bool) override {
int threadIdx = tbb::task_arena::current_thread_index();
int thrIdx = _streamId * _threadsPerStream + threadIdx + _threadBindingOffset;
// pin thread to the vacant slot
PinThreadToVacantCore(thrIdx, _threadBindingStep, _ncpus, _mask);
}
void on_scheduler_exit(bool) override {
// reset the thread's mask (to the original process mask)
PinCurrentThreadByMask(_ncpus, _mask);
}
~PinningObserver() {
observe(false);
}
};
#endif // IE_THREAD != IE_THREAD_TBB
struct Stream {
int _streamId = 0;
int _numaNodeId = 0;
#if IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO
std::unique_ptr<tbb::task_arena> _taskArena;
std::unique_ptr<PinningObserver> _pinningObserver;
#endif
};
struct CPUStreamsExecutor::Impl {
std::string _name;
std::vector<std::thread> _threads;
std::mutex _mutex;
std::condition_variable _queueCondVar;
std::queue<Task> _taskQueue;
bool _isStopped = false;
int _ncpus = 0;
CpuSet _processMask;
ThreadLocal<Stream*> _localStream;
};
int CPUStreamsExecutor::GetStreamId() {
auto stream = _impl->_localStream.local();
if (nullptr == stream) THROW_IE_EXCEPTION << "Not in the stream thread";
return stream->_streamId;
}
int CPUStreamsExecutor::GetNumaNodeId() {
auto stream = _impl->_localStream.local();
if (nullptr == stream) THROW_IE_EXCEPTION << "Not in the stream thread";
return stream->_numaNodeId;
}
CPUStreamsExecutor::CPUStreamsExecutor(const IStreamsExecutor::Config& config) :
_impl{new Impl} {
IE_ASSERT(config._streams > 0);
_impl->_name = config._name;
auto numaNodes = getAvailableNUMANodes();
IE_ASSERT(!numaNodes.empty());
if (ThreadBindingType::CORES == config._threadBindingType) {
std::tie(_impl->_processMask, _impl->_ncpus) = GetProcessMask();
}
for (auto streamId = 0; streamId < config._streams; ++streamId) {
_impl->_threads.emplace_back([=] {
annotateSetThreadName((_impl->_name + "_" + std::to_string(streamId)).c_str());
Stream stream;
stream._streamId = streamId;
stream._numaNodeId = numaNodes.at(streamId/((config._streams + numaNodes.size() - 1)/numaNodes.size()));
struct Stream {
#if IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO
auto concurrency = (0 == config._threadsPerStream) ? tbb::task_arena::automatic : config._threadsPerStream;
if (ThreadBindingType::NUMA == config._threadBindingType) {
stream._taskArena.reset(new tbb::task_arena(tbb::task_arena::constraints(stream._numaNodeId, concurrency)));
} else if ((0 != config._threadsPerStream) || ThreadBindingType::CORES == config._threadBindingType) {
stream._taskArena.reset(new tbb::task_arena(concurrency));
if (ThreadBindingType::CORES == config._threadBindingType) {
if (nullptr != _impl->_processMask) {
stream._pinningObserver.reset(new PinningObserver{*stream._taskArena,
_impl->_processMask,
_impl->_ncpus,
stream._streamId,
config._threadsPerStream,
config._threadBindingStep,
config._threadBindingOffset});
struct Observer: public tbb::task_scheduler_observer {
CpuSet _mask;
int _ncpus = 0;
int _threadBindingStep = 0;
int _offset = 0;
Observer(tbb::task_arena& arena,
CpuSet mask,
int ncpus,
const int streamId,
const int threadsPerStream,
const int threadBindingStep,
const int threadBindingOffset) :
tbb::task_scheduler_observer(arena),
_mask{std::move(mask)},
_ncpus(ncpus),
_threadBindingStep(threadBindingStep),
_offset{streamId * threadsPerStream + threadBindingOffset} {
}
void on_scheduler_entry(bool) override {
PinThreadToVacantCore(_offset + tbb::task_arena::current_thread_index(), _threadBindingStep, _ncpus, _mask);
}
void on_scheduler_exit(bool) override {
PinCurrentThreadByMask(_ncpus, _mask);
}
~Observer() override = default;
};
#endif
explicit Stream(Impl* impl) :
_impl(impl) {
{
std::lock_guard<std::mutex> lock{_impl->_streamIdMutex};
if (_impl->_streamIdQueue.empty()) {
_streamId = _impl->_streamId++;
} else {
_streamId = _impl->_streamIdQueue.front();
_impl->_streamIdQueue.pop();
}
}
_numaNodeId = _impl->_usedNumaNodes.at(
(_streamId % _impl->_config._streams)/
((_impl->_config._streams + _impl->_usedNumaNodes.size() - 1)/_impl->_usedNumaNodes.size()));
#if IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO
auto concurrency = (0 == _impl->_config._threadsPerStream) ? tbb::task_arena::automatic : _impl->_config._threadsPerStream;
if (ThreadBindingType::NUMA == _impl->_config._threadBindingType) {
_taskArena.reset(new tbb::task_arena{tbb::task_arena::constraints{_numaNodeId, concurrency}});
} else if ((0 != _impl->_config._threadsPerStream) || (ThreadBindingType::CORES == _impl->_config._threadBindingType)) {
_taskArena.reset(new tbb::task_arena{concurrency});
if (ThreadBindingType::CORES == _impl->_config._threadBindingType) {
CpuSet processMask;
int ncpus = 0;
std::tie(processMask, ncpus) = GetProcessMask();
if (nullptr != processMask) {
_observer.reset(new Observer{*_taskArena,
std::move(processMask),
ncpus,
_streamId,
_impl->_config._threadsPerStream,
_impl->_config._threadBindingStep,
_impl->_config._threadBindingOffset});
_observer->observe(true);
}
}
}
#elif IE_THREAD == IE_THREAD_OMP
omp_set_num_threads(config._threadsPerStream);
if (!checkOpenMpEnvVars(false) && (ThreadBindingType::NONE != config._threadBindingType)) {
if (nullptr != _impl->_processMask) {
parallel_nt(config._threadsPerStream, [&] (int threadIndex, int threadsPerStream) {
int thrIdx = stream._streamId * threadsPerStream + threadIndex + config._threadBindingOffset;
PinThreadToVacantCore(thrIdx, config._threadBindingStep, _impl->_ncpus, _impl->_processMask);
omp_set_num_threads(_impl->_config._threadsPerStream);
if (!checkOpenMpEnvVars(false) && (ThreadBindingType::NONE != _impl->_config._threadBindingType)) {
CpuSet processMask;
int ncpus = 0;
std::tie(processMask, ncpus) = GetProcessMask();
if (nullptr != processMask) {
parallel_nt(_impl->_config._threadsPerStream, [&] (int threadIndex, int threadsPerStream) {
int thrIdx = _streamId * _impl->_config._threadsPerStream + threadIndex + _impl->_config._threadBindingOffset;
PinThreadToVacantCore(thrIdx, _impl->_config._threadBindingStep, ncpus, processMask);
});
}
}
#elif IE_THREAD == IE_THREAD_SEQ
if (ThreadBindingType::NUMA == config._threadBindingType) {
PinCurrentThreadToSocket(stream._numaNodeId);
} else if (ThreadBindingType::CORES == config._threadBindingType) {
PinThreadToVacantCore(stream._streamId + config._threadBindingOffset, config._threadBindingStep, _impl->_ncpus, _impl->_processMask);
if (ThreadBindingType::NUMA == _impl->_config._threadBindingType) {
PinCurrentThreadToSocket(_numaNodeId);
} else if (ThreadBindingType::CORES == _impl->_config._threadBindingType) {
CpuSet processMask;
int ncpus = 0;
std::tie(processMask, ncpus) = GetProcessMask();
if (nullptr != processMask) {
PinThreadToVacantCore(_streamId + _impl->_config._threadBindingOffset, _impl->_config._threadBindingStep, ncpus, processMask);
}
}
#endif
_impl->_localStream.local() = &stream;
for (bool stopped = false; !stopped;) {
Task currentTask;
{ // waiting for the new task or for stop signal
std::unique_lock<std::mutex> lock(_impl->_mutex);
_impl->_queueCondVar.wait(lock, [&] { return !_impl->_taskQueue.empty() || (stopped = _impl->_isStopped); });
if (!_impl->_taskQueue.empty()) {
currentTask = std::move(_impl->_taskQueue.front());
_impl->_taskQueue.pop();
}
}
if (currentTask) {
}
~Stream() {
{
std::lock_guard<std::mutex> lock{_impl->_streamIdMutex};
_impl->_streamIdQueue.push(_streamId);
}
#if IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO
if (nullptr != stream._taskArena) {
stream._taskArena->execute(std::move(currentTask));
} else {
currentTask();
}
#else
currentTask();
#endif
}
if (nullptr != _observer) {
_observer->observe(false);
}
});
#endif
}
Impl* _impl = nullptr;
int _streamId = 0;
int _numaNodeId = 0;
bool _execute = false;
std::queue<Task> _taskQueue;
#if IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO
std::unique_ptr<tbb::task_arena> _taskArena;
std::unique_ptr<Observer> _observer;
#endif
};
explicit Impl(const Config& config) :
_config{config},
_streams([this] {
return std::make_shared<Impl::Stream>(this);
}) {
auto numaNodes = getAvailableNUMANodes();
std::copy_n(std::begin(numaNodes),
std::min(std::max(static_cast<std::size_t>(1),
static_cast<std::size_t>(_config._streams)),
numaNodes.size()),
std::back_inserter(_usedNumaNodes));
for (auto streamId = 0; streamId < _config._streams; ++streamId) {
_threads.emplace_back([this, streamId] {
annotateSetThreadName((_config._name + "_" + std::to_string(streamId)).c_str());
for (bool stopped = false; !stopped;) {
Task task;
{
std::unique_lock<std::mutex> lock(_mutex);
_queueCondVar.wait(lock, [&] { return !_taskQueue.empty() || (stopped = _isStopped); });
if (!_taskQueue.empty()) {
task = std::move(_taskQueue.front());
_taskQueue.pop();
}
}
if (task) {
Execute(task, *(_streams.local()));
}
}
});
}
}
void Enqueue(Task task) {
{
std::lock_guard<std::mutex> lock(_mutex);
_taskQueue.emplace(std::move(task));
}
_queueCondVar.notify_one();
}
void Execute(const Task& task, Stream& stream) {
#if IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO
auto& arena = stream._taskArena;
if (nullptr != arena) {
arena->execute(std::move(task));
} else {
task();
}
#else
task();
#endif
}
void Defer(Task task) {
auto& stream = *(_streams.local());
stream._taskQueue.push(std::move(task));
if (!stream._execute) {
stream._execute = true;
try {
while (!stream._taskQueue.empty()) {
Execute(stream._taskQueue.front(), stream);
stream._taskQueue.pop();
}
} catch(...) {}
stream._execute = false;
}
}
Config _config;
std::mutex _streamIdMutex;
int _streamId = 0;
std::queue<int> _streamIdQueue;
std::vector<std::thread> _threads;
std::mutex _mutex;
std::condition_variable _queueCondVar;
std::queue<Task> _taskQueue;
bool _isStopped = false;
std::vector<int> _usedNumaNodes;
ThreadLocal<std::shared_ptr<Stream>> _streams;
};
int CPUStreamsExecutor::GetStreamId() {
auto stream = _impl->_streams.local();
return stream->_streamId;
}
int CPUStreamsExecutor::GetNumaNodeId() {
auto stream = _impl->_streams.local();
return stream->_numaNodeId;
}
CPUStreamsExecutor::CPUStreamsExecutor(const IStreamsExecutor::Config& config) :
_impl{new Impl{config}} {
}
CPUStreamsExecutor::~CPUStreamsExecutor() {
@ -191,12 +247,16 @@ CPUStreamsExecutor::~CPUStreamsExecutor() {
}
}
void CPUStreamsExecutor::Execute(Task task) {
_impl->Defer(std::move(task));
}
void CPUStreamsExecutor::run(Task task) {
{
std::lock_guard<std::mutex> lock(_impl->_mutex);
_impl->_taskQueue.emplace(std::move(task));
if (0 == _impl->_config._streams) {
_impl->Defer(std::move(task));
} else {
_impl->Enqueue(std::move(task));
}
_impl->_queueCondVar.notify_one();
}
} // namespace InferenceEngine

2
inference-engine/src/inference_engine/threading/ie_istreams_executor.cpp
View File

@ -126,4 +126,4 @@ Parameter IStreamsExecutor::Config::GetConfig(const std::string& key) {
return {};
}
} // namespace InferenceEngine
} // namespace InferenceEngine

2
inference-engine/src/inference_engine/xml_parse_utils.cpp
View File

@ -76,7 +76,7 @@ unsigned int XMLParseUtils::GetUIntAttr(const pugi::xml_node& node, const char*
std::string XMLParseUtils::GetStrAttr(const pugi::xml_node& node, const char* str) {
auto attr = node.attribute(str);
if (attr.empty())
THROW_IE_EXCEPTION << "node <" << node.name() << "> is missing mandatory attribute: " << str << " at offset "
THROW_IE_EXCEPTION << "node <" << node.name() << "> is missing mandatory attribute: '" << str << "' at offset "
<< node.offset_debug();
return attr.value();
}

49
inference-engine/src/ir_readers/ie_ir_parser.cpp
View File

@ -344,29 +344,40 @@ std::shared_ptr<ngraph::Node> V10Parser::createNode(const std::vector<ngraph::Ou
}
std::shared_ptr<ngraph::Node> ngraphNode;
if (opsets.count(params.version)) {
auto opset = opsets.at(params.version);
for (const auto& creator : creators) {
if (creator->shouldCreate(params.type)) {
// Try to create operation from creators
for (const auto& creator : creators) {
if (creator->shouldCreate(params.type)) {
bool useCreator = false;
// Check that opset is registered
useCreator |= opsets.find(params.version) == opsets.end();
if (!useCreator) {
// Check that creator can create operation with the version from opset
const auto opset = opsets.at(params.version);
// Opset should contains the same version of operation or doesn't contain operation with current type
useCreator |= opset.contains_type(creator->getNodeType()) || !opset.contains_type(params.type);
}
if (useCreator)
ngraphNode = creator->createLayer(inputs, node, weights, params);
break;
}
}
if (!ngraphNode) {
if (!opset.contains_type(params.type)) {
THROW_IE_EXCEPTION << "Opset " << params.version << " doesn't contain the operation with type: " << params.type;
}
ngraphNode = std::shared_ptr<ngraph::Node>(opset.create(params.type));
ngraphNode->set_arguments(inputs);
XmlDeserializer visitor(node);
if (ngraphNode->visit_attributes(visitor))
ngraphNode->constructor_validate_and_infer_types();
break;
}
}
// Try to create operation from loaded opsets
if (!ngraphNode && opsets.count(params.version)) {
auto opset = opsets.at(params.version);
if (!opset.contains_type(params.type)) {
THROW_IE_EXCEPTION << "Opset " << params.version << " doesn't contain the operation with type: " << params.type;
}
ngraphNode = std::shared_ptr<ngraph::Node>(opset.create(params.type));
ngraphNode->set_arguments(inputs);
XmlDeserializer visitor(node);
if (ngraphNode->visit_attributes(visitor))
ngraphNode->constructor_validate_and_infer_types();
}
// Create GenericIE operation for backward compatibility
if (!ngraphNode && (params.version == "experimental" || params.version == "extension")) {
// Try to create Generic node for backward compatibility
std::map<std::string, Parameter> parameters;

12
inference-engine/src/ir_readers/ie_ir_parser.hpp
View File

@ -140,8 +140,7 @@ private:
const GenericLayerParams& layerParsePrms) = 0;
bool shouldCreate(const std::string& nodeType) const;
std::shared_ptr<ngraph::Node> createOptionalParameter(const GenericLayerParams::LayerPortData& port);
virtual ngraph::NodeTypeInfo getNodeType() const = 0;
};
template <class T>
@ -151,6 +150,9 @@ private:
std::shared_ptr<ngraph::Node> createLayer(const ngraph::OutputVector& inputs, const pugi::xml_node& node,
const Blob::CPtr& weights,
const GenericLayerParams& layerParsePrms) override;
ngraph::NodeTypeInfo getNodeType() const override {
return T::type_info;
}
};
std::shared_ptr<ngraph::Node> createNode(const ngraph::OutputVector& inputs, const pugi::xml_node& node,
@ -203,6 +205,12 @@ private:
std::vector<size_t> shape;
if (!getParameters<size_t>(node.child("data"), name, shape)) return;
static_cast<ngraph::Strides&>(*a) = ngraph::Strides(shape);
} else if (auto a = ngraph::as_type<ngraph::AttributeAdapter<ngraph::op::TopKSortType>>(&adapter)) {
if (!getStrAttribute(node.child("data"), name, val)) return;
static_cast<ngraph::op::TopKSortType&>(*a) = ngraph::as_enum<ngraph::op::TopKSortType>(val);
} else if (auto a = ngraph::as_type<ngraph::AttributeAdapter<ngraph::op::TopKMode>>(&adapter)) {
if (!getStrAttribute(node.child("data"), name, val)) return;
static_cast<ngraph::op::TopKMode&>(*a) = ngraph::as_enum<ngraph::op::TopKMode>(val);
}
}
void on_adapter(const std::string& name, ngraph::ValueAccessor<double>& adapter) override {

15
inference-engine/src/legacy_api/src/convert_function_to_cnn_network.cpp
View File

@ -334,6 +334,15 @@ InferenceEngine::details::CNNLayerCreator::CNNLayerCreator(const std::shared_ptr
res->params = params;
return res;
});
addSpecificCreator({"ScatterElementsUpdate"}, [](const std::shared_ptr<::ngraph::Node>& node,
const std::map<std::string, std::string> params) -> CNNLayerPtr {
LayerParams attrs = {node->get_friendly_name(), node->description(),
details::convertPrecision(node->get_output_element_type(0))};
auto res = std::make_shared<ScatterElementsUpdateLayer>(attrs);
res->params = params;
return res;
});
}
CNNLayerPtr InferenceEngine::details::CNNLayerCreator::create() {
@ -599,8 +608,10 @@ std::shared_ptr<CNNNetworkImpl> convertFunctionToICNNNetwork(const std::shared_p
}
size_t inputCount(0);
for (size_t i = 0; i < layer->get_input_size(); i++) {
const auto &input = layer->get_inputs()[i];
if (isInternalLayer(input.get_output().get_node(), op_names, keep_constants)) continue;
const auto &constant = ngraph::as_type_ptr<ngraph::op::Constant>(layer->get_inputs()[i].get_output().get_node());
if (constant && isInternalConstLayer(constant, layer, keep_constants)) {
continue;
}
inputCount++;
}
cnnLayer->insData.resize(inputCount);

3
inference-engine/src/legacy_api/src/ie_deprecated.cpp
View File

@ -13,6 +13,7 @@ IE_SUPPRESS_DEPRECATED_START
namespace InferenceEngine {
Precision CNNNetwork::getPrecision() const {
if (actual == nullptr) THROW_IE_EXCEPTION << "CNNNetwork was not initialized.";
return actual->getPrecision();
}
@ -41,7 +42,7 @@ StatusCode ICNNNetwork::AddExtension(const IShapeInferExtensionPtr& extension, R
};
void CNNNetwork::AddExtension(InferenceEngine::IShapeInferExtensionPtr extension) {
CALL_STATUS_FNC(AddExtension, extension);
CALL_STATUS_FNC(AddExtension, extension);
}
CNNLayer::CNNLayer(const LayerParams& prms)

8
inference-engine/src/legacy_api/src/ie_layer_validators.cpp
View File

@ -3163,12 +3163,12 @@ void ScatterElementsUpdateValidator::checkShapes(const CNNLayer* layer, const ve
THROW_IE_EXCEPTION << layer->name << " Incorrect number of 'updates' tensors dimension";
Precision inIdxPrecision = layer->insData[INDICES].lock()->getTensorDesc().getPrecision();
if (inIdxPrecision != Precision::FP32 && inIdxPrecision != Precision::I32)
THROW_IE_EXCEPTION << layer->name << " Incorrect input 'Indices' precision. Only FP32 or I32 are supported!";
if (inIdxPrecision != Precision::FP32 && inIdxPrecision != Precision::I32 && inIdxPrecision != Precision::I64)
THROW_IE_EXCEPTION << layer->name << " Incorrect input 'Indices' precision. Only FP32 or I32 or I64 are supported!";
Precision inAxisPrecision = layer->insData[AXIS].lock()->getTensorDesc().getPrecision();
if (inAxisPrecision != Precision::FP32 && inAxisPrecision != Precision::I32)
THROW_IE_EXCEPTION << layer->name << " Incorrect input 'Axis' precision. Only FP32 or I32 are supported!";
if (inAxisPrecision != Precision::FP32 && inAxisPrecision != Precision::I32 && inIdxPrecision != Precision::I64)
THROW_IE_EXCEPTION << layer->name << " Incorrect input 'Axis' precision. Only FP32 or I32 or I64 are supported!";
if (layer->insData[DATA].lock()->getTensorDesc().getPrecision() !=
layer->insData[UPDATES].lock()->getTensorDesc().getPrecision())

1
inference-engine/src/legacy_api/src/shape_infer/built-in/ie_tensor_iterator_shape_infer.hpp
View File

@ -92,6 +92,7 @@ public:
if (!_body_reshaper)
THROW_IE_EXCEPTION << "Request of apply reshape results while shape infer was not finished";
_body_reshaper->apply();
_body_reshaper.reset(); // WA: reset _body_reshaper to release ownership for input data
}
private:

2
inference-engine/src/legacy_api/src/shape_infer/ie_reshaper.hpp
View File

@ -80,7 +80,7 @@ public:
/**
* @brief Perform shape inference for the given input shapes but not apply it.
* In case of cusses call apply() method.
* In case of success call apply() method.
* @param inputShapes - Map of input names (data) to their input shapes.
* @throws exception if shape infer failed without corruption of original shapes
*/

24
inference-engine/src/low_precision_transformations/include/low_precision_transformations/power.hpp Normal file
View File

@ -0,0 +1,24 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <algorithm>
#include "ie_layers.h"
#include "low_precision_transformations/transformation_context.hpp"
#include "low_precision_transformations/layer_transformation.hpp"
namespace InferenceEngine {
namespace details {
class INFERENCE_ENGINE_API_CLASS(PowerTransformation) : public LayerTransformation {
public:
PowerTransformation(const Params& params) : LayerTransformation(params) {}
~PowerTransformation() override {}
void transform(TransformationContext& context, CNNLayer& layer) const override;
bool canBeTransformed(const TransformationContext& context, const CNNLayer& layer) const override;
};
} // namespace details
} // namespace InferenceEngine

75
inference-engine/src/low_precision_transformations/src/power.cpp Normal file
View File

@ -0,0 +1,75 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "low_precision_transformations/power.hpp"
#include <algorithm>
#include <details/caseless.hpp>
#include <string>
#include <memory>
#include <vector>
#include "low_precision_transformations/common/ie_lpt_exception.hpp"
#include "low_precision_transformations/network_helper.hpp"
using namespace InferenceEngine;
using namespace InferenceEngine::details;
bool PowerTransformation::canBeTransformed(const TransformationContext& context, const CNNLayer& layer) const {
if (!LayerTransformation::canBeTransformed(context, layer)) {
return false;
}
if (layer.insData.size() != 1) {
THROW_IE_LPT_EXCEPTION(layer) << "layer inputs '" << layer.insData.size() << "' is not correct";
}
if (!CaselessEq<std::string>()(layer.type, "Power")) {
THROW_IE_LPT_EXCEPTION(layer) << "layer '" << layer.name << "' is not correct";
}
const PowerLayer* powerLayer = dynamic_cast<const PowerLayer*>(&layer);
if (powerLayer == nullptr) {
THROW_IE_LPT_EXCEPTION(layer) << "unexpected Power layer type";
}
if (powerLayer->power != 1.f) {
return false;
}
const CNNLayerPtr parent = CNNNetworkHelper::getParent(layer, 0);
return !(parent->type != "ScaleShift");
}
void PowerTransformation::transform(TransformationContext& context, CNNLayer& layer) const {
if (!canBeTransformed(context, layer)) {
return;
}
const PowerLayer* powerLayer = dynamic_cast<const PowerLayer*>(&layer);
if (powerLayer == nullptr) {
THROW_IE_LPT_EXCEPTION(layer) << "unexpected Power layer type";
}
const CNNLayerPtr parent = CNNNetworkHelper::getParent(layer, 0);
Blob::Ptr weightsBlob = CNNNetworkHelper::getBlob(parent, "weights");
auto wBuffer = weightsBlob->buffer().as<float*>();
for (size_t channel = 0ul; channel < weightsBlob->size(); ++channel) {
wBuffer[channel] = wBuffer[channel] * powerLayer->scale;
}
Blob::Ptr shiftsBlob = CNNNetworkHelper::getBlob(parent, "biases");
auto sBuffer = shiftsBlob->buffer().as<float*>();
for (size_t channel = 0ul; channel < shiftsBlob->size(); ++channel) {
sBuffer[channel] = sBuffer[channel] * powerLayer->scale + powerLayer->offset;
}
const std::vector<CNNLayerPtr> children = CNNNetworkHelper::getChildren(layer);
CNNNetworkHelper::removeLayer(context.network, std::make_shared<CNNLayer>(layer));
context.removeLayer(layer);
if (children.empty()) {
const std::string originalName = layer.name;
CNNNetworkHelper::renameLayer(context.network, parent->name, layer.name);
}
}

5
inference-engine/src/low_precision_transformations/src/scaleshift_to_convolution.cpp
View File

@ -17,7 +17,7 @@
using namespace InferenceEngine;
using namespace InferenceEngine::details;
static const std::unordered_set<std::string> defaultIgnoreWithParents = {
static const char * defaultIgnoreWithParents[] = {
"Convolution",
"FakeQuantize"
};
@ -25,7 +25,8 @@ static const std::unordered_set<std::string> defaultIgnoreWithParents = {
ScaleShiftToConvolutionTransformation::ScaleShiftToConvolutionTransformation(const Params& params) :
WeightableLayerTransformation(params),
groupSize(1ul),
ignoreWithParents(defaultIgnoreWithParents) {
ignoreWithParents(defaultIgnoreWithParents, defaultIgnoreWithParents +
sizeof(defaultIgnoreWithParents) / sizeof(defaultIgnoreWithParents[0])) {
}
void ScaleShiftToConvolutionTransformation::transform(TransformationContext& context, CNNLayer& layer) const {

4
inference-engine/src/low_precision_transformations/src/transformer.cpp
View File

@ -34,6 +34,7 @@
#include "low_precision_transformations/permute.hpp"
#include "low_precision_transformations/pooling.hpp"
#include "low_precision_transformations/resample.hpp"
#include "low_precision_transformations/power.hpp"
#include "low_precision_transformations/reshape.hpp"
#include "low_precision_transformations/scaleshift_to_convolution.hpp"
#include "low_precision_transformations/squeeze.hpp"
@ -186,7 +187,8 @@ LowPrecisionTransformations LowPrecisionTransformer::getAllTransformations(const
{ "ReLU", LayerTransformationPtr(new ActivationTransformation(params)) },
{ "MVN", LayerTransformationPtr(new MvnTransformation(params)) },
{ "Eltwise", LayerTransformationPtr(new EltwiseTransformation(params)) },
{ "Resample", LayerTransformationPtr(new ResampleTransformation(params)) }
{ "Resample", LayerTransformationPtr(new ResampleTransformation(params)) },
{ "Power", LayerTransformationPtr(new PowerTransformation(params)) }
}),
std::map<std::string, LayerTransformationPtr>({
{ "FakeQuantize", LayerTransformationPtr(new FuseFakeQuantizeAndScaleShiftTransformation(params)) },

14
inference-engine/src/mkldnn_plugin/CMakeLists.txt
View File

@ -133,15 +133,15 @@ file(GLOB HEADERS
addVersionDefines(mkldnn_plugin.cpp CI_BUILD_NUMBER MKL_VERSION)
include_directories(
${IE_MAIN_SOURCE_DIR}/include
$<TARGET_PROPERTY:inference_engine_plugin_api,INTERFACE_INCLUDE_DIRECTORIES>
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/mkldnn
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_BINARY_DIR}/include)
include_directories(SYSTEM
${IE_MAIN_SOURCE_DIR}/thirdparty/mkl-dnn/src/common
${IE_MAIN_SOURCE_DIR}/thirdparty/mkl-dnn/src/cpu
${IE_MAIN_SOURCE_DIR}/thirdparty/mkl-dnn/include
${CMAKE_BINARY_DIR}/include/
)
${IE_MAIN_SOURCE_DIR}/thirdparty/mkl-dnn/include)
if (GEMM STREQUAL "MKL")
log_rpath_from_dir(MKL "${MKL}/lib")
@ -167,8 +167,8 @@ add_library(mkldnn_plugin_layers_no_opt_s OBJECT ${CROSS_COMPILED_SOURCES})
set_ie_threading_interface_for(mkldnn_plugin_layers_no_opt_s)
target_compile_definitions(mkldnn_plugin_layers_no_opt_s PRIVATE "USE_STATIC_IE;IMPLEMENT_INFERENCE_ENGINE_PLUGIN")
set(object_libraries mkldnn_plugin_layers_no_opt)
set(mkldnn_plugin_object_libraries mkldnn_plugin_layers_no_opt_s)
list(APPEND object_libraries mkldnn_plugin_layers_no_opt)
list(APPEND mkldnn_plugin_object_libraries mkldnn_plugin_layers_no_opt_s)
# SSE 4.2 optimized layers

6
inference-engine/src/mkldnn_plugin/bf16transformer.cpp
View File

@ -55,6 +55,9 @@ void BF16Transformer::convertToBFloat16(InferenceEngine::CNNNetwork &network) {
InputsDataMap inputs = network.getInputsInfo();
OutputsDataMap outputs = network.getOutputsInfo();
for (auto iter : sortedLayers) {
if (_skipmarking.find(iter->type) != _skipmarking.end()) {
continue;
}
for (size_t o = 0; o < iter->outData.size(); o++) {
if (inputs.find(iter->outData[o]->getName()) == inputs.end()
&& outputs.find(iter->outData[o]->getName()) == outputs.end()
@ -109,6 +112,9 @@ void BF16Transformer::optimizeToFloat(InferenceEngine::CNNNetwork &network) {
// 2b. go over all unknown layers for this algo and mark them as fp32 and add to the toAnalyzeTensors
// 2c. go over all inputs to _initbf16 and if they are fp32 - add them to the toAnalyzeTensors
for (auto iter : sortedLayers) {
if (_skipmarking.find(iter->type) != _skipmarking.end()) {
continue;
}
if (_initbf16.find(iter->type) == _initbf16.end()
&& _complementbf16.find(iter->type) == _complementbf16.end()
&& _multiinput.find(iter->type) == _multiinput.end()) {

5
inference-engine/src/mkldnn_plugin/bf16transformer.h
View File

@ -15,9 +15,12 @@ class BF16Transformer {
const InferenceEngine::details::caseless_set<std::string> _initbf16 =
{ "convolution", "fullyconnected", "innerproduct" };
const InferenceEngine::details::caseless_set<std::string> _complementbf16 =
{ "relu", "pooling", "norm", "gather" };
{ "relu", "tanh", "elu", "square", "abs", "sqrt", "linear", "bounded_relu", "soft_relu", "logistic",
"exp", "gelu", "clamp", "swish", "prelu", "pooling", "norm", "gather" };
const InferenceEngine::details::caseless_set<std::string> _multiinput =
{ "concat", "eltwise" };
const InferenceEngine::details::caseless_set<std::string> _skipmarking =
{ "const" };
/**
* Tries to mark tensor as FP32 by analyzing of local consumers of the tensor. Do not mark if

64
inference-engine/src/mkldnn_plugin/mkldnn_exec_network.cpp
View File

@ -41,25 +41,7 @@ MKLDNNExecNetwork::CreateInferRequestImpl(InferenceEngine::InputsDataMap network
MKLDNNExecNetwork::MKLDNNExecNetwork(const InferenceEngine::ICNNNetwork &network,
const Config &cfg,
const MKLDNNExtensionManager::Ptr& extMgr) :
InferenceEngine::ExecutableNetworkThreadSafeDefault([&] ()->ITaskExecutor::Ptr {
ExecutorManager *executorManager = ExecutorManager::getInstance();
if (cfg.exclusiveAsyncRequests) {
// special case when all InferRequests are muxed into a single queue
return executorManager->getExecutor("CPU");;
} else {
const int env_threads = parallel_get_env_threads();
const auto& numa_nodes = getAvailableNUMANodes();
const auto numa_nodes_num = numa_nodes.size();
auto streamExecutorConfig = cfg.streamExecutorConfig;
// use logical cores only for single-socket targets in throughput mode
const int hw_cores = streamExecutorConfig._streams > 1 && numa_nodes_num == 1 ? parallel_get_max_threads() : getNumberOfCPUCores();
const int threads = streamExecutorConfig._threads ? streamExecutorConfig._threads : (env_threads ? env_threads : hw_cores);
streamExecutorConfig._threadsPerStream = std::max(1, threads/streamExecutorConfig._streams);
streamExecutorConfig._name = "CPUStreamsExecutor";
return executorManager->getIdleCPUStreamsExecutor(streamExecutorConfig);
}
} ()),
InferenceEngine::ExecutableNetworkThreadSafeDefault{nullptr, nullptr},
extensionManager(extMgr),
_cfg{cfg},
_name{network.getName()} {
@ -101,7 +83,21 @@ MKLDNNExecNetwork::MKLDNNExecNetwork(const InferenceEngine::ICNNNetwork &network
LayerTransformation::Params(params).setPrecisionsOnActivations({ Precision::U8 }),
"ScaleShift"));
transformer.transform(*_clonedNetwork);
if (with_cpu_x86_bfloat16()) {
// Check if network is INT8 or Binary.
// BF16 transformations were disabled since CPU plug-in doesn't support mixed precision execution:
// BF16 + INT8 or BF16 + BIN.
bool isFloatModel = true;
CNNNetworkIterator i(&network);
while (i != CNNNetworkIterator()) {
if (CaselessEq<std::string>()((*i)->type, "FakeQuantize")) {
isFloatModel = false;
break;
}
i++;
}
if (with_cpu_x86_bfloat16() && isFloatModel) {
BF16Transformer bf16Transformer;
CNNNetwork cnnetwork(_clonedNetwork);
if (cfg.enforceBF16 == true) {
@ -126,6 +122,30 @@ MKLDNNExecNetwork::MKLDNNExecNetwork(const InferenceEngine::ICNNNetwork &network
}
}
if (cfg.exclusiveAsyncRequests) {
// special case when all InferRequests are muxed into a single queue
_taskExecutor = ExecutorManager::getInstance()->getExecutor("CPU");
} else {
const int env_threads = parallel_get_env_threads();
const auto& numa_nodes = getAvailableNUMANodes();
const auto numa_nodes_num = numa_nodes.size();
auto streamExecutorConfig = cfg.streamExecutorConfig;
// use logical cores only for single-socket targets in throughput mode
const int hw_cores = streamExecutorConfig._streams > 1 && numa_nodes_num == 1 ? parallel_get_max_threads() : getNumberOfCPUCores();
const int threads = streamExecutorConfig._threads ? streamExecutorConfig._threads : (env_threads ? env_threads : hw_cores);
streamExecutorConfig._threadsPerStream = streamExecutorConfig._streams
? std::max(1, threads/streamExecutorConfig._streams)
: threads;
streamExecutorConfig._name = "CPUStreamsExecutor";
_taskExecutor = ExecutorManager::getInstance()->getIdleCPUStreamsExecutor(streamExecutorConfig);
}
if (0 != cfg.streamExecutorConfig._streams) {
_callbackExecutor = ExecutorManager::getInstance()->getIdleCPUStreamsExecutor(
IStreamsExecutor::Config{"CPUCallbackExecutor", 1, 0, IStreamsExecutor::ThreadBindingType::NONE});
} else {
_callbackExecutor = _taskExecutor;
}
_graphs = decltype(_graphs){[&] {
// TODO: Remove `cloneNet` to `localNetwork` when `MKLDNNGraph::CreateGraph`
// is fixed and does not change content of network passed (CVS-26420)
@ -230,7 +250,9 @@ void MKLDNNExecNetwork::GetMetric(const std::string &name, Parameter &result, Re
Config engConfig = _graphs.begin()->get()->getProperty();
auto option = engConfig._config.find(CONFIG_KEY(CPU_THROUGHPUT_STREAMS));
IE_ASSERT(option != engConfig._config.end());
result = IE_SET_METRIC(OPTIMAL_NUMBER_OF_INFER_REQUESTS, static_cast<unsigned int>(std::stoi(option->second)));
auto streams = std::stoi(option->second);
result = IE_SET_METRIC(OPTIMAL_NUMBER_OF_INFER_REQUESTS, static_cast<unsigned int>(
streams ? streams : 1));
} else {
THROW_IE_EXCEPTION << "Unsupported ExecutableNetwork metric: " << name;
}

10
inference-engine/src/mkldnn_plugin/mkldnn_graph_optimizer.cpp
View File

@ -704,7 +704,6 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndActivation(MKLDNNGraph &graph) {
return activationNode &&
(activationNode->getAlgorithm() == eltwise_relu ||
(conv->getCnnLayer()->precision == Precision::FP32 &&
conv->getCnnLayer()->insData[0].lock()->getPrecision() != Precision::BF16 &&
isOneOf(activationNode->getAlgorithm(), {eltwise_elu, eltwise_logistic, eltwise_bounded_relu, eltwise_clamp, eltwise_swish})));
};
@ -778,7 +777,6 @@ void MKLDNNGraphOptimizer::FuseFullyConnectedAndSimpleOperation(MKLDNNGraph &gra
auto isSutableParentNode = [](MKLDNNNodePtr node) {
return node->getType() == FullyConnected &&
node->getCnnLayer()->insData[0].lock()->getPrecision() != Precision::BF16 &&
node->getChildEdges().size() == 1;
};
@ -850,9 +848,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndDepthwise(MKLDNNGraph &graph) {
bool isSutableConv = (node->getType() == Convolution) &&
node->getCnnLayer()->precision == Precision::FP32;
bool isSutableBinConv = node->getType() == BinaryConvolution;
return (isSutableConv || isSutableBinConv) && node->getChildEdges().size() == 1 &&
!(node->getCnnLayer()->insData[0].lock()->getPrecision() == Precision::BF16 &&
node->getCnnLayer()->outData[0]->getPrecision() == Precision::FP32);
return (isSutableConv || isSutableBinConv) && node->getChildEdges().size() == 1;
};
auto isSutableChildNode = [](MKLDNNNodePtr node) {
@ -1125,9 +1121,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndSimpleOperation(MKLDNNGraph &graph)
auto isSutableParentNode = [](MKLDNNNodePtr node) {
return node->getType() == Convolution &&
node->getChildEdges().size() == 1 &&
node->getCnnLayer()->precision == Precision::FP32 &&
!(node->getCnnLayer()->insData[0].lock()->getPrecision() == Precision::BF16 &&
node->getCnnLayer()->outData[0]->getPrecision() == Precision::FP32);
node->getCnnLayer()->precision == Precision::FP32;
};
auto isSutableChildNode = [&](MKLDNNNodePtr node) {

5
inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp
View File

@ -19,6 +19,7 @@
#include <transformations/common_optimizations/common_optimizations.hpp>
#include <transformations/convert_opset1_to_legacy/convert_opset1_to_legacy.hpp>
#include <transformations/convert_opset2_to_opset1/convert_opset2_to_opset1.hpp>
#include <transformations/convert_opset3_to_opset2/convert_opset3_to_opset2.hpp>
#include <ngraph/opsets/opset1.hpp>
#include <ngraph/opsets/opset2.hpp>
#include <ngraph/op/fused/gelu.hpp>
@ -55,10 +56,11 @@ Engine::Engine() {
Engine::~Engine() {
ExecutorManager::getInstance()->clear("CPUStreamsExecutor");
ExecutorManager::getInstance()->clear("CPUCallbackExecutor");
}
InferenceEngine::ExecutableNetworkInternal::Ptr
Engine::LoadExeNetworkImpl(const ICore * /*core*/, const InferenceEngine::ICNNNetwork &network, const std::map<std::string, std::string> &config) {
Engine::LoadExeNetworkImpl(const InferenceEngine::ICNNNetwork &network, const std::map<std::string, std::string> &config) {
// verification of supported input
InferenceEngine::InputsDataMap _networkInputs;
network.getInputsInfo(_networkInputs);
@ -100,6 +102,7 @@ Engine::LoadExeNetworkImpl(const ICore * /*core*/, const InferenceEngine::ICNNNe
// Note: instead of running all Conversion Transformations you can make up your own transformation pipeline
ngraph::pass::CommonOptimizations().run_on_function(nGraphFunc);
ngraph::pass::ConvertOpSet3ToOpSet2(transformations_callback).run_on_function(nGraphFunc);
ngraph::pass::ConvertOpSet2ToOpSet1(transformations_callback).run_on_function(nGraphFunc);
ngraph::pass::ConvertOpSet1ToLegacy(transformations_callback).run_on_function(nGraphFunc);
clonedNetwork = InferenceEngine::details::convertFunctionToICNNNetwork(nGraphFunc, *clonedNetwork);

7
inference-engine/src/mkldnn_plugin/mkldnn_plugin.h
View File

@ -71,14 +71,11 @@ public:
~Engine() override;
InferenceEngine::ExecutableNetworkInternal::Ptr
LoadExeNetworkImpl(const InferenceEngine::ICore * core, const InferenceEngine::ICNNNetwork &network,
LoadExeNetworkImpl(const InferenceEngine::ICNNNetwork &network,
const std::map<std::string, std::string> &config) override;
void AddExtension(InferenceEngine::IExtensionPtr extension) override;
/**
* @deprecated
* @param config
*/
void SetConfig(const std::map<std::string, std::string> &config) override;
InferenceEngine::Parameter GetConfig(const std::string& name, const std::map<std::string, InferenceEngine::Parameter>& options) const override;

9
inference-engine/src/mkldnn_plugin/nodes/list.hpp
View File

@ -35,9 +35,6 @@ struct ExtensionsHolder {
std::map<std::string, IShapeInferImpl::Ptr> si_list;
};
template <mkldnn::impl::cpu::cpu_isa_t T>
class TExtensionsHolder : public ExtensionsHolder {};
template<mkldnn::impl::cpu::cpu_isa_t Type>
class MKLDNNExtensions : public IExtension {
public:
@ -89,11 +86,11 @@ public:
}
static std::shared_ptr<ExtensionsHolder> GetExtensionsHolder() {
static std::shared_ptr<TExtensionsHolder<Type>> localHolder;
static std::shared_ptr<ExtensionsHolder> localHolder;
if (localHolder == nullptr) {
localHolder = std::make_shared<TExtensionsHolder<Type>>();
localHolder = std::make_shared<ExtensionsHolder>();
}
return std::dynamic_pointer_cast<ExtensionsHolder>(localHolder);
return localHolder;
}
private:

2
inference-engine/src/mkldnn_plugin/nodes/mkldnn_permute_node.cpp
View File

@ -812,7 +812,7 @@ static void permute_to_04123(int MB, MKLDNNMemoryPtr& srcMemPtr, MKLDNNMemoryPtr
});
}
std::multimap<InferenceEngine::SizeVector, MKLDNNPermuteNode::PermuteImpl> MKLDNNPermuteNode::OptimizedCases = {
const std::multimap<InferenceEngine::SizeVector, MKLDNNPermuteNode::PermuteImpl> MKLDNNPermuteNode::OptimizedCases = {
{{0, 2, 3, 1}, MKLDNNPermuteNode::PermuteImpl(permute_to_0231, [](int MB, MKLDNNMemoryPtr& srcMemPtr, MKLDNNMemoryPtr& dstMemPtr) {
return true;
})}, // NCHW -> NHWC case

2
inference-engine/src/mkldnn_plugin/nodes/mkldnn_permute_node.h
View File

@ -68,7 +68,7 @@ private:
isApplicable isValidParams;
};
static std::multimap<InferenceEngine::SizeVector, PermuteImpl> OptimizedCases;
static const std::multimap<InferenceEngine::SizeVector, PermuteImpl> OptimizedCases;
std::shared_ptr<jit_uni_permute_kernel> permute_kernel;
};

41
inference-engine/src/mkldnn_plugin/nodes/mkldnn_pooling_node.cpp
View File

@ -67,21 +67,23 @@ void MKLDNNPoolingNode::getSupportedDescriptors() {
invertVectorCopyUtoI(poolingLayer->_stride, stride);
invertVectorCopyUtoI(poolingLayer->_kernel, kernel);
auto allPads = getPaddings(*poolingLayer);
invertVectorCopyUtoI(allPads.begin, paddingL);
invertVectorCopyUtoI(allPads.end, paddingR);
invertVectorCopyUtoI(allPads.begin, data_pad_begin);
invertVectorCopyUtoI(allPads.end, data_pad_end);
effective_pad_begin = data_pad_begin;
effective_pad_end.resize(data_pad_end.size());
auto parentDims = getParentEdgeAt(0)->getDims();
auto childDims = getChildEdgeAt(0)->getDims();
if ((parentDims.ndims() < 4) || (parentDims.ndims() > 5))
THROW_IE_EXCEPTION << "Pooling layer. Unsupported mode. Only 4D and 5D blobs are supported as input.";
for (int i = 0; i < paddingR.size(); i++) {
for (int i = 0; i < effective_pad_end.size(); i++) {
int krn = kernel[i];
int src = getParentEdgeAt(0)->getDims()[2 + i];
int dst = getChildEdgeAt(0)->getDims()[2 + i];
int calc_dst = (src - krn + paddingL[i]) / stride[i] + 1;
paddingR[i] = (dst - calc_dst) * stride[i];
int calc_dst = (src - krn + data_pad_begin[i]) / stride[i] + 1;
effective_pad_end[i] = (dst - calc_dst) * stride[i];
}
if (inputPrecision == Precision::I8 || inputPrecision == Precision::U8) {
// i8 layers supports only ndhwc and nhwc layouts
@ -138,13 +140,20 @@ void MKLDNNPoolingNode::createDescriptor(const std::vector<InferenceEngine::Tens
algorithm alg;
if (type == PoolingLayer::PoolType::AVG) {
bool not_zero_l = false;
for (auto lr : paddingL) {
for (auto lr : data_pad_begin) {
if (lr) {
not_zero_l = true;
break;
}
}
if (!exclude_pad && not_zero_l)
bool not_zero_r = false;
for (auto pr : data_pad_end) {
if (pr) {
not_zero_r = true;
break;
}
}
if (!exclude_pad && (not_zero_l || not_zero_r))
alg = pooling_avg_include_padding;
else
alg = pooling_avg_exclude_padding;
@ -158,24 +167,20 @@ void MKLDNNPoolingNode::createDescriptor(const std::vector<InferenceEngine::Tens
std::shared_ptr<pooling_forward::desc> desc_ptr(
new pooling_forward::desc(prop_kind::forward_scoring, alg,
in_candidate, out_candidate,
stride, kernel, paddingL, paddingR,
stride, kernel, effective_pad_begin, effective_pad_end,
mkldnn::padding_kind::zero));
bool not_zero_r = false;
for (auto pr : paddingR) {
if (pr) {
not_zero_r = true;
break;
}
}
if (alg == pooling_avg_include_padding && not_zero_r) {
if (alg == pooling_avg_include_padding) {
// In case of AVG including paddings the norm coeff should be calculated
// with tacking into account original pads. So we need to restore
// original values (R_padding = L_padding).
// original values for end paddings.
//
// WA. Because mkldnn uses different formula to calculate AVG norm coeff
// in compare with Caffe. In mkldnn coeff is always 1/(KH*KW)
for (int i = 0; i < paddingL.size(); i++) desc_ptr->data.padding[1][i] = paddingL[i];
for (int i = 0; i < data_pad_end.size(); i++) {
if (data_pad_end[i] != effective_pad_end[i])
desc_ptr->data.padding[1][i] = static_cast<ptrdiff_t>(data_pad_end[i]);
}
}
descs.emplace_back(desc_ptr);

15
inference-engine/src/mkldnn_plugin/nodes/mkldnn_pooling_node.h
View File

@ -34,10 +34,21 @@ private:
InferenceEngine::PoolingLayer::PoolType type = InferenceEngine::PoolingLayer::MAX;
bool exclude_pad = false;
std::vector<ptrdiff_t> stride;
std::vector<ptrdiff_t> paddingL;
std::vector<ptrdiff_t> paddingR;
std::vector<ptrdiff_t> kernel;
/// Effective padding. Used to define correct output shape by MKLDNN
/// reshape formula: (iw - kernel + pad_l + pad_r) / strides[i - 2] + 1
/// should be passed into pooling desc constructor.
std::vector<ptrdiff_t> effective_pad_begin;
std::vector<ptrdiff_t> effective_pad_end;
/// Effective pad value. Describe how much zero element added to input
/// data tensor. May be less than "Effective padding" values.
/// If pooling window is out of this padding, the region of averaging
/// is decreased.
std::vector<ptrdiff_t> data_pad_begin;
std::vector<ptrdiff_t> data_pad_end;
InferenceEngine::Precision inputPrecision = InferenceEngine::Precision::FP32;
InferenceEngine::Precision outputPrecision = InferenceEngine::Precision::FP32;

16
inference-engine/src/mkldnn_plugin/nodes/mkldnn_resample_node.cpp
View File

@ -39,7 +39,7 @@ struct jit_uni_resample_nearest_kernel_f32 : public jit_uni_resample_nearest_ker
DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_resample_nearest_kernel_f32)
explicit jit_uni_resample_nearest_kernel_f32(jit_resample_config_params jcp, const mkldnn_primitive_attr &attr)
: jit_uni_resample_nearest_kernel(jcp, attr), jit_generator() {
: jit_uni_resample_nearest_kernel(jcp, attr), jit_generator() {
const auto &p = attr_.post_ops_;
for (int i = 0; i < p.len_; i++) {
auto &post_op = p.entry_[i];
@ -321,6 +321,14 @@ void MKLDNNResampleNode::initSupportedPrimitiveDescriptors() {
}
}
if (inputPrecision == Precision::BF16) {
inputPrecision = Precision::FP32;
}
if (outputPrecision == Precision::BF16) {
outputPrecision = Precision::FP32;
}
auto inputDataType = MKLDNNExtensionUtils::IEPrecisionToDataType(inputPrecision);
auto outputDataType = MKLDNNExtensionUtils::IEPrecisionToDataType(outputPrecision);
@ -587,7 +595,7 @@ void MKLDNNResampleNode::execute(mkldnn::stream strm) {
// f32 and no fused, f32->input is f32, no fuse->output is f32
void MKLDNNResampleNode::NearestNeighbor_PLN(const float *in_ptr_, float *out_ptr_, int B, int C, int ID, int IH, int IW,
float fx, float fy, float fz, int OD, int OH, int OW) {
float fx, float fy, float fz, int OD, int OH, int OW) {
std::vector<int> index_buffer(OD * OH * OW);
for (int oz = 0; oz < OD; oz++) {
float iz = oz * fz;
@ -640,7 +648,7 @@ void MKLDNNResampleNode::NearestNeighbor_PLN(const float *in_ptr_, float *out_pt
// int8->input may be int8, fused->output may be int8
template <typename in_data_t, typename out_data_t>
void MKLDNNResampleNode::NearestNeighbor_BLK(const in_data_t *in_ptr_, out_data_t *out_ptr_, int B, int C, int ID, int IH, int IW,
float fx, float fy, float fz, int OD, int OH, int OW) {
float fx, float fy, float fz, int OD, int OH, int OW) {
std::vector<int> index_d(OD);
std::vector<int> index_h(OH);
std::vector<int> index_w(OW);
@ -792,7 +800,7 @@ static inline float triangleCoeff(float x) {
template <typename in_data_t, typename out_data_t>
void MKLDNNResampleNode::LinearInterpolation(const in_data_t *in_ptr_, out_data_t *out_ptr_, int B, int C, int ID, int IH, int IW,
float fx, float fy, float fz, int OD, int OH, int OW, int kernel_width, bool antialias) {
float fx, float fy, float fz, int OD, int OH, int OW, int kernel_width, bool antialias) {
if (IW == OW && IH == OH && ID == OD) {
size_t size = B * C * ID * IH * IW;
if (input_prec == Precision::FP32) {

26
inference-engine/src/multi_device/CMakeLists.txt Normal file
View File

@ -0,0 +1,26 @@
# Copyright (C) 2018-2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
set (TARGET_NAME "MultiDevicePlugin")
if(ENABLE_LTO)
ie_enable_lto()
endif()
file(GLOB SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/*.cpp
)
file(GLOB HEADERS
${CMAKE_CURRENT_SOURCE_DIR}/*.hpp
)
ie_add_plugin(NAME ${TARGET_NAME}
DEVICE_NAME "MULTI"
SOURCES ${SOURCES} ${HEADERS}
VERSION_DEFINES_FOR multi_device.cpp)
target_link_libraries(${TARGET_NAME} PRIVATE inference_engine)
set_ie_threading_interface_for(${TARGET_NAME})

548
inference-engine/src/multi_device/multi_device.cpp Normal file
View File

@ -0,0 +1,548 @@
// Copyright (C) 2018-2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#include <string>
#include <vector>
#include <iostream>
#include <memory>
#include <utility>
#include <map>
#include <unordered_map>
#include "ie_metric_helpers.hpp"
#include <ie_api.h>
#include <cpp_interfaces/base/ie_plugin_base.hpp>
#include <cpp_interfaces/base/ie_infer_async_request_base.hpp>
#include <multi-device/multi_device_config.hpp>
#include <ie_plugin_config.hpp>
#include "multi_device.hpp"
namespace MultiDevicePlugin {
using namespace InferenceEngine;
// ------------------------------MultiDeviceInferRequest----------------------------
MultiDeviceInferRequest::MultiDeviceInferRequest(const InputsDataMap& networkInputs,
const OutputsDataMap& networkOutputs)
: InferRequestInternal(networkInputs, networkOutputs) {
// Allocate all input blobs
for (const auto &it : networkInputs) {
Layout l = it.second->getLayout();
Precision p = it.second->getPrecision();
SizeVector dims = it.second->getTensorDesc().getDims();
TensorDesc desc = TensorDesc(p, dims, l);
_inputs[it.first] = make_blob_with_precision(desc);
_inputs[it.first]->allocate();
}
// Allocate all output blobs
for (const auto &it : networkOutputs) {
Layout l = it.second->getLayout();
Precision p = it.second->getPrecision();
SizeVector dims = it.second->getTensorDesc().getDims();
TensorDesc desc = TensorDesc(p, dims, l);
_outputs[it.first] = make_blob_with_precision(desc);
_outputs[it.first]->allocate();
}
}
void MultiDeviceInferRequest::SetBlobsToAnotherRequest(InferRequest& req) {
for (const auto &it : _networkInputs) {
Blob::Ptr blob;
auto &name = it.first;
// this request is already in BUSY state, so using the internal functions safely
GetBlob(name.c_str(), blob);
req.SetBlob(name.c_str(), blob);
}
for (const auto &it : _networkOutputs) {
Blob::Ptr blob;
auto &name = it.first;
// this request is already in BUSY state, so using the internal functions safely
GetBlob(name.c_str(), blob);
req.SetBlob(name.c_str(), blob);
}
}
MultiDeviceAsyncInferRequest::MultiDeviceAsyncInferRequest(
const MultiDeviceInferRequest::Ptr& inferRequest,
const bool needPerfCounters,
const MultiDeviceExecutableNetwork::Ptr& multiDeviceExecutableNetwork,
const ITaskExecutor::Ptr& callbackExecutor) :
AsyncInferRequestThreadSafeDefault(inferRequest, nullptr, callbackExecutor),
_multiDeviceExecutableNetwork{multiDeviceExecutableNetwork},
_inferRequest{inferRequest},
_needPerfCounters{needPerfCounters} {
struct ThisRequestExecutor : public ITaskExecutor {
explicit ThisRequestExecutor(MultiDeviceAsyncInferRequest* _this_) : _this{_this_} {}
void run(Task task) override {
auto workerInferRequest = _this->_workerInferRequest;
workerInferRequest->_task = std::move(task);
workerInferRequest->_inferRequest.StartAsync();
};
MultiDeviceAsyncInferRequest* _this = nullptr;
};
_pipeline = {
{_multiDeviceExecutableNetwork, [this] {
_workerInferRequest = MultiDeviceExecutableNetwork::_thisWorkerInferRequest;
_inferRequest->SetBlobsToAnotherRequest(_workerInferRequest->_inferRequest);
}},
{std::make_shared<ThisRequestExecutor>(this), [this] {
auto status = _workerInferRequest->_status;
if (InferenceEngine::StatusCode::OK != status) {
if (nullptr != InferenceEngine::CurrentException()) {
std::rethrow_exception(InferenceEngine::CurrentException());
} else {
THROW_IE_EXCEPTION << InferenceEngine::details::as_status << status;
}
}
if (_needPerfCounters) {
_perfMap = _workerInferRequest->_inferRequest.GetPerformanceCounts();
}
}}
};
}
void MultiDeviceAsyncInferRequest::Infer_ThreadUnsafe() {
InferUsingAsync();
}
void MultiDeviceAsyncInferRequest::GetPerformanceCounts_ThreadUnsafe(std::map<std::string, InferenceEngineProfileInfo> &perfMap) const {
perfMap = std::move(_perfMap);
}
MultiDeviceAsyncInferRequest::~MultiDeviceAsyncInferRequest() {
StopAndWait();
}
// ------------------------------MultiDeviceExecutableNetwork----------------------------
thread_local MultiDeviceExecutableNetwork::WorkerInferRequest* MultiDeviceExecutableNetwork::_thisWorkerInferRequest = nullptr;
struct IdleGuard {
explicit IdleGuard(MultiDeviceExecutableNetwork::WorkerInferRequest* workerInferRequestPtr,
MultiDeviceExecutableNetwork::NotBusyWorkerRequests& notBusyWorkerRequests) :
_workerInferRequestPtr{workerInferRequestPtr},
_notBusyWorkerRequests{&notBusyWorkerRequests} {
}
~IdleGuard() {
if (nullptr != _notBusyWorkerRequests) {
_notBusyWorkerRequests->push(_workerInferRequestPtr);
}
}
MultiDeviceExecutableNetwork::NotBusyWorkerRequests* Release() {
auto notBusyWorkerRequests = _notBusyWorkerRequests;
_notBusyWorkerRequests = nullptr;
return notBusyWorkerRequests;
}
MultiDeviceExecutableNetwork::WorkerInferRequest* _workerInferRequestPtr = nullptr;
MultiDeviceExecutableNetwork::NotBusyWorkerRequests* _notBusyWorkerRequests = nullptr;
};
MultiDeviceExecutableNetwork::MultiDeviceExecutableNetwork(const DeviceMap<InferenceEngine::ExecutableNetwork>& networksPerDevice,
const DeviceMap<DeviceInformation>& networkDevices,
const std::unordered_map<std::string, InferenceEngine::Parameter>& config,
const bool needPerfCounters) :
InferenceEngine::ExecutableNetworkThreadSafeDefault(nullptr, std::make_shared<InferenceEngine::ImmediateExecutor>()),
_devicePriorities{networkDevices},
_networksPerDevice{networksPerDevice},
_config{config},
_needPerfCounters{needPerfCounters} {
_taskExecutor.reset();
for (auto&& networkValue : _networksPerDevice) {
auto& device = networkValue.first;
auto& network = networkValue.second;
auto itNumRequests = _devicePriorities.find(device);
unsigned int optimalNum = 0;
try {
optimalNum = network.GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
} catch (const details::InferenceEngineException &iie) {
THROW_IE_EXCEPTION
<< "Every device used with the Multi-Device should "
<< "support OPTIMAL_NUMBER_OF_INFER_REQUESTS ExecutableNetwork metric. "
<< "Failed to query the metric for the " << device << " with error:" << iie.what();
}
const auto numRequests = (_devicePriorities.end() == itNumRequests ||
itNumRequests->second.numRequestsPerDevices == -1) ? optimalNum : itNumRequests->second.numRequestsPerDevices;
auto& workerRequests = _workerRequests[device];
auto& idleWorkerRequests = _idleWorkerRequests[device];
workerRequests.resize(numRequests);
auto* idleWorkerRequestsPtr = &(idleWorkerRequests);
for (auto&& workerRequest : workerRequests) {
workerRequest._inferRequest = network.CreateInferRequest();
auto* workerRequestPtr = &workerRequest;
idleWorkerRequests.push(workerRequestPtr);
workerRequest._inferRequest.SetCompletionCallback<std::function<void(InferRequest, StatusCode)>>(
[workerRequestPtr, this, device, idleWorkerRequestsPtr] (InferRequest , StatusCode status) mutable {
IdleGuard idleGuard{workerRequestPtr, *idleWorkerRequestsPtr};
workerRequestPtr->_status = status;
{
auto capturedTask = std::move(workerRequestPtr->_task);
capturedTask();
}
if (!_terminate) {
idleGuard.Release()->push(workerRequestPtr);
ScheduleToWorkerInferRequest();
}
});
}
}
}
void MultiDeviceExecutableNetwork::ScheduleToWorkerInferRequest() {
auto devices = [&] {
std::lock_guard<std::mutex> lock(_mutex);
return _devicePriorities;
}();
for (auto&& device : devices) {
auto& idleWorkerRequests = _idleWorkerRequests[device.first];
WorkerInferRequest* workerRequestPtr = nullptr;
if (idleWorkerRequests.try_pop(workerRequestPtr)) {
IdleGuard idleGuard{workerRequestPtr, idleWorkerRequests};
Task inferPipelineTask;
if (_inferPipelineTasks.try_pop(inferPipelineTask)) {
_thisWorkerInferRequest = workerRequestPtr;
inferPipelineTask();
idleGuard.Release();
break;
}
}
}
}
void MultiDeviceExecutableNetwork::run(Task inferPipelineTask) {
if (!_terminate) {
_inferPipelineTasks.push(std::move(inferPipelineTask));
ScheduleToWorkerInferRequest();
}
}
MultiDeviceExecutableNetwork::~MultiDeviceExecutableNetwork() {
{
std::lock_guard<std::mutex> lock(_mutex);
_devicePriorities.clear();
}
_terminate = true;
/* NOTE: The only threads that use `MultiDeviceExecutableNetwork` Context are those that are used by Worker infer requests.
* But AsyncInferRequest destructor should waits for all asynchronous tasks that are used by the request
*/
_workerRequests.clear();
}
InferenceEngine::InferRequestInternal::Ptr MultiDeviceExecutableNetwork::CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs) {
return std::make_shared<MultiDeviceInferRequest>(networkInputs, networkOutputs);
}
void MultiDeviceExecutableNetwork::CreateInferRequest(IInferRequest::Ptr& asyncRequest) {
auto syncRequestImpl = CreateInferRequestImpl(_networkInputs, _networkOutputs);
syncRequestImpl->setPointerToExecutableNetworkInternal(shared_from_this());
auto asyncTreadSafeImpl = std::make_shared<MultiDeviceAsyncInferRequest>(std::static_pointer_cast<MultiDeviceInferRequest>(syncRequestImpl),
_needPerfCounters,
std::static_pointer_cast<MultiDeviceExecutableNetwork>(shared_from_this()),
_callbackExecutor);
asyncRequest.reset(new InferRequestBase<MultiDeviceAsyncInferRequest>(asyncTreadSafeImpl), [](IInferRequest *p) { p->Release(); });
asyncTreadSafeImpl->SetPointerToPublicInterface(asyncRequest);
}
void MultiDeviceExecutableNetwork::SetConfig(const std::map<std::string, InferenceEngine::Parameter> &config,
InferenceEngine::ResponseDesc * /* resp */) {
auto priorities = config.find(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES);
if (priorities == config.end() || config.size() > 1) {
THROW_IE_EXCEPTION << NOT_IMPLEMENTED_str <<
"The only config supported for the Network's SetConfig is MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES";
} else {
auto multiPlugin = std::dynamic_pointer_cast<MultiDeviceInferencePlugin>(this->_plugin);
assert(multiPlugin != nullptr);
auto metaDevices = multiPlugin->ParseMetaDevices(priorities->second, {});
if (std::any_of(metaDevices.begin(), metaDevices.end(), [](const std::pair<DeviceName, DeviceInformation> & kvp) {
return kvp.second.numRequestsPerDevices != -1;
})) {
THROW_IE_EXCEPTION << NOT_IMPLEMENTED_str << "You can only change device priorities but not number of requests"
<<" with the Network's SetConfig(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES!";
}
{
std::lock_guard<std::mutex> lock{_mutex};
for (auto && device : metaDevices) {
if (_devicePriorities.find(device.first) == _devicePriorities.end()) {
THROW_IE_EXCEPTION << NOT_FOUND_str << "You can only change device priorities but not add new devices with"
<< " the Network's SetConfig(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES." << device.first <<
" device was not in the original device list!";
}
}
_devicePriorities = metaDevices;
// update value in config
_config[MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES] = priorities->second;
}
}
}
void MultiDeviceExecutableNetwork::GetConfig(const std::string &name, InferenceEngine::Parameter &result,
InferenceEngine::ResponseDesc * /* resp */) const {
auto res = _config.find(name);
if (res != _config.end()) {
result = res->second;
} else {
THROW_IE_EXCEPTION << NOT_FOUND_str << name <<" not found in the ExecutableNetwork config";
}
}
void MultiDeviceExecutableNetwork::GetMetric(const std::string &name, Parameter &result, ResponseDesc *resp) const {
if (name == METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)) {
unsigned int res = 0u;
for (auto n : _networksPerDevice) {
try {
res += n.second.GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
} catch (const details::InferenceEngineException &iie) {
THROW_IE_EXCEPTION
<< "Every device used with the Multi-Device should "
<< "support OPTIMAL_NUMBER_OF_INFER_REQUESTS ExecutableNetwork metric. "
<< "Failed to query the metric for the " << n.first << " with error:" << iie.what();
}
}
result = IE_SET_METRIC(OPTIMAL_NUMBER_OF_INFER_REQUESTS, res);
} else if (name == METRIC_KEY(NETWORK_NAME)) {
auto it = _networksPerDevice.begin();
IE_ASSERT(it != _networksPerDevice.end());
result = IE_SET_METRIC(NETWORK_NAME, it->second.GetMetric(
METRIC_KEY(NETWORK_NAME)).as<std::string>());
} else if (name == METRIC_KEY(SUPPORTED_METRICS)) {
result = IE_SET_METRIC(SUPPORTED_METRICS, {
METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS),
METRIC_KEY(SUPPORTED_METRICS),
METRIC_KEY(NETWORK_NAME),
METRIC_KEY(SUPPORTED_CONFIG_KEYS)
});
} else if (name == METRIC_KEY(SUPPORTED_CONFIG_KEYS)) {
std::vector<std::string> configKeys = { MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES };
result = IE_SET_METRIC(SUPPORTED_CONFIG_KEYS, configKeys);
} else {
THROW_IE_EXCEPTION << "Unsupported Network metric: " << name;
}
}
// ------------------------------MultiDeviceInferencePlugin----------------------------
namespace {
std::map<std::string, std::string> mergeConfigs(std::map<std::string, std::string> config,
const std::map<std::string, std::string> & local) {
for (auto && kvp : local) {
config[kvp.first] = kvp.second;
}
return config;
}
} // namespace
std::map<std::string, std::string> MultiDeviceInferencePlugin::GetSupportedConfig(
const std::map<std::string, std::string> & config, const std::string & deviceName) const {
std::vector<std::string> supportedConfigKeys = GetCore()->GetMetric(deviceName, METRIC_KEY(SUPPORTED_CONFIG_KEYS));
std::map<std::string, std::string> supportedConfig;
for (auto&& key : supportedConfigKeys) {
auto itKey = config.find(key);
if (config.end() != itKey) {
supportedConfig[key] = itKey->second;
}
}
return supportedConfig;
}
DeviceMap<DeviceInformation> MultiDeviceInferencePlugin::ParseMetaDevices(const std::string& priorities,
const std::map<std::string, std::string> & config) const {
DeviceMap<DeviceInformation> metaDevices;
// parsing the string and splitting to tokens
std::vector<std::string> devicesWithRequests;
// parsing the string and splitting the comma-separated tokens
std::string::size_type i = 0;
std::string::size_type idelimeter;
while ((idelimeter = priorities.find(',', i)) != std::string::npos) {
devicesWithRequests.push_back(priorities.substr(i, idelimeter - i));
i = idelimeter + 1;
}
// last token in the string (which has no comma after that)
devicesWithRequests.push_back(priorities.substr(i, priorities.length() - i));
auto getDeviceConfig = [&] (const DeviceName & deviceWithID) {
DeviceIDParser deviceParser(deviceWithID);
std::string deviceName = deviceParser.getDeviceName();
std::map<std::string, std::string> tconfig = mergeConfigs(_config, config);
// set device ID if any
std::string deviceIDLocal = deviceParser.getDeviceID();
if (!deviceIDLocal.empty()) {
tconfig[PluginConfigParams::KEY_DEVICE_ID] = deviceIDLocal;
}
return GetSupportedConfig(tconfig, deviceName);
};
for (auto && d : devicesWithRequests) {
auto openingBracket = d.find_first_of('(');
auto closingBracket = d.find_first_of(')', openingBracket);
auto device_name = d.substr(0, openingBracket);
int numRequests = -1;
if (closingBracket != std::string::npos && openingBracket < closingBracket) {
numRequests = std::stol(d.substr(openingBracket + 1, closingBracket - 1));
if (numRequests <= 0) {
THROW_IE_EXCEPTION << "Priority value for '" << device_name << "' must be > 0, while " << numRequests
<< "is passed";
}
}
// create meta device
metaDevices[device_name] = { getDeviceConfig(device_name), numRequests };
}
return metaDevices;
}
Parameter MultiDeviceInferencePlugin::GetConfig(const std::string& name,
const std::map<std::string, Parameter> & options) const {
if (name == MULTI_CONFIG_KEY(DEVICE_PRIORITIES)) {
auto it = _config.find(MULTI_CONFIG_KEY(DEVICE_PRIORITIES));
if (it == _config.end()) {
THROW_IE_EXCEPTION << "Value for KEY_MULTI_DEVICE_PRIORITIES is not set";
} else {
return { it->second };
}
} else {
THROW_IE_EXCEPTION << "Unsupported config key: " << name;
}
}
void MultiDeviceInferencePlugin::SetConfig(const std::map<std::string, std::string> & config) {
for (auto && kvp : config) {
_config[kvp.first] = kvp.second;
}
}
IE_SUPPRESS_DEPRECATED_START
INFERENCE_PLUGIN_API(InferenceEngine::StatusCode) CreatePluginEngine(
InferenceEngine::IInferencePlugin *&plugin,
InferenceEngine::ResponseDesc *resp) noexcept {
try {
plugin = make_ie_compatible_plugin(
{{2, 1},
CI_BUILD_NUMBER,
"MultiDevicePlugin"}, std::make_shared<MultiDeviceInferencePlugin>());
return OK;
}
catch (std::exception &ex) {
return DescriptionBuffer(GENERAL_ERROR, resp) << ex.what();
}
}
IE_SUPPRESS_DEPRECATED_END
MultiDeviceInferencePlugin::MultiDeviceInferencePlugin() {
_pluginName = "MULTI";
}
InferenceEngine::Parameter MultiDeviceInferencePlugin::GetMetric(const std::string& name,
const std::map<std::string, InferenceEngine::Parameter> & options) const {
if (name == METRIC_KEY(SUPPORTED_METRICS)) {
std::vector<std::string> metrics;
metrics.push_back(METRIC_KEY(SUPPORTED_METRICS));
metrics.push_back(METRIC_KEY(FULL_DEVICE_NAME));
metrics.push_back(METRIC_KEY(SUPPORTED_CONFIG_KEYS));
IE_SET_METRIC_RETURN(SUPPORTED_METRICS, metrics);
} else if (name == METRIC_KEY(FULL_DEVICE_NAME)) {
std::string name = { "MULTI" };
IE_SET_METRIC_RETURN(FULL_DEVICE_NAME, name);
} else if (name == METRIC_KEY(SUPPORTED_CONFIG_KEYS)) {
std::vector<std::string> configKeys = { MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES };
IE_SET_METRIC_RETURN(SUPPORTED_CONFIG_KEYS, configKeys);
} else {
THROW_IE_EXCEPTION << "Unsupported metric key " << name;
}
}
ExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadExeNetworkImpl(const ICNNNetwork &network,
const std::map<std::string, std::string>& config) {
if (GetCore() == nullptr) {
THROW_IE_EXCEPTION << "Please, work with MULTI device via InferencEngine::Core object";
}
// TODO: do we really need a clone?
ICNNNetwork::Ptr clonedNetwork = cloneNet(network);
auto fullConfig = mergeConfigs(_config, config);
auto priorities = fullConfig.find(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES);
if (priorities == fullConfig.end()) {
THROW_IE_EXCEPTION << "KEY_MULTI_DEVICE_PRIORITIES key is not set for MULTI device";
}
DeviceMap<DeviceInformation> metaDevices = ParseMetaDevices(priorities->second, fullConfig);
// collect the settings that are applicable to the devices we are loading the network to
std::unordered_map<std::string, InferenceEngine::Parameter> multiNetworkConfig;
multiNetworkConfig.insert(*priorities);
DeviceMap<ExecutableNetwork> executableNetworkPerDevice;
for (auto& p : metaDevices) {
auto & deviceName = p.first;
auto & metaDevice = p.second;
auto & deviceConfig = metaDevice.config;
executableNetworkPerDevice.insert({ deviceName, GetCore()->LoadNetwork(CNNNetwork{clonedNetwork}, deviceName, deviceConfig) });
multiNetworkConfig.insert(deviceConfig.begin(), deviceConfig.end());
}
if (executableNetworkPerDevice.empty())
THROW_IE_EXCEPTION << NOT_FOUND_str << "Failed to load Executable network to any device "
<< "that the MULTI device is initialized to work with";
auto perfConfig = fullConfig.find(PluginConfigParams::KEY_PERF_COUNT);
bool enablePerfCounters = (fullConfig.end() != perfConfig) && (perfConfig->second == PluginConfigParams::YES);
return std::make_shared<MultiDeviceExecutableNetwork>(executableNetworkPerDevice,
metaDevices,
multiNetworkConfig,
enablePerfCounters);
}
void MultiDeviceInferencePlugin::QueryNetwork(const ICNNNetwork& network,
const std::map<std::string, std::string>& config,
QueryNetworkResult& queryResult) const {
if (GetCore() == nullptr) {
THROW_IE_EXCEPTION << "Please, work with MULTI device via InferencEngine::Core object";
}
queryResult.rc = StatusCode::OK;
queryResult.supportedLayersMap.clear();
auto fullConfig = mergeConfigs(_config, config);
auto priorities = fullConfig.find(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES);
if (priorities == fullConfig.end()) {
THROW_IE_EXCEPTION << "KEY_MULTI_DEVICE_PRIORITIES key is not set for MULTI device";
}
DeviceMap<DeviceInformation> metaDevices = ParseMetaDevices(priorities->second, fullConfig);
std::map<std::string, QueryNetworkResult> queryResults;
for (auto&& value : metaDevices) {
auto& deviceName = value.first;
auto& metaDevice = value.second;
queryResults[deviceName] = GetCore()->QueryNetwork(network, deviceName, metaDevice.config);
}
details::CNNNetworkIterator i(&network);
while (i != details::CNNNetworkIterator()) {
CNNLayer::Ptr layer = *i;
bool layerIsInQueryResultsForAllDevices = std::all_of(std::begin(queryResults), std::end(queryResults),
[&](const std::map<std::string, QueryNetworkResult>::value_type& qr) {
return qr.second.supportedLayersMap.end() != qr.second.supportedLayersMap.find(layer->name);});
if (layerIsInQueryResultsForAllDevices) {
queryResult.supportedLayersMap[layer->name] = GetName();
}
i++;
}
}
} // namespace MultiDevicePlugin

176
inference-engine/src/multi_device/multi_device.hpp Normal file
View File

@ -0,0 +1,176 @@
// Copyright (C) 2018-2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#pragma once
#include <atomic>
#include <mutex>
#include <queue>
#include <unordered_map>
#include <map>
#include <vector>
#include <utility>
#include <memory>
#include <string>
#include <cpp/ie_plugin_cpp.hpp>
#include <ie_plugin_dispatcher.hpp>
#include <cpp_interfaces/impl/ie_plugin_internal.hpp>
#include <cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp>
#include <cpp_interfaces/impl/ie_infer_async_request_thread_safe_default.hpp>
#include "ie_iinfer_request.hpp"
#include "details/ie_exception_conversion.hpp"
#include <ie_parallel.hpp>
#if (IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO)
#include <tbb/concurrent_queue.h>
#endif
namespace MultiDevicePlugin {
using DeviceName = std::string;
struct DeviceInformation {
std::map<std::string, std::string> config;
int numRequestsPerDevices;
};
template<typename T>
using DeviceMap = std::unordered_map<DeviceName, T>;
class MultiDeviceInferRequest : public InferenceEngine::InferRequestInternal {
public:
using Ptr = std::shared_ptr<MultiDeviceInferRequest>;
explicit MultiDeviceInferRequest(const InferenceEngine::InputsDataMap& networkInputs,
const InferenceEngine::OutputsDataMap& networkOutputs);
void GetPerformanceCounts(std::map<std::string, InferenceEngineProfileInfo>&) const override {
THROW_IE_EXCEPTION << NOT_IMPLEMENTED_str;
}
void InferImpl() override {
THROW_IE_EXCEPTION << NOT_IMPLEMENTED_str;
}
// Multi-Device impl specific: sets the data (blobs from the device-less requets to the specific device request)
void SetBlobsToAnotherRequest(InferenceEngine::InferRequest& req);
};
#if ((IE_THREAD == IE_THREAD_TBB) || (IE_THREAD == IE_THREAD_TBB_AUTO))
template <typename T>
using ThreadSafeQueue = tbb::concurrent_queue<T>;
#else
template <typename T>
class ThreadSafeQueue {
public:
void push(T value) {
std::lock_guard<std::mutex> lock(_mutex);
_queue.push(std::move(value));
}
bool try_pop(T& value) {
std::lock_guard<std::mutex> lock(_mutex);
if (!_queue.empty()) {
value = std::move(_queue.front());
_queue.pop();
return true;
} else {
return false;
}
}
bool empty() {
std::lock_guard<std::mutex> lock(_mutex);
return _queue.empty();
}
protected:
std::queue<T> _queue;
std::mutex _mutex;
};
#endif
class MultiDeviceExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadSafeDefault,
public ITaskExecutor {
public:
using Ptr = std::shared_ptr<MultiDeviceExecutableNetwork>;
struct WorkerInferRequest {
InferenceEngine::InferRequest _inferRequest;
Task _task;
InferenceEngine::StatusCode _status = InferenceEngine::StatusCode::OK;
};
using NotBusyWorkerRequests = ThreadSafeQueue<WorkerInferRequest*>;
explicit MultiDeviceExecutableNetwork(const DeviceMap<InferenceEngine::ExecutableNetwork>& networksPerDevice,
const DeviceMap<DeviceInformation>& networkDevices,
const std::unordered_map<std::string, InferenceEngine::Parameter>& config,
const bool needPerfCounters = false);
void SetConfig(const std::map<std::string, InferenceEngine::Parameter> &config, InferenceEngine::ResponseDesc *resp) override;
void GetConfig(const std::string &name, InferenceEngine::Parameter &result, InferenceEngine::ResponseDesc *resp) const override;
void GetMetric(const std::string &name, InferenceEngine::Parameter &result, InferenceEngine::ResponseDesc *resp) const override;
void run(Task inferTask) override;
void CreateInferRequest(InferenceEngine::IInferRequest::Ptr& asyncRequest) override;
InferenceEngine::InferRequestInternal::Ptr CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs) override;
~MultiDeviceExecutableNetwork() override;
void ScheduleToWorkerInferRequest();
static thread_local WorkerInferRequest* _thisWorkerInferRequest;
std::atomic_bool _terminate = {false};
std::mutex _mutex;
DeviceMap<DeviceInformation> _devicePriorities;
DeviceMap<InferenceEngine::ExecutableNetwork> _networksPerDevice;
ThreadSafeQueue<Task> _inferPipelineTasks;
DeviceMap<NotBusyWorkerRequests> _idleWorkerRequests;
DeviceMap<std::vector<WorkerInferRequest>> _workerRequests;
std::unordered_map<std::string, InferenceEngine::Parameter> _config;
bool _needPerfCounters = false;
};
class MultiDeviceAsyncInferRequest : public InferenceEngine::AsyncInferRequestThreadSafeDefault {
public:
using Ptr = std::shared_ptr<MultiDeviceAsyncInferRequest>;
explicit MultiDeviceAsyncInferRequest(const MultiDeviceInferRequest::Ptr& inferRequest,
const bool needPerfCounters,
const MultiDeviceExecutableNetwork::Ptr& multiDeviceExecutableNetwork,
const InferenceEngine::ITaskExecutor::Ptr& callbackExecutor);
void Infer_ThreadUnsafe() override;
void GetPerformanceCounts_ThreadUnsafe(std::map<std::string, InferenceEngineProfileInfo> &_perfMap) const override;
~MultiDeviceAsyncInferRequest() override;
protected:
MultiDeviceExecutableNetwork::Ptr _multiDeviceExecutableNetwork;
MultiDeviceInferRequest::Ptr _inferRequest;
std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> _perfMap;
bool _needPerfCounters = false;
MultiDeviceExecutableNetwork::WorkerInferRequest* _workerInferRequest = nullptr;
};
class MultiDeviceInferencePlugin : public InferenceEngine::InferencePluginInternal {
public:
MultiDeviceInferencePlugin();
~MultiDeviceInferencePlugin() override = default;
InferenceEngine::ExecutableNetworkInternal::Ptr LoadExeNetworkImpl(const InferenceEngine::ICNNNetwork& network,
const std::map<std::string, std::string>& config) override;
void SetConfig(const std::map<std::string, std::string>& config) override;
Parameter GetConfig(const std::string& name,
const std::map<std::string, Parameter> & options) const override;
void QueryNetwork(const InferenceEngine::ICNNNetwork& network,
const std::map<std::string, std::string>& config,
InferenceEngine::QueryNetworkResult& res) const override;
InferenceEngine::Parameter GetMetric(const std::string& name,
const std::map<std::string, InferenceEngine::Parameter>& options) const override;
DeviceMap<DeviceInformation> ParseMetaDevices(const std::string & devicesRequestsCfg,
const std::map<std::string, std::string> & config) const;
protected:
std::map<std::string, std::string> GetSupportedConfig(const std::map<std::string, std::string>& config,
const DeviceName & deviceName) const;
};
} // namespace MultiDevicePlugin

26
inference-engine/src/plugin_api/cpp_interfaces/impl/ie_plugin_internal.hpp
View File

@ -102,7 +102,7 @@ public:
_core = core;
}
const ICore* GetCore() const noexcept override {
ICore* GetCore() const noexcept override {
return _core;
}
@ -137,7 +137,6 @@ public:
THROW_IE_EXCEPTION << NOT_IMPLEMENTED_str;
}
RemoteContext::Ptr GetDefaultContext() override {
THROW_IE_EXCEPTION << NOT_IMPLEMENTED_str;
}
@ -154,7 +153,7 @@ protected:
* @param config string-string map of config parameters relevant only for this load operation
* @return Shared pointer to the ExecutableNetwork object
*/
virtual ExecutableNetworkInternal::Ptr LoadExeNetworkImpl(const ICore* core, const ICNNNetwork& network,
virtual ExecutableNetworkInternal::Ptr LoadExeNetworkImpl(const ICNNNetwork& network,
const std::map<std::string, std::string>& config) = 0;
/**
@ -163,16 +162,13 @@ protected:
* @note The function is used in
* InferencePluginInternal::LoadNetwork(const ICNNNetwork&, const std::map<std::string, std::string>&, RemoteContext::Ptr)
* which performs common steps first and calls this plugin-dependent method implementation after.
* @param core A pointer to ICore interface.
* @param network A network object
* @param context A remote context
* @param config string-string map of config parameters relevant only for this load operation
* @return Shared pointer to the ExecutableNetwork object
*/
virtual ExecutableNetworkInternal::Ptr LoadExeNetworkImpl(const ICore* core, const ICNNNetwork& network,
RemoteContext::Ptr context,
virtual ExecutableNetworkInternal::Ptr LoadExeNetworkImpl(const ICNNNetwork& network, RemoteContext::Ptr context,
const std::map<std::string, std::string>& config) {
(void)core;
(void)network;
(void)context;
(void)config;
@ -190,21 +186,21 @@ protected:
void cloneAndCreateExecutableNetwork(IExecutableNetwork::Ptr& executableNetwork, const ICNNNetwork& network,
const std::map<std::string, std::string>& config,
RemoteContext::Ptr context = nullptr) {
InputsDataMap networkInputs;
OutputsDataMap networkOutputs;
InputsDataMap networkInputs, networkInputsCloned;
OutputsDataMap networkOutputs, networkOutputsCloned;
network.getInputsInfo(networkInputs);
network.getOutputsInfo(networkOutputs);
copyInputOutputInfo(networkInputs, networkOutputs, _networkInputs, _networkOutputs);
copyInputOutputInfo(networkInputs, networkOutputs, networkInputsCloned, networkOutputsCloned);
ExecutableNetworkInternal::Ptr impl;
if (nullptr == context) {
impl = LoadExeNetworkImpl(GetCore(), network, config);
impl = LoadExeNetworkImpl(network, config);
} else {
impl = LoadExeNetworkImpl(GetCore(), network, context, config);
impl = LoadExeNetworkImpl(network, context, config);
}
impl->setNetworkInputs(_networkInputs);
impl->setNetworkOutputs(_networkOutputs);
impl->setNetworkInputs(networkInputsCloned);
impl->setNetworkOutputs(networkOutputsCloned);
impl->SetPointerToPluginInternal(shared_from_this());
executableNetwork.reset(new ExecutableNetworkBase<ExecutableNetworkInternal>(impl), [](details::IRelease* p) {
@ -243,8 +239,6 @@ protected:
}
std::string _pluginName; //!< A device name that plugins enables
InferenceEngine::InputsDataMap _networkInputs; //!< Holds information about network inputs info
InferenceEngine::OutputsDataMap _networkOutputs; //!< Holds information about network outputs data
std::map<std::string, std::string> _config; //!< A map config keys -> values
ICore* _core = nullptr; //!< A pointer to ICore interface
};

2
inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iplugin_internal.hpp
View File

@ -217,7 +217,7 @@ public:
* @brief Gets reference to ICore interface
* @return Reference to ICore interface
*/
virtual const ICore* GetCore() const noexcept = 0;
virtual ICore* GetCore() const noexcept = 0;
/**
* @brief Queries a plugin about support layers in network

50
inference-engine/src/plugin_api/ie_icore.hpp
View File

@ -32,6 +32,7 @@ public:
virtual std::shared_ptr<ITaskExecutor> GetTaskExecutor() const = 0;
/**
* @deprecated Use ICore::GetMetric, ICore::LoadNetwork, ICore::QueryNetwork instead
* @brief Returns reference to plugin by a device name
* @param deviceName - a name of device
* @return Reference to plugin
@ -55,6 +56,55 @@ public:
*/
virtual CNNNetwork ReadNetwork(const std::string& modelPath, const std::string& binPath) const = 0;
/**
* @brief Creates an executable network from a network object.
*
* Users can create as many networks as they need and use
* them simultaneously (up to the limitation of the hardware resources)
*
* @param network CNNNetwork object acquired from Core::ReadNetwork
* @param deviceName Name of device to load network to
* @param config Optional map of pairs: (config parameter name, config parameter value) relevant only for this load
* operation
* @return An executable network reference
*/
virtual ExecutableNetwork LoadNetwork(const CNNNetwork& network, const std::string& deviceName,
const std::map<std::string, std::string>& config = {}) = 0;
/**
* @brief Creates an executable network from a previously exported network
* @param deviceName Name of device load executable network on
* @param networkModel network model stream
* @param config Optional map of pairs: (config parameter name, config parameter value) relevant only for this load
* operation*
* @return An executable network reference
*/
virtual ExecutableNetwork ImportNetwork(std::istream& networkModel, const std::string& deviceName = {},
const std::map<std::string, std::string>& config = {}) = 0;
/**
* @brief Query device if it supports specified network with specified configuration
*
* @param deviceName A name of a device to query
* @param network Network object to query
* @param config Optional map of pairs: (config parameter name, config parameter value)
* @return An object containing a map of pairs a layer name -> a device name supporting this layer.
*/
virtual QueryNetworkResult QueryNetwork(const ICNNNetwork& network, const std::string& deviceName,
const std::map<std::string, std::string>& config) const = 0;
/**
* @brief Gets general runtime metric for dedicated hardware.
*
* The method is needed to request common device properties
* which are executable network agnostic. It can be device name, temperature, other devices-specific values.
*
* @param deviceName - A name of a device to get a metric value.
* @param name - metric name to request.
* @return Metric value corresponding to metric key.
*/
virtual Parameter GetMetric(const std::string& deviceName, const std::string& name) const = 0;
/**
* @brief Default virtual destructor
*/

6
inference-engine/src/plugin_api/ie_profiling.hpp
View File

@ -138,7 +138,7 @@ inline static void annotateEnd(IttTaskHandles& h, IttBlock&) {
#define IE_STR(x) IE_STR_(x)
#define IE_STR_(x) #x
class ProfilingTask;
struct ProfilingTask;
struct IttStatic {};
@ -179,8 +179,8 @@ struct ProfilingTask {
}
private:
friend void annotateBegin(IttStatic&, IttProfilingTask& t);
friend void annotateEnd(IttStatic&, IttProfilingTask& t);
friend void annotateBegin(IttStatic&, IttProfilingTask& t);
friend void annotateEnd(IttStatic&, IttProfilingTask& t);
std::string name;
#ifdef ENABLE_PROFILING_ITT

2
inference-engine/src/plugin_api/threading/ie_cpu_streams_executor.hpp
View File

@ -44,6 +44,8 @@ public:
void run(Task task) override;
void Execute(Task task) override;
int GetStreamId() override;
int GetNumaNodeId() override;

6
inference-engine/src/plugin_api/threading/ie_istreams_executor.hpp
View File

@ -121,6 +121,12 @@ public:
* @return `ID` of current NUMA Node, or throws exceptions if called not from stream thread
*/
virtual int GetNumaNodeId() = 0;
/**
* @brief Execute the task in the current thread using streams executor configuration and constraints
* @param task A task to start
*/
virtual void Execute(Task task) = 0;
};

6
inference-engine/src/preprocessing/CMakeLists.txt
View File

@ -17,15 +17,15 @@ file(GLOB LIBRARY_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp)
#
function(ie_avx512_core_optimization_flags flags)
if(WIN32)
if(CMAKE_CXX_COMPILER_ID STREQUAL Intel)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
set(${flags} "/QxCORE-AVX512" PARENT_SCOPE)
elseif(CMAKE_CXX_COMPILER_ID MATCHES MSVC)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set(${flags} "/arch:AVX512" PARENT_SCOPE)
else()
message(WARNING "Unsupported CXX compiler ${CMAKE_CXX_COMPILER_ID}")
endif()
else()
if(CMAKE_CXX_COMPILER_ID STREQUAL Intel)
if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
set(${flags} "-xCORE-AVX512" PARENT_SCOPE)
else()
set(${flags} "-mavx512f -mavx512bw -mavx512dq -mfma" PARENT_SCOPE)

12
inference-engine/src/transformations/include/transformations/common_optimizations/common_optimizations_tbl.hpp
View File

@ -14,4 +14,14 @@
// #include <transformations/transformations_tbl.hpp>
// #undef NGRAPH_PASS
NGRAPH_PASS(NopElimination, ::ngraph::pass)
// This pass must be called first in pipeline
NGRAPH_PASS(InitNodeInfo, ::ngraph::pass)
NGRAPH_PASS(ConvertPriorBox, ::ngraph::pass) // WA: ConvertPriorBox must be executed before CF
NGRAPH_PASS(ConstantFolding, ::ngraph::pass)
NGRAPH_PASS(RemoveFilteringBoxesBySize, ::ngraph::pass) // Resolves dynamism (replaces NonZero), CF needed
NGRAPH_PASS(ConstantFolding, ::ngraph::pass)
NGRAPH_PASS(StridedSliceOptimization, ::ngraph::pass) // depends on CF
NGRAPH_PASS(NopElimination, ::ngraph::pass) // may introduce fake dynamism
NGRAPH_PASS(AlgebraicSimplification, ::ngraph::pass) // may introduce fake dynamism
NGRAPH_PASS(ConstantFolding, ::ngraph::pass)
NGRAPH_PASS(ConvertScatterElementsToScatter, ::ngraph::pass) // partially depends on CF

2
inference-engine/src/transformations/include/transformations/convert_opset1_to_legacy/convert_opset1_to_legacy_tbl.hpp
View File

@ -14,8 +14,6 @@
// #include <transformations/transformations_tbl.hpp>
// #undef NGRAPH_PASS
NGRAPH_PASS(InitNodeInfo, ::ngraph::pass)
NGRAPH_PASS(ConvertPriorBox, ::ngraph::pass)
NGRAPH_PASS(ConstantFolding, ::ngraph::pass)
NGRAPH_PASS(ConvertReduceToPooling, ::ngraph::pass)
NGRAPH_PASS(ConvertMod, ::ngraph::pass)

31
inference-engine/src/transformations/include/transformations/convert_opset3_to_opset2/convert_broadcast3.hpp Normal file
View File

@ -0,0 +1,31 @@
// Copyright (C) 2018-2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <vector>
#include <memory>
#include <ie_api.h>
#include <ngraph/pass/graph_rewrite.hpp>
#include "transformations/utils/pass_param.hpp"
namespace ngraph {
namespace pass {
class INFERENCE_ENGINE_API_CLASS(ConvertBroadcast3);
} // namespace pass
} // namespace ngraph
class ngraph::pass::ConvertBroadcast3: public ngraph::pass::GraphRewrite, public ngraph::pass::PassParam {
public:
ConvertBroadcast3() : GraphRewrite() {
convert_broadcast3();
}
private:
void convert_broadcast3();
};

31
inference-engine/src/transformations/include/transformations/convert_opset3_to_opset2/convert_nms3.hpp Normal file
View File

@ -0,0 +1,31 @@
// Copyright (C) 2018-2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <vector>
#include <memory>
#include <ie_api.h>
#include <ngraph/pass/graph_rewrite.hpp>
#include "transformations/utils/pass_param.hpp"
namespace ngraph {
namespace pass {
class INFERENCE_ENGINE_API_CLASS(ConvertNMS3);
} // namespace pass
} // namespace ngraph
class ngraph::pass::ConvertNMS3: public ngraph::pass::GraphRewrite, public ngraph::pass::PassParam {
public:
ConvertNMS3() : GraphRewrite() {
convert_nms3();
}
private:
void convert_nms3();
};

Some files were not shown because too many files have changed in this diff Show More