natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Docs to Sphinx (#8151) 

* docs to sphinx

* Update GPU.md

* Update CPU.md

* Update AUTO.md

* Update performance_int8_vs_fp32.md

* update

* update md

* updates

* disable doc ci

* disable ci

* fix index.rst

Co-authored-by: Andrey Zaytsev <andrey.zaytsev@intel.com>
This commit is contained in:
Nikolay Tyukaev 2021-10-24 17:43:00 +03:00 committed by GitHub
parent d7a917d92c
commit 361dec7362
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
636 changed files with 20601 additions and 14376 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/build_doc.yml vendored
View File

@ -33,10 +33,10 @@ jobs:
run: |
mkdir build
cd build
cmake -DENABLE_DOCS=ON ..
echo "TBD"
- name: Build doc
run: cmake --build . --target openvino_docs
run: echo "TBD"
working-directory: build
- name: 'Upload doc'

2
.gitignore vendored
View File

@ -2,6 +2,8 @@
_*
# but ensure we don't skip __init__.py
!__init__.py
# and sphinx documentation folders
!docs/_*
# developer tools
*.idea

397
docs/CMakeLists.txt
View File

@ -44,327 +44,190 @@ if(NOT ENABLE_DOCKER)
endforeach()
endif()
set(LINKCHECKER_PY "" CACHE FILEPATH "Path to linkchecker.py for documentation check")
set(OMZ_DOCS_DIR "" CACHE PATH "Path to open_model_zoo documentation")
set(WORKBENCH_DOCS_DIR "" CACHE PATH "Path to workbench documentation")
set(POT_DOCS_DIR "" CACHE PATH "Path to post-training-compression-tool documentation")
set(GST_DOCS_DIR "" CACHE PATH "Path to gst-video-analytics documentation")
set(LINKCHECKER_PY "" CACHE FILEPATH "Path to linkchecker.py for documentation check dir.")
set(OMZ_DOCS_DIR "" CACHE PATH "Path to open_model_zoo documentation dir.")
set(WORKBENCH_DOCS_DIR "" CACHE PATH "Path to workbench documentation dir.")
set(POT_DOCS_DIR "" CACHE PATH "Path to post-training-compression-tool documentation dir.")
set(GST_DOCS_DIR "" CACHE PATH "Path to gst-video-analytics documentation dir.")
set(GITHUB_API_TOKEN "" CACHE PATH "Path to file containing github api token.")
set(GRAPH_CSV_DIR "" CACHE PATH "Path to the folder containing csv data for rendering graphs.")
function(build_docs)
find_package(Doxygen REQUIRED dot)
find_package(PythonInterp 3 REQUIRED)
find_package(LATEX REQUIRED)
execute_process(
COMMAND ${PYTHON_EXECUTABLE} -m pip show lxml
RESULT_VARIABLE PIP_EXIT_CODE
OUTPUT_QUIET
)
if (NOT ${PIP_EXIT_CODE} EQUAL 0)
message(FATAL_ERROR "lxml package is not installed. Please use \"pip install lxml\".")
find_program(DOXYREST_EXECUTABLE NAMES doxyrest)
if (NOT DOXYREST_EXECUTABLE)
message(FATAL_ERROR "No doxyrest found. Documentation output is not available")
endif()
set(DOCS_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}")
set(DOXYGEN_DIR "${OpenVINO_MAIN_SOURCE_DIR}/docs/doxygen")
set(IE_SOURCE_DIR "${OpenVINO_MAIN_SOURCE_DIR}/inference-engine")
set(PYTHON_API_IN "${IE_SOURCE_DIR}/ie_bridges/python/src/openvino/inference_engine/ie_api.pyx")
set(PYTHON_API_OUT "${DOCS_BUILD_DIR}/python_api/ie_api.pyx")
set(C_API "${IE_SOURCE_DIR}/ie_bridges/c/include")
set(PLUGIN_API_DIR "${DOCS_BUILD_DIR}/IE_PLUGIN_DG")
set(NGRAPH_DIR "${OpenVINO_MAIN_SOURCE_DIR}/ngraph")
set(NGRAPH_PY_DIR "${NGRAPH_DIR}/python/src/ngraph/")
set(NGRAPH_CPP_DIR "${NGRAPH_DIR}/core/include/" "${NGRAPH_DIR}/frontend/onnx_import/include")
set(DOCS_SOURCE_DIR "${OpenVINO_MAIN_SOURCE_DIR}/docs")
set(DOXYGEN_DIR "${DOCS_SOURCE_DIR}/doxygen")
# API INPUT
set(NGRAPH_DIR "${OpenVINO_MAIN_SOURCE_DIR}/ngraph")
# markdown docs
set(MARKDOWN_INPUT "${DOCS_BUILD_DIR}")
# IE C++ API
set(IE_SOURCE_DIR "${OpenVINO_MAIN_SOURCE_DIR}/inference-engine")
# IE C API
set(IE_C_API "${IE_SOURCE_DIR}/ie_bridges/c/include")
# Preprocessing scripts
set(DOXY_MD_FILTER "${DOXYGEN_DIR}/doxy_md_filter.py")
set(DOXY_LAYOUT_SCRIPT "${DOXYGEN_DIR}/build_main_layout.py")
set(PYNGRAPH_REF_SCRIPT "${DOXYGEN_DIR}/pyngraph_ref.py")
set(DOXY_LOG_SCRIPT "${DOXYGEN_DIR}/log.py")
set(PYX_FILTER "${DOXYGEN_DIR}/pyx_filter.py")
# assets dir
set(ASSETS_DIR "${DOXYGEN_DIR}/assets")
# header and footer
set(HEADER_SOURCE "${DOXYGEN_DIR}/header.html.in")
set(FOOTER_SOURCE "${DOXYGEN_DIR}/footer.html.in")
set(HEADER_BUILD "${DOCS_BUILD_DIR}/header.html")
set(FOOTER_BUILD "${DOCS_BUILD_DIR}/footer.html")
configure_file(${HEADER_SOURCE} ${HEADER_BUILD} @ONLY)
configure_file(${FOOTER_SOURCE} ${FOOTER_BUILD} @ONLY)
file(GLOB_RECURSE doc_source_files
LIST_DIRECTORIES true RELATIVE ${OpenVINO_MAIN_SOURCE_DIR}
"${OpenVINO_MAIN_SOURCE_DIR}/docs/*.md"
"${OpenVINO_MAIN_SOURCE_DIR}/docs/*.png"
"${OpenVINO_MAIN_SOURCE_DIR}/docs/*.gif"
"${OpenVINO_MAIN_SOURCE_DIR}/docs/*.jpg"
"${OpenVINO_MAIN_SOURCE_DIR}/docs/*.svg"
"${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.md"
"${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.png"
"${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.gif"
"${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.jpg"
"${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.svg")
configure_file(${PYTHON_API_IN} ${PYTHON_API_OUT} @ONLY)
set(NGRAPH_CPP_CONFIG_SOURCE "${DOXYGEN_DIR}/ngraph_cpp_api.config")
set(NGRAPH_PY_CONFIG_SOURCE "${DOXYGEN_DIR}/ngraph_py_api.config")
set(IE_CONFIG_SOURCE "${DOXYGEN_DIR}/ie_docs.config")
set(C_CONFIG_SOURCE "${DOXYGEN_DIR}/ie_c_api.config")
set(PY_CONFIG_SOURCE "${DOXYGEN_DIR}/ie_py_api.config")
set(PLUGIN_CONFIG_SOURCE "${DOXYGEN_DIR}/ie_plugin_api.config")
set(NGRAPH_CPP_CONFIG_BUILD "${DOCS_BUILD_DIR}/ngraph_cpp_api.config")
set(NGRAPH_PY_CONFIG_BUILD "${DOCS_BUILD_DIR}/ngraph_py_api.config")
set(IE_CONFIG_BUILD "${DOCS_BUILD_DIR}/ie_docs.config")
set(C_CONFIG_BUILD "${DOCS_BUILD_DIR}/ie_c_api.config")
set(PY_CONFIG_BUILD "${DOCS_BUILD_DIR}/ie_py_api.config")
set(PLUGIN_CONFIG_BUILD "${DOCS_BUILD_DIR}/ie_plugin_api.config")
set(NGRAPH_CPP_LAYOUT_SOURCE "${DOXYGEN_DIR}/ngraph_cpp_api.xml")
set(NGRAPH_PY_LAYOUT_SOURCE "${DOXYGEN_DIR}/ngraph_py_api.xml")
set(IE_LAYOUT_SOURCE "${DOXYGEN_DIR}/ie_docs.xml")
set(OPENVINO_LAYOUT_SOURCE "${DOXYGEN_DIR}/openvino_docs.xml")
set(C_LAYOUT_SOURCE "${DOXYGEN_DIR}/ie_c_api.xml")
set(PY_LAYOUT_SOURCE "${DOXYGEN_DIR}/ie_py_api.xml")
set(PLUGIN_LAYOUT_SOURCE "${DOXYGEN_DIR}/ie_plugin_api.xml")
set(NGRAPH_CPP_LAYOUT_BUILD "${DOCS_BUILD_DIR}/ngraph_cpp_api.xml")
set(NGRAPH_PY_LAYOUT_BUILD "${DOCS_BUILD_DIR}/ngraph_py_api.xml")
set(IE_LAYOUT_BUILD "${DOCS_BUILD_DIR}/ie_docs.xml")
set(OPENVINO_LAYOUT_BUILD "${DOCS_BUILD_DIR}/openvino_docs.xml")
set(C_LAYOUT_BUILD "${DOCS_BUILD_DIR}/ie_c_api.xml")
set(PY_LAYOUT_BUILD "${DOCS_BUILD_DIR}/ie_py_api.xml")
set(PLUGIN_LAYOUT_BUILD "${DOCS_BUILD_DIR}/ie_plugin_api.xml")
set(PREPARE_XML_SCRIPT "${DOXYGEN_DIR}/prepare_xml.py")
set(REMOVE_XML_SCRIPT "${DOXYGEN_DIR}/remove_xml.py")
set(COPY_IMAGES_SCRIPT "${DOXYGEN_DIR}/copy_images.py")
set(DOXYGEN_MAPPING_SCRIPT "${DOXYGEN_DIR}/create_mapping.py")
set(DOXYGEN_MAPPING_FILE "${DOCS_BUILD_DIR}/mapping.json")
# out dirs
set(OUTPUT_DIRECTORY "${DOCS_BUILD_DIR}/html")
set(IE_OUTPUT "${OUTPUT_DIRECTORY}")
set(C_OUTPUT "${OUTPUT_DIRECTORY}/ie_c_api")
set(PY_OUTPUT "${OUTPUT_DIRECTORY}/ie_python_api")
set(PLUGIN_OUTPUT "${OUTPUT_DIRECTORY}/ie_plugin_api")
set(NGRAPH_CPP_OUTPUT "${OUTPUT_DIRECTORY}/ngraph_cpp_api")
set(NGRAPH_PY_OUTPUT "${OUTPUT_DIRECTORY}/ngraph_python_api")
set(XML_OUTPUT "${DOCS_BUILD_DIR}/xml")
set(RST_OUTPUT "${DOCS_BUILD_DIR}/rst")
set(SPHINX_OUTPUT "${DOCS_BUILD_DIR}/_build")
# Tables of contents
configure_file(${NGRAPH_CPP_LAYOUT_SOURCE} ${NGRAPH_CPP_LAYOUT_BUILD} @ONLY)
configure_file(${NGRAPH_PY_LAYOUT_SOURCE} ${NGRAPH_PY_LAYOUT_BUILD} @ONLY)
configure_file(${IE_LAYOUT_SOURCE} ${IE_LAYOUT_BUILD} @ONLY)
configure_file(${OPENVINO_LAYOUT_SOURCE} ${OPENVINO_LAYOUT_BUILD} @ONLY)
configure_file(${C_LAYOUT_SOURCE} ${C_LAYOUT_BUILD} @ONLY)
configure_file(${PY_LAYOUT_SOURCE} ${PY_LAYOUT_BUILD} @ONLY)
configure_file(${PLUGIN_LAYOUT_SOURCE} ${PLUGIN_LAYOUT_BUILD} @ONLY)
# Sphinx folders, doxyrest templates and config
set(SPHINX_CONF_IN "${DOCS_SOURCE_DIR}/conf.py")
set(SPHINX_CONF_OUT "${RST_OUTPUT}/conf.py")
set(SPHINX_STATIC_IN "${DOCS_SOURCE_DIR}/_static")
set(SPHINX_STATIC_OUT "${RST_OUTPUT}/_static")
set(SPHINX_INDEX_IN "${DOCS_SOURCE_DIR}/index.rst")
set(SPHINX_INDEX_OUT "${RST_OUTPUT}/index.rst")
set(API_DOCS_IN "${DOCS_SOURCE_DIR}/api")
set(API_DOCS_OUT "${RST_OUTPUT}/api")
set(DOXYREST_IN "${DOCS_SOURCE_DIR}/doxyrest")
set(DOXYREST_OUT "${DOCS_BUILD_DIR}/doxyrest")
set(DOXYREST_SPHINX_IN "${DOCS_SOURCE_DIR}/doxyrest-sphinx")
set(DOXYREST_SPHINX_OUT "${RST_OUTPUT}/doxyrest-sphinx")
set(DOXYREST_CONFIG_IN "${DOCS_SOURCE_DIR}/doxyrest-config.lua")
set(DOXYREST_CONFIG_OUT "${DOCS_BUILD_DIR}/doxyrest-config.lua")
configure_file(${DOXYREST_CONFIG_IN} ${DOXYREST_CONFIG_OUT} @ONLY)
configure_file(${SPHINX_CONF_IN} ${SPHINX_CONF_OUT} @ONLY)
# Doxygen config files
configure_file(${NGRAPH_CPP_CONFIG_SOURCE} ${NGRAPH_CPP_CONFIG_BUILD} @ONLY)
configure_file(${NGRAPH_PY_CONFIG_SOURCE} ${NGRAPH_PY_CONFIG_BUILD} @ONLY)
configure_file(${IE_CONFIG_SOURCE} ${IE_CONFIG_BUILD} @ONLY)
configure_file(${C_CONFIG_SOURCE} ${C_CONFIG_BUILD} @ONLY)
configure_file(${PY_CONFIG_SOURCE} ${PY_CONFIG_BUILD} @ONLY)
configure_file(${PLUGIN_CONFIG_SOURCE} ${PLUGIN_CONFIG_BUILD} @ONLY)
# Doxygen config
set(DOXYFILE_SOURCE "${DOXYGEN_DIR}/Doxyfile.config")
set(DOXYFILE_BUILD "${DOCS_BUILD_DIR}/Doxyfile.config")
configure_file(${DOXYFILE_SOURCE} ${DOXYFILE_BUILD} @ONLY)
# Preprocessing scripts
set(DOXY_MD_FILTER "${DOXYGEN_DIR}/doxy_md_filter.py")
set(PYX_FILTER "${DOXYGEN_DIR}/pyx_filter.py")
list(APPEND commands COMMAND ${PYTHON_EXECUTABLE} ${DOXY_MD_FILTER}
--input_dir=${OpenVINO_MAIN_SOURCE_DIR}
--output_dir=${DOCS_BUILD_DIR}/openvino
--exclude_dir=${CMAKE_CURRENT_BINARY_DIR}
--exclude_dir=${OpenVINO_MAIN_SOURCE_DIR}/out)
# nGraph C++ API
# include additional repositories
add_custom_target(ngraph_cpp_api
COMMAND ${CMAKE_COMMAND} -E copy_directory ${ASSETS_DIR} ${NGRAPH_CPP_OUTPUT}/assets
COMMAND ${DOXYGEN_EXECUTABLE} ${NGRAPH_CPP_CONFIG_BUILD}
WORKING_DIRECTORY ${DOCS_BUILD_DIR}
VERBATIM)
# openvino notebooks dir
if(GITHUB_API_TOKEN)
set(NBDOC_SCRIPT "${DOCS_SOURCE_DIR}/nbdoc/nbdoc.py")
list(APPEND commands
COMMAND ${PYTHON_EXECUTABLE} "${NBDOC_SCRIPT}" "${GITHUB_API_TOKEN}" "${RST_OUTPUT}/notebooks"
)
endif()
# nGraph Python API
add_custom_target(ngraph_py_api
COMMAND ${CMAKE_COMMAND} -E copy_directory ${ASSETS_DIR} ${NGRAPH_PY_OUTPUT}/assets
COMMAND ${DOXYGEN_EXECUTABLE} ${NGRAPH_PY_CONFIG_BUILD}
WORKING_DIRECTORY ${DOCS_BUILD_DIR}
VERBATIM)
# C API
add_custom_target(c_api
COMMAND ${CMAKE_COMMAND} -E copy_directory ${ASSETS_DIR} ${C_OUTPUT}/assets
COMMAND ${DOXYGEN_EXECUTABLE} ${C_CONFIG_BUILD}
WORKING_DIRECTORY ${DOCS_BUILD_DIR}
COMMENT "Generating C API Reference"
VERBATIM)
# Python API
add_custom_target(py_api
COMMAND ${CMAKE_COMMAND} -E copy_directory ${ASSETS_DIR} ${PY_OUTPUT}/assets
COMMAND ${DOXYGEN_EXECUTABLE} ${PY_CONFIG_BUILD}
WORKING_DIRECTORY ${DOCS_BUILD_DIR}
COMMENT "Generating Python API Reference"
VERBATIM)
add_custom_command(TARGET py_api
PRE_BUILD
COMMAND ${PYTHON_EXECUTABLE} ${PYX_FILTER} ${PYTHON_API_OUT}
COMMENT "Pre-process Python API")
# Preprocess docs
add_custom_target(preprocess_docs
COMMENT "Pre-process docs"
VERBATIM)
# ovino doc files
file(GLOB_RECURSE ovino_doc_files
LIST_DIRECTORIES true RELATIVE ${OpenVINO_MAIN_SOURCE_DIR}
"${OpenVINO_MAIN_SOURCE_DIR}/docs/*.md"
"${OpenVINO_MAIN_SOURCE_DIR}/docs/*.png"
"${OpenVINO_MAIN_SOURCE_DIR}/docs/*.gif"
"${OpenVINO_MAIN_SOURCE_DIR}/docs/*.jpg"
"${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.md"
"${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.png"
"${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.gif"
"${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.jpg")
foreach(source_file ${ovino_doc_files})
list(APPEND commands COMMAND ${CMAKE_COMMAND} -E copy
"${OpenVINO_MAIN_SOURCE_DIR}/${source_file}" "${DOCS_BUILD_DIR}/openvino/${source_file}")
endforeach()
if(GRAPH_CSV_DIR)
set(GRAPH_CSV_DIR_OUT "${RST_OUTPUT}/csv")
list(APPEND commands
COMMAND ${CMAKE_COMMAND} -E copy_directory "${GRAPH_CSV_DIR}" "${GRAPH_CSV_DIR_OUT}"
)
endif()
# omz doc files
if(EXISTS "${OMZ_DOCS_DIR}")
get_filename_component(OMZ_DOCS_DIR "${OMZ_DOCS_DIR}" ABSOLUTE)
file(GLOB_RECURSE omz_doc_files
LIST_DIRECTORIES true RELATIVE ${OMZ_DOCS_DIR}
"${OMZ_DOCS_DIR}/*.md"
"${OMZ_DOCS_DIR}/*.png"
"${OMZ_DOCS_DIR}/*.gif"
"${OMZ_DOCS_DIR}/*.jpg")
foreach(source_file ${omz_doc_files})
list(APPEND commands COMMAND ${CMAKE_COMMAND} -E copy
"${OMZ_DOCS_DIR}/${source_file}" "${DOCS_BUILD_DIR}/omz/${source_file}")
endforeach()
configure_file("${OMZ_DOCS_DIR}/omz_docs.xml" "${DOCS_BUILD_DIR}/omz_docs.xml" @ONLY)
list(APPEND commands
COMMAND ${PYTHON_EXECUTABLE} ${OMZ_DOCS_DIR}/ci/prepare-documentation.py ${CMAKE_BINARY_DIR}/open_model_zoo)
list(APPEND commands COMMAND ${PYTHON_EXECUTABLE} ${DOXY_MD_FILTER}
--input_dir=${CMAKE_BINARY_DIR}/open_model_zoo
--output_dir=${DOCS_BUILD_DIR}/open_model_zoo)
endif()
# workbench doc files
if(EXISTS "${WORKBENCH_DOCS_DIR}")
get_filename_component(WORKBENCH_DOCS_DIR "${WORKBENCH_DOCS_DIR}" ABSOLUTE)
file(GLOB_RECURSE workbench_doc_files
LIST_DIRECTORIES true RELATIVE ${WORKBENCH_DOCS_DIR}
"${WORKBENCH_DOCS_DIR}/*.md"
"${WORKBENCH_DOCS_DIR}/*.png"
"${WORKBENCH_DOCS_DIR}/*.gif"
"${WORKBENCH_DOCS_DIR}/*.jpg")
foreach(source_file ${workbench_doc_files})
list(APPEND commands COMMAND ${CMAKE_COMMAND} -E copy
"${WORKBENCH_DOCS_DIR}/${source_file}" "${DOCS_BUILD_DIR}/workbench/${source_file}")
endforeach()
configure_file("${WORKBENCH_DOCS_DIR}/docs/Workbench_DG/workbench_docs.xml" "${DOCS_BUILD_DIR}/workbench_docs.xml" @ONLY)
list(APPEND commands COMMAND ${PYTHON_EXECUTABLE} ${DOXY_MD_FILTER}
--input_dir=${WORKBENCH_DOCS_DIR}
--output_dir=${DOCS_BUILD_DIR}/workbench)
endif()
# pot doc files
if(EXISTS "${POT_DOCS_DIR}")
get_filename_component(POT_DOCS_DIR "${POT_DOCS_DIR}" ABSOLUTE)
file(GLOB_RECURSE pot_doc_files
LIST_DIRECTORIES true RELATIVE ${POT_DOCS_DIR}
"${POT_DOCS_DIR}/*.md"
"${POT_DOCS_DIR}/*.png"
"${POT_DOCS_DIR}/*.gif"
"${POT_DOCS_DIR}/*.jpg")
foreach(source_file ${pot_doc_files})
list(APPEND commands COMMAND ${CMAKE_COMMAND} -E copy
"${POT_DOCS_DIR}/${source_file}" "${DOCS_BUILD_DIR}/pot/${source_file}")
endforeach()
configure_file("${POT_DOCS_DIR}/docs/pot_docs.xml" "${DOCS_BUILD_DIR}/pot_docs.xml" @ONLY)
list(APPEND commands COMMAND ${PYTHON_EXECUTABLE} ${DOXY_MD_FILTER}
--input_dir=${POT_DOCS_DIR}
--output_dir=${DOCS_BUILD_DIR}/pot)
endif()
# gst doc files
if(EXISTS "${GST_DOCS_DIR}")
get_filename_component(GST_DOCS_DIR "${GST_DOCS_DIR}" ABSOLUTE)
file(GLOB_RECURSE gst_doc_files
LIST_DIRECTORIES true RELATIVE ${GST_DOCS_DIR}
"${GST_DOCS_DIR}/*.md"
"${GST_DOCS_DIR}/*.png"
"${GST_DOCS_DIR}/*.gif"
"${GST_DOCS_DIR}/*.jpg")
foreach(source_file ${gst_doc_files})
list(APPEND commands COMMAND ${CMAKE_COMMAND} -E copy
"${GST_DOCS_DIR}/${source_file}" "${DOCS_BUILD_DIR}/gst/${source_file}")
endforeach()
list(APPEND commands COMMAND ${PYTHON_EXECUTABLE} ${DOXY_MD_FILTER}
--input_dir=${GST_DOCS_DIR}
--output_dir=${DOCS_BUILD_DIR}/gst)
endif()
add_custom_target(preprocess_docs
COMMENT "Preprocess documentation"
VERBATIM)
# Preprocess docs
add_custom_command(TARGET preprocess_docs
PRE_BUILD
${commands}
COMMAND ${PYTHON_EXECUTABLE} ${DOXY_LAYOUT_SCRIPT} --openvino ${OPENVINO_LAYOUT_BUILD}
COMMAND ${PYTHON_EXECUTABLE} ${DOXY_MD_FILTER} ${DOCS_BUILD_DIR}
COMMENT "Pre-process markdown and image links")
# IE dev guide and C++ API
add_custom_target(ie_docs
DEPENDS ngraph_cpp_api preprocess_docs
COMMAND ${CMAKE_COMMAND} -E copy_directory ${ASSETS_DIR} ${IE_OUTPUT}/assets
COMMAND ${DOXYGEN_EXECUTABLE} ${IE_CONFIG_BUILD}
WORKING_DIRECTORY ${DOCS_BUILD_DIR}
VERBATIM)
# Plugin API
add_custom_target(plugin_api
DEPENDS ngraph_cpp_api ie_docs
COMMAND ${CMAKE_COMMAND} -E copy_directory ${ASSETS_DIR} ${PLUGIN_OUTPUT}/assets
COMMAND ${DOXYGEN_EXECUTABLE} ${PLUGIN_CONFIG_BUILD}
WORKING_DIRECTORY ${DOCS_BUILD_DIR}
COMMENT "Generating Plugin API Reference"
VERBATIM)
# Umbrella OpenVINO target
add_custom_target(openvino_docs
DEPENDS ngraph_cpp_api ngraph_py_api c_api py_api ie_docs plugin_api
COMMENT "Generating OpenVINO documentation"
VERBATIM)
set_target_properties(openvino_docs ie_docs c_api py_api preprocess_docs plugin_api
ngraph_py_api ngraph_cpp_api
PROPERTIES FOLDER docs)
add_custom_command(TARGET openvino_docs
POST_BUILD
COMMAND ${PYTHON_EXECUTABLE} ${DOXY_LOG_SCRIPT} --log "${DOCS_BUILD_DIR}/ie_docs.log"
--include_omz $<BOOL:${OMZ_DOCS_DIR}>
--include_wb $<BOOL:${WORKBENCH_DOCS_DIR}>
--include_pot $<BOOL:${POT_DOCS_DIR}>
--include_gst $<BOOL:${GST_DOCS_DIR}>
COMMENT "Parse doxygen log to find errors."
${commands}
WORKING_DIRECTORY ${DOCS_BUILD_DIR}
COMMENT "Preprocess documentation"
VERBATIM)
# added linkcheker
add_custom_target(doxygen_xml
DEPENDS preprocess_docs
COMMAND ${PYTHON_EXECUTABLE} ${REMOVE_XML_SCRIPT} ${XML_OUTPUT}
COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYFILE_BUILD}
WORKING_DIRECTORY ${DOCS_BUILD_DIR}
COMMENT "Generate doxygen XML output"
VERBATIM)
# Post-process docs
add_custom_command(TARGET doxygen_xml
POST_BUILD
COMMAND ${PYTHON_EXECUTABLE} ${PREPARE_XML_SCRIPT} ${XML_OUTPUT}
COMMAND ${CMAKE_COMMAND} -E copy_directory ${DOXYREST_IN} ${DOXYREST_OUT}
COMMAND ${DOXYREST_EXECUTABLE} -c ${DOXYREST_CONFIG_OUT}
COMMAND ${PYTHON_EXECUTABLE} ${COPY_IMAGES_SCRIPT} ${XML_OUTPUT} ${RST_OUTPUT}
COMMAND ${PYTHON_EXECUTABLE} ${DOXYGEN_MAPPING_SCRIPT} ${XML_OUTPUT} ${DOCS_BUILD_DIR} ${OpenVINO_MAIN_SOURCE_DIR}/../
COMMAND ${CMAKE_COMMAND} -E copy ${SPHINX_INDEX_IN} ${SPHINX_INDEX_OUT}
COMMAND ${CMAKE_COMMAND} -E copy_directory ${API_DOCS_IN} ${API_DOCS_OUT}
COMMAND ${CMAKE_COMMAND} -E copy_directory ${DOXYREST_IN} ${DOXYREST_OUT}
COMMAND ${CMAKE_COMMAND} -E copy_directory ${DOXYREST_SPHINX_IN} ${DOXYREST_SPHINX_OUT}
COMMAND ${CMAKE_COMMAND} -E copy_directory ${SPHINX_STATIC_IN} ${SPHINX_STATIC_OUT}
COMMENT "Prepare xml"
VERBATIM)
add_custom_target(sphinx_docs
DEPENDS doxygen_xml
COMMAND sphinx-build -b html ${RST_OUTPUT} ${SPHINX_OUTPUT}
WORKING_DIRECTORY ${RST_OUTPUT}
VERBATIM)
set_target_properties(doxygen_xml sphinx_docs
PROPERTIES FOLDER docs)
if(EXISTS "${LINKCHECKER_PY}")
add_custom_target(docs_check
COMMAND ${PYTHON_EXECUTABLE} "${LINKCHECKER_PY}" -v "${DOCS_BUILD_DIR}/html/"
COMMENT "Check links in generated documentation"
WORKING_DIRECTORY "${DOCS_BUILD_DIR}"
VERBATIM)
set_target_properties(docs_check PROPERTIES FOLDER docs)
endif()
find_program(browser NAMES xdg-open)
if(browser)
add_custom_target(ie_docs_open
COMMAND ${browser} "${OpenVINO_MAIN_SOURCE_DIR}/docs/html/index.html"
DEPENDS ie_docs
COMMAND ${browser} "${SPHINX_OUTPUT}/index.html"
DEPENDS sphinx_docs
COMMENT "Open OpenVINO documentation"
VERBATIM)
set_target_properties(ie_docs_open PROPERTIES FOLDER docs)

43
docs/HOWTO/Custom_Layers_Guide.md
View File

@ -131,14 +131,25 @@ Firstly, open the model in the TensorBoard or other TensorFlow* model visualizat
batch dimension because the value for the batch dimension is not hardcoded in the model. Model Optimizer need to set all
dynamic dimensions to some specific value to create the IR, therefore specify the command line parameter `-b 1` to set
the batch dimension equal to 1. The actual batch size dimension can be changed at runtime using the Inference Engine API
described in the [Using Shape Inference](../IE_DG/ShapeInference.md). Also refer to
[Converting a Model Using General Conversion Parameters](../MO_DG/prepare_model/convert_model/Converting_Model_General.md)
and [Convert Your TensorFlow* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md)
described in the [Using Shape Inference](../IE_DG/ShapeInference.md). Also refer to the General Conversion Parameters section in [Converting a Model to Intermediate Representation (IR)](../MO_DG/prepare_model/convert_model/Converting_Model.md) and [Convert Your TensorFlow* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md)
for more details and command line parameters used for the model conversion.
```bash
./<MO_INSTALL_DIR>/mo.py --input_model <PATH_TO_MODEL>/wnet_20.pb -b 1
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
python3 mo.py --input_model <PATH_TO_MODEL>/wnet_20.pb -b 1
.. tab:: pip installation
.. code-block:: sh
mo --input_model <PATH_TO_MODEL>/wnet_20.pb -b 1
@endsphinxdirective
> **NOTE:** This conversion guide is applicable for the 2021.3 release of OpenVINO and that starting from 2021.4
> the OpenVINO supports this model out of the box.
@ -257,9 +268,21 @@ The implementation should be saved to the file `mo_extensions/front/tf/ComplexAb
@snippet ComplexAbs.py complex_abs:transformation
Now it is possible to convert the model using the following command line:
```bash
./<MO_INSTALL_DIR>/mo.py --input_model <PATH_TO_MODEL>/wnet_20.pb -b 1 --extensions mo_extensions/
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
python3 mo.py --input_model <PATH_TO_MODEL>/wnet_20.pb -b 1 --extensions mo_extensions/
.. tab:: pip installation
.. code-block:: sh
mo --input_model <PATH_TO_MODEL>/wnet_20.pb -b 1 --extensions mo_extensions/
@endsphinxdirective
The sub-graph corresponding to the originally non-supported one is depicted in the image below:
@ -350,7 +373,7 @@ python3 mri_reconstruction_demo.py \
## Converting Models:
- [Convert Your Caffe* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_Caffe.md)
- [Convert Your Kaldi* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md)
- [Convert Your TensorFlow* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md)
- [Convert Your MXNet* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_MxNet.md)
- [Convert Your Kaldi* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md)
- [Convert Your ONNX* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md)

10
docs/IE_DG/Bfloat16Inference.md
View File

@ -1,5 +1,13 @@
# Bfloat16 Inference {#openvino_docs_IE_DG_Bfloat16Inference}
## Bfloat16 Inference Usage (C++)
@sphinxdirective
.. raw:: html
<div id="switcher-cpp" class="switcher-anchor">C++</div>
@endsphinxdirective
## Disclaimer
Inference Engine with the bfloat16 inference implemented on CPU must support the native `avx512_bf16` instruction and therefore the bfloat16 data format.
@ -37,7 +45,7 @@ Bfloat16 data usage provides the following benefits that increase performance:
4. Reduced size of data in memory, as a result, larger models fit in the same memory bounds.
5. Reduced amount of data that must be transferred, as a result, reduced data transition time.
For default optimization on CPU, source model is converted from FP32 or FP16 to BF16 and executed internally on platforms with native BF16 support. In this case, `KEY_ENFORCE_BF16` is set to `YES`.
For default optimization on CPU, source model is converted from FP32 or FP16 to BF16 and executed internally on platforms with native BF16 support. This is done by setting `KEY_ENFORCE_BF16` `YES` in the `PluginConfigParams` for `GetConfig()`.
The code below demonstrates how to check if the key is set:
@snippet snippets/Bfloat16Inference1.cpp part1

146
docs/IE_DG/Deep_Learning_Inference_Engine_DevGuide.md
View File

@ -1,117 +1,57 @@
# Inference Engine Developer Guide {#openvino_docs_IE_DG_Deep_Learning_Inference_Engine_DevGuide}
This Guide provides an overview of the Inference Engine describing the typical workflow for performing inference of a pre-trained and optimized deep learning model and a set of sample applications.
@sphinxdirective
> **NOTE:** Before you perform inference with the Inference Engine, your models should be converted to the Inference Engine format using the Model Optimizer or built directly in runtime using nGraph API. To learn about how to use Model Optimizer, refer to the [Model Optimizer Developer Guide](../MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). To learn about the pre-trained and optimized models delivered with the OpenVINO™ toolkit, refer to [Pre-Trained Models](@ref omz_models_group_intel).
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_IE_DG_Integrate_with_customer_application_new_API
openvino_docs_deployment_optimization_guide_dldt_optimization_guide
openvino_docs_deployment_optimization_guide_dldt_optimization_guide_additional
openvino_docs_IE_DG_API_Changes
openvino_docs_IE_DG_Memory_primitives
openvino_docs_IE_DG_InferenceEngine_QueryAPI
openvino_docs_IE_DG_Model_caching_overview
openvino_docs_IE_DG_Extensibility_DG_Intro
openvino_docs_IE_DG_Migration_CoreAPI
openvino_inference_engine_ie_bridges_python_docs_api_overview
openvino_docs_IE_DG_ONNX_Support
openvino_docs_IE_DG_OnnxImporterTutorial
openvino_docs_IE_DG_DynamicBatching
openvino_docs_IE_DG_ShapeInference
openvino_docs_IE_DG_Bfloat16Inference
openvino_docs_IE_DG_Int8Inference
openvino_docs_IE_DG_network_state_intro
openvino_docs_IE_DG_Known_Issues_Limitations
openvino_docs_IE_DG_Glossary
After you have used the Model Optimizer to create an Intermediate Representation (IR), use the Inference Engine to infer the result for a given input data.
@endsphinxdirective
Inference Engine is a set of C++ libraries providing a common API to deliver inference solutions on the platform of your choice: CPU, GPU, or VPU. Use the Inference Engine API to read the Intermediate Representation, set the input and output formats, and execute the model on devices. While the C++ libraries is the primary implementation, C libraries and Python bindings are also available.
## Introduction
Inference Engine is a set of C++ libraries with C and Python bindings providing a common API to deliver inference solutions on the platform of your choice. Use the Inference Engine API to read the Intermediate Representation (IR), ONNX and execute the model on devices.
For Intel® Distribution of OpenVINO™ toolkit, Inference Engine binaries are delivered within release packages.
Inference Engine uses a plugin architecture. Inference Engine plugin is a software component that contains complete implementation for inference on a certain Intel® hardware device: CPU, GPU, VPU, etc. Each plugin implements the unified API and provides additional hardware-specific APIs.
The scheme below illustrates the typical workflow for deploying a trained deep learning model:
The open source version is available in the [OpenVINO™ toolkit GitHub repository](https://github.com/openvinotoolkit/openvino) and can be built for supported platforms using the <a href="https://github.com/openvinotoolkit/openvino/wiki/BuildingCode">Inference Engine Build Instructions</a>.
![](img/BASIC_FLOW_IE_C.svg)
To learn about how to use the Inference Engine API for your application, see the [Integrating Inference Engine in Your Application](Integrate_with_customer_application_new_API.md) documentation.
\\* _nGraph_ is the internal graph representation in the OpenVINO™ toolkit. Use it to [build a model from source code](https://docs.openvinotoolkit.org/latest/openvino_docs_nGraph_DG_build_function.html).
For complete API Reference, see the [Inference Engine API References](./api_references.html) section.
Inference Engine uses a plugin architecture. Inference Engine plugin is a software component that contains complete implementation for inference on a certain Intel&reg; hardware device: CPU, GPU, VPU, etc. Each plugin implements the unified API and provides additional hardware-specific APIs.
## Video
## Modules in the Inference Engine component
### Core Inference Engine Libraries
@sphinxdirective
Your application must link to the core Inference Engine libraries:
* Linux* OS:
- `libinference_engine.so`, which depends on `libinference_engine_transformations.so`, `libtbb.so`, `libtbbmalloc.so` and `libngraph.so`
* Windows* OS:
- `inference_engine.dll`, which depends on `inference_engine_transformations.dll`, `tbb.dll`, `tbbmalloc.dll` and `ngraph.dll`
* macOS*:
- `libinference_engine.dylib`, which depends on `libinference_engine_transformations.dylib`, `libtbb.dylib`, `libtbbmalloc.dylib` and `libngraph.dylib`
.. list-table::
The required C++ header files are located in the `include` directory.
* - .. raw:: html
This library contains the classes to:
* Create Inference Engine Core object to work with devices and read network (InferenceEngine::Core)
* Manipulate network information (InferenceEngine::CNNNetwork)
* Execute and pass inputs and outputs (InferenceEngine::ExecutableNetwork and InferenceEngine::InferRequest)
### Plugin Libraries to Read a Network Object
Starting from 2020.4 release, Inference Engine introduced a concept of `CNNNetwork` reader plugins. Such plugins can be automatically dynamically loaded by Inference Engine in runtime depending on file format:
* Linux* OS:
- `libinference_engine_ir_reader.so` to read a network from IR
- `libinference_engine_onnx_reader.so` to read a network from ONNX model format
* Windows* OS:
- `inference_engine_ir_reader.dll` to read a network from IR
- `inference_engine_onnx_reader.dll` to read a network from ONNX model format
### Device-Specific Plugin Libraries
For each supported target device, Inference Engine provides a plugin — a DLL/shared library that contains complete implementation for inference on this particular device. The following plugins are available:
| Plugin | Device Type |
| ------- | ----------------------------- |
|CPU | Intel® Xeon® with Intel® AVX2 and AVX512, Intel® Core™ Processors with Intel® AVX2, Intel® Atom® Processors with Intel® SSE |
|GPU | Intel® Processor Graphics, including Intel® HD Graphics and Intel® Iris® Graphics |
|MYRIAD | Intel® Neural Compute Stick 2 powered by the Intel® Movidius™ Myriad™ X |
|GNA | Intel&reg; Speech Enabling Developer Kit, Amazon Alexa* Premium Far-Field Developer Kit, Intel&reg; Pentium&reg; Silver J5005 Processor, Intel&reg; Pentium&reg; Silver N5000 Processor, Intel&reg; Celeron&reg; J4005 Processor, Intel&reg; Celeron&reg; J4105 Processor, Intel&reg; Celeron&reg; Processor N4100, Intel&reg; Celeron&reg; Processor N4000, Intel&reg; Core&trade; i3-8121U Processor, Intel&reg; Core&trade; i7-1065G7 Processor, Intel&reg; Core&trade; i7-1060G7 Processor, Intel&reg; Core&trade; i5-1035G4 Processor, Intel&reg; Core&trade; i5-1035G7 Processor, Intel&reg; Core&trade; i5-1035G1 Processor, Intel&reg; Core&trade; i5-1030G7 Processor, Intel&reg; Core&trade; i5-1030G4 Processor, Intel&reg; Core&trade; i3-1005G1 Processor, Intel&reg; Core&trade; i3-1000G1 Processor, Intel&reg; Core&trade; i3-1000G4 Processor |
|HETERO | Automatic splitting of a network inference between several devices (for example if a device doesn't support certain layers|
|MULTI | Simultaneous inference of the same network on several devices in parallel|
The table below shows the plugin libraries and additional dependencies for Linux, Windows and macOS platforms.
| Plugin | Library name for Linux | Dependency libraries for Linux | Library name for Windows | Dependency libraries for Windows | Library name for macOS | Dependency libraries for macOS |
|--------|-----------------------------|-------------------------------------------------------------|--------------------------|--------------------------------------------------------------------------------------------------------|------------------------------|---------------------------------------------|
| CPU | `libMKLDNNPlugin.so` | `libinference_engine_lp_transformations.so` | `MKLDNNPlugin.dll` | `inference_engine_lp_transformations.dll` | `libMKLDNNPlugin.so` | `inference_engine_lp_transformations.dylib` |
| GPU | `libclDNNPlugin.so` | `libinference_engine_lp_transformations.so`, `libOpenCL.so` | `clDNNPlugin.dll` | `OpenCL.dll`, `inference_engine_lp_transformations.dll` | Is not supported | - |
| MYRIAD | `libmyriadPlugin.so` | `libusb.so`, | `myriadPlugin.dll` | `usb.dll` | `libmyriadPlugin.so` | `libusb.dylib` |
| HDDL | `libHDDLPlugin.so` | `libbsl.so`, `libhddlapi.so`, `libmvnc-hddl.so` | `HDDLPlugin.dll` | `bsl.dll`, `hddlapi.dll`, `json-c.dll`, `libcrypto-1_1-x64.dll`, `libssl-1_1-x64.dll`, `mvnc-hddl.dll` | Is not supported | - |
| GNA | `libGNAPlugin.so` | `libgna.so`, | `GNAPlugin.dll` | `gna.dll` | Is not supported | - |
| HETERO | `libHeteroPlugin.so` | Same as for selected plugins | `HeteroPlugin.dll` | Same as for selected plugins | `libHeteroPlugin.so` | Same as for selected plugins |
| MULTI | `libMultiDevicePlugin.so` | Same as for selected plugins | `MultiDevicePlugin.dll` | Same as for selected plugins | `libMultiDevicePlugin.so` | Same as for selected plugins |
> **NOTE**: All plugin libraries also depend on core Inference Engine libraries.
Make sure those libraries are in your computer's path or in the place you pointed to in the plugin loader. Make sure each plugin's related dependencies are in the:
* Linux: `LD_LIBRARY_PATH`
* Windows: `PATH`
* macOS: `DYLD_LIBRARY_PATH`
On Linux and macOS, use the script `bin/setupvars.sh` to set the environment variables.
On Windows, run the `bin\setupvars.bat` batch file to set the environment variables.
To learn more about supported devices and corresponding plugins, see the [Supported Devices](supported_plugins/Supported_Devices.md) chapter.
## Common Workflow for Using the Inference Engine API
The common workflow contains the following steps:
1. **Create Inference Engine Core object** - Create an `InferenceEngine::Core` object to work with different devices, all device plugins are managed internally by the `Core` object. Register extensions with custom nGraph operations (`InferenceEngine::Core::AddExtension`).
2. **Read the Intermediate Representation** - Using the `InferenceEngine::Core` class, read an Intermediate Representation file into an object of the `InferenceEngine::CNNNetwork` class. This class represents the network in the host memory.
3. **Prepare inputs and outputs format** - After loading the network, specify input and output precision and the layout on the network. For these specification, use the `InferenceEngine::CNNNetwork::getInputsInfo()` and `InferenceEngine::CNNNetwork::getOutputsInfo()`.
4. Pass per device loading configurations specific to this device (`InferenceEngine::Core::SetConfig`), and register extensions to this device (`InferenceEngine::Core::AddExtension`).
5. **Compile and Load Network to device** - Use the `InferenceEngine::Core::LoadNetwork()` method with specific device (e.g. `CPU`, `GPU`, etc.) to compile and load the network on the device. Pass in the per-target load configuration for this compilation and load operation.
6. **Set input data** - With the network loaded, you have an `InferenceEngine::ExecutableNetwork` object. Use this object to create an `InferenceEngine::InferRequest` in which you signal the input buffers to use for input and output. Specify a device-allocated memory and copy it into the device memory directly, or tell the device to use your application memory to save a copy.
7. **Execute** - With the input and output memory now defined, choose your execution mode:
* Synchronously - `InferenceEngine::InferRequest::Infer()` method. Blocks until inference is completed.
* Asynchronously - `InferenceEngine::InferRequest::StartAsync()` method. Check status with the `InferenceEngine::InferRequest::Wait()` method (0 timeout), wait, or specify a completion callback.
8. **Get the output** - After inference is completed, get the output memory or read the memory you provided earlier. Do this with the `InferenceEngine::IInferRequest::GetBlob()` method.
## Video: Inference Engine Concept
<iframe width="560" height="315" src="https://www.youtube.com/embed/e6R13V8nbak" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
## Further Reading
For more details on the Inference Engine API, refer to the [Integrating Inference Engine in Your Application](Integrate_with_customer_application_new_API.md) documentation.
<iframe height="315" width="100%"
src="https://www.youtube.com/embed/e6R13V8nbak">
</iframe>
* - **Inference Engine Concept**. Duration: 3:43
@endsphinxdirective

24
docs/IE_DG/DynamicBatching.md
View File

@ -1,12 +1,18 @@
Using Dynamic Batching {#openvino_docs_IE_DG_DynamicBatching}
======================
# Using Dynamic Batching {#openvino_docs_IE_DG_DynamicBatching}
Dynamic Batching feature allows you to dynamically change batch size for inference calls
within preset batch size limit.
This feature might be useful when batch size is unknown beforehand, and using extra large batch size is
undesired or impossible due to resource limitations.
For example, face detection with person age, gender, or mood recognition is a typical usage scenario.
## Dynamic Batching Introduction (C++)
@sphinxdirective
.. raw:: html
<div id="switcher-cpp" class="switcher-anchor">C++</div>
@endsphinxdirective
The Dynamic Batching feature allows you to dynamically change batch size for inference calls
within a preset batch size limit.
This feature might be useful when batch size is unknown beforehand and using an extra-large batch size is
undesirable or impossible due to resource limitations.
For example, applying face detection and then mood labeling to a video, you won't know in advance how many frames will contain a face when you pass inferencing results to a secondary model.
## Usage
@ -14,7 +20,7 @@ You can activate Dynamic Batching by setting <code>KEY_DYN_BATCH_ENABLED</code>
passed to the plugin while loading a network.
This configuration creates an <code>ExecutableNetwork</code> object that will allow setting batch size
dynamically in all of its infer requests using <code>SetBatch()</code> method.
The batch size that was set in passed <code>CNNNetwork</code> object will be used as a maximum batch size limit.
The batch size that was set in the passed <code>CNNNetwork</code> object will be used as a maximum batch size limit.
Here is a code example:
@ -23,7 +29,7 @@ Here is a code example:
## Limitations
Currently, certain limitations for using Dynamic Batching exist:
Currently, there are certain limitations for using Dynamic Batching:
* Use Dynamic Batching with CPU and GPU plugins only.

17
docs/IE_DG/Extensibility_DG/AddingNGraphOps.md
View File

@ -1,6 +1,8 @@
# Custom nGraph Operation {#openvino_docs_IE_DG_Extensibility_DG_AddingNGraphOps}
# Custom nGraph Operations {#openvino_docs_IE_DG_Extensibility_DG_AddingNGraphOps}
The Inference Engine Extension API allows you to register operation sets (opsets) with custom nGraph operations to support models with operations that OpenVINO™ does not support out-of-the-box.
Inference Engine Extension API allows you to register operation sets (opsets) with custom nGraph operations to support models with operations which OpenVINO™ does not support out-of-the-box.
Besides creating custom nGraph operations, to [support custom operations](../../HOWTO/Custom_Layers_Guide.md) in your model you must also create a Model Optimizer extension for the custom operations and an Inference Engine device plugin extension for the device you will use for inference.
## Operation Class
@ -26,8 +28,8 @@ Based on that, declaration of an operation class can look as follows:
The provided implementation has several fields:
* `add` of type `int64_t` is an attribute of a custom operation.
* `type_info` of type `ngraph::NodeTypeInfo` defines the type and version of an operation.
* `add` of type `int64_t` is an attribute of a custom operation
* `type_info` of type `ngraph::NodeTypeInfo` defines type and version of an operation
### Operation Constructors
@ -67,14 +69,13 @@ To add custom operations to the [Extension](Extension.md) class, create an opera
@snippet template_extension/extension.cpp extension:getOpSets
This method returns a map of opsets that exist in the extension library.
nGraph provides an opset mechanism to group operations into clusters. S. Different opsets distinguish between different versions of one operation.
This method returns a map of opsets that exist in the [extension library](Extension.md).
nGraph provides an opset mechanism to group operations into clusters. Different opsets distinguish between different versions of one operation.
When specifying opset names, follow the rules below:
* Use unique opset names.
* Do not use the following built-in opset names: `extension`, `experimental`, `opset1`, `opset2`, `opset3`, ... , `opsetN`.
* Make sure that the Model Optimizer and your extension use the same opset names.
* [Make sure that the Model Optimizer](../../HOWTO/Custom_Layers_Guide.md) and your extension use the same opset names.
* IR v10 operations have the mandatory `version` attribute specifying the opset.
Operations from the default opset cannot be redefined.

4
docs/IE_DG/Extensibility_DG/Building.md
View File

@ -2,13 +2,13 @@
Inference Engine build infrastructure provides the Inference Engine Package for application development.
To build an extension library, use the following CMake script:
To configure the build of your extension library, use the following CMake script:
@snippet template_extension/CMakeLists.txt cmake:extension
This CMake script finds the Inference Engine and nGraph using the `find_package` CMake command.
To build an extension library, run the commands below:
To build the extension library, run the commands below:
```sh
$ cd template_extension

16
docs/IE_DG/Extensibility_DG/CPU_Kernel.md
View File

@ -1,6 +1,8 @@
# How to Implement Custom CPU Operations {#openvino_docs_IE_DG_Extensibility_DG_CPU_Kernel}
# CPU Kernel Custom Operations {#openvino_docs_IE_DG_Extensibility_DG_CPU_Kernel}
The primary means of the performance of the CPU codepath in the Inference Engine is the Intel® Math Kernel Library for Deep Neural Networks (Intel® MKL-DNN), and new CPU kernels extend the Inference Engine plugin for the Intel MKL-DNN. Implementing the InferenceEngine::ILayerExecImpl defines a general CPU-side extension. There are no Intel MKL-DNN specifics in the way you need to implement a kernel.
To enable operations not supported by OpenVINO™ out of the box, you need a custom extension for Model Optimizer, a custom nGraph operation set, and a custom kernel for the device you will target. This page describes custom kernel support for the CPU device.
The primary means of the performance of the CPU codepath in the Inference Engine is the Intel® Math Kernel Library for Deep Neural Networks (Intel® MKL-DNN), and new CPU kernels extend the Inference Engine plugin for the Intel MKL-DNN. Implementing the InferenceEngine::ILayerExecImpl API call defines a general CPU-side extension. There are no Intel MKL-DNN specifics in the way you need to implement a kernel.
## Implementation Class
@ -20,31 +22,32 @@ The provided implementation has several fields:
### Constructor of Implementation
An implementation constructor checks parameters of an nGraph operation, stores required attributes, and stores an error message in the case of an error.
An implementation constructor checks parameters of an nGraph operation, stores required attributes, and stores an error message in case of an error.
@snippet template_extension/cpu_kernel.cpp cpu_implementation:ctor
### `getSupportedConfigurations`
InferenceEngine::ILayerExecImpl::getSupportedConfigurations method returns all supported configuration formats (input/output tensor layouts) for your implementation. To specify formats of data, use InferenceEngine::TensorDesc. Refer to the [Memory Primitives](../Memory_primitives.md) section for instructions.
The InferenceEngine::ILayerExecImpl::getSupportedConfigurations method returns all supported configuration formats (input/output tensor layouts) for your implementation. To specify formats of data, use InferenceEngine::TensorDesc. Refer to the [Memory Primitives](../Memory_primitives.md) section for instructions.
@snippet template_extension/cpu_kernel.cpp cpu_implementation:getSupportedConfigurations
### `init`
InferenceEngine::ILayerExecImpl::init method gets a runtime-selected configuration from a vector that is populated from the `getSupportedConfigurations` method and checks the parameters:
The InferenceEngine::ILayerExecImpl::init method gets a runtime-selected configuration from a vector that is populated from the `getSupportedConfigurations` method and checks the parameters:
@snippet template_extension/cpu_kernel.cpp cpu_implementation:init
### `execute`
InferenceEngine::ILayerExecImpl::execute method accepts and processes the actual tenors as input/output blobs:
The InferenceEngine::ILayerExecImpl::execute method accepts and processes the actual tensors as input/output blobs:
@snippet template_extension/cpu_kernel.cpp cpu_implementation:execute
## Register Implementation in `Extension` Class
To register custom kernel implementation in the [Extension](Extension.md) class, implement the following methods:
* <a href="#getImpTypes">getImplTypes</a>
* <a href="#getImplementation">getImplementation</a>
@ -66,4 +69,3 @@ InferenceEngine::IExtension::getImplementation returns the kernel implementation
Use the `AddExtension` method of the general plugin interface to load your primitives:
@snippet snippets/CPU_Kernel.cpp part0

21
docs/IE_DG/Extensibility_DG/Custom_ONNX_Ops.md
View File

@ -1,7 +1,7 @@
# Custom ONNX* Operators {#openvino_docs_IE_DG_Extensibility_DG_Custom_ONNX_Ops}
The ONNX\* importer provides a mechanism to register custom ONNX operators based on predefined or custom nGraph operations.
The function responsible for registering a new operator is called `ngraph::onnx_import::register_operator` and is defined in `onnx_import/onnx_utils.hpp`.
The function responsible for registering a new operator is called `ngraph::onnx_import::register_operator` and is defined in [`onnx_import/onnx_utils.hpp`](https://docs.openvinotoolkit.org/latest/ngraph_cpp_api/onnx__utils_8hpp_source.html).
## Register Custom ONNX Operator Based on Predefined nGraph Operations
@ -14,18 +14,22 @@ x < 0 => f(x) = x * beta
where `alpha` and `beta` are float constants.
1. Include headers:
@snippet onnx_custom_op/onnx_custom_op.cpp onnx_custom_op:headers
2. Register the CustomRelu operator in the ONNX importer:
@snippet onnx_custom_op/onnx_custom_op.cpp onnx_custom_op:register_operator
The `register_operator` function takes four arguments: op_type, opset version, domain, and a function object.
The function object is a user-defined function that takes `ngraph::onnx_import::Node` as an input and based on that, returns a graph with nGraph operations.
The `ngraph::onnx_import::Node` class represents a node in an ONNX model. It provides functions to fetch input node(s) using `get_ng_inputs`, attribute value using `get_attribute_value`, and many more. See `onnx_import/core/node.hpp` for full class declaration.
The `ngraph::onnx_import::Node` class represents a node in an ONNX model. It provides functions to fetch input node(s) using `get_ng_inputs`, attribute value using `get_attribute_value`, and many more. See [`onnx_import/core/node.hpp`](https://docs.openvinotoolkit.org/latest/ngraph_cpp_api/core_2include_2ngraph_2node_8hpp_source.html) for full class declaration.
New operator registration must happen before an ONNX model is read. For example, if an model uses the `CustomRelu` operator, call `register_operator("CustomRelu", ...)` before InferenceEngine::Core::ReadNetwork.
Reregistering ONNX operators within the same process is supported. If you register an existing operator, you get a warning.
The example below demonstrates an exemplary model that requires a previously created `CustomRelu` operator:
@snippet onnx_custom_op/onnx_custom_op.cpp onnx_custom_op:model
@ -33,27 +37,30 @@ To create a graph with nGraph operations, visit [Custom nGraph Operations](Addin
For a complete list of predefined nGraph operators, visit [Available Operations Sets](../../ops/opset.md).
If you do not need an operator anymore, unregister it by calling `unregister_operator`. The function takes three arguments: `op_type`, `version`, and `domain`.
@snippet onnx_custom_op/onnx_custom_op.cpp onnx_custom_op:unregister_operator
## Register Custom ONNX Operator Based on Custom nGraph Operations
The same principles apply when registering a custom ONNX operator based on custom nGraph operations.
This example shows how to register a custom ONNX operator based on `Operation` presented in [this tutorial](AddingNGraphOps.md), which is used in [TemplateExtension](Extension.md).
This example shows how to register a custom ONNX operator based on `Operation` presented in [this tutorial](AddingNGraphOps.md), which is used in [TemplateExtension](Extension.md):
@snippet template_extension/extension.cpp extension:ctor
Here, the `register_operator` function is called in the constructor of Extension. The constructor makes sure that the function is called before InferenceEngine::Core::ReadNetwork, because InferenceEngine::Core::AddExtension must be called before a model with a custom operator is read.
The example below demonstrates how to unregister an operator from the destructor of Extension:
@snippet template_extension/extension.cpp extension:dtor
> **REQUIRED**: It is mandatory to unregister a custom ONNX operator if it is defined in a dynamic shared library.
## Requirements for Building with CMake
A program that uses the `register_operator` functionality requires `ngraph` and `onnx_importer` libraries in addition to the Inference Engine.
The `onnx_importer` is a component of the `ngraph` package , so `find_package(ngraph REQUIRED COMPONENTS onnx_importer)` can find both.
The `ngraph` package exposes two variables, `${NGRAPH_LIBRARIES}` and `${ONNX_IMPORTER_LIBRARIES}`, which reference the `ngraph` and `onnx_importer` libraries.
Those variables need to be passed to the `target_link_libraries` command in the CMakeLists.txt file.
A program that uses the `register_operator` functionality requires `ngraph::ngraph` and `ngraph::onnx_ngraph_frontend` libraries in addition to the Inference Engine.
The `onnx_ngraph_frontend` is a component of the `ngraph` package, so `find_package(ngraph REQUIRED COMPONENTS onnx_ngraph_frontend)` can find both.
Those libraries need to be passed to the `target_link_libraries` command in the CMakeLists.txt file.
See CMakeLists.txt below for reference:
@snippet onnx_custom_op/CMakeLists.txt cmake:onnx_custom_op

5
docs/IE_DG/Extensibility_DG/Extension.md
View File

@ -25,6 +25,5 @@ Also, an `Extension` object should implement the following methods:
Implement the InferenceEngine::IExtension::getOpSets method if the extension contains custom layers.
Read [Custom nGraph Operation](AddingNGraphOps.md) for more information.
To understand how to integrate execution kernels to the extension library, read the [documentation about development of custom CPU kernels](CPU_Kernel.md).
To understand how to register custom ONNX operator to the extension library, read the [documentation about custom ONNX operators](Custom_ONNX_Ops.md).
To integrate execution kernels to the extension library, read [How to Implement Custom CPU Operations](CPU_Kernel.md).
To register a custom ONNX\* operator to the extension library, read [Custom ONNX Operators](Custom_ONNX_Ops.md).

21
docs/IE_DG/Extensibility_DG/GPU_Kernel.md
View File

@ -1,15 +1,17 @@
# How to Implement Custom GPU Operations {#openvino_docs_IE_DG_Extensibility_DG_GPU_Kernel}
The GPU codepath abstracts many details about OpenCL\*. You need to provide the kernel code in OpenCL C and the configuration file that connects the kernel and its parameters to the parameters of the operation.
To enable operations not supported by OpenVINO™ out of the box, you need a custom extension for Model Optimizer, a custom nGraph operation set, and a custom kernel for the device you will target. This page describes custom kernel support for the GPU device.
There are two options of using the custom operation configuration file:
The GPU codepath abstracts many details about OpenCL\*. You need to provide the kernel code in OpenCL C and an XML configuration file that connects the kernel and its parameters to the parameters of the operation.
There are two options for using the custom operation configuration file:
* Include a section with your kernels into the global automatically-loaded `cldnn_global_custom_kernels/cldnn_global_custom_kernels.xml` file, which is hosted in the `<INSTALL_DIR>/deployment_tools/inference_engine/bin/intel64/{Debug/Release}` folder
* Call the `InferenceEngine::Core::SetConfig()` method from your application with the `InferenceEngine::PluginConfigParams::KEY_CONFIG_FILE` key and the configuration file name as a value before loading the network that uses custom operations to the plugin:
@snippet snippets/GPU_Kernel.cpp part0
All Inference Engine samples, except the trivial `hello_classification`,
All Inference Engine samples, except the trivial `hello_classification`, and most Open Model Zoo demos
feature a dedicated command-line option `-c` to load custom kernels. For example, to load custom operations for the classification sample, run the command below:
```sh
$ ./classification_sample -m <path_to_model>/bvlc_alexnet_fp16.xml -i ./validation_set/daily/227x227/apron.bmp -d GPU
@ -132,8 +134,8 @@ queuing an OpenCL program for execution.
## Example Configuration File
The following code sample provides an example configuration file in the
`.xml` format. For information on the configuration file structure, see
The following code sample provides an example configuration file in XML
format. For information on the configuration file structure, see
[Configuration File Format](#config-file-format).
```xml
<CustomLayer name="ReLU" type="SimpleGPU" version="1">
@ -208,12 +210,12 @@ __kernel void example_relu_kernel(
}
```
> **NOTE:** As described in the previous section, all things like
> **NOTE:** As described in the previous section, all items like
> `INPUT0_TYPE` are actually defined as OpenCL (pre-)compiler inputs by
> the Inference Engine for efficiency reasons. See [Debugging
> Tips](#debugging-tips) for information on debugging the results.
> **NOTE**: Several GPU-targeted kernels are also added to the binaries upon samples compilation
> **NOTE**: Several GPU-targeted kernels are also added to the binaries upon compilation of samples
> so that the sample application can easy load them.
> Refer to the `cldnn_global_custom_kernels` folder in the GPU plugin installation directory.
@ -221,10 +223,11 @@ __kernel void example_relu_kernel(
* **Using `printf` in the OpenCL™ Kernels**.
To debug the specific values, you can use `printf` in your kernels.
However, be careful: for instance, do not output excessively
as it would generate too much data. The `printf` output is typical, so
However, be careful not to output excessively, which
could generate too much data. The `printf` output is typical, so
your output can be truncated to fit the buffer. Also, because of
buffering, you actually get an entire buffer of output when the
execution ends.<br>
For more information, refer to the [printf
Function](https://www.khronos.org/registry/OpenCL/sdk/1.2/docs/man/xhtml/printfFunction.html).

22
docs/IE_DG/Extensibility_DG/Intro.md
View File

@ -1,27 +1,23 @@
# Inference Engine Extensibility Mechanism {#openvino_docs_IE_DG_Extensibility_DG_Intro}
Inference Engine Extensibility API enables you to add support of custom operations to the Inference Engine.
Extension should contain operation sets with custom operations and execution kernels for custom operations.
Physically, an extension library can be represented as a dynamic library exporting the single `CreateExtension` function
that creates a new extension instance.
If your model contains operations not normally supported by OpenVINO, the Inference Engine Extensibility API lets you add support for those custom operations in a library containing custom nGraph operation sets, corresponding extensions to the Model Optimizer, and a device plugin extension. See the overview in the [Custom Operations Guide](../../HOWTO/Custom_Layers_Guide.md) to learn how these work together.
To load the Extensibility library to the `InferenceEngine::Core` object, use the
`InferenceEngine::Core::AddExtension` method.
To load the Extensibility library to the `InferenceEngine::Core` object, use the `InferenceEngine::Core::AddExtension` method.
## Inference Engine Extension Library
Inference Engine Extension dynamic library contains the following components:
An Inference Engine Extension dynamic library contains the following components:
* [Extension Library](Extension.md):
- Contains custom operation sets.
- Provides CPU implementations for custom operations.
- Contains custom operation sets
- Provides CPU implementations for custom operations
* [Custom nGraph Operation](AddingNGraphOps.md):
- Enables the use of `InferenceEngine::Core::ReadNetwork` to read Intermediate Representation (IR) with unsupported
operations.
- Enables the creation of `ngraph::Function` with unsupported operations.
- Provides a shape inference mechanism for custom operations.
operations
- Enables the creation of `ngraph::Function` with unsupported operations
- Provides a shape inference mechanism for custom operations
> **NOTE**: This documentation is written based on the `Template extension`, which demonstrates extension development details. Find the complete code of the `Template extension`, which is fully compilable and up-to-date, at `<dldt source tree>/docs/template_extension`.
> **NOTE**: This documentation is written based on the [`Template extension`](https://github.com/openvinotoolkit/openvino/tree/master/docs/template_extension), which demonstrates extension development details. You can review the complete code, which is fully compilable and up-to-date, to see how it works.
## Execution Kernels

40
docs/IE_DG/Extensibility_DG/VPU_Kernel.md
View File

@ -1,10 +1,12 @@
# How to Implement Custom Layers for VPU (Intel® Neural Compute Stick 2) {#openvino_docs_IE_DG_Extensibility_DG_VPU_Kernel}
To enable operations not supported by OpenVINO™ out of the box, you need a custom extension for Model Optimizer, a custom nGraph operation set, and a custom kernel for the device you will target. This page describes custom kernel support for one the VPU, the Intel® Neural Compute Stick 2 device, which uses the MYRIAD device plugin.
> **NOTES:**
> * OpenCL\* custom layer support is available in the preview mode.
> * This section assumes you are familiar with developing kernels using OpenCL.
To customize your topology with an OpenCL layer, follow the steps below:
To customize your topology with an OpenCL layer, carry out the tasks described on this page:
1. Write and compile your OpenCL code with the standalone offline OpenCL compiler (`clc`).
2. Write a configuration file to bind the OpenCL kernel to the topology file (`.xml`) of the model IR.
@ -13,11 +15,11 @@ To customize your topology with an OpenCL layer, follow the steps below:
## Compile OpenCL code for VPU (Intel® Neural Compute Stick 2)
> **NOTE:** OpenCL compiler, targeting Intel® Neural Compute Stick 2 for the SHAVE* processor only, is redistributed with OpenVINO.
OpenCL support is provided by ComputeAorta*, and is distributed under a license agreement between Intel® and Codeplay* Software Ltd.
OpenCL support is provided by ComputeAorta* and is distributed under a license agreement between Intel® and Codeplay* Software Ltd.
The OpenCL toolchain for the Intel® Neural Compute Stick 2 supports offline compilation only, so first compile OpenCL C code using the standalone `clc` compiler. You can find the compiler binary at `<INSTALL_DIR>/deployment_tools/tools/cl_compiler`.
> **NOTE:** By design, custom OpenCL layers support any OpenCL kernels written with 1.2 version assumed. It also supports half float extension and is optimized for this type, because it is a native type for Intel® Movidius™ VPUs.
> **NOTE:** By design, custom OpenCL layers support any OpenCL kernels written assuming OpenCL version 1.2. It also supports half float extension and is optimized for this type, because it is a native type for Intel® Movidius™ VPUs.
1. Prior to running a compilation, make sure that the following variables are set:
* `SHAVE_MA2X8XLIBS_DIR=<INSTALL_DIR>/deployment_tools/tools/cl_compiler/lib/`
@ -25,19 +27,19 @@ The OpenCL toolchain for the Intel® Neural Compute Stick 2 supports offline com
* `SHAVE_MYRIAD_LD_DIR=<INSTALL_DIR>/deployment_tools/tools/cl_compiler/bin/`
* `SHAVE_MOVIASM_DIR=<INSTALL_DIR>/deployment_tools/tools/cl_compiler/bin/`
2. Run the compilation with the command below. You should use `--strip-binary-header` to make an OpenCL runtime-agnostic binary runnable with the Inference Engine.
```bash
cd <INSTALL_DIR>/deployment_tools/tools/cl_compiler/bin
./clc --strip-binary-header custom_layer.cl -o custom_layer.bin
```
```bash
cd <INSTALL_DIR>/deployment_tools/tools/cl_compiler/bin
./clc --strip-binary-header custom_layer.cl -o custom_layer.bin
```
## Write a Configuration File
To tie the topology IR for a layer you customize, prepare a configuration file, so that the Inference Engine can find parameters for your kernel and the execution work grid is described.
For example, given the following OpenCL kernel signature:
For example, consider the following OpenCL kernel signature:
```cpp
__kernel void reorg_nhwc(__global const half *src, __global half *out, int w, int h, int c, int stride);
```
Configuration file for this kernel might be the following:
A configuration file for this kernel might be the following:
```xml
<CustomLayer name="ReorgYolo" type="MVCL" version="1">
<Kernel entry="reorg_nhwc">
@ -62,7 +64,7 @@ Each custom layer is described with the `CustomLayer` node. It has the following
- Sub-node `Kernel` must contain the following attributes:
- `entry` – The name of your kernel function as you defined it in a source file. In the example above, it is `reorg_nhwc`.
- Node `Source` must contain the following attributes:
- `filename` – The path to a compiled binary relative to the `.xml` binding file.
- `filename` – The path to a compiled binary relative to the XML configuration file.
- Sub-node `Parameters` – Describes parameters bindings. For more information, see the description below.
- Sub-node `WorkSizes` – Describes local and global work group sizes and the source for dimension deduction as a pair `direction,port`. In the example above, the work group is described relatively to the dimension of the input tensor that comes through port 0 in the IR. `global` and `local` work group configurations support any simple math expressions with +,-,\*,/, and () from `B`(batch), `Y`(height), `X`(width) and `F`(channels).
- Sub-node `Where` – Allows to customize bindings with the `key="value"` attribute. For example, to substitute only 3x3 convolutions, write `<Where kernel="3,3"/>` in the binding xml.
@ -70,8 +72,8 @@ Each custom layer is described with the `CustomLayer` node. It has the following
Parameter description supports `Tensor` of one of tensor types such as `input`, `output`, `input_buffer`, `output_buffer` or `data`, `Scalar`, or `Data` nodes and has the following format:
- Each `Tensor` node of `input` or `output` type must contain the following attributes:
- `arg-name` – The name of a kernel parameter in the kernel signature.
- `type` – Node type: `input` or `output` as in the IR.
- `port-index` – A number of input/output ports as in the IR.
- `type` – Node type: `input` or `output` as specified in the IR.
- `port-index` – A number of input/output ports as specified in the IR.
- `format` – The channel order in the tensor. Optional conversion layers are generated if the custom layer format is not compatible with formats of neighboring layers. `BFXY`, `BYXF`, and `ANY` formats are supported currently.
- Each `Tensor` node of `input_buffer` or `output_buffer` type must contain the following attributes:
- `arg-name` – The name of a kernel parameter in the kernel signature.
@ -417,7 +419,7 @@ This decreases the execution time up to 40% against the best performing vectoriz
stalls completely on memory access without any prefetch. The same recommendation is applicable for scalar load/store
from/to a `__blobal` pointer since work-group copying could be done in a vector fashion.
10. Use a manual DMA extension. Local (on-chip) memory throughput is up to 24x higher than DDR throughput. Starting from OpenVINO™ 2020.1, VPU OpenCL features manual-DMA kernel extension to copy sub-tensor used by work group into local memory and performing compute without DDR evolved. Here is the simple GRN kernel implementation that runs over DDR. Local size is equal to (width of the input tensor, 1, 1) to define a large enough work group to get code automatically vectorized and unrolled, while global size is (width of the input tensor, height of the input tensor, 1):
10. Use a manual DMA extension. Local (on-chip) memory throughput is up to 24x higher than DDR throughput. Starting from OpenVINO™ 2020.1, VPU OpenCL features manual-DMA kernel extension to copy sub-tensor used by work group into local memory and performing compute without DDR evolved. Here is the simple GRN kernel implementation that runs over DDR. Local size is in the form (width of the input tensor, 1, 1) to define a large enough work group to get code automatically vectorized and unrolled, while global size is (width of the input tensor, height of the input tensor, 1):
```cpp
__kernel void grn_NCHW(
__global const half* restrict src_data,
@ -444,7 +446,9 @@ from/to a `__blobal` pointer since work-group copying could be done in a vector
}
}
```
This kernel can be rewritten to introduce special data binding `__dma_preload` and `__dma_postwrite intrinsics`. This means that instead of one kernel, a group of three kernels should be implemented: `kernelName`, `__dma_preload_kernelName`, and `__dma_postwrite_kernelName`. `__dma_preload_kernelName` for a particular work group `n` is guaranteed to be executed before the `n`-th work group itself, while `__dma_postwrite_kernelName` is guaranteed to be executed after a corresponding work group. You can define one of those functions that are intended to be used to copy data from-to `__global` and `__local` memory. The syntactics requires exact functional signature match. The example below illustrates how to prepare your kernel for manual-DMA.
```cpp
__kernel void __dma_preload_grn_NCHW(
__global const half* restrict src,
@ -453,7 +457,7 @@ This kernel can be rewritten to introduce special data binding `__dma_preload` a
__local half* restrict local_dst,
int C,
float bias)
{
{
// ToDO: copy required piece of src tensor into local_src
}
@ -478,9 +482,9 @@ This kernel can be rewritten to introduce special data binding `__dma_preload` a
{
// same as the example above
}
```
GRN kernel operates on channel-major tensors to compute average over full channel range and then normalizes input elements to produce the output.
As a part of manual DMA extension, a group of work group copy functions are introduced in addition to `async_work_group_copy`, which is also mapped to DMA call.
```
The GRN kernel operates on channel-major tensors to compute average over full channel range and then normalizes input elements to produce the output.
As a part of the manual DMA extension, a group of work group copy functions are introduced in addition to `async_work_group_copy`, which is also mapped to a DMA call.
Here is the list of supported functions:
```cpp
@ -613,7 +617,7 @@ __kernel void grn_NCHW(
Note the `get_local_size` and `get_local_id` usage inside the kernel. 21x speedup is expected for a kernel on enet-curbs setup because it was completely limited by memory usage.
An alternative method of using DMA is to use work item copy extension. Those functions are executed inside a kernel and requires work groups equal to single work item.
An alternative method to using DMA is to use work item copy extension. Those functions are executed inside a kernel and requires work groups equal to single work item.
Here is the list of supported work item functions:
```cpp

33
docs/IE_DG/InferenceEngine_QueryAPI.md
View File

@ -1,12 +1,20 @@
Introduction to Inference Engine Device Query API {#openvino_docs_IE_DG_InferenceEngine_QueryAPI}
===============================
# Introduction to Inference Engine Device Query API {#openvino_docs_IE_DG_InferenceEngine_QueryAPI}
This section provides a high-level description of the process of querying of different device properties and configuration values.
Refer to the [Hello Query Device Sample](../../inference-engine/samples/hello_query_device/README.md) sources and [Multi-Device Plugin guide](supported_plugins/MULTI.md) for example of using the Inference Engine Query API in user applications.
## The Inference Engine Query API (C++)
@sphinxdirective
.. raw:: html
<div id="switcher-cpp" class="switcher-anchor">C++</div>
@endsphinxdirective
The OpenVINO™ toolkit supports inferencing with several types of devices (processors or accelerators).
This page provides a high-level description of the process of querying device properties and configuration values at runtime.
Refer to the [Hello Query Device Sample](../../inference-engine/samples/hello_query_device/README.md) sources and [Multi-Device Plugin guide](supported_plugins/MULTI.md) for examples of using the Inference Engine Query API in user applications.
## Using the Inference Engine Query API in Your Code
The Inference Engine `Core` class provides the following API to query device information, set or get different device configuration properties:
The Inference Engine `Core` class provides the following [API](https://docs.openvinotoolkit.org/latest/classInferenceEngine_1_1Core.html) to query device information, set or get different device configuration properties:
* <code>InferenceEngine::Core::GetAvailableDevices</code> - Provides a list of available devices. If there are more than one instance of a specific device, the devices are enumerated with `.suffix` where `suffix` is a unique string identifier. The device name can be passed to all methods of the `InferenceEngine::Core` class that work with devices, for example `InferenceEngine::Core::LoadNetwork`.
* <code>InferenceEngine::Core::GetMetric</code> - Provides information about specific device.
@ -26,6 +34,7 @@ The `InferenceEngine::ExecutableNetwork` class is also extended to support the Q
@snippet snippets/InferenceEngine_QueryAPI0.cpp part0
The function returns list of available devices, for example:
```
MYRIAD.1.2-ma2480
MYRIAD.1.4-ma2480
@ -39,13 +48,13 @@ GPU.1
Each device name can then be passed to:
* `InferenceEngine::Core::LoadNetwork` to load the network to a specific device.
* `InferenceEngine::Core::GetMetric` to get common or device specific metrics.
* All other methods of the `Core` class that accept `deviceName`.
* `InferenceEngine::Core::LoadNetwork` to load the network to a specific device
* `InferenceEngine::Core::GetMetric` to get common or device specific metrics
* All other methods of the `Core` class that accept `deviceName`
### GetConfig()
The code below demonstrates how to understand whether `HETERO` device dumps `.dot` files with split graphs during the split stage:
The code below demonstrates how to understand whether the `HETERO` device dumps GraphViz `.dot` files with split graphs during the split stage:
@snippet snippets/InferenceEngine_QueryAPI1.cpp part1
@ -59,17 +68,17 @@ For documentation about common configuration keys, refer to `ie_plugin_config.hp
A returned value appears as follows: `Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz`.
> **NOTE**: All metrics have specific type, which is specified during metric instantiation. The list of common device-agnostic metrics can be found in `ie_plugin_config.hpp`. Device specific metrics (for example, for `HDDL`, `MYRIAD` devices) can be found in corresponding plugin folders.
> **NOTE**: All metrics have a type, which is specified during metric instantiation. The list of common device-agnostic metrics can be found in `ie_plugin_config.hpp`. Device specific metrics (for example, for HDDL or MYRIAD devices) can be found in corresponding plugin folders.
## Query API in the ExecutableNetwork Class
### GetMetric()
The method is used to get executable network specific metric such as `METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)`:
The method is used to get an executable network specific metric such as `METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)`:
@snippet snippets/InferenceEngine_QueryAPI3.cpp part3
Or the current temperature of `MYRIAD` device:
Or the current temperature of the `MYRIAD` device:
@snippet snippets/InferenceEngine_QueryAPI4.cpp part4

75
docs/IE_DG/Int8Inference.md
View File

@ -1,6 +1,6 @@
# Low-Precision 8-bit Integer Inference {#openvino_docs_IE_DG_Int8Inference}
## Supported devices
## Disclaimer
Low-precision 8-bit inference is optimized for:
- Intel® architecture processors with the following instruction set architecture extensions:
@ -11,43 +11,54 @@ Low-precision 8-bit inference is optimized for:
- Intel® processor graphics:
- Intel® Iris® Xe Graphics
- Intel® Iris® Xe MAX Graphics
- A model must be quantized. You can use a quantized model from [OpenVINO™ Toolkit Intel's Pre-Trained Models](@ref omz_models_group_intel) or quantize a model yourself. For quantization, you can use the:
- [Post-Training Optimization Tool](@ref pot_docs_LowPrecisionOptimizationGuide) delivered with the Intel® Distribution of OpenVINO™ toolkit release package.
- [Neural Network Compression Framework](https://www.intel.com/content/www/us/en/artificial-intelligence/posts/openvino-nncf.html) available on GitHub: https://github.com/openvinotoolkit/nncf
## Low-Precision 8-bit Integer Inference Workflow
## Introduction
8-bit computations (referred to as `int8`) offer better performance compared to the results of inference in higher precision (for example, `fp32`), because they allow loading more data into a single processor instruction. Usually the cost for significant boost is reduced accuracy. However, it is proved that an accuracy drop can be negligible and depends on task requirements, so that the application engineer can set up the maximum accuracy drop that is acceptable.
For 8-bit integer computation, a model must be quantized. You can use a quantized model from [OpenVINO™ Toolkit Intel's Pre-Trained Models](@ref omz_models_group_intel) or quantize a model yourself. For quantization, you can use the following:
- [Post-Training Optimization Tool](@ref pot_docs_LowPrecisionOptimizationGuide) delivered with the Intel® Distribution of OpenVINO™ toolkit release package
- [Neural Network Compression Framework](https://www.intel.com/content/www/us/en/artificial-intelligence/posts/openvino-nncf.html) available on GitHub: https://github.com/openvinotoolkit/nncf
For 8-bit integer computations, a model must be quantized. Quantized models can be downloaded from [Overview of OpenVINO™ Toolkit Intel's Pre-Trained Models](@ref omz_models_group_intel). If the model is not quantized, you can use the [Post-Training Optimization Tool](@ref pot_README) to quantize the model. The quantization process adds [FakeQuantize](../ops/quantization/FakeQuantize_1.md) layers on activations and weights for most layers. Read more about mathematical computations in the [Uniform Quantization with Fine-Tuning](https://github.com/openvinotoolkit/nncf/blob/develop/docs/compression_algorithms/Quantization.md).
The quantization process adds [FakeQuantize](../ops/quantization/FakeQuantize_1.md) layers on activations and weights for most layers. Read more about mathematical computations in the [Uniform Quantization with Fine-Tuning](https://github.com/openvinotoolkit/nncf/blob/develop/docs/compression_algorithms/Quantization.md).
When you pass the quantized IR to the OpenVINO™ plugin, the plugin automatically recognizes it as a quantized model and performs 8-bit inference. Note, if you pass a quantized model to another plugin that does not support 8-bit inference but supports all operations from the model, the model is inferred in precision that this plugin supports.
When you pass the quantized IR to the OpenVINO™ plugin, the plugin automatically recognizes it as a quantized model and performs 8-bit inference. Note that if you pass a quantized model to another plugin that does not support 8-bit inference but supports all operations from the model, the model is inferred in precision that this plugin supports.
In *Runtime stage*, the quantized model is loaded to the plugin. The plugin uses the `Low Precision Transformation` component to update the model to infer it in low precision:
- Update `FakeQuantize` layers to have quantized output tensors in a low precision range and add dequantization layers to compensate the update. Dequantization layers are pushed through as many layers as possible to have more layers in low precision. After that, most layers quantized input tensors in the low precision range and can be inferred in low precision. Ideally, dequantization layers should be fused in the next `FakeQuantize` layer.
- Quantize weights and store them in `Constant` layers.
At runtime, the quantized model is loaded to the plugin. The plugin uses the `Low Precision Transformation` component to update the model to infer it in low precision:
- Update `FakeQuantize` layers to have quantized output tensors in low-precision range and add dequantization layers to compensate for the update. Dequantization layers are pushed through as many layers as possible to have more layers in low precision. After that, most layers have quantized input tensors in low-precision range and can be inferred in low precision. Ideally, dequantization layers should be fused in the next `FakeQuantize` layer.
- Weights are quantized and stored in `Constant` layers.
## Prerequisites
Let's explore the quantized [TensorFlow* implementation of ResNet-50](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public/resnet-50-tf) model. Use the [Model Downloader](@ref omz_tools_downloader) tool to download the `fp16` model from [OpenVINO™ Toolkit - Open Model Zoo repository](https://github.com/openvinotoolkit/open_model_zoo):
```sh
cd $INTEL_OPENVINO_DIR/deployment_tools/tools/model_downloader
./downloader.py --name resnet-50-tf --precisions FP16-INT8 --output_dir <your_model_directory>
```
After that, you should quantize the model by the [Model Quantizer](@ref omz_tools_downloader) tool. For the dataset, you can choose to download the ImageNet dataset from [here](https://www.image-net.org/download.php).
```sh
./quantizer.py --model_dir --name public/resnet-50-tf --dataset_dir <DATASET_DIR> --precisions=FP16-INT8
```
## Inference
Let's explore quantized [TensorFlow* implementation of the ResNet-50](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public/resnet-50-tf) model. Use [Model Downloader](@ref omz_tools_downloader) to download the `FP16` model from [OpenVINO™ Toolkit - Open Model Zoo repository](https://github.com/openvinotoolkit/open_model_zoo):
```sh
<omz_dir>//tools/downloader/downloader.py --name resnet-50-tf --precisions FP16-INT8
```
After that you should quantize the model with the [Model Quantizer](@ref omz_tools_downloader) tool.
```sh
<omz_dir>//tools/downloader/quantizer.py --model_dir public/resnet-50-tf --dataset_dir <DATASET_DIR> --precisions=FP16-INT8
```
The simplest way to infer the model and collect performance counters is the [C++ Benchmark Application](../../inference-engine/samples/benchmark_app/README.md).
```sh
./benchmark_app -m resnet-50-tf.xml -d CPU -niter 1 -api sync -report_type average_counters -report_folder pc_report_dir
```
If you infer the model with the Inference Engine CPU plugin and collect performance counters, all operations (except the last non-quantized SoftMax) are executed in INT8 precision.
If you infer the model with the OpenVINO™ CPU plugin and collect performance counters, all operations (except the last non-quantized SoftMax) are executed in INT8 precision.
## Results analysis
## Low-Precision 8-bit Integer Inference Workflow
For 8-bit integer computations, a model must be quantized. Quantized models can be downloaded from [Overview of OpenVINO™ Toolkit Intel's Pre-Trained Models](@ref omz_models_group_intel). If the model is not quantized, you can use the [Post-Training Optimization Tool](@ref pot_README) to quantize the model. The quantization process adds [FakeQuantize](../ops/quantization/FakeQuantize_1.md) layers on activations and weights for most layers. Read more about mathematical computations in the [Uniform Quantization with Fine-Tuning](https://github.com/openvinotoolkit/nncf/blob/develop/docs/compression_algorithms/Quantization.md).
8-bit inference pipeline includes two stages (also refer to the figure below):
1. *Offline stage*, or *model quantization*. During this stage, [FakeQuantize](../ops/quantization/FakeQuantize_1.md) layers are added before most layers to have quantized tensors before layers in a way that low-precision accuracy drop for 8-bit integer inference satisfies the specified threshold. The output of this stage is a quantized model. Quantized model precision is not changed, quantized tensors are in original precision range (`fp32`). `FakeQuantize` layer has `levels` attribute which defines quants count. Quants count defines precision which is used during inference. For `int8` range `levels` attribute value has to be 255 or 256. To quantize the model, you can use the [Post-Training Optimization Tool](@ref pot_README) delivered with the Intel® Distribution of OpenVINO™ toolkit release package.
When you pass the quantized IR to the OpenVINO™ plugin, the plugin automatically recognizes it as a quantized model and performs 8-bit inference. Note, if you pass a quantized model to another plugin that does not support 8-bit inference but supports all operations from the model, the model is inferred in precision that this plugin supports.
2. *Runtime stage*. This stage is an internal procedure of the OpenVINO™ plugin. During this stage, the quantized model is loaded to the plugin. The plugin uses `Low Precision Transformation` component to update the model to infer it in low precision:
- Update `FakeQuantize` layers to have quantized output tensors in low precision range and add dequantization layers to compensate the update. Dequantization layers are pushed through as many layers as possible to have more layers in low precision. After that, most layers have quantized input tensors in low precision range and can be inferred in low precision. Ideally, dequantization layers should be fused in the next `FakeQuantize` layer.
- Weights are quantized and stored in `Constant` layers.
![int8_flow]
## Performance Counters
Information about layer precision is stored in the performance counters that are
available from the Inference Engine API. For example, the part of performance counters table for quantized [TensorFlow* implementation of ResNet-50](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public/resnet-50-tf) model inference on [CPU Plugin](supported_plugins/CPU.md) looks as follows:
@ -65,12 +76,12 @@ available from the Inference Engine API. For example, the part of performance co
| resnet\_model/add\_5/fq\_input\_1 | NOT\_RUN | FakeQuantize | undef | 0 | 0 |
> The `exeStatus` column of the table includes possible values:
> - `EXECUTED` - layer was executed by standalone primitive,
> - `NOT_RUN` - layer was not executed by standalone primitive or was fused with another operation and executed in another layer primitive.
>
> The `execType` column of the table includes inference primitives with specific suffixes. The layers have the following marks:
> * Suffix `I8` for layers that had 8-bit data type input and were computed in 8-bit precision
> * Suffix `FP32` for layers computed in 32-bit precision
The `exeStatus` column of the table includes possible values:
- `EXECUTED` - layer was executed by standalone primitive,
- `NOT_RUN` - layer was not executed by standalone primitive or was fused with another operation and executed in another layer primitive.
The `execType` column of the table includes inference primitives with specific suffixes. The layers have the following marks:
* Suffix `I8` for layers that had 8-bit data type input and were computed in 8-bit precision
* Suffix `FP32` for layers computed in 32-bit precision
All `Convolution` layers are executed in int8 precision. Rest layers are fused into Convolutions using post operations optimization technique, which is described in [Internal CPU Plugin Optimizations](supported_plugins/CPU.md).
All `Convolution` layers are executed in int8 precision. Rest layers are fused into Convolutions using post operations optimization technique, which is described in [Internal CPU Plugin Optimizations](supported_plugins/CPU.md).

519
docs/IE_DG/Integrate_with_customer_application_new_API.md
View File

@ -1,129 +1,250 @@
Integrate the Inference Engine with Your Application {#openvino_docs_IE_DG_Integrate_with_customer_application_new_API}
===============================
# Integrate Inference Engine {#openvino_docs_IE_DG_Integrate_with_customer_application_new_API}
This section provides a high-level description of the process of integrating the Inference Engine into your application.
Refer to the [Hello Classification Sample](../../inference-engine/samples/hello_classification/README.md) sources
for example of using the Inference Engine in applications.
## Integrate Inference Engine with Your C++ Application
## Use the Inference Engine API in Your Code
@sphinxdirective
.. raw:: html
The core `libinference_engine.so` library implements loading and parsing a model Intermediate Representation (IR), and triggers inference using a specified device. The core library has the following API:
<div id="switcher-cpp" class="switcher-anchor">C++</div>
@endsphinxdirective
* `InferenceEngine::Core`
* `InferenceEngine::Blob`, `InferenceEngine::TBlob`,
`InferenceEngine::NV12Blob`
* `InferenceEngine::BlobMap`
* `InferenceEngine::InputsDataMap`, `InferenceEngine::InputInfo`,
* `InferenceEngine::OutputsDataMap`
The following diagram illustrates the typical Inference Engine С++ API workflow:
C++ Inference Engine API wraps the capabilities of core library:
![ie_api_flow_cpp]
* `InferenceEngine::CNNNetwork`
* `InferenceEngine::ExecutableNetwork`
* `InferenceEngine::InferRequest`
Read the sections below to learn about each item.
## Integration Steps
> **NOTE**: Before start using Inference Engine, make sure you set all environment variables during the installation. If you did not, follow the instructions from the _Set the Environment Variables_ section in the installation guides:
> * [For Windows* 10](../install_guides/installing-openvino-windows.md)
> * [For Linux*](../install_guides/installing-openvino-linux.md)
> * [For macOS*](../install_guides/installing-openvino-macos.md)
> * To build an open source version, use the [Inference Engine Build Instructions](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode).
Integration process includes the following steps:
![integration_process]
### Link with Inference Library
1) **Create Inference Engine Core** to manage available devices and read network objects:
1. **Create a structure** for the project:
``` sh
project/
├── CMakeLists.txt - CMake file to build
├── ... - Additional folders like includes/
└── src/ - source folder
└── main.cpp
build/ - build directory
...
```
2. **Include Inference Engine, nGraph and OpenCV libraries** in `project/CMakeLists.txt`
[OpenCV](https://docs.opencv.org/master/db/df5/tutorial_linux_gcc_cmake.html) integration is needed mostly for pre-processing input data and nGraph for more complex applications using [nGraph API](../nGraph_DG/nGraph_dg.md).
``` cmake
cmake_minimum_required(VERSION 3.0.0)
project(project_name)
find_package(ngraph REQUIRED)
find_package(InferenceEngine REQUIRED)
find_package(OpenCV REQUIRED)
add_executable(${PROJECT_NAME} src/main.cpp)
target_link_libraries(${PROJECT_NAME} PRIVATE ${InferenceEngine_LIBRARIES} ${OpenCV_LIBS} ${NGRAPH_LIBRARIES})
```
### Use Inference Engine API to Implement Inference Pipeline
This section provides step-by-step instructions to implement a typical inference pipeline with the Inference Engine C++ API:
![ie_api_use_cpp]
#### Step 1. Create Inference Engine Core
Use the following code to create Inference Engine Core to manage available devices and read network objects:
@snippet snippets/Integrate_with_customer_application_new_API.cpp part0
2) **Read a model IR** created by the Model Optimizer (.xml is supported format):
#### Step 2 (Optional). Configure Input and Output of the Model
@snippet snippets/Integrate_with_customer_application_new_API.cpp part1
@sphinxdirective
.. raw:: html
**Or read the model from ONNX format** (.onnx and .prototxt are supported formats). You can find more information about the ONNX format support in the document [ONNX format support in the OpenVINO™](./ONNX_Support.md).
<div class="collapsible-section">
@endsphinxdirective
@snippet snippets/Integrate_with_customer_application_new_API.cpp part2
Optionally, configure input and output of the model using the steps below:
3) **Configure input and output**. Request input and output information using `InferenceEngine::CNNNetwork::getInputsInfo()`, and `InferenceEngine::CNNNetwork::getOutputsInfo()`
methods:
1. Load a model to a Core object:
@sphinxdirective
.. tab:: IR
.. code-block:: c
auto network = core.ReadNetwork("model.xml");
@snippet snippets/Integrate_with_customer_application_new_API.cpp part3
.. tab:: ONNX
.. code-block:: c
auto network = core.ReadNetwork("model.onnx");
Optionally, set the number format (precision) and memory layout for inputs and outputs. Refer to the
[Supported configurations](supported_plugins/Supported_Devices.md) chapter to choose the relevant configuration.
You can find more information about the ONNX format support in the document `ONNX format support in the OpenVINO™ <https://docs.openvinotoolkit.org/latest/openvino_docs_IE_DG_ONNX_Support.html>`_
.. tab:: nGraph
.. code-block:: c
std::shared_ptr<Function> createNetwork() {
// To construct a network, please follow
// https://docs.openvinotoolkit.org/latest/openvino_docs_nGraph_DG_build_function.html
}
auto network = CNNNetwork(createNetwork());
You can also allow input of any size. To do this, mark each input as resizable by setting a desired resize algorithm (e.g. `BILINEAR`) inside of the appropriate input info.
@endsphinxdirective
Basic color format conversions are supported as well. By default, the Inference Engine assumes
that the input color format is `BGR` and color format conversions are disabled. The Inference
Engine supports the following color format conversions:
* `RGB->BGR`
* `RGBX->BGR`
* `BGRX->BGR`
* `NV12->BGR`
2. Request input and output information using `InferenceEngine::CNNNetwork::getInputsInfo()`, and `InferenceEngine::CNNNetwork::getOutputsInfo()` methods:
```cpp
/** Take information about all topology inputs **/
InferenceEngine::InputsDataMap input_info = network.getInputsInfo();
/** Iterate over all input info**/
for (auto &item : input_info) {
auto input_data = item.second;
// Add your input configuration steps here
}
/** Take information about all topology outputs **/
InferenceEngine::OutputsDataMap output_info = network.getOutputsInfo();
/** Iterate over all output info**/
for (auto &item : output_info) {
auto output_data = item.second;
// Add your output configuration steps here
}
```
Configuring options:
1. **Set precision** (number format): FP16, FP32, INT8, etc. Refer to the Supported Configurations section on the [Supported Devices](supported_plugins/Supported_Devices.md) page to choose the relevant configuration.<br>
For input (*iterate over all input info*):
```cpp
input_data->setPrecision(InferenceEngine::Precision::U8);
```
For output (*iterate over all output info*):
```cpp
output_data->setPrecision(InferenceEngine::Precision::FP32);
```
**By default**, the input and output precision is set to `Precision::FP32`.
where `X` is a channel that will be ignored during inference. To enable the conversions, set a
desired color format (for example, `RGB`) for each input inside of the appropriate input info.
2. **Set layout** (NCHW, ).<br>
For input (*iterate over all input info*):
```cpp
input_data->setLayout(InferenceEngine::Layout::NCHW);
```
**By default**, the input layout is set to `Layout::NCHW`.<br>
For output (*iterate over all output info*):
```cpp
output_data->setLayout(InferenceEngine::Layout::NC);
```
**By default**, the output layout depends on a number of its dimensions:<br>
|Number of dimensions | 5 | 4 | 3 | 2 | 1 |
|:--------------------|-------|------|-----|----|----|
|Layout | NCDHW | NCHW | CHW | NC | C |
3. **Set resize algorithm for inputs** (Bilinear). You can allow input of any size. To do this, mark each input as resizable by setting a desired resize algorithm (e.g. `BILINEAR`) inside of the appropriate input info (*Iterate over all input info*):
```cpp
input_data->getPreProcess().setResizeAlgorithm(InferenceEngine::RESIZE_BILINEAR);
```
**By default**, no resize algorithm is set for inputs.
If you want to run inference for multiple images at once, you can use the built-in batch
pre-processing functionality.
4. **Set color format** (BGR, RGB, NV12). Basic color format conversions are supported as well. **By default**, the Inference Engine assumes that the input color format is BGR and color format conversions are disabled. Set `ColorFormat::RAW` input color format if the input does not need color conversions. The Inference Engine supports the following color format conversions:
* RGB->BGR
* RGBX->BGR
* BGRX->BGR
* NV12->BGR
where X is a channel that will be ignored during inference. To enable the conversions, set a desired color format (for example, RGB) for each input inside of the appropriate input info (*iterate over all input info*):
```cpp
input_data->getPreProcess().setColorFormat(InferenceEngine::ColorFormat::RGB);
```
> **NOTE**: NV12 input color format pre-processing differs from other color conversions. In case of NV12, Inference Engine expects two separate image planes (Y and UV). You must use a specific `InferenceEngine::NV12Blob` object instead of default blob object and set this blob to the Inference Engine Infer Request using `InferenceEngine::InferRequest::SetBlob()`. Refer to [Hello NV12 Input Classification C++ Sample](../../inference-engine/samples/hello_nv12_input_classification/README.md) for more details.
5. **Run on multiple images** with setting batch. If you want to run inference for multiple images at once, you can use the built-in batch pre-processing functionality.
**NOTE** : Batch pre-processing is not supported if input color format is set to `ColorFormat::NV12`.
> **NOTE**: Batch pre-processing is not supported if input color format is set to `ColorFormat::NV12`.
@sphinxdirective
.. raw:: html
You can use the following code snippet to configure input and output:
</div>
@endsphinxdirective
@snippet snippets/Integrate_with_customer_application_new_API.cpp part4
#### Step 3. Load the Model to the Device
> **NOTE**: NV12 input color format pre-processing differs from other color conversions. In case of NV12,
> Inference Engine expects two separate image planes (Y and UV). You must use a specific
> `InferenceEngine::NV12Blob` object instead of default blob object and set this blob to
> the Inference Engine Infer Request using `InferenceEngine::InferRequest::SetBlob()`.
> Refer to [Hello NV12 Input Classification C++ Sample](../../inference-engine/samples/hello_nv12_input_classification/README.md)
> for more details.
Load the model to the device using `InferenceEngine::Core::LoadNetwork()`:
If you skip this step, the default values are set:
* no resize algorithm is set for inputs
* input color format - `ColorFormat::RAW` meaning that input does not need color
conversions
* input and output precision - `Precision::FP32`
* input layout - `Layout::NCHW`
* output layout depends on number of its dimensions:
@sphinxdirective
.. tab:: IR
|Number of dimensions | 5 | 4 | 3 | 2 | 1 |
|:--------------------|-------|------|-----|----|----|
|Layout | NCDHW | NCHW | CHW | NC | C |
.. code-block:: c
4) **Load the model** to the device using `InferenceEngine::Core::LoadNetwork()`:
executable_network = core.LoadNetwork("model.xml", "CPU");
@snippet snippets/Integrate_with_customer_application_new_API.cpp part5
.. tab:: ONNX
It creates an executable network from a network object. The executable network is associated with single hardware device.
It is possible to create as many networks as needed and to use them simultaneously (up to the limitation of the hardware resources).
Third parameter is a configuration for plugin. It is map of pairs: (parameter name, parameter value). Choose device from
[Supported devices](supported_plugins/Supported_Devices.md) page for more details about supported configuration parameters.
.. code-block:: c
executable_network = core.LoadNetwork("model.onnx", "CPU");
.. tab:: nGraph
.. code-block:: c
std::shared_ptr<Function> createNetwork() {
// To construct a network, please follow
// https://docs.openvinotoolkit.org/latest/openvino_docs_nGraph_DG_build_function.html
}
auto network = CNNNetwork(createNetwork());
executable_network = core.LoadNetwork(network, "CPU");
.. tab:: Model From Step 2
Follow this step only if you went through optional "Step 2 (Optional). Configure Input and Output of the Model", otherwise use another tab for your model type: IR (OpenVINO Intermediate Representation), ONNX or nGraph.
.. code-block:: c
executable_network = core.LoadNetwork(network, "CPU");
@endsphinxdirective
It creates an executable network from a network object. The executable network is associated with single hardware device.
It is possible to create as many networks as needed and to use them simultaneously (up to the limitation of the hardware resources).
Third parameter is a configuration for plugin. It is map of pairs: (parameter name, parameter value). Choose device from
[Supported devices](supported_plugins/Supported_Devices.md) page for more details about supported configuration parameters.
@snippet snippets/Integrate_with_customer_application_new_API.cpp part6
5) **Create an infer request**:
#### Step 4. Create an Inference Request
Create an infer request using the following code:
@snippet snippets/Integrate_with_customer_application_new_API.cpp part7
6) **Prepare input**. You can use one of the following options to prepare input:
* **Optimal way for a single network.** Get blobs allocated by an infer request using `InferenceEngine::InferRequest::GetBlob()` and feed an image and the input data to the blobs. In this case, input data must be aligned (resized manually) with a given blob size and have a correct color format.
#### Step 5. Prepare Input
@snippet snippets/Integrate_with_customer_application_new_API.cpp part8
You can use one of the following options to prepare input:
* **Optimal way for a cascade of networks (output of one network is input for another).** Get output blob from the first request using `InferenceEngine::InferRequest::GetBlob()` and set it as input for the second request using `InferenceEngine::InferRequest::SetBlob()`.
* **Optimal way for a single network.** Get blobs allocated by an infer request using `InferenceEngine::InferRequest::GetBlob()` and feed an image and the input data to the blobs. In this case, input data must be aligned (resized manually) with a given blob size and have a correct color format.
@snippet snippets/Integrate_with_customer_application_new_API.cpp part9
@snippet snippets/Integrate_with_customer_application_new_API.cpp part8
* **Optimal way to handle ROI (a ROI object located inside of input of one network is input for another).** It is possible to re-use shared input by several networks. You do not need to allocate separate input blob for a network if it processes a ROI object located inside of already allocated input of a previous network. For instance, when first network detects objects on a video frame (stored as input blob) and second network accepts detected bounding boxes (ROI inside of the frame) as input. In this case, it is allowed to re-use pre-allocated input blob (used by first network) by second network and just crop ROI without allocation of new memory using `InferenceEngine::make_shared_blob()` with passing of `InferenceEngine::Blob::Ptr` and `InferenceEngine::ROI` as parameters.
* **Optimal way for a cascade of networks (output of one network is input for another).** Get output blob from the first request using `InferenceEngine::InferRequest::GetBlob()` and set it as input for the second request using `InferenceEngine::InferRequest::SetBlob()`.
@snippet snippets/Integrate_with_customer_application_new_API.cpp part10
@snippet snippets/Integrate_with_customer_application_new_API.cpp part9
Make sure that shared input is kept valid during execution of each network. Otherwise, ROI blob may be corrupted if the original input blob (that ROI is cropped from) has already been rewritten.
* **Optimal way to handle ROI (a ROI object located inside of input of one network is input for another).** It is possible to re-use shared input by several networks. You do not need to allocate separate input blob for a network if it processes a ROI object located inside of already allocated input of a previous network. For instance, when first network detects objects on a video frame (stored as input blob) and second network accepts detected bounding boxes (ROI inside of the frame) as input. In this case, it is allowed to re-use pre-allocated input blob (used by first network) by second network and just crop ROI without allocation of new memory using `InferenceEngine::make_shared_blob()` with passing of `InferenceEngine::Blob::Ptr` and `InferenceEngine::ROI` as parameters.
* Allocate input blobs of the appropriate types and sizes, feed an image and the input data to the blobs, and call `InferenceEngine::InferRequest::SetBlob()` to set these blobs for an infer request:
@snippet snippets/Integrate_with_customer_application_new_API.cpp part10
@snippet snippets/Integrate_with_customer_application_new_API.cpp part11
Make sure that shared input is kept valid during execution of each network. Otherwise, ROI blob may be corrupted if the original input blob (that ROI is cropped from) has already been rewritten.
A blob can be filled before and after `SetBlob()`.
* Allocate input blobs of the appropriate types and sizes, feed an image and the input data to the blobs, and call `InferenceEngine::InferRequest::SetBlob()` to set these blobs for an infer request:
@snippet snippets/Integrate_with_customer_application_new_API.cpp part11
A blob can be filled before and after `SetBlob()`.
> **NOTE:**
>
@ -141,22 +262,28 @@ Make sure that shared input is kept valid during execution of each network. Othe
> corresponding values of the read network. No pre-processing will happen for this blob. If you
> call `GetBlob()` after `SetBlob()`, you will get the blob you set in `SetBlob()`.
7) **Do inference** by calling the `InferenceEngine::InferRequest::StartAsync` and `InferenceEngine::InferRequest::Wait` methods for asynchronous request:
#### Step 6. Start Inference
@snippet snippets/Integrate_with_customer_application_new_API.cpp part12
Start inference in asynchronous or synchronous mode. Async API usage can improve overall frame-rate of the application, because rather than wait for inference to complete, the app can continue doing things on the host, while accelerator is busy.
or by calling the `InferenceEngine::InferRequest::Infer` method for synchronous request:
* For synchronous inference request:
```cpp
infer_request.Infer();
```
@snippet snippets/Integrate_with_customer_application_new_API.cpp part13
* For asynchronous inference request:
```cpp
infer_request.StartAsync();
infer_request.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY);
```
`StartAsync` returns immediately and starts inference without blocking main thread, `Infer` blocks main thread and returns when inference is completed. Call `Wait` for waiting result to become available for asynchronous request.
`StartAsync` returns immediately and starts inference without blocking main thread, `Infer` blocks
main thread and returns when inference is completed. Call `Wait` for waiting result to become available for asynchronous request.
There are three ways to use it:
* specify maximum duration in milliseconds to block for. The method is blocked until the specified timeout has elapsed, or the result becomes available, whichever comes first.
* `InferenceEngine::InferRequest::WaitMode::RESULT_READY` - waits until inference result becomes available
* `InferenceEngine::InferRequest::WaitMode::STATUS_ONLY` - immediately returns request status.It does not
block or interrupts current thread.
There are three ways to use it:
* specify maximum duration in milliseconds to block for. The method is blocked until the specified timeout has elapsed, or the result becomes available, whichever comes first.
* `InferenceEngine::InferRequest::WaitMode::RESULT_READY` - waits until inference result becomes available
* `InferenceEngine::InferRequest::WaitMode::STATUS_ONLY` - immediately returns request status.It does not
block or interrupts current thread.
Both requests are thread-safe: can be called from different threads without fearing corruption and failures.
@ -165,42 +292,21 @@ Multiple requests for single `ExecutableNetwork` are executed sequentially one b
While request is ongoing, all its methods except `InferenceEngine::InferRequest::Wait` would throw an
exception.
8) Go over the output blobs and **process the results**.
Note that casting `Blob` to `TBlob` via `std::dynamic_pointer_cast` is not the recommended way. It's better to access data via the `buffer()` and `as()` methods as follows:
#### Step 7. Process the Inference Results
Go over the output blobs and process the inference results. Note that casting `Blob` to `TBlob` via `std::dynamic_pointer_cast` is not the recommended way. It's better to access data via the `buffer()` and `as()` methods as follows:
@snippet snippets/Integrate_with_customer_application_new_API.cpp part14
## Build Your Application
### Build Your Application
For details about building your application, refer to the CMake files for the sample applications.
All samples source code is located in the `<INSTALL_DIR>/openvino/inference_engine/samples` directory, where `INSTALL_DIR` is the OpenVINO™ installation directory.
### CMake project creation
To build your project using CMake with the default build tools currently available on your machine, execute the following commands:
1. **Create a structure** for the project:
``` sh
project/
├── CMakeLists.txt - CMake file to build
├── ... - Additional folders like includes/
└── src/ - source folder
└── main.cpp
build/ - build directory
...
```
> **NOTE**: Make sure you set environment variables first by running `<INSTALL_DIR>/bin/setupvars.sh` (or `setupvars.bat` for Windows). Otherwise the `InferenceEngine_DIR` and `OpenCV_DIR` variables won't be configured properly to pass `find_package` calls.
2. **Include Inference Engine, nGraph and OpenCV libraries** in `project/CMakeLists.txt`
[OpenCV](https://docs.opencv.org/master/db/df5/tutorial_linux_gcc_cmake.html) integration is needed mostly for pre-processing input data and ngraph for more complex applications using [ngraph API](../nGraph_DG/nGraph_dg.md).
``` cmake
cmake_minimum_required(VERSION 3.0.0)
project(project_name)
find_package(ngraph REQUIRED)
find_package(InferenceEngine REQUIRED)
find_package(OpenCV REQUIRED)
add_executable(${PROJECT_NAME} src/main.cpp)
target_link_libraries(${PROJECT_NAME} PRIVATE ${InferenceEngine_LIBRARIES} ${OpenCV_LIBS} ${NGRAPH_LIBRARIES})
```
3. **To build your project** using CMake with the default build tools currently available on your machine, execute the following commands:
> **NOTE**: Make sure you set environment variables first by running `<INSTALL_DIR>/bin/setupvars.sh` (or setupvars.bat for Windows)`. Otherwise the `InferenceEngine_DIR` and `OpenCV_DIR` variables won't be configured properly to pass `find_package` calls.
```sh
cd build/
cmake ../project
@ -210,6 +316,179 @@ It's allowed to specify additional build options (e.g. to build CMake project on
### Run Your Application
Before running, make sure you completed **Set the Environment Variables** section in [OpenVINO Installation](../../inference-engine/samples/hello_nv12_input_classification/README.md) document so that the application can find the libraries.
> **NOTE**: Before running, make sure you completed **Set the Environment Variables** section in [OpenVINO Installation](../../inference-engine/samples/hello_nv12_input_classification/README.md) document so that the application can find the libraries.
[integration_process]: img/integration_process.png
To run compiled applications on Microsoft* Windows* OS, make sure that Microsoft* Visual C++ 2017
Redistributable and Intel® C++ Compiler 2017 Redistributable packages are installed and
`<INSTALL_DIR>/bin/intel64/Release/*.dll` files are placed to the
application folder or accessible via `%PATH%` environment variable.
## Integrate Inference Engine with Your Python Application
@sphinxdirective
.. raw:: html
<div id="switcher-python" class="switcher-anchor">Python</div>
@endsphinxdirective
This document explains how to integrate and use the Inference Engine API with your Python application.
The following diagram illustrates the typical Inference Engine Python API workflow:
![ie_api_flow_python]
Read the sections below to learn about each item.
### Import Inference Module
To make use of the Inference Engine functionality, import IECore to your application:
```py
from openvino.inference_engine import IECore
```
### Use Inference Engine API
This section provides step-by-step instructions to implement a typical inference pipeline with the Inference Engine API:
![ie_api_use_python]
#### Step 1. Create Inference Engine Core
Use the following code to create Inference Engine Core to manage available devices and read network objects:
```py
ie = IECore()
```
#### Step 2 (Optional). Read model. Configure Input and Output of the Model
@sphinxdirective
.. raw:: html
<div class="collapsible-section">
@endsphinxdirective
Optionally, configure input and output of the model using the steps below:
1. Read model
@sphinxdirective
.. tab:: IR
.. code-block:: python
net = ie.read_network(model="model.xml")
.. tab:: ONNX
.. code-block:: python
net = ie.read_network(model="model.onnx")
.. tab:: nGraph
.. code-block:: python
#Basic example of nGraph model creation
param = Parameter(Type.f32, Shape([1, 3, 22, 22]))
relu = ng.relu(param)
func = Function([relu], [param], 'test')
caps = Function.to_capsule(func)
net = IENetwork(caps)
@endsphinxdirective
2. Request input and output information using input_info, outputs
```py
inputs = net.input_info
input_name = next(iter(net.input_info))
outputs = net.outputs
output_name = next(iter(net.outputs))
```
Information for this input layer is stored in input_info. The next cell prints the input layout, precision and shape.
```py
print("Inputs:")
for name, info in net.input_info.items():
print("\tname: {}".format(name))
print("\tshape: {}".format(info.tensor_desc.dims))
print("\tlayout: {}".format(info.layout))
print("\tprecision: {}\n".format(info.precision))
```
This cell output tells us that the model expects inputs with a shape of [1,3,224,224], and that this is in NCHW layout. This means that the model expects input data with a batch size (N) of 1, 3 channels (C), and images of a height (H) and width (W) of 224. The input data is expected to be of FP32 (floating point) precision.
Getting the output layout, precision and shape is similar to getting the input layout, precision and shape.
```py
print("Outputs:")
for name, info in net.outputs.items():
print("\tname: {}".format(name))
print("\tshape: {}".format(info.shape))
print("\tlayout: {}".format(info.layout))
print("\tprecision: {}\n".format(info.precision))
```
This cell output shows that the model returns outputs with a shape of [1, 1001], where 1 is the batch size (N) and 1001 the number of classes (C). The output is returned as 32-bit floating point.
@sphinxdirective
.. raw:: html
</div>
@endsphinxdirective
#### Step 3. Load model to the Device
Load the model to the device using `load_network()`:
@sphinxdirective
.. tab:: IR
.. code-block:: python
exec_net = ie.load_network(network= "model.xml", device_name="CPU")
.. tab:: ONNX
.. code-block:: python
exec_net = ie.load_network(network= "model.onnx", device_name="CPU")
.. tab:: Model from step 2
.. code-block:: python
exec_net = ie.load_network(network=net, device_name="CPU")
@endsphinxdirective
This example is designed for CPU device, refer to the [Supported Devices](../IE_DG/supported_plugins/Supported_Devices.md) page to read about more devices.
#### Step 4. Prepare input
```py
import cv2
import numpy as np
image = cv2.imread("image.png")
# Resize with OpenCV your image if needed to match with net input shape
# N, C, H, W = net.input_info[input_name].tensor_desc.dims
# image = cv2.resize(src=image, dsize=(W, H))
# Converting image to NCHW format with FP32 type
input_data = np.expand_dims(np.transpose(image, (2, 0, 1)), 0).astype(np.float32)
```
#### Step 5. Start Inference
```py
result = exec_net.infer({input_name: input_data})
```
#### Step 6. Process the Inference Results
```py
output = result[output_name]
```
### Run Your Application
Congratulations, you have made your first Python application with OpenVINO™ toolkit, now you may run it.
[ie_api_flow_cpp]: img/BASIC_IE_API_workflow_Cpp.svg
[ie_api_use_cpp]: img/IMPLEMENT_PIPELINE_with_API_C.svg
[ie_api_flow_python]: img/BASIC_IE_API_workflow_Python.svg
[ie_api_use_python]: img/IMPLEMENT_PIPELINE_with_API_Python.svg

2
docs/IE_DG/Intro_to_Performance.md
View File

@ -1,4 +1,4 @@
# Introduction to the Performance Topics {#openvino_docs_IE_DG_Intro_to_Performance}
# Runtime Optimization Guide {#openvino_docs_IE_DG_Intro_to_Performance}
This section is a shorter version of the
[Optimization Guide](../optimization_guide/dldt_optimization_guide.md) for the Intel® Distribution of OpenVINO™ Toolkit.

2
docs/IE_DG/Introduction.md
View File

@ -72,8 +72,8 @@ For the list of supported models refer to the framework or format specific page:
* [Supported Caffe* models](../MO_DG/prepare_model/convert_model/Convert_Model_From_Caffe.md)
* [Supported TensorFlow* models](../MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md)
* [Supported MXNet* models](../MO_DG/prepare_model/convert_model/Convert_Model_From_MxNet.md)
* [Supported ONNX* models](../MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md)
* [Supported Kaldi* models](../MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md)
* [Supported ONNX* models](../MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md)
## Intermediate Representation

32
docs/IE_DG/Known_Issues_Limitations.md
View File

@ -3,7 +3,8 @@
## Multiple OpenMP Loadings
If the application uses the Inference Engine with third-party components that depend on Intel OpenMP, multiple loadings of the libiomp library may occur and cause OpenMP runtime initialization conflicts. This may happen, for example, if the application uses Intel® Math Kernel Library (Intel® MKL) through the “Single Dynamic Library” (<code>libmkl_rt.so</code>) mechanism and calls Intel MKL after loading the Inference Engine plugin.
The error log looks as follows:
The error log looks like this:
```sh
OMP: Error #15: Initializing libiomp5.so, but found libiomp5.so already initialized.
OMP: Hint: This means that multiple copies of the OpenMP runtime have been linked into the program. That is dangerous, since it can degrade performance or cause incorrect results. The best thing to do is to ensure that only a single OpenMP runtime is linked into the process, e.g. by avoiding static linking of the OpenMP runtime in any library. As an unsafe, unsupported, undocumented workaround you can set the environment variable KMP_DUPLICATE_LIB_OK=TRUE to allow the program to continue to execute, but that may cause crashes or silently produce incorrect results. For more information, please see http://www.intel.com/software/products/support/.
@ -12,20 +13,20 @@ OMP: Hint: This means that multiple copies of the OpenMP runtime have been linke
Possible workarounds:
* Preload the OpenMP runtime using the <code>LD_PRELOAD</code> variable:
```sh
LD_PRELOAD=<path_to_libiomp5.so> <path_to your_executable>
```
```sh
LD_PRELOAD=<path_to_libiomp5.so> <path_to your_executable>
```
This eliminates multiple loadings of libiomp, and makes all the components use this specific version of OpenMP.
* Alternatively, you can set <code>KMP_DUPLICATE_LIB_OK=TRUE</code>. However, performance degradation or results incorrectness may occur in this case.
* Alternatively, you can set <code>KMP_DUPLICATE_LIB_OK=TRUE</code>. However, performance degradation or incorrect results may occur in this case.
## Old proto compiler breaks protobuf library
With python protobuf library version 3.5.1 the following incompatibility can happen.
The known case is for Cent OS 7.4
With python protobuf library version 3.5.1, the following incompatibility can happen.
The known case is for Cent OS 7.4.
The error log looks as follows:
The error log looks like this:
```sh
File "../lib64/python3.5/site-packages/google/protobuf/descriptor.py", line 829, in _new_
@ -33,25 +34,24 @@ return _message.default_pool.AddSerializedFile(serialized_pb)
TypeError: expected bytes, str found
```
Possible workaround is to upgrade default protobuf compiler (libprotoc 2.5.0) to newer version, for example
libprotoc 2.6.1.
A possible workaround is to upgrade default protobuf compiler (libprotoc 2.5.0) to newer version, for example libprotoc 2.6.1.
[protobuf_issue]: https://github.com/google/protobuf/issues/4272
## Dynamic batching
Refer to the **Limitations** section of [Dynamic batching page](DynamicBatching.md)
Refer to the **Limitations** section of the [Dynamic batching page](DynamicBatching.md).
## Static Shape Infer
Refer to the **Limitations** section of [Static Shape Infer page](ShapeInference.md)
Refer to the **Limitations** section of the [Static Shape Infer page](ShapeInference.md).
## Image Pre-Processing Performance Optimization Issue
As described in [documentation for new API](Integrate_with_customer_application_new_API.md), you can set an image blob of any size to an
infer request using resizable input. Resize is executed during inference using configured resize algorithm.
As described in [documentation for the new API](Integrate_with_customer_application_new_API.md), you can set an image blob of any size to an
infer request using resizable input. Resize is executed during inference using the configured resize algorithm.
But currently resize algorithms are not completely optimized. So expect performance degradation if resizable input is
specified and an input blob (to be resized) is set (`SetBlob()` is used). Required performance is met for
But currently, resize algorithms are not completely optimized. So expect performance degradation if resizable input is
specified and an input blob (to be resized) is set using `SetBlob()`. The best performance is for the
[CPU](supported_plugins/CPU.md) plugin only (because enabled openMP* provides parallelism).
Another limitation is that currently, resize algorithms support NCHW layout only. So if you set NHWC layout for an input

23
docs/IE_DG/Memory_primitives.md
View File

@ -1,5 +1,12 @@
Inference Engine Memory primitives {#openvino_docs_IE_DG_Memory_primitives}
=====================================================================
# Inference Engine Memory Primitives {#openvino_docs_IE_DG_Memory_primitives}
## Inference Memory Primitives (C++)
@sphinxdirective
.. raw:: html
<div id="switcher-cpp" class="switcher-anchor">C++</div>
@endsphinxdirective
## Blobs
@ -18,7 +25,7 @@ InferenceEngine::Blob::Ptr blob = InferenceEngine::make_shared_blob<float>(tdesc
This class allows to create planar layouts using the standard formats (like <code>InferenceEngine::Layout::NCDHW</code>, <code>InferenceEngine::Layout::NCHW</code>, <code>InferenceEngine::Layout::NC</code>, <code>InferenceEngine::Layout::C</code> and etc) and also non-planar layouts using <code>InferenceEngine::BlockingDesc</code>.
In order to create a complex layout you should use <code>InferenceEngine::BlockingDesc</code> which allows to define the blocked memory with offsets and strides.
In order to create a complex layout you should use <code>InferenceEngine::BlockingDesc</code>, which allows you to define the blocked memory with offsets and strides.
## Examples
@ -27,13 +34,12 @@ In order to create a complex layout you should use <code>InferenceEngine::Blocki
InferenceEngine::BlockingDesc({1, 20, 20, 25}, {0, 2, 3, 1}); // or
InferenceEngine::BlockingDesc({1, 20, 20, 25}, InferenceEngine::Layout::NHWC);
</pre>
2. If you have a memory with real dimensions {N: 1, C: 25, H: 20, W: 20} but with channels which are blocked by 8, you can define it using next parameters:<br/>
2. If you have a memory with real dimensions {N: 1, C: 25, H: 20, W: 20} but with channels that are blocked by 8, you can define it using next parameters:<br/>
<pre class="brush:cpp">
InferenceEngine::BlockingDesc({1, 4, 20, 20, 8}, {0, 1, 2, 3, 1})
</pre>
3. Also you can set strides and offsets if layout contains it.
4. If you have a complex blob layout and you don't want to calculate the real offset to data you can use methods
<code>InferenceEngine::TensorDesc::offset(size_t l)</code> or <code>InferenceEngine::TensorDesc::offset(SizeVector v)</code>.<br/>
4. If you have a complex blob layout and you don't want to calculate the real offset to data you can use the <code>InferenceEngine::TensorDesc::offset(size_t l)</code> or <code>InferenceEngine::TensorDesc::offset(SizeVector v)</code> methods.<br/>
For example:
<pre class="brush:cpp">
InferenceEngine::BlockingDesc blk({1, 4, 20, 20, 8}, {0, 1, 2, 3, 1});
@ -43,8 +49,7 @@ tdesc.offset(1); // = 8
tdesc.offset({0, 0, 0, 2}); // = 16
tdesc.offset({0, 1, 0, 2}); // = 17
</pre>
5. If you would like to create a TensorDesc with a planar format and for N dimensions (N can be different 1, 2, 4 and etc), you can use the method
<code>InferenceEngine::TensorDesc::getLayoutByDims</code>.
5. If you would like to create a TensorDesc with a planar format and for N dimensions (N can be different 1, 2, 4 and etc), you can use the <code>InferenceEngine::TensorDesc::getLayoutByDims</code> method.
<pre class="brush:cpp">
InferenceEngine::TensorDesc::getLayoutByDims({1}); // InferenceEngine::Layout::C
InferenceEngine::TensorDesc::getLayoutByDims({1, 2}); // InferenceEngine::Layout::NC
@ -52,4 +57,4 @@ InferenceEngine::TensorDesc::getLayoutByDims({1, 2, 3, 4}); // InferenceEngine::
InferenceEngine::TensorDesc::getLayoutByDims({1, 2, 3}); // InferenceEngine::Layout::BLOCKED
InferenceEngine::TensorDesc::getLayoutByDims({1, 2, 3, 4, 5}); // InferenceEngine::Layout::NCDHW
InferenceEngine::TensorDesc::getLayoutByDims({1, 2, 3, 4, 5, ...}); // InferenceEngine::Layout::BLOCKED
</pre>
</pre>

446
docs/IE_DG/Model_Downloader.md Normal file
View File

@ -0,0 +1,446 @@
# Model Downloader {#openvino_docs_IE_DG_Tools_Model_Downloader}
This directory contains scripts that automate certain model-related tasks
based on configuration files in the models' directories.
* Model Downloader: `downloader.py` downloads model files from online sources
and, if necessary, patches them to make them more usable with Model
Optimizer;
* Model Converter: `converter.py` converts the models that are not in the
Inference Engine IR format into that format using Model Optimizer.
* Model Quantizer: `quantizer.py` quantizes full-precision models in the IR
format into low-precision versions using Post-Training Optimization Toolkit.
* Model Information Dumper: `info_dumper.py` prints information about the models
in a stable machine-readable format.
> **TIP**: You can quick start with the Model Downloader inside the OpenVINO™ Deep Learning Workbench (DL Workbench). DL Workbench is the OpenVINO™ toolkit UI that enables you to import a model, analyze its performance and accuracy, visualize the outputs, optimize and prepare the model for deployment on various Intel® platforms.
## Prerequisites
1. Install Python (version 3.6 or higher)
2. Install the tools' dependencies with the following command:
```sh
python3 -mpip install --user -r ./requirements.in
```
For the model converter, you will also need to install the OpenVINO&trade;
toolkit and the prerequisite libraries for Model Optimizer. See the
[OpenVINO toolkit documentation](https://docs.openvinotoolkit.org/) for details.
To convert models from certain frameworks, you will also need to install
additional dependencies.
@sphinxdirective
.. tab:: Caffe2
.. code-block:: python
python3 -mpip install --user -r ./requirements-caffe2.in
.. tab:: Pytorch
.. code-block:: python
python3 -mpip install --user -r ./requirements-pytorch.in
.. tab:: TensorFlow
.. code-block:: python
python3 -mpip install --user -r ./requirements-tensorflow.in
@endsphinxdirective
## Model Downloader
The basic usage is to run the script like this:
```sh
./downloader.py --all
```
This will download all models. The `--all` option can be replaced with
other filter options to download only a subset of models. See the "Shared options"
section.
### Model Downloader Starting Parameters
@sphinxdirective
+---------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------+
| Parameter | Explanation | Example |
+===========================+==================================================================================================================================================================================================================================================================================================================================================================================================+=====================================================================================+
| ``-o``/``--output_dir`` | By default, the script will download models into a directory tree rooted in the current directory. Use this parameter to download into a different directory. | ``./downloader.py --all --output_dir my/download/directory`` |
+---------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------+
| ``--precisions`` | Specify comma separated precisions of weights to be downloaded | ``./downloader.py --name face-detection-retail-0004 --precisions FP16,FP16-INT8`` |
+---------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------+
| ``--num_attempts`` | By default, the script will attempt to download each file only once. Use this parameter to change that and increase the robustness of the download process | ``./downloader.py --all --num_attempts 5 # attempt each download five times`` |
+---------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------+
| ``--cache_dir`` | Make the script use the specified directory as a cache. The script will place a copy of each downloaded file in the cache, or, if it is already there, retrieve it from the cache instead of downloading it again. The cache format is intended to remain compatible in future Open Model Zoo versions, so you can use a cache to avoid redownloading most files when updating Open Model Zoo. | ``./downloader.py --all --cache_dir my/cache/directory`` |
+---------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------+
| ``-j``/``--jobs`` | The script downloads files for multiple models concurrently. | ``./downloader.py --all -j8 # download up to 8 models at a time`` |
+---------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------+
| ``--progress_format`` | By default, the script outputs progress information as unstructured, human-readable text. Use this option, if you want to consume progress information programmatically. | ``./downloader.py --all --progress_format=json`` |
+---------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------+
@endsphinxdirective
When this option is set to `json`, the script's standard output is replaced by
a machine-readable progress report, whose format is documented in the
"JSON progress report format" section. This option does not affect errors and
warnings, which will still be printed to the standard error stream in a
human-readable format.
You can also set this option to `text` to explicitly request the default text
format.
See the "Shared options" section for information on other options accepted by
the script.
### JSON progress report format
This section documents the format of the progress report produced by the script
when the `--progress_format=json` option is specified.
The report consists of a sequence of events, where each event is represented
by a line containing a JSON-encoded object. Each event has a member with the
name `$type` whose value determines the type of the event, as well as which
additional members it contains.
@sphinxdirective
+------------------------------------+-------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Event type | Additional members | Explanation |
+====================================+=========================================================================+================================================================================================================================================================================================================================================================================================================================================+
| ``model_download_begin`` | ``model`` (string), ``num_files`` (integer) | The script started downloading the model named by ``model``. ``num_files`` is the number of files that will be downloaded for this model. This event will always be followed by a corresponding ``model_download_end`` event. |
+------------------------------------+-------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| ``model_download_end`` | ``model`` (string), ``successful`` (boolean) | The script stopped downloading the model named by ``model``. ``successful`` is true if every file was downloaded successfully. |
+------------------------------------+-------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| ``model_file_download_begin`` | ``model`` (string), ``model_file`` (string), ``size`` (integer) | The script started downloading the file named by ``model_file`` of the model named by ``model``. ``size`` is the size of the file in bytes. This event will always occur between ``model_download_begin`` and ``model_download_end`` events for the model, and will always be followed by a corresponding ``model_file_download_end`` event. |
+------------------------------------+-------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| ``model_file_download_end`` | ``model`` (string), ``model_file`` (string), ``successful`` (boolean) | The script stopped downloading the file named by ``model_file`` of the model named by ``model``. ``successful`` is true if the file was downloaded successfully. |
+------------------------------------+-------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| ``model_file_download_progress`` | ``model`` (string), ``model_file`` (string), ``size`` (integer) | The script downloaded ``size`` bytes of the file named by ``model_file`` of the model named by ``model`` so far. Note that ``size`` can decrease in a subsequent event if the download is interrupted and retried. This event will always occur between ``model_file_download_begin`` and ``model_file_download_end`` events for the file. |
+------------------------------------+-------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| ``model_postprocessing_begin`` | ``model`` | The script started post-download processing on the model named by ``model``. This event will always be followed by a corresponding ``model_postprocessing_end`` event. |
+------------------------------------+-------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| ``model_postprocessing_end`` | ``model`` | The script stopped post-download processing on the model named by ``model``. |
+------------------------------------+-------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
@endsphinxdirective
Additional event types and members may be added in the future.
Tools parsing the machine-readable format should avoid relying on undocumented details.
In particular:
* Tools should not assume that any given event will occur for a given model/file
(unless specified otherwise above) or will only occur once.
* Tools should not assume that events will occur in a certain order beyond
the ordering constraints specified above. In particular, when the `--jobs` option
is set to a value greater than 1, event sequences for different files or models
may get interleaved.
## Model Converter
The basic usage is to run the script like this:
```sh
./converter.py --all
```
This will convert all models into the Inference Engine IR format. Models that
were originally in that format are ignored. Models in PyTorch and Caffe2 formats will be
converted in ONNX format first.
The `--all` option can be replaced with other filter options to convert only
a subset of models. See the "Shared options" section.
### Model Converter Starting Parameters
@sphinxdirective
+-----------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------+
| Parameter | Explanation | Example |
+=============================+==================================================================================================================================================================================================================================================================+==================================================================================================+
| ``-d``/``--download_dir`` | The current directory must be the root of a download tree created by the model downloader. Use this parameter to specify a different download tree path. | ``./converter.py --all --download_dir my/download/directory`` |
+-----------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------+
| ``-o``/``--output_dir`` | By default, the script will download models into a directory tree rooted in the current directory. Use this parameter to download into a different directory. Note: models in intermediate format are placed to this directory too. | ``./converter.py --all --output_dir my/output/directory`` |
+-----------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------+
| ``--precisions`` | By default, the script will produce models in every precision that is supported for conversion. Use this parameter to only produce models in a specific precision. If the specified precision is not supported for a model, that model will be skipped. | ``./converter.py --all --precisions=FP16`` |
+-----------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------+
| ``--add_mo_arg`` | Add extra Model Optimizer arguments to the ones specified in the model configuration. The option can be repeated to add multiple arguments | ``./converter.py --name=caffenet --add_mo_arg=--reverse_input_channels --add_mo_arg=--silent`` |
+-----------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------+
| ``-j``/``--jobs`` | Run multiple conversion commands concurrently. The argument to the option must be either a maximum number of concurrently executed commands, or "auto", in which case the number of CPUs in the system is used. By default, all commands are run sequentially. | ``./converter.py --all -j8 # run up to 8 commands at a time`` |
+-----------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------+
| ``--dry_run`` | Print the conversion commands without actually running them.. | ``./converter.py --all --dry_run`` |
+-----------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------+
| ``-p``/``--python`` | By default, the script will run Model Optimizer using the same Python executable that was used to run the script itself. Apply this parameter to use a different Python executable. | ``./converter.py --all --python my/python`` |
+-----------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------+
@endsphinxdirective
The Python script will attempt to locate Model Optimizer using several methods:
1. If the `--mo` option was specified, then its value will be used as the path
to the script to run:
```sh
./converter.py --all --mo my/openvino/path/model_optimizer/mo.py
```
2. Otherwise, if the selected Python executable can import the `mo` package,
then that package will be used.
3. Otherwise, if the OpenVINO&trade; toolkit's `setupvars.sh`/`setupvars.bat`
script has been executed, the environment variables set by that script will
be used to locate Model Optimizer within the toolkit.
4. Otherwise, the script will fail.
See the "Shared options" section for information on other options accepted by
the script.
## Model Quantizer
Before you run the model quantizer, you must prepare a directory with
the datasets required for the quantization process. This directory will be
referred to as `<DATASET_DIR>` below. You can find more detailed information
about dataset preparation in the [Dataset Preparation Guide](../../data/datasets.md).
The basic usage is to run the script like this:
```sh
./quantizer.py --all --dataset_dir <DATASET_DIR>
```
This will quantize all models for which quantization is supported. Other models
are ignored.
The `--all` option can be replaced with other filter options to quantize only
a subset of models. See the "Shared options" section.
### Model Quantizer Starting Parameters
@sphinxdirective
+---------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+
| Parameter | Explanation | Example |
+===========================+=====================================================================================================================================================================================================================================================================================================================+=========================================================================================+
| ``--model_dir`` | The current directory must be the root of a tree of model files create by the model converter. Use this parameter to specify a different model tree path | ``./quantizer.py --all --dataset_dir <DATASET_DIR> --model_dir my/model/directory`` |
+---------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+
| ``-o``/``--output_dir`` | By default, the script will download models into a directory tree rooted in the current directory. Use this parameter to download into a different directory. Note: models in intermediate format are placed to this directory too. | ``./quantizer.py --all --dataset_dir <DATASET_DIR> --output_dir my/output/directory`` |
+---------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+
| ``--precisions`` | By default, the script will produce models in every precision that is supported as a quantization output. Use this parameter to only produce models in a specific precision. | ``./quantizer.py --all --dataset_dir <DATASET_DIR> --precisions=FP16-INT8`` |
+---------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+
| ``--target_device`` | It's possible to specify a target device for Post-Training Optimization Toolkitto optimize for. The supported values are those accepted by the "target\_device" option in Post-Training Optimization Toolkit's config files. If this option is unspecified, Post-Training Optimization Toolkit's default is used. | ``../quantizer.py --all --dataset_dir <DATASET_DIR> --target_device VPU`` |
+---------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+
| ``--dry_run`` | The script can print the quantization commands without actually running them. With this option specified, the configuration file for Post-Training Optimization Toolkit will still be created, so that you can inspect it. | ``./quantizer.py --all --dataset_dir <DATASET_DIR> --dry_run`` |
+---------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+
| ``-p``/``--python`` | By default, the script will run Model Optimizer using the same Python executable that was used to run the script itself. Apply this parameter to use a different Python executable. | ``./quantizer.py --all --dataset_dir <DATASET_DIR> --python my/python`` |
+---------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+
@endsphinxdirective
The script will attempt to locate Post-Training Optimization Toolkit using several methods:
1. If the `--pot` option was specified, then its value will be used as the path
to the script to run:
```sh
./quantizer.py --all --dataset_dir <DATASET_DIR> --pot my/openvino/path/post_training_optimization_toolkit/main.py
```
2. Otherwise, if the selected Python executable can import the `pot` package,
then that package will be used.
3. Otherwise, if the OpenVINO&trade; toolkit's `setupvars.sh`/`setupvars.bat`
script has been executed, the environment variables set by that script will
be used to locate Post-Training Optimization Toolkit within the OpenVINO toolkit.
4. Otherwise, the script will fail.
See the "Shared options" section for information on other options accepted by
the script.
## Model Information Dumper
The basic usage is to run the script like this:
```sh
./info_dumper.py --all
```
This will print to standard output information about all models.
The only options accepted by the script are those described in the "Shared options"
section.
The script's output is a JSON array, each element of which is a JSON object
describing a single model. Each such object has the following keys:
@sphinxdirective
+--------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Parameter | Explanation |
+======================================+=====================================================================================================================================================================================================================================================================================+
| ``name`` | the identifier of the model, as accepted by the ``--name`` option. |
+--------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| ``composite_model_name`` | the identifier of the composite model name, if the model is a part of composition of several models (e.g. encoder-decoder), otherwise - ``null`` |
+--------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| ``description`` | text describing the model. Paragraphs are separated by line feed characters. |
+--------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| ``framework`` | a string identifying the framework whose format the model is downloaded in. Current possible values are ``dldt`` (Inference Engine IR), ``caffe``, ``caffe2``, ``mxnet``, ``onnx``, ``pytorch`` and ``tf`` (TensorFlow). Additional possible values might be added in the future. |
+--------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| ``license_url`` | a URL for the license that the model is distributed under. |
+--------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| ``quantization_output_precisions`` | the list of precisions that the model can be quantized to by the model quantizer. Current possible values are ``FP16-INT8`` and ``FP32-INT8``; additional possible values might be added in the future. |
+--------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| ``quantization_output_precisions`` | the list of precisions that the model can be quantized to by the model quantizer. Current possible values are ``FP16-INT8`` and ``FP32-INT8``; additional possible values might be added in the future. |
+--------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| ``subdirectory`` | the subdirectory of the output tree into which the downloaded or converted files will be placed by the downloader or the converter, respectively. |
+--------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
@endsphinxdirective
* `precisions`: the list of precisions that the model has IR files for. For models downloaded
in a format other than the Inference Engine IR format, these are the precisions that the model
converter can produce IR files in. Current possible values are:
* `FP16`
* `FP16-INT1`
* `FP16-INT8`
* `FP32`
* `FP32-INT1`
* `FP32-INT8`
Additional possible values might be added in the future.
* `task_type`: a string identifying the type of task that the model performs.
are:
@sphinxdirective
.. raw:: html
<div class="collapsible-section" data-title="Current possible values">
@endsphinxdirective
* `action_recognition`
* `classification`
* `colorization`
* `detection`
* `face_recognition`
* `feature_extraction`
* `head_pose_estimation`
* `human_pose_estimation`
* `image_inpainting`
* `image_processing`
* `image_translation`
* `instance_segmentation`
* `machine_translation`
* `monocular_depth_estimation`
* `named_entity_recognition`
* `noise_suppression`
* `object_attributes`
* `optical_character_recognition`
* `place_recognition`
* `question_answering`
* `salient_object_detection`
* `semantic_segmentation`
* `sound_classification`
* `speech_recognition`
* `style_transfer`
* `text_to_speech`
* `time_series`
* `token_recognition`
@sphinxdirective
.. raw:: html
</div>
@endsphinxdirective
Additional possible values might be added in the future.
## Shared options
The are certain options that all tools accept.
`-h`/`--help` can be used to print a help message:
```sh
./TOOL.py --help
```
There are several mutually exclusive filter options that select the models the
tool will process:
@sphinxdirective
+--------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| Parameter | Explanation | Example |
+==============+===================================================================================================================================================================================================================================================================================+===========================================+
| ``--all`` | Selects all models | ``./TOOL.py --all`` |
+--------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| ``--name`` | takes a comma-separated list of patterns and selects models that match at least one of these patterns. The patterns may contain shell-style wildcards. For composite models, the name of composite model is accepted, as well as the names of individual models it consists of. | ``./TOOL.py --name 'mtcnn,densenet-*'`` |
+--------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
@endsphinxdirective
See https://docs.python.org/3/library/fnmatch.html for a full description of
the pattern syntax.
`--list` takes a path to a file that must contain a list of patterns and
selects models that match at least one of those patterns.
For composite models, the name of composite model is accepted, as well as the names
of individual models it consists of.
```sh
./TOOL.py --list my.lst
```
The file must contain one pattern per line. The pattern syntax is the same
as for the `--name` option. Blank lines and comments starting with `#` are
ignored. For example:
```
mtcnn # get all three models: mtcnn-o, mtcnn-p, mtcnn-r
densenet-* # get all DenseNet variants
```
To see the available models, you can use the `--print_all` option. When this
option is specified, the tool will print all model names defined in the
configuration file and exit:
```
$ ./TOOL.py --print_all
action-recognition-0001-decoder
action-recognition-0001-encoder
age-gender-recognition-retail-0013
driver-action-recognition-adas-0002-decoder
driver-action-recognition-adas-0002-encoder
emotions-recognition-retail-0003
face-detection-adas-0001
face-detection-retail-0004
face-detection-retail-0005
[...]
```
Either `--print_all` or one of the filter options must be specified.

50
docs/IE_DG/Model_caching_overview.md
View File

@ -1,52 +1,57 @@
# Model Caching Overview {#openvino_docs_IE_DG_Model_caching_overview}
## Introduction
## Introduction (C++)
As described in [Inference Engine Developer Guide](Deep_Learning_Inference_Engine_DevGuide.md), common application flow consists of the following steps:
@sphinxdirective
.. raw:: html
1. **Create Inference Engine Core object**
<div id="switcher-cpp" class="switcher-anchor">C++</div>
@endsphinxdirective
2. **Read the Intermediate Representation** - Read an Intermediate Representation file into an object of the `InferenceEngine::CNNNetwork`
As described in the [Inference Engine Developer Guide](Deep_Learning_Inference_Engine_DevGuide.md), common application flow consists of the following steps:
3. **Prepare inputs and outputs**
1. **Create Inference Engine Core object**: First step to manage available devices and read network objects
4. **Set configuration** Pass device-specific loading configurations to the device
2. **Read the Intermediate Representation**: Read an Intermediate Representation file into an object of the `InferenceEngine::CNNNetwork`
5. **Compile and Load Network to device** - Use the `InferenceEngine::Core::LoadNetwork()` method with specific device
3. **Prepare inputs and outputs**: If needed, manipulate precision, memory layout, size or color format
6. **Set input data**
4. **Set configuration**: Pass device-specific loading configurations to the device
7. **Execute**
5. **Compile and Load Network to device**: Use the `InferenceEngine::Core::LoadNetwork()` method with a specific device
Step #5 can potentially perform several time-consuming device-specific optimizations and network compilations,
and such delays can lead to bad user experience on application startup. To avoid this, some devices offer
Import/Export network capability, and it is possible to either use [Compile tool](../../inference-engine/tools/compile_tool/README.md)
6. **Set input data**: Specify input blob
7. **Execute**: Carry out inference and process results
Step 5 can potentially perform several time-consuming device-specific optimizations and network compilations,
and such delays can lead to a bad user experience on application startup. To avoid this, some devices offer
import/export network capability, and it is possible to either use the [Compile tool](../../inference-engine/tools/compile_tool/README.md)
or enable model caching to export compiled network automatically. Reusing cached networks can significantly reduce load network time.
## Set "CACHE_DIR" config option to enable model caching
To enable model caching, the application must specify the folder where to store cached blobs. It can be done like this
To enable model caching, the application must specify a folder to store cached blobs, which is done like this:
@snippet snippets/InferenceEngine_Caching0.cpp part0
With this code, if device supports Import/Export network capability, cached blob is automatically created inside the `myCacheFolder` folder
CACHE_DIR config is set to the Core object. If device does not support Import/Export capability, cache is just not created and no error is thrown
With this code, if the device specified by `LoadNetwork` supports import/export network capability, a cached blob is automatically created inside the `myCacheFolder` folder.
CACHE_DIR config is set to the Core object. If the device does not support import/export capability, cache is not created and no error is thrown.
Depending on your device, total time for loading network on application startup can be significantly reduced.
Please also note that very first LoadNetwork (when cache is not yet created) takes slightly longer time to 'export' compiled blob into a cache file
Also note that the very first LoadNetwork (when cache is not yet created) takes slightly longer time to "export" the compiled blob into a cache file:
![caching_enabled]
## Even faster: use LoadNetwork(modelPath)
In some cases, applications do not need to customize inputs and outputs every time. Such applications always
In some cases, applications do not need to customize inputs and outputs every time. Such an application always
call `cnnNet = ie.ReadNetwork(...)`, then `ie.LoadNetwork(cnnNet, ..)` and it can be further optimized.
For such cases, more convenient API to load network in one call is introduced in the 2021.4 release.
For these cases, the 2021.4 release introduces a more convenient API to load the network in a single call, skipping the export step:
@snippet snippets/InferenceEngine_Caching1.cpp part1
With enabled model caching, total load time is even smaller - in case that ReadNetwork is optimized as well
With model caching enabled, total load time is even smaller, if ReadNetwork is optimized as well.
@snippet snippets/InferenceEngine_Caching2.cpp part2
@ -55,11 +60,10 @@ With enabled model caching, total load time is even smaller - in case that ReadN
## Advanced examples
Not every device supports network import/export capability, enabling of caching for such devices do not have any effect.
Not every device supports network import/export capability. For those that don't, enabling caching has no effect.
To check in advance if a particular device supports model caching, your application can use the following code:
@snippet snippets/InferenceEngine_Caching3.cpp part3
[caching_enabled]: ../img/caching_enabled.png
[caching_times]: ../img/caching_times.png

41
docs/IE_DG/ONNX_Support.md
View File

@ -1,7 +1,15 @@
# ONNX format support in the OpenVINO™ {#openvino_docs_IE_DG_ONNX_Support}
# ONNX Native Format Support {#openvino_docs_IE_DG_ONNX_Support}
Starting from the 2020.4 release, OpenVINO™ supports reading native ONNX models.
`Core::ReadNetwork()` method provides a uniform way to read models from IR or ONNX format, it is a recommended approach to reading models.
## Introduction (C++)
@sphinxdirective
.. raw:: html
<div id="switcher-cpp" class="switcher-anchor">C++</div>
@endsphinxdirective
Starting with the 2020.4 release, OpenVINO™ supports reading native ONNX models in addition to [converting ONNX models](https://docs.openvinotoolkit.org/latest/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_ONNX.html) with the Model Optimizer.
The `Core::ReadNetwork()` method provides a uniform way to read models from IR or ONNX format and is the recommended approach for reading models.
Example:
```cpp
@ -9,12 +17,12 @@ InferenceEngine::Core core;
auto network = core.ReadNetwork("model.onnx");
```
**Reshape feature:**
## Reshape Feature
OpenVINO™ doesn't provide a mechanism to specify pre-processing (like mean values subtraction, reverse input channels) for the ONNX format.
If an ONNX model contains dynamic shapes for input, please use the `CNNNetwork::reshape` method for shape specialization.
**Weights saved in external files:**
## Weights Saved in External Files
OpenVINO™ supports ONNX models that store weights in external files. It is especially useful for models larger than 2GB because of protobuf limitations.
To read such models, use the `ReadNetwork` overload which takes `modelPath` as input parameter (both `std::string` and `std::wstring`).
@ -23,19 +31,14 @@ Otherwise, a runtime exception is thrown.
Reading models with external weights is not supported by the `ReadNetwork(const std::string& model, const Blob::CPtr& weights)` overload.
Paths to external weight files are saved in an ONNX model; these paths are relative to the model's directory path.
It means that if a model is located at:
`home/user/workspace/models/model.onnx`
and a file that contains external weights:
`home/user/workspace/models/data/weights.bin`
the path saved in model should be:
`data/weights.bin`.
It means that if a model is located at `home/user/workspace/models/model.onnx` and a file that contains external weights is in `home/user/workspace/models/data/weights.bin`, then the path saved in the model should be:
`data/weights.bin`
**NOTE**
> **NOTE**: A single model can use many external weights files.
* A single model can use many external weights files.
* Data of many tensors can be stored in a single external weights file (it is processed using offset and length values, which can be also saved in a model).
> **NOTE**: Data of many tensors can be stored in a single external weights file (it is processed using offset and length values, which can be also saved in a model).
The described mechanism is the only possibility to read weights from external files. The following input parameters of the `ReadNetwork` function overloads are NOT supported for ONNX models and should be passed as empty:
The described mechanism is the only way to read weights from external files. The following input parameters of the `ReadNetwork` function overloads are **not** supported for ONNX models and should be passed as empty:
* `const std::wstring& binPath`
* `const std::string& binPath`
* `const Blob::CPtr& weights`
@ -43,8 +46,8 @@ The described mechanism is the only possibility to read weights from external fi
You can find more details about the external data mechanism in [ONNX documentation](https://github.com/onnx/onnx/blob/master/docs/ExternalData.md).
To convert a model to use the external data feature, you can use [ONNX helper functions](https://github.com/onnx/onnx/blob/master/onnx/external_data_helper.py).
**Unsupported types of tensors:**
### Unsupported Types of Tensors
* `string`,
* `complex64`,
* `complex128`.
* `string`
* `complex64`
* `complex128`

186
docs/IE_DG/Operations_specifications.md Normal file
View File

@ -0,0 +1,186 @@
# Operations Specifications {#openvino_docs_operations_specifications}
@sphinxdirective
.. toctree::
:maxdepth: 1
openvino_docs_ops_arithmetic_Abs_1
openvino_docs_ops_arithmetic_Acos_1
openvino_docs_ops_arithmetic_Acosh_3
openvino_docs_ops_pooling_AdaptiveAvgPool_8
openvino_docs_ops_pooling_AdaptiveMaxPool_8
openvino_docs_ops_arithmetic_Add_1
openvino_docs_ops_arithmetic_Asin_1
openvino_docs_ops_arithmetic_Asinh_3
openvino_docs_ops_infrastructure_Assign_3
openvino_docs_ops_arithmetic_Atan_1
openvino_docs_ops_arithmetic_Atanh_3
openvino_docs_ops_pooling_AvgPool_1
openvino_docs_ops_normalization_BatchNormInference_1
openvino_docs_ops_normalization_BatchNormInference_5
openvino_docs_ops_movement_BatchToSpace_2
openvino_docs_ops_convolution_BinaryConvolution_1
openvino_docs_ops_movement_Broadcast_1
openvino_docs_ops_movement_Broadcast_3
openvino_docs_ops_condition_Bucketize_3
openvino_docs_ops_sequence_CTCGreedyDecoder_1
openvino_docs_ops_sequence_CTCGreedyDecoderSeqLen_6
openvino_docs_ops_arithmetic_Ceiling_1
openvino_docs_ops_activation_Clamp_1
openvino_docs_ops_movement_Concat_1
openvino_docs_ops_infrastructure_Constant_1
openvino_docs_ops_type_ConvertLike_1
openvino_docs_ops_type_Convert_1
openvino_docs_ops_convolution_ConvolutionBackpropData_1
openvino_docs_ops_convolution_Convolution_1
openvino_docs_ops_arithmetic_Cos_1
openvino_docs_ops_arithmetic_Cosh_1
openvino_docs_ops_sequence_CTCLoss_4
openvino_docs_ops_arithmetic_CumSum_3
openvino_docs_ops_convolution_DeformableConvolution_8
openvino_docs_ops_detection_DeformablePSROIPooling_1
openvino_docs_ops_movement_DepthToSpace_1
openvino_docs_ops_detection_DetectionOutput_1
openvino_docs_ops_signals_DFT_7
openvino_docs_ops_arithmetic_Divide_1
openvino_docs_ops_matrix_Einsum_7
openvino_docs_ops_activation_Elu_1
openvino_docs_ops_sparse_EmbeddingBagOffsetsSum_3
openvino_docs_ops_sparse_EmbeddingBagPackedSum_3
openvino_docs_ops_sparse_EmbeddingSegmentsSum_3
openvino_docs_ops_comparison_Equal_1
openvino_docs_ops_arithmetic_Erf_1
openvino_docs_ops_activation_Exp_1
openvino_docs_ops_detection_ExperimentalDetectronDetectionOutput_6
openvino_docs_ops_detection_ExperimentalDetectronGenerateProposalsSingleImage_6
openvino_docs_ops_detection_ExperimentalDetectronPriorGridGenerator_6
openvino_docs_ops_detection_ExperimentalDetectronROIFeatureExtractor_6
openvino_docs_ops_sort_ExperimentalDetectronTopKROIs_6
openvino_docs_ops_movement_ExtractImagePatches_3
openvino_docs_ops_quantization_FakeQuantize_1
openvino_docs_ops_arithmetic_FloorMod_1
openvino_docs_ops_arithmetic_Floor_1
openvino_docs_ops_normalization_GRN_1
openvino_docs_ops_sequence_GRUCell_3
openvino_docs_ops_sequence_GRUSequence_5
openvino_docs_ops_movement_GatherTree_1
openvino_docs_ops_movement_Gather_1
openvino_docs_ops_movement_Gather_7
openvino_docs_ops_movement_Gather_8
openvino_docs_ops_movement_GatherElements_6
openvino_docs_ops_movement_GatherND_5
openvino_docs_ops_activation_GELU_2
openvino_docs_ops_activation_GELU_7
openvino_docs_ops_comparison_GreaterEqual_1
openvino_docs_ops_comparison_Greater_1
openvino_docs_ops_convolution_GroupConvolutionBackpropData_1
openvino_docs_ops_convolution_GroupConvolution_1
openvino_docs_ops_activation_HardSigmoid_1
openvino_docs_ops_activation_HSigmoid_5
openvino_docs_ops_activation_HSwish_4
openvino_docs_ops_signals_IDFT_7
openvino_docs_ops_condition_If_8
openvino_docs_ops_image_Interpolate_1
openvino_docs_ops_image_Interpolate_4
openvino_docs_ops_normalization_LRN_1
openvino_docs_ops_sequence_LSTMCell_1
openvino_docs_ops_sequence_LSTMSequence_1
openvino_docs_ops_comparison_LessEqual_1
openvino_docs_ops_comparison_Less_1
openvino_docs_ops_arithmetic_Log_1
openvino_docs_ops_logical_LogicalAnd_1
openvino_docs_ops_logical_LogicalNot_1
openvino_docs_ops_logical_LogicalOr_1
openvino_docs_ops_logical_LogicalXor_1
openvino_docs_ops_activation_LogSoftmax_5
openvino_docs_ops_infrastructure_Loop_5
openvino_docs_ops_normalization_MVN_1
openvino_docs_ops_normalization_MVN_6
openvino_docs_ops_matrix_MatMul_1
openvino_docs_ops_sort_MatrixNonMaxSuppression_8
openvino_docs_ops_pooling_MaxPool_1
openvino_docs_ops_arithmetic_Maximum_1
openvino_docs_ops_arithmetic_Minimum_1
openvino_docs_ops_activation_Mish_4
openvino_docs_ops_arithmetic_Mod_1
openvino_docs_ops_sort_MulticlassNonMaxSuppression_8
openvino_docs_ops_arithmetic_Multiply_1
openvino_docs_ops_arithmetic_Negative_1
openvino_docs_ops_sort_NonMaxSuppression_1
openvino_docs_ops_sort_NonMaxSuppression_3
openvino_docs_ops_sort_NonMaxSuppression_4
openvino_docs_ops_sort_NonMaxSuppression_5
openvino_docs_ops_condition_NonZero_3
openvino_docs_ops_normalization_NormalizeL2_1
openvino_docs_ops_comparison_NotEqual_1
openvino_docs_ops_sequence_OneHot_1
openvino_docs_ops_activation_PReLU_1
openvino_docs_ops_detection_PSROIPooling_1
openvino_docs_ops_movement_Pad_1
openvino_docs_ops_infrastructure_Parameter_1
openvino_docs_ops_arithmetic_Power_1
openvino_docs_ops_detection_PriorBoxClustered_1
openvino_docs_ops_detection_PriorBox_1
openvino_docs_ops_detection_Proposal_1
openvino_docs_ops_detection_Proposal_4
openvino_docs_ops_generation_RandomUniform_8
openvino_docs_ops_generation_Range_1
openvino_docs_ops_generation_Range_4
openvino_docs_ops_infrastructure_ReadValue_3
openvino_docs_ops_activation_ReLU_1
openvino_docs_ops_reduction_ReduceL1_4
openvino_docs_ops_reduction_ReduceL2_4
openvino_docs_ops_reduction_ReduceLogicalAnd_1
openvino_docs_ops_reduction_ReduceLogicalOr_1
openvino_docs_ops_reduction_ReduceMax_1
openvino_docs_ops_reduction_ReduceMean_1
openvino_docs_ops_reduction_ReduceMin_1
openvino_docs_ops_reduction_ReduceProd_1
openvino_docs_ops_reduction_ReduceSum_1
openvino_docs_ops_detection_RegionYolo_1
openvino_docs_ops_detection_ReorgYolo_1
openvino_docs_ops_shape_Reshape_1
openvino_docs_ops_infrastructure_Result_1
openvino_docs_ops_movement_Reverse_1
openvino_docs_ops_movement_ReverseSequence_1
openvino_docs_ops_sequence_RNNCell_3
openvino_docs_ops_sequence_RNNSequence_5
openvino_docs_ops_detection_ROIAlign_3
openvino_docs_ops_detection_ROIPooling_1
openvino_docs_ops_movement_Roll_7
openvino_docs_ops_arithmetic_Round_5
openvino_docs_ops_movement_ScatterElementsUpdate_3
openvino_docs_ops_movement_ScatterNDUpdate_3
openvino_docs_ops_movement_ScatterUpdate_3
openvino_docs_ops_condition_Select_1
openvino_docs_ops_activation_Selu_1
openvino_docs_ops_shape_ShapeOf_1
openvino_docs_ops_shape_ShapeOf_3
openvino_docs_ops_movement_ShuffleChannels_1
openvino_docs_ops_activation_Sigmoid_1
openvino_docs_ops_arithmetic_Sign_1
openvino_docs_ops_arithmetic_Sin_1
openvino_docs_ops_arithmetic_Sinh_1
openvino_docs_ops_activation_SoftMax_1
openvino_docs_ops_activation_SoftPlus_4
openvino_docs_ops_movement_SpaceToBatch_2
openvino_docs_ops_movement_SpaceToDepth_1
openvino_docs_ops_movement_Split_1
openvino_docs_ops_arithmetic_Sqrt_1
openvino_docs_ops_arithmetic_SquaredDifference_1
openvino_docs_ops_shape_Squeeze_1
openvino_docs_ops_movement_StridedSlice_1
openvino_docs_ops_arithmetic_Subtract_1
openvino_docs_ops_activation_Swish_4
openvino_docs_ops_arithmetic_Tan_1
openvino_docs_ops_arithmetic_Tanh_1
openvino_docs_ops_infrastructure_TensorIterator_1
openvino_docs_ops_movement_Tile_1
openvino_docs_ops_sort_TopK_1
openvino_docs_ops_sort_TopK_3
openvino_docs_ops_movement_Transpose_1
openvino_docs_ops_shape_Unsqueeze_1
openvino_docs_ops_movement_VariadicSplit_1
@endsphinxdirective

33
docs/IE_DG/Samples_Overview.md
View File

@ -1,5 +1,36 @@
# Inference Engine Samples {#openvino_docs_IE_DG_Samples_Overview}
@sphinxdirective
.. toctree::
:maxdepth: 1
:hidden:
openvino_inference_engine_samples_classification_sample_async_README
openvino_inference_engine_ie_bridges_python_sample_classification_sample_async_README
openvino_inference_engine_samples_hello_classification_README
openvino_inference_engine_ie_bridges_c_samples_hello_classification_README
openvino_inference_engine_ie_bridges_python_sample_hello_classification_README
openvino_inference_engine_samples_hello_reshape_ssd_README
openvino_inference_engine_ie_bridges_python_sample_hello_reshape_ssd_README
openvino_inference_engine_samples_hello_nv12_input_classification_README
openvino_inference_engine_ie_bridges_c_samples_hello_nv12_input_classification_README
openvino_inference_engine_samples_hello_query_device_README
openvino_inference_engine_ie_bridges_python_sample_hello_query_device_README
openvino_inference_engine_samples_ngraph_function_creation_sample_README
openvino_inference_engine_ie_bridges_python_sample_ngraph_function_creation_sample_README
openvino_inference_engine_samples_object_detection_sample_ssd_README
openvino_inference_engine_ie_bridges_python_sample_object_detection_sample_ssd_README
openvino_inference_engine_ie_bridges_c_samples_object_detection_sample_ssd_README
openvino_inference_engine_samples_speech_sample_README
openvino_inference_engine_ie_bridges_python_sample_speech_sample_README
openvino_inference_engine_samples_style_transfer_sample_README
openvino_inference_engine_ie_bridges_python_sample_style_transfer_sample_README
openvino_inference_engine_samples_benchmark_app_README
openvino_inference_engine_tools_benchmark_tool_README
@endsphinxdirective
The Inference Engine sample applications are simple console applications that show how to utilize specific Inference Engine capabilities within an application, assist developers in executing specific tasks such as loading a model, running inference, querying specific device capabilities and etc.
After installation of Intel® Distribution of OpenVINO™ toolkit, С, C++ and Python* sample applications are available in the following directories, respectively:
@ -109,7 +140,7 @@ for the debug configuration — in `<path_to_build_directory>/intel64/Debug/`.
The recommended Windows* build environment is the following:
* Microsoft Windows* 10
* Microsoft Visual Studio* 2017, or 2019. Make sure that C++ CMake tools for Windows is [enabled](https://docs.microsoft.com/en-us/cpp/build/cmake-projects-in-visual-studio?view=msvc-160#:~:text=The%20Visual%20C%2B%2B%20Tools%20for,Visual%20Studio%20generators%20are%20supported).
* Microsoft Visual Studio* 2017, or 2019
* CMake* version 3.10 or higher
> **NOTE**: If you want to use Microsoft Visual Studio 2019, you are required to install CMake 3.14.

85
docs/IE_DG/ShapeInference.md
View File

@ -1,27 +1,37 @@
Using Shape Inference {#openvino_docs_IE_DG_ShapeInference}
==========================================
# Using the Reshape Inference Feature {#openvino_docs_IE_DG_ShapeInference}
OpenVINO™ provides the following methods for runtime model reshaping:
## Introduction (C++)
* **Set a new input shape** with the `InferenceEngine::CNNNetwork::reshape` method.<br>
The `InferenceEngine::CNNNetwork::reshape` method updates input shapes and propagates them down to the outputs of the model through all intermediate layers.
@sphinxdirective
.. raw:: html
<div id="switcher-cpp" class="switcher-anchor">C++</div>
@endsphinxdirective
OpenVINO™ provides two methods for runtime model reshaping: setting a new input shape and setting a new batch dimension value.
## Set a new input shape** with the `InferenceEngine::CNNNetwork::reshape` method
The `InferenceEngine::CNNNetwork::reshape` method updates input shapes and propagates them down to the outputs of the model through all intermediate layers.
> **NOTES**:
> - Starting with the 2021.1 release, the Model Optimizer converts topologies keeping shape-calculating sub-graphs by default, which enables correct shape propagation during reshaping in most cases.
> - Older versions of IRs are not guaranteed to reshape successfully. Please regenerate them with the Model Optimizer of the latest version of OpenVINO™.<br>
> - If an ONNX model does not have a fully defined input shape and the model was imported with the ONNX importer, reshape the model before loading it to the plugin.
* **Set a new batch dimension value** with the `InferenceEngine::CNNNetwork::setBatchSize` method.<br>
The meaning of a model batch may vary depending on the model design.
This method does not deduce batch placement for inputs from the model architecture.
It assumes that the batch is placed at the zero index in the shape for all inputs and uses the `InferenceEngine::CNNNetwork::reshape` method to propagate updated shapes through the model.
## Set a new batch dimension value with the `InferenceEngine::CNNNetwork::setBatchSize` method.
The method transforms the model before a new shape propagation to relax a hard-coded batch dimension in the model, if any.
The meaning of a model batch may vary depending on the model design.
This method does not deduce batch placement for inputs from the model architecture.
It assumes that the batch is placed at the zero index in the shape for all inputs and uses the `InferenceEngine::CNNNetwork::reshape` method to propagate updated shapes through the model.
Use `InferenceEngine::CNNNetwork::reshape` instead of `InferenceEngine::CNNNetwork::setBatchSize` to set new input shapes for the model in case the model has:
* Multiple inputs with different zero-index dimension meanings
* Input without a batch dimension
* 0D, 1D, or 3D shape
The method transforms the model before a new shape propagation to relax a hard-coded batch dimension in the model, if any.
Use `InferenceEngine::CNNNetwork::reshape` instead of `InferenceEngine::CNNNetwork::setBatchSize` to set new input shapes for the model if the model has one of the following:
* Multiple inputs with different zero-index dimension meanings
* Input without a batch dimension
* 0D, 1D, or 3D shape
The `InferenceEngine::CNNNetwork::setBatchSize` method is a high-level API method that wraps the `InferenceEngine::CNNNetwork::reshape` method call and works for trivial models from the batch placement standpoint.
Use `InferenceEngine::CNNNetwork::reshape` for other models.
@ -37,10 +47,11 @@ Inference Engine takes three kinds of a model description as an input, which are
3. [nGraph function](../nGraph_DG/nGraph_dg.md) through the constructor of `InferenceEngine::CNNNetwork`
`InferenceEngine::CNNNetwork` keeps an `ngraph::Function` object with the model description internally.
The object should have fully defined input shapes to be successfully loaded to the Inference Engine plugins.
To resolve undefined input dimensions of a model, call the `CNNNetwork::reshape` method providing new input shapes before loading to the Inference Engine plugin.
The object should have fully-defined input shapes to be successfully loaded to Inference Engine plugins.
To resolve undefined input dimensions of a model, call the `CNNNetwork::reshape` method to provide new input shapes before loading to the Inference Engine plugin.
Run the following code right after `InferenceEngine::CNNNetwork` creation to explicitly check for model input names and shapes:
```cpp
CNNNetwork network = ... // read IR / ONNX model or create from nGraph::Function explicitly
const auto parameters = network.getFunction()->get_parameters();
@ -55,10 +66,30 @@ To feed input data of a shape that is different from the model input shape, resh
Once the input shape of `InferenceEngine::CNNNetwork` is set, call the `InferenceEngine::Core::LoadNetwork` method to get an `InferenceEngine::ExecutableNetwork` object for inference with updated shapes.
There are other approaches to reshape the model during the stage of <a href="_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html#when_to_specify_input_shapes">IR generation</a> or [nGraph::Function creation](../nGraph_DG/build_function.md).
There are other approaches to reshape the model during the stage of <a href="_docs_MO_DG_prepare_model_convert_model_Converting_Model.html#when_to_specify_input_shapes">IR generation</a> or [nGraph::Function creation](../nGraph_DG/build_function.md).
Practically, some models are not ready to be reshaped. In this case, a new input shape cannot be set with the Model Optimizer or the `InferenceEngine::CNNNetwork::reshape` method.
## Usage of Reshape Method <a name="usage_of_reshape_method"></a>
The primary method of the feature is `InferenceEngine::CNNNetwork::reshape`.
It gets new input shapes and propagates it from input to output for all intermediates layers of the given network.
The method takes `InferenceEngine::ICNNNetwork::InputShapes` - a map of pairs: name of input data and its dimension.
The algorithm for resizing network is the following:
1) **Collect the map of input names and shapes from Intermediate Representation (IR)** using helper method `InferenceEngine::CNNNetwork::getInputShapes`
2) **Set new input shapes**
3) **Call reshape**
Here is a code example:
@snippet snippets/ShapeInference.cpp part0
The Shape Inference feature is used in [Smart Classroom Demo](@ref omz_demos_smart_classroom_demo_cpp).
## Troubleshooting Reshape Errors
Operation semantics may impose restrictions on input shapes of the operation.
@ -82,26 +113,6 @@ For example, Object Detection models from TensorFlow have resizing restrictions
To keep the model valid after the reshape, choose a new input shape that satisfies conditions listed in the `pipeline.config` file.
For details, refer to the <a href="_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Object_Detection_API_Models.html#tf_od_custom_input_shape">Tensorflow Object Detection API models resizing techniques</a>.
## Usage of Reshape Method <a name="usage_of_reshape_method"></a>
The primary method of the feature is `InferenceEngine::CNNNetwork::reshape`.
It gets new input shapes and propagates it from input to output for all intermediates layers of the given network.
The method takes `InferenceEngine::ICNNNetwork::InputShapes` - a map of pairs: name of input data and its dimension.
The algorithm for resizing network is the following:
1) **Collect the map of input names and shapes from Intermediate Representation (IR)** using helper method `InferenceEngine::CNNNetwork::getInputShapes`
2) **Set new input shapes**
3) **Call reshape**
Here is a code example:
@snippet snippets/ShapeInference.cpp part0
Shape Inference feature is used in [Smart Classroom Demo](@ref omz_demos_smart_classroom_demo_cpp).
## Extensibility
Inference Engine provides a special mechanism that allows to add the support of shape inference for custom operations.

32
docs/IE_DG/img/BASIC_FLOW_IE_C.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 49 KiB

32
docs/IE_DG/img/BASIC_FLOW_MO.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 49 KiB

19
docs/IE_DG/img/BASIC_FLOW_MO_simplified.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 42 KiB

15
docs/IE_DG/img/BASIC_IE_API_workflow_Cpp.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 47 KiB

10
docs/IE_DG/img/BASIC_IE_API_workflow_Python.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 32 KiB

63
docs/IE_DG/img/DEVELOPMENT_FLOW_V3_crunch.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 246 KiB

29
docs/IE_DG/img/IMPLEMENT_PIPELINE_with_API_C.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 96 KiB

26
docs/IE_DG/img/IMPLEMENT_PIPELINE_with_API_Python.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 79 KiB

60
docs/IE_DG/img/LATENCY_VS_THROUGHPUT.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 118 KiB

3
docs/IE_DG/img/Latency_Throughput.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:17298610e06aed2ccfdad44c22a84a410ff4511456b28f6953091d9e593ab7fd
size 53668

22
docs/IE_DG/img/WHAT_TO_USE.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 203 KiB

3
docs/IE_DG/img/cpu_int8_flow.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:83bcd7888d3843ddfd9a601288627e98f5874290c00b9988bf1beac9209f2e8d
size 79741

3
docs/IE_DG/img/cpu_streams_explained.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d5cf2212b3634a264722b386899197a7f0fa56fbdad97c017d2733cc0d2694d4
size 105457

3
docs/IE_DG/img/development_deployment.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:dc7766d83d95b40e1dc6b5a63f7a801adf9780664f9ae6d4c88676c2f5b88688
size 107123

179
docs/IE_DG/img/diagram_workflow.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 162 KiB

3
docs/IE_DG/img/ie_api_cpp.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c5bcecb8ddef4fdc797474da92ab6a9977a43d0d68e6fec75b2e0a41441042c2
size 22993

3
docs/IE_DG/img/ie_api_integration_cpp.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9831807da09a8b1d75b5c20e5390526efdbacc7ccf9120221225d633abeb13be
size 47476

3
docs/IE_DG/img/ie_api_integration_python.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4e9464a77d206d7fd642110a8c409f90b3885fb85e7d132012e86a5edfc72aeb
size 41636

3
docs/IE_DG/img/ie_api_python.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:72b8f5803b63465d349e5757fe59938269619f761d9a67dcfc03ab8e54bc0c3d
size 19158

3
docs/IE_DG/img/ie_workflow_steps.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:62dac902ffb5f26ee034203910a77112ca87c7fc80a511cbd500930bb2919118
size 17983

3
docs/IE_DG/img/integration_process.png
View File

@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9fff52e5faaf108371db87e53959453216554152b15ca0432b1541f94def297e
size 19145

3
docs/IE_DG/img/latency_throughput_1.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:17298610e06aed2ccfdad44c22a84a410ff4511456b28f6953091d9e593ab7fd
size 53668

3
docs/IE_DG/img/output_trimmed.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e47c2b259bea6d3539f0c4556de4cc3a07f6d60af54e1cf32002b4a7bc2cc90a
size 25231

3
docs/IE_DG/img/resnet_269.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:92d36b9527a3e316cd9eb2b6f5054c312466df004e4aa9c3458e165330bc6561
size 24157

3
docs/IE_DG/img/vtune_async.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c47ede993681ba3f0a3e3f4274369ee1854365b1bcd1b5cb0f649a781fdf51bd
size 6215

3
docs/IE_DG/img/vtune_option.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4a82b414dbc4f7ce2eae625bb7c9c7b88c154a7c476374683dd9886564560f67
size 7951

3
docs/IE_DG/img/vtune_regular.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9a4fce51076df19fbca04a36d6886765771f8ffc174bebbd751bfc77d91ab1f2
size 7081

3
docs/IE_DG/img/vtune_timeline.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c514316f78f04e8c000f6b95dc579d8c63c57f00c4c980ea4d358a6a4f1b9d7e
size 8744

3
docs/IE_DG/img/vtune_topdown_view.jpg Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:40c4b9096ef264807d930fe64d427f53a69ce2247c836415e64c5aa72d9f245e
size 36468

3
docs/IE_DG/img/workflow_steps.png
View File

@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5e22bc22d614c7335ae461a8ce449ea8695973d755faca718cf74b95972c94e2
size 19773

152
docs/IE_DG/integrate_with_customer_application_python.md Normal file
View File

@ -0,0 +1,152 @@
# Integrate Inference Engine with Your Python Application {#openvino_docs_IE_DG_integrate_with_customer_application_python}
This document explains how to integrate and use the Inference Engine API with your Python application.
The following diagram illustrates the typical Inference Engine Python API workflow:
![ie_api_flow_python]
Read the sections below to learn about each item.
## Link with Inference Engine Library
To make use of the Inference Engine functionality, import IECore to your application:
```py
from openvino.inference_engine import IECore
```
## Use Inference Engine API Implement Inference Pipeline
This section provides step-by-step instructions to implement a typical inference pipeline with the Inference Engine Python API:
![ie_api_use_python]
### Step 1. Create Inference Engine Core
Use the following code to create Inference Engine Core to manage available devices and read network objects:
```py
ie = IECore()
```
### Step 2 (Optional). Read model. Configure Input and Output of the Model
@sphinxdirective
.. raw:: html
<div class="collapsible-section">
@endsphinxdirective
Optionally, configure input and output of the model using the steps below:
1. Read model
@sphinxdirective
.. tab:: IR
.. code-block:: python
net = ie.read_network(model="model.xml")
.. tab:: ONNX
.. code-block:: python
net = ie.read_network(model="model.onnx")
.. tab:: nGraph
.. code-block:: python
// TBD
@endsphinxdirective
2. Request input and output information using input_info, outputs
```py
input_name = next(iter(net.input_info))
output_name = next(iter(net.outputs))
```
Information for this input layer is stored in input_info. The next cell prints the input layout, precision and shape.
```py
print(f"input layout: {net.input_info[input_layer].layout}")
print(f"input precision: {net.input_info[input_layer].precision}")
print(f"input shape: {net.input_info[input_layer].tensor_desc.dims}")
```
This cell output tells us that the model expects inputs with a shape of [1,3,224,224], and that this is in NCHW layout. This means that the model expects input data with a batch size (N) of 1, 3 channels (C), and images of a height (H) and width (W) of 224. The input data is expected to be of FP32 (floating point) precision.
Getting the output layout, precision and shape is similar to getting the input layout, precision and shape.
```py
print(f"output layout: {net.outputs[output_layer].layout}")
print(f"output precision: {net.outputs[output_layer].precision}")
print(f"output shape: {net.outputs[output_layer].shape}")
```
This cell output shows that the model returns outputs with a shape of [1, 1001], where 1 is the batch size (N) and 1001 the number of classes (C). The output is returned as 32-bit floating point.
@sphinxdirective
.. raw:: html
</div>
@endsphinxdirective
### Step 3. Load model to the Device
Load the model to the device using `load_network()`:
@sphinxdirective
.. tab:: IR
.. code-block:: python
exec_net = ie.load_network(network= "model.xml", device_name="CPU")
.. tab:: ONNX
.. code-block:: python
exec_net = ie.load_network(network= "model.onnx", device_name="CPU")
.. tab:: nGraph
.. code-block:: python
// TBD
.. tab:: Model from Step 2
.. code-block:: python
exec_net = ie.load_network(network=net, device_name="CPU")
@endsphinxdirective
### Step 4. Prepare input
```py
import cv2
import numpy as np
image = cv2.imread("image.png")
# Resize with OpenCV your image if needed to match with net input shape
# res_image = cv2.resize(src=image, dsize=(W, H))
# Converting image to NCHW format with FP32 type
input_data = np.expand_dims(np.transpose(image, (2, 0, 1)), 0).astype(np.float32)
```
### Step 5. Start Inference
```py
input_name = next(iter(net.input_info))
result = exec_net.infer({input_name: input_data})
```
### Step 6. Process the Inference Results
```py
output_name = next(iter(net.outputs))
output = result[output_name]
```
## Run Application
[ie_api_flow_python]: img/ie_api_python.png
[ie_api_use_python]: img/ie_api_integration_python.png

10
docs/IE_DG/network_state_intro.md
View File

@ -7,7 +7,7 @@ This section describes how to work with stateful networks in OpenVINO toolkit, s
The section additionally provides small examples of stateful network and code to infer it.
## What is a Stateful Network
## What is a Stateful Network?
Several use cases require processing of data sequences. When length of a sequence is known and small enough,
we can process it with RNN like networks that contain a cycle inside. But in some cases, like online speech recognition of time series
@ -316,7 +316,7 @@ InferenceEngine::lowLatency2(cnnNetwork, false);
![low_latency_limitation_2](./img/low_latency_limitation_2.png)
**Current solution:** Trim non-reshapable layers via [ModelOptimizer CLI](../MO_DG/prepare_model/convert_model/Converting_Model_General.md) `--input`, `--output`. For example, the parameter and the problematic constant in the picture above can be trimmed using the following command line option:
**Current solution:** Trim non-reshapable layers via [ModelOptimizer CLI](../MO_DG/prepare_model/convert_model/Converting_Model.md) `--input`, `--output`. For example, the parameter and the problematic constant in the picture above can be trimmed using the following command line option:
`--input Reshape_layer_name`. The problematic constant can be also replaced using ngraph, as shown in the example below.
```cpp
@ -403,7 +403,7 @@ InferenceEngine::LowLatency(cnnNetwork);
4. Use state API. See sections [OpenVINO state API](#openvino-state-api), [Example of stateful network inference](#example-of-stateful-network-inference).
### Known Limitations
### Known Limitations for LowLatency [DEPRECATED]
1. Parameters connected directly to ReadValues (States) after the transformation is applied are not allowed.
Unnecessary parameters may remain on the graph after applying the transformation. The automatic handling of this case inside the transformation is not possible now. Such Parameters should be removed manually from `ngraph::Function` or replaced with a Constant.
@ -411,7 +411,7 @@ InferenceEngine::LowLatency(cnnNetwork);
![low_latency_limitation_1](./img/low_latency_limitation_1.png)
**Current solutions:**
* Replace Parameter with Constant (freeze) with the value [0, 0, 0 … 0] via [ModelOptimizer CLI](../MO_DG/prepare_model/convert_model/Converting_Model_General.md) `--input` or `--freeze_placeholder_with_value`.
* Replace Parameter with Constant (freeze) with the value [0, 0, 0 … 0] via [ModelOptimizer CLI](../MO_DG/prepare_model/convert_model/Converting_Model.md) `--input` or `--freeze_placeholder_with_value`.
* Use ngraph API to replace Parameter with Constant.
```cpp
@ -436,7 +436,7 @@ InferenceEngine::LowLatency(cnnNetwork);
![low_latency_limitation_2](./img/low_latency_limitation_2.png)
**Current solution:** trim non-reshapable layers via [ModelOptimizer CLI](../MO_DG/prepare_model/convert_model/Converting_Model_General.md) `--input`, `--output`. For example, we can trim the Parameter and the problematic Constant in the picture above, using the following command line option:
**Current solution:** trim non-reshapable layers via [ModelOptimizer CLI](../MO_DG/prepare_model/convert_model/Converting_Model.md) `--input`, `--output`. For example, we can trim the Parameter and the problematic Constant in the picture above, using the following command line option:
`--input Reshape_layer_name`. We can also replace the problematic Constant using ngraph, as shown in the example below.
```cpp

24
docs/IE_DG/protecting_model_guide.md
View File

@ -1,7 +1,7 @@
# Using Encrypted Models with OpenVINO&trade; {#openvino_docs_IE_DG_protecting_model_guide}
Deploying deep-learning capabilities to edge devices can present security
challenges. For example, ensuring inference integrity or providing copyright
challenges, for example, ensuring inference integrity or providing copyright
protection of your deep-learning models.
One possible solution is to use cryptography to protect models as they are
@ -14,43 +14,41 @@ This guide demonstrates how to use OpenVINO securely with protected models.
## Secure Model Deployment
After a model is optimized by the OpenVINO Model Optimizer, it's then deployed
After a model is optimized by the OpenVINO Model Optimizer, it's deployed
to target devices in the Intermediate Representation (IR) format. An optimized
model is stored on an edge device and executed by the Inference Engine.
model is stored on an edge device and executed by the Inference Engine.
(ONNX and nGraph models can also be read natively by the Inference Engine.)
To protect deep-learning models, you can encrypt an optimized model before
deploying it to the edge device. The edge device should keep the stored model
protected at all times and have the model decrypted **in runtime only** for use
by the Inference Engine.
![deploy_encrypted_model]
![deploy_encrypted_model](img/deploy_encrypted_model.png)
## Loading Encrypted Models
The OpenVINO Inference Engine requires model decryption before loading. Allocate
a temporary memory block for model decryption, and use
`InferenceEngine::Core::ReadNetwork` method to load the model from memory buffer.
For more information, see the `InferenceEngine::Core` Class
Reference Documentation.
a temporary memory block for model decryption and use the
`InferenceEngine::Core::ReadNetwork` method to load the model from a memory buffer.
For more information, see the `InferenceEngine::Core` Class Reference Documentation.
@snippet snippets/protecting_model_guide.cpp part0
Hardware-based protection, such as Intel&reg; Software Guard Extensions
(Intel&reg; SGX), can be utilized to protect decryption operation secrets and
Hardware-based protection such as Intel&reg; Software Guard Extensions
(Intel&reg; SGX) can be utilized to protect decryption operation secrets and
bind them to a device. For more information, go to [Intel&reg; Software Guard
Extensions](https://software.intel.com/en-us/sgx).
Use `InferenceEngine::Core::ReadNetwork()` to set model representations and
weights respectively.
Currently there are no possibility to read external weights from memory for ONNX models.
Currently there is no way to read external weights from memory for ONNX models.
The `ReadNetwork(const std::string& model, const Blob::CPtr& weights)` function
should be called with `weights` passed as an empty `Blob`.
@snippet snippets/protecting_model_guide.cpp part1
[deploy_encrypted_model]: img/deploy_encrypted_model.png
## Additional Resources
- Intel® Distribution of OpenVINO™ toolkit home page: [https://software.intel.com/en-us/openvino-toolkit](https://software.intel.com/en-us/openvino-toolkit)

66
docs/IE_DG/supported_plugins/AUTO.md
View File

@ -1,6 +1,12 @@
# Auto-Device Plugin {#openvino_docs_IE_DG_supported_plugins_AUTO}
## Auto-Device Plugin Execution
## Auto-Device Plugin Execution (C++)
@sphinxdirective
.. raw:: html
<div id="switcher-cpp" class="switcher-anchor">C++</div>
@endsphinxdirective
Auto-device is a new special "virtual" or "proxy" device in the OpenVINO™ toolkit.
@ -12,14 +18,13 @@ for example, precisions. Then Auto-device assigns inference requests to the sele
From the application point of view, this is just another device that handles all accelerators in full system.
With the 2021.4 release, Auto-device setup is done in three major steps:
* Step 1: Configure each device as usual (for example, via the conventional <code>SetConfig</code> method)
* Step 2: Load a network to the Auto-device plugin. This is the only change needed in your application
* Step 3: Just like with any other executable network (resulted from <code>LoadNetwork</code>), create as many requests as needed to saturate the devices.
These steps are covered below in details.
1. Step 1: Configure each device as usual (for example, via the conventional <code>SetConfig</code> method)
2. Step 2: Load a network to the Auto-device plugin. This is the only change needed in your application
3. Step 3: Just like with any other executable network (resulted from <code>LoadNetwork</code>), create as many requests as needed to saturate the devices.
These steps are covered below in detail.
## Defining and Configuring the Auto-Device Plugin
Following the OpenVINO notions of “devices”, the Auto-device has “AUTO” name. The only configuration option for Auto-device is a limited device list:
Following the OpenVINO notions of "devices", the Auto-device has "AUTO" name. The only configuration option for Auto-device is a limited device list:
| Parameter name | Parameter values | Default | Description |
| :--- | :--- | :--- |:-----------------------------------------------------------------------------|
@ -37,7 +42,9 @@ There are two ways to use Auto-device:
@snippet snippets/AUTO1.cpp part1
Auto-device supports query device optimization capabilities in metric;
> **NOTE:** The Inference Engine lets you use "GPU" as an alias for "GPU.0" in function calls.
Auto-device supports query device optimization capabilities in metric:
| Parameter name | Parameter values |
| :--- | :--- |
@ -61,14 +68,16 @@ Available devices:
Device: GPU.1
```
### Default Auto-Device selecting logic
### Default Auto-Device selecting logic
With the 2021.4 release, Auto-Device selects the most suitable device with following default logic:
1. Check if dGPU, iGPU and CPU device are available
2. Get the precision of the input model, such as FP32
3. According to the priority of dGPU, iGPU and CPU (in this order), if the device supports the precision of input network, select it as the most suitable device
1. Check if dGPU (discrete Intel® GPU), iGPU (integrated Intel® GPU) and CPU device are available.
2. Get the precision of the input model, such as FP32.
3. According to the priority of dGPU, iGPU and CPU (in this order), if the device supports the precision of input network, select it as the most suitable device.
For example, CPU, dGPU and iGPU can support below precision and optimization capabilities:
#### Examples
For example, CPU, dGPU, and iGPU can support the precision and optimization capabilities below:
| Device | OPTIMIZATION_CAPABILITIES |
| :--- | :--- |
@ -76,22 +85,22 @@ For example, CPU, dGPU and iGPU can support below precision and optimization cap
| dGPU | FP32 BIN BATCHED_BLOB FP16 INT8 |
| iGPU | FP32 BIN BATCHED_BLOB FP16 INT8 |
When application use Auto-device to run FP16 IR on system with CPU, dGPU and iGPU, Auto-device will offload this workload to dGPU.
When an application uses Auto-device to run FP16 IR on system with CPU, dGPU and iGPU, the plugin will offload this workload to dGPU.
When application use Auto-device to run FP16 IR on system with CPU and iGPU, Auto-device will offload this workload to iGPU.
When an application uses Auto-device to run FP16 IR on system with CPU and iGPU, the plugin will offload this workload to iGPU.
When application use Auto-device to run WINOGRAD-enabled IR on system with CPU, dGPU and iGPU, Auto-device will offload this workload to CPU.
When an application uses Auto-device to run WINOGRAD-enabled IR on system with CPU, dGPU and iGPU, the plugin will offload this workload to CPU.
In any case, when loading the network to dGPU or iGPU fails, the networks falls back to CPU as the last choice.
Whenever loading a network to dGPU or iGPU fails, the plugin falls back to CPU as the last choice.
### Limit Auto Target Devices Logic
According to the Auto-device selection logic from the previous section,
the most suitable device from available devices to load mode as follows:
According to the Auto-device selection logic from the previous section, tell the Inference Engine
to use the most suitable device from available devices as follows:
@snippet snippets/AUTO2.cpp part2
Another way to load mode to device from limited choice of devices is with Auto-device:
You can also use the Auto-device plugin to choose a device from a limited choice of devices, in this example CPU and GPU:
@snippet snippets/AUTO3.cpp part3
@ -108,21 +117,22 @@ allowing the Auto-device plugin to parse and apply it to the right devices. See
## Using the Auto-Device with OpenVINO Samples and Benchmark App
Note that every OpenVINO sample that supports "-d" (which stands for "device") command-line option transparently accepts the Auto-device.
The Benchmark Application is the best example of the optimal usage of the Auto-device.
Note that every OpenVINO sample that supports the "-d" (which stands for "device") command-line option transparently accepts the Auto-device.
The Benchmark Application is the best example of optimal usage of the Auto-device.
You do not need to set the number of requests and CPU threads, as the application provides optimal out-of-the-box performance.
Below is the example command-line to evaluate AUTO performance with that:
Below is an example command to evaluate AUTO performance with the Benchmark application:
```sh
./benchmark_app –d AUTO –m <model> -i <input> -niter 1000
```
You can also use the auto-device with limit device choice:
You can also use the Auto-device plugin with limited device choice:
```sh
./benchmark_app –d AUTO:CPU,GPU –m <model> -i <input> -niter 1000
```
Note that the default CPU stream is 1 if using “-d AUTO”.
Note that you can use the FP16 IR to work with auto-device.
Also note that no demos are (yet) fully optimized for the auto-device, by means of selecting the most suitable device,
using the GPU streams/throttling, and so on.
**NOTES:**
* The default CPU stream is 1 if using “-d AUTO”.
* You can use the FP16 IR to work with Auto-device.
* No demos are fully optimized for Auto-device yet to select the most suitable device,
use GPU streams/throttling, and so on.

61
docs/IE_DG/supported_plugins/CPU.md
View File

@ -1,8 +1,8 @@
CPU Plugin {#openvino_docs_IE_DG_supported_plugins_CPU}
=======
## Introducing CPU Plugin
The CPU plugin was developed in order to provide opportunity for high performance scoring of neural networks on CPU, using the Intel® Math Kernel Library for Deep Neural Networks (Intel® MKL-DNN).
## Introducing the CPU Plugin
The CPU plugin was developed to achieve high performance of neural networks on CPU, using the Intel® Math Kernel Library for Deep Neural Networks (Intel® MKL-DNN).
Currently, the CPU plugin uses Intel® Threading Building Blocks (Intel® TBB) in order to parallelize calculations. Please refer to the [Optimization Guide](../../optimization_guide/dldt_optimization_guide.md) for associated performance considerations.
@ -10,56 +10,59 @@ The set of supported layers can be expanded with [the Extensibility mechanism](.
## Supported Platforms
OpenVINO™ toolkit is officially supported and validated on the following platforms:
OpenVINO™ toolkit, including the CPU plugin, is officially supported and validated on the following platforms:
| Host | OS (64-bit) |
| :--- | :--- |
| Development | Ubuntu* 18.04, CentOS* 7.5, MS Windows* 10 |
| Target | Ubuntu* 18.04, CentOS* 7.5, MS Windows* 10 |
| Development | Ubuntu* 18.04 or 20.04, CentOS* 7.6, MS Windows* 10, macOS* 10.15 |
| Target | Ubuntu* 18.04 or 20.04, CentOS* 7.6, MS Windows* 10, macOS* 10.15 |
The CPU Plugin supports inference on Intel® Xeon® with Intel® Advanced Vector Extensions 2 (Intel® AVX2), Intel® Advanced Vector Extensions 512 (Intel® AVX-512), and AVX512_BF16, Intel® Core™
The CPU plugin supports inference on Intel® Xeon® with Intel® Advanced Vector Extensions 2 (Intel® AVX2), Intel® Advanced Vector Extensions 512 (Intel® AVX-512), and AVX512_BF16, Intel® Core™
Processors with Intel® AVX2, Intel Atom® Processors with Intel® Streaming SIMD Extensions (Intel® SSE).
You can use `-pc` the flag for samples to know which configuration is used by some layer.
This flag shows execution statistics that you can use to get information about layer name,
execution status, layer type, execution time, and the type of the execution primitive.
You can use the `-pc` flag for samples to know which configuration is used by a layer.
This flag shows execution statistics that you can use to get information about layer name, layer type,
execution status, execution time, and the type of the execution primitive.
## Internal CPU Plugin Optimizations
CPU plugin supports several graph optimization algorithms, such as fusing or removing layers.
The CPU plugin supports several graph optimization algorithms, such as fusing or removing layers.
Refer to the sections below for details.
> **NOTE**: For layer descriptions, see the [IR Notation Reference](../../ops/opset.md).
### Lowering Inference Precision
CPU plugin follows default optimization approach. This approach means that inference is made with lower precision if it is possible on a given platform to reach better performance with acceptable range of accuracy.
The CPU plugin follows a default optimization approach. This approach means that inference is made with lower precision if it is possible on a given platform to reach better performance with an acceptable range of accuracy.
> **NOTE**: For details, see the [Using Bfloat16 Inference](../Bfloat16Inference.md).
### Fusing Convolution and Simple Layers
Merge of a Convolution layer and any of the simple layers listed below:
Merge of a convolution layer and any of the simple layers listed below:
- Activation: ReLU, ELU, Sigmoid, Clamp
- Depthwise: ScaleShift, PReLU
- FakeQuantize
> **NOTE**: You can have any number and order of simple layers.
A combination of a Convolution layer and simple layers results in a single fused layer called
*Convolution*:
A combination of a convolution layer and simple layers results in a single fused layer called
*Convolution*:
![conv_simple_01]
### Fusing Pooling and FakeQuantize Layers
A combination of Pooling and FakeQuantize layers results in a single fused layer called *Pooling*:
![pooling_fakequant_01]
### Fusing FullyConnected and Activation Layers
A combination of FullyConnected and Activation layers results in a single fused layer called
*FullyConnected*:
*FullyConnected*:
![fullyconnected_activation_01]
@ -76,16 +79,19 @@ layer and simple layers results in a single layer called *Convolution* (or *Bina
### Fusing Convolution and Sum Layers
A combination of Convolution, Simple, and Eltwise layers with the sum operation results in a single layer called *Convolution*:
A combination of convolution, simple, and Eltwise layers with the sum operation results in a single layer called *Convolution*:
![conv_sum_relu_01]
### Fusing a Group of Convolutions
If a topology contains the following pipeline, a CPU plugin merges Split, Convolution, and Concatenation layers into a single Convolution layer with the group parameter:
> **NOTE**: Parameters of the Convolution layers must coincide.
If a topology contains the following pipeline, a CPU plugin merges split, convolution, and concatenation layers into a single convolution layer with the group parameter:
![group_convolutions_01]
> **NOTE**: Parameters of the convolution layers must coincide.
### Removing a Power Layer
CPU plugin removes a Power layer from a topology if it has the following parameters:
@ -97,7 +103,7 @@ CPU plugin removes a Power layer from a topology if it has the following paramet
## Supported Configuration Parameters
The plugin supports the configuration parameters listed below.
All parameters must be set with the <code>InferenceEngine::Core::LoadNetwork()</code> method.
All parameters must be set with the `InferenceEngine::Core::LoadNetwork()` method.
When specifying key values as raw strings (that is, when using Python API), omit the `KEY_` prefix.
Refer to the OpenVINO samples for usage examples: [Benchmark App](../../../inference-engine/samples/benchmark_app/README.md).
@ -105,18 +111,17 @@ These are general options, also supported by other plugins:
| Parameter name | Parameter values | Default | Description |
| :--- | :--- | :--- | :----------------------------------------------------------------------------------------------------------------------------|
| `KEY_EXCLUSIVE_ASYNC_REQUESTS` | `YES`/`NO` | `NO` | Forces async requests (also from different executable networks) to execute serially. This prevents potential oversubscription|
| `KEY_PERF_COUNT` | `YES`/`NO` | `NO` | Enables gathering performance counters |
| KEY_EXCLUSIVE_ASYNC_REQUESTS | YES/NO | NO | Forces async requests (also from different executable networks) to execute serially. This prevents potential oversubscription|
| KEY_PERF_COUNT | YES/NO | NO | Enables gathering performance counters |
CPU-specific settings:
| Parameter name | Parameter values | Default | Description |
| :--- | :--- | :--- |:-----------------------------------------------------------------------------|
| `KEY_CPU_THREADS_NUM` | `positive integer values`| `0` | Specifies the number of threads that CPU plugin should use for inference. Zero (default) means using all (logical) cores|
| `KEY_CPU_BIND_THREAD` | `YES`/`NUMA`/`NO` | `YES` | Binds inference threads to CPU cores. 'YES' (default) binding option maps threads to cores - this works best for static/synthetic scenarios like benchmarks. The 'NUMA' binding is more relaxed, binding inference threads only to NUMA nodes, leaving further scheduling to specific cores to the OS. This option might perform better in the real-life/contended scenarios. Note that for the latency-oriented cases (number of the streams is less or equal to the number of NUMA nodes, see below) both YES and NUMA options limit number of inference threads to the number of hardware cores (ignoring hyper-threading) on the multi-socket machines. |
| `KEY_CPU_THROUGHPUT_STREAMS` | `KEY_CPU_THROUGHPUT_NUMA`, `KEY_CPU_THROUGHPUT_AUTO`, or `positive integer values`| `1` | Specifies number of CPU "execution" streams for the throughput mode. Upper bound for the number of inference requests that can be executed simultaneously. All available CPU cores are evenly distributed between the streams. The default value is 1, which implies latency-oriented behavior for single NUMA-node machine, with all available cores processing requests one by one. On the multi-socket (multiple NUMA nodes) machine, the best latency numbers usually achieved with a number of streams matching the number of NUMA-nodes. <br>`KEY_CPU_THROUGHPUT_NUMA` creates as many streams as needed to accommodate NUMA and avoid associated penalties.<br>`KEY_CPU_THROUGHPUT_AUTO` creates bare minimum of streams to improve the performance; this is the most portable option if you don't know how many cores your target machine has (and what would be the optimal number of streams). Note that your application should provide enough parallel slack (for example, run many inference requests) to leverage the throughput mode. <br> Non-negative integer value creates the requested number of streams. If a number of streams is 0, no internal streams are created and user threads are interpreted as stream master threads.|
| `KEY_ENFORCE_BF16` | `YES`/`NO`| `YES` | The name for setting to execute in bfloat16 precision whenever it is possible. This option lets plugin know to downscale the precision where it sees performance benefits from bfloat16 execution. Such option does not guarantee accuracy of the network, you need to verify the accuracy in this mode separately, based on performance and accuracy results. It should be your decision whether to use this option or not. |
| Parameter name | Parameter values | Default | Description |
| :--- | :--- | :--- | :--- |
| KEY_CPU_THREADS_NUM | positive integer values| 0 | Specifies the number of threads that CPU plugin should use for inference. Zero (default) means using all (logical) cores|
| KEY_CPU_BIND_THREAD | YES/NUMA/NO | YES | Binds inference threads to CPU cores. 'YES' (default) binding option maps threads to cores - this works best for static/synthetic scenarios like benchmarks. The 'NUMA' binding is more relaxed, binding inference threads only to NUMA nodes, leaving further scheduling to specific cores to the OS. This option might perform better in the real-life/contended scenarios. Note that for the latency-oriented cases (number of the streams is less or equal to the number of NUMA nodes, see below) both YES and NUMA options limit number of inference threads to the number of hardware cores (ignoring hyper-threading) on the multi-socket machines. |
| KEY_CPU_THROUGHPUT_STREAMS | KEY_CPU_THROUGHPUT_NUMA, KEY_CPU_THROUGHPUT_AUTO, or positive integer values| 1 | Specifies number of CPU "execution" streams for the throughput mode. Upper bound for the number of inference requests that can be executed simultaneously. All available CPU cores are evenly distributed between the streams. The default value is 1, which implies latency-oriented behavior for single NUMA-node machine, with all available cores processing requests one by one. On the multi-socket (multiple NUMA nodes) machine, the best latency numbers usually achieved with a number of streams matching the number of NUMA-nodes. <br>KEY_CPU_THROUGHPUT_NUMA creates as many streams as needed to accommodate NUMA and avoid associated penalties.<br>KEY_CPU_THROUGHPUT_AUTO creates bare minimum of streams to improve the performance; this is the most portable option if you don't know how many cores your target machine has (and what would be the optimal number of streams). Note that your application should provide enough parallel slack (for example, run many inference requests) to leverage the throughput mode. <br> Non-negative integer value creates the requested number of streams. If a number of streams is 0, no internal streams are created and user threads are interpreted as stream master threads.|
| KEY_ENFORCE_BF16 | YES/NO| YES | The name for setting to execute in bfloat16 precision whenever it is possible. This option lets plugin know to downscale the precision where it sees performance benefits from bfloat16 execution. Such option does not guarantee accuracy of the network, you need to verify the accuracy in this mode separately, based on performance and accuracy results. It should be your decision whether to use this option or not. |
> **NOTE**: To disable all internal threading, use the following set of configuration parameters: `KEY_CPU_THROUGHPUT_STREAMS=0`, `KEY_CPU_THREADS_NUM=1`, `KEY_CPU_BIND_THREAD=NO`.

22
docs/IE_DG/supported_plugins/GNA.md
View File

@ -2,11 +2,11 @@
## Introducing the GNA Plugin
Intel® Gaussian & Neural Accelerator is a low-power neural coprocessor for continuous inference at the edge.
The Intel® Gaussian & Neural Accelerator is a low-power neural coprocessor for continuous inference at the edge.
Intel® GNA is not intended to replace classic inference devices such as
The Intel® GNA is not intended to replace classic inference devices such as a
CPU, graphics processing unit (GPU), or vision processing unit (VPU). It is designed for offloading
continuous inference workloads including but not limited to noise reduction or speech recognition
continuous inference workloads such as noise reduction or speech recognition
to save power and free CPU resources.
The GNA plugin provides a way to run inference on Intel® GNA, as well as in the software execution mode on CPU.
@ -86,7 +86,7 @@ Intel® GNA essentially operates in the low-precision mode, which represents a m
Unlike other plugins supporting low-precision execution, the GNA plugin can calculate quantization factors at the model loading time, so you can run a model without calibration using the [Post-Training Optimizaton Tool](@ref pot_README).
However, this mode may not provide satisfactory accuracy because the internal quantization algorithm is based on heuristics which may or may not be efficient, depending on the model and dynamic range of input data.
Starting with 2021.4 release of OpenVINO™, GNA plugin users are encouraged to use the [POT API Usage sample for GNA](@ref pot_sample_speech_README) to get a model with quantization hints based on statistics for the provided dataset.
Starting with 2021.4 release of OpenVINO, GNA plugin users are encouraged to use the [POT API Usage sample for GNA](@ref pot_sample_speech_README) to get a model with quantization hints based on statistics for the provided dataset.
## <a name="execution-modes">Execution Modes</a>
@ -97,7 +97,6 @@ Starting with 2021.4 release of OpenVINO™, GNA plugin users are encouraged to
| `GNA_HW` | Uses Intel® GNA if available, otherwise raises an error. |
| `GNA_SW` | *Deprecated*. Executes the GNA-compiled graph on CPU performing calculations in the same precision as the Intel® GNA, but not in the bit-exact mode. |
| `GNA_SW_EXACT` | Executes the GNA-compiled graph on CPU performing calculations in the same precision as the Intel® GNA in the bit-exact mode. |
| `GNA_HW_WITH_SW_FBACK` | Uses Intel® GNA if available, otherwise raises an error. If the HW queue is not empty, automatically falls back to CPU in the bit-exact mode. |
| `GNA_SW_FP32` | Executes the GNA-compiled graph on CPU but substitutes parameters and calculations from low precision to floating point (`FP32`). |
## Supported Configuration Parameters
@ -190,19 +189,6 @@ executableNet.SetConfig(newConfig);
```
2. Resubmit and switch back to GNA_HW expecting that the competing application has finished.
> **NOTE:** This method is deprecated since a new automatic QoS mode has been introduced in 2021.4.1 release of OpenVINO™ (see below).
## GNA3 Automatic QoS Feature on Windows*
Starting with 2021.4.1 release of OpenVINO and 03.00.00.1363 version of Windows* GNA driver, a new execution mode (GNA_HW_WITH_SW_FBACK) is introduced
to assure that workloads satisfy real-time execution. In this mode, the GNA driver automatically falls back on CPU for a particular infer request
if the HW queue is not empty, so there is no need for explicitly switching between GNA and CPU.
**NOTE:** Due to the "first come - first served" nature of GNA driver and the QoS feature, this mode may lead to increased CPU consumption
if there are several clients using GNA simultaneously.
Even a lightweight competing infer request which has not been cleared at the time when the user's GNA client process makes its request,
can cause the user's request to be executed on CPU, thereby unnecessarily increasing CPU utilization and power.
## See Also
* [Supported Devices](Supported_Devices.md)

12
docs/IE_DG/supported_plugins/GPU.md
View File

@ -1,6 +1,17 @@
GPU Plugin {#openvino_docs_IE_DG_supported_plugins_GPU}
=======
@sphinxdirective
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_IE_DG_supported_plugins_GPU_RemoteBlob_API
@endsphinxdirective
The GPU plugin uses the Intel® Compute Library for Deep Neural Networks (clDNN) to infer deep neural networks.
clDNN is an open source performance library for Deep Learning (DL) applications intended for acceleration of Deep Learning Inference on Intel® Processor Graphics including Intel® HD Graphics, Intel® Iris® Graphics, Intel® Iris® Xe Graphics, and Intel® Iris® Xe MAX graphics.
For an in-depth description of clDNN, see [Inference Engine source files](https://github.com/openvinotoolkit/openvino/tree/master/inference-engine/src/cldnn_engine) and [Accelerate Deep Learning Inference with Intel® Processor Graphics](https://software.intel.com/en-us/articles/accelerating-deep-learning-inference-with-intel-processor-graphics).
@ -99,7 +110,6 @@ The plugin supports the configuration parameters listed below.
All parameters must be set before calling <code>InferenceEngine::Core::LoadNetwork()</code> in order to take effect.
When specifying key values as raw strings (that is, when using Python API), omit the `KEY_` prefix.
| Parameter Name | Parameter Values | Default | Description |
|---------------------|-----------------------------|-----------------|-----------------------------------------------------------|
| `KEY_CACHE_DIR` | `"<cache_dir>"` | `""` | Specifies a directory where compiled OCL binaries can be cached. First model loading generates the cache, and all subsequent LoadNetwork calls use precompiled kernels which significantly improves load time. If empty - caching is disabled |

4
docs/IE_DG/supported_plugins/GPU_RemoteBlob_API.md
View File

@ -4,13 +4,13 @@ Remote Blob API of GPU Plugin {#openvino_docs_IE_DG_supported_plugins_GPU_Remote
The GPU plugin implementation of the `RemoteContext` and `RemoteBlob` interfaces supports GPU
pipeline developers who need video memory sharing and interoperability with existing native APIs
such as OpenCL\*, Microsoft DirectX\*, or VAAPI\*.
Using these interfaces allows to avoid any memory copy overhead when plugging the OpenVINO™ inference
Using these interfaces allows you to avoid any memory copy overhead when plugging the OpenVINO™ inference
into an existing GPU pipeline. It also enables OpenCL kernels participating in the pipeline to become
native buffer consumers or producers of the OpenVINO™ inference.
Since the GPU plugin works on top of the clDNN library, the functionality above is also implemented
using OpenCL and its sharing extensions provided by Intel®.
There are two interoperability scenarios that are supported for the Remote Blob API:
There are two interoperability scenarios supported by the Remote Blob API:
* GPU plugin context and memory objects can be constructed from low-level device, display, or memory
handles and used to create the OpenVINO™ `ExecutableNetwork` or `Blob` class.

14
docs/IE_DG/supported_plugins/HDDL.md
View File

@ -6,27 +6,27 @@ The Inference Engine HDDL plugin was developed for inference with neural network
## Configuring the HDDL Plugin
To configure your Intel® Vision Accelerator Design With Intel® Movidius™ on supported OSs, refer to the Steps for Intel® Vision Accelerator Design with Intel® Movidius™ VPUs section in the installation guides for [Linux](../../install_guides/installing-openvino-linux.md) or [Windows](../../install_guides/installing-openvino-windows.md).
To configure your Intel® Vision Accelerator Design With Intel® Movidius™ on supported operating systems, refer to the Steps for Intel® Vision Accelerator Design with Intel® Movidius™ VPUs section in the installation guides for [Linux](../../install_guides/installing-openvino-linux.md) or [Windows](../../install_guides/installing-openvino-windows.md).
## Supported networks
For the "Supported Networks", please reference to [MYRIAD Plugin](MYRIAD.md)
To see the list of supported networks for the HDDL plugin, refer to the list on the [MYRIAD Plugin page](MYRIAD.md).
## Supported Configuration Parameters
See VPU common configuration parameters for the [VPU Plugins](VPU.md).
When specifying key values as raw strings (that is, when using Python API), omit the `KEY_` prefix.
See VPU common configuration parameters for [VPU Plugins](VPU.md).
When specifying key values as raw strings (that is, when using the Python API), omit the `KEY_` prefix.
In addition to common parameters for MYRIAD plugin and HDDL plugin, HDDL plugin accepts the following options:
In addition to common parameters for both VPU plugins, the HDDL plugin accepts the following options:
| Parameter Name | Parameter Values | Default | Description |
| :--- | :--- | :--- | :--- |
| KEY_PERF_COUNT | YES/NO | NO | Enable performance counter option. |
| KEY_PERF_COUNT | YES/NO | NO | Enable performance counter option. |
| KEY_VPU_HDDL_GRAPH_TAG | string | empty string | Allows to execute network on specified count of devices. |
| KEY_VPU_HDDL_STREAM_ID | string | empty string | Allows to execute inference on a specified device. |
| KEY_VPU_HDDL_DEVICE_TAG | string | empty string | Allows to allocate/deallocate networks on specified devices. |
| KEY_VPU_HDDL_BIND_DEVICE | YES/NO | NO | Whether the network should bind to a device. Refer to vpu_plugin_config.hpp. |
| KEY_VPU_HDDL_RUNTIME_PRIORITY | signed int | 0 | Specify the runtime priority of a device among all devices that running a same network Refer to vpu_plugin_config.hpp. |
| KEY_VPU_HDDL_RUNTIME_PRIORITY | signed int | 0 | Specify the runtime priority of a device among all devices running the same network. Refer to vpu_plugin_config.hpp. |
## See Also

89
docs/IE_DG/supported_plugins/HETERO.md
View File

@ -1,7 +1,12 @@
Heterogeneous Plugin {#openvino_docs_IE_DG_supported_plugins_HETERO}
=======
# Heterogeneous Plugin {#openvino_docs_IE_DG_supported_plugins_HETERO}
## Introducing the Heterogeneous Plugin
## Introducing the Heterogeneous Plugin (C++)
@sphinxdirective
.. raw:: html
<div id="switcher-cpp" class="switcher-anchor">C++</div>
@endsphinxdirective
The heterogeneous plugin enables computing for inference on one network on several devices.
The purposes of executing networks in heterogeneous mode:
@ -9,29 +14,40 @@ The purposes of executing networks in heterogeneous mode:
* Utilize all available hardware more efficiently during one inference
The execution through heterogeneous plugin can be divided to two independent steps:
* Setting of affinity to layers
* Setting of affinity to layers (in manual mode or automatically using fallback policy)
* Loading a network to the Heterogeneous plugin, splitting the network to parts, and executing them through the plugin
These steps are decoupled. The setting of affinity can be done automatically using fallback policy or in manual mode.
The fallback policy for automatic affinity causes "greedy" behavior and assigns all layers that can be executed on certain device according to the priorities you specify (for example, `HETERO:GPU,CPU`).
Automatic policy does not take into account plugin peculiarities such as the inability to infer some layers without other special layers placed before or after that layer. The plugin is responsible for solving such cases. If the device plugin does not support the subgraph topology constructed by the Heterogeneous plugin, then you should set affinity manually.
The fallback automatic policy causes "greedy" behavior and assigns all layers that can be executed on certain device according to the priorities you specify (for example, `HETERO:GPU,CPU`).
Automatic policy does not take into account plugin peculiarities such as the inability to infer some layers without other special layers placed before or after that layer. The plugin is responsible for solving such cases. If the device plugin does not support the subgraph topology constructed by the Hetero plugin, then you should set affinity manually.
### Details of Splitting Network and Execution
During loading of the network to the Heterogeneous plugin, the network is divided into separate parts and loaded to dedicated plugins.
Intermediate blobs between these subgraphs are allocated automatically in the most efficient way.
## Sample Usage
Inference Engine sample programs can use the Heterogeneous plugin used with the `-d` option:
```sh
./object_detection_sample_ssd -m <path_to_model>/ModelSSD.xml -i <path_to_pictures>/picture.jpg -d HETERO:GPU,CPU
```
where:
- `HETERO` stands for the Heterogeneous plugin
- `GPU,CPU` points to fallback policy with priority on GPU and fallback to CPU
You can point more than two devices: `-d HETERO:MYRIAD,GPU,CPU`
Some of the topologies are not friendly to heterogeneous execution on some devices or cannot be executed in such mode at all.
Examples of such networks are networks having activation layers which are not supported on primary device.
If transmitting data from one part of a network to another part in heterogeneous mode takes more time than in normal mode, it may not make sense to execute them in heterogeneous mode.
In this case, you can define heaviest part manually and set the affinity to avoid sending data back and forth many times during one inference.
## Annotation of Layers per Device and Default Fallback Policy
Default fallback policy decides which layer goes to which device automatically according to the support in dedicated plugins (FPGA, GPU, CPU, MYRIAD).
Default fallback policy decides which layer goes to which device automatically according to the support in dedicated plugins (GPU, CPU, MYRIAD).
Another way to annotate a network is to set affinity manually using <code>ngraph::Node::get_rt_info</code> with key `"affinity"`:
@snippet snippets/HETERO0.cpp part0
The fallback policy does not work if even one layer has an initialized affinity. The sequence should be calling of automating affinity settings and then fix manually.
The fallback policy does not work if even one layer has an initialized affinity. The sequence should be to call automating affinity settings and then fix manually.
> **NOTE**: If you set affinity manually, be careful at the current moment Inference Engine plugins don't support constant (`Constant`->`Result`) and empty (`Parameter`->`Result`) networks. Please avoid such subgraphs when you set affinity manually.
> **NOTE**: If you set affinity manually, be careful because currently Inference Engine plugins don't support constant (`Constant`->`Result`) and empty (`Parameter`->`Result`) networks. Please avoid such subgraphs when you set affinity manually.
@snippet snippets/HETERO1.cpp part1
@ -39,56 +55,47 @@ If you rely on the default affinity distribution, you can avoid calling <code>In
@snippet snippets/HETERO2.cpp part2
> **NOTE**: `InferenceEngine::Core::QueryNetwork` does not depend on affinities set by a user, but queries for layer support based on device capabilities.
> **NOTE**: `InferenceEngine::Core::QueryNetwork` does not depend on affinities set by a user. Instead, it queries for layer support based on device capabilities.
### Handling Difficult Topologies
Some topologies are not friendly to heterogeneous execution on some devices or cannot be executed at all with this plugin
Examples are networks having activation layers that are not supported on the primary device.
If transmitting data from one part of a network to another part in heterogeneous mode takes more time than in normal mode, it may not make sense to execute them in heterogeneous mode.
In this case, you can define the heaviest part manually and set the affinity to avoid sending data back and forth many times during one inference.
## Details of Splitting Network and Execution
During loading of the network to heterogeneous plugin, network is divided to separate parts and loaded to dedicated plugins.
Intermediate blobs between these sub graphs are allocated automatically in the most efficient way.
## Execution Precision
Precision for inference in heterogeneous plugin is defined by
* Precision of IR.
* Ability of final plugins to execute in precision defined in IR
Examples:
Example:
* If you want to execute GPU with CPU fallback with FP16 on GPU, you need to use only FP16 IR.
* If you want to execute on FPGA with CPU fallback, you can use any precision for IR. The execution on FPGA is defined by bitstream, the execution on CPU happens in FP32.
Samples can be used with the following command:
## Analyzing Performance Heterogeneous Execution
After enabling the <code>KEY_HETERO_DUMP_GRAPH_DOT</code> config key (shown in code snippet below), you can dump GraphViz* `.dot` files with annotations of devices per layer.
```sh
./object_detection_sample_ssd -m <path_to_model>/ModelSSD.xml -i <path_to_pictures>/picture.jpg -d HETERO:FPGA,CPU
```
where:
- `HETERO` stands for heterogeneous plugin
- `FPGA,CPU` points to fallback policy with priority on FPGA and fallback to CPU
You can point more than two devices: `-d HETERO:FPGA,GPU,CPU`
## Analyzing Heterogeneous Execution
After enabling of <code>KEY_HETERO_DUMP_GRAPH_DOT</code> config key, you can dump GraphViz* `.dot` files with annotations of devices per layer.
Heterogeneous plugin can generate two files:
* `hetero_affinity_<network name>.dot` - annotation of affinities per layer. This file is written to the disk only if default fallback policy was executed
* `hetero_subgraphs_<network name>.dot` - annotation of affinities per graph. This file is written to the disk during execution of <code>ICNNNetwork::LoadNetwork()</code> for heterogeneous plugin
The Heterogeneous plugin can generate two files:
* `hetero_affinity_<network name>.dot`: Annotation of affinities per layer. This file is written to the disk only if default fallback policy was executed
* `hetero_subgraphs_<network name>.dot`: Annotation of affinities per graph. This file is written to the disk during execution of <code>ICNNNetwork::LoadNetwork()</code> for the Heterogeneous plugin.
@snippet snippets/HETERO3.cpp part3
You can use GraphViz* utility or converters to `.png` formats. On Ubuntu* operating system, you can use the following utilities:
You can use GraphViz* utility or converters to `.png` formats. On the Ubuntu* operating system, you can use the following utilities:
* `sudo apt-get install xdot`
* `xdot hetero_subgraphs.dot`
You can use performance data (in samples, it is an option `-pc`) to get performance data on each subgraph.
Here is an example of the output: for Googlenet v1 running on FPGA with fallback to CPU:
Here is an example of the output: for Googlenet v1 running on MYRIAD with fallback to CPU:
```cpp
subgraph1: 1. input preprocessing (mean data/FPGA):EXECUTED layerType: realTime: 129 cpu: 129 execType:
subgraph1: 1. input preprocessing (mean data/MYRIAD):EXECUTED layerType: realTime: 129 cpu: 129 execType:
subgraph1: 2. input transfer to DDR:EXECUTED layerType: realTime: 201 cpu: 0 execType:
subgraph1: 3. FPGA execute time:EXECUTED layerType: realTime: 3808 cpu: 0 execType:
subgraph1: 3. MYRIAD execute time:EXECUTED layerType: realTime: 3808 cpu: 0 execType:
subgraph1: 4. output transfer from DDR:EXECUTED layerType: realTime: 55 cpu: 0 execType:
subgraph1: 5. FPGA output postprocessing:EXECUTED layerType: realTime: 7 cpu: 7 execType:
subgraph1: 5. MYRIAD output postprocessing:EXECUTED layerType: realTime: 7 cpu: 7 execType:
subgraph1: 6. copy to IE blob:EXECUTED layerType: realTime: 2 cpu: 2 execType:
subgraph2: out_prob: NOT_RUN layerType: Output realTime: 0 cpu: 0 execType: unknown
subgraph2: prob: EXECUTED layerType: SoftMax realTime: 10 cpu: 10 execType: ref

53
docs/IE_DG/supported_plugins/MULTI.md
View File

@ -2,19 +2,26 @@
## Introducing the Multi-Device Plugin
The Multi-Device plugin automatically assigns inference requests to available computational devices to execute the requests in parallel. Potential gains are as follows:
@sphinxdirective
.. raw:: html
<div id="switcher-cpp" class="switcher-anchor">C++</div>
@endsphinxdirective
The Multi-Device plugin automatically assigns inference requests to available computational devices to execute the requests in parallel. By contrast, the Heterogeneous plugin can run different layers on different devices but not in parallel. Potential gains with Multi-Device are as follows:
* Improved throughput that multiple devices can deliver (compared to single-device execution)
* More consistent performance, since the devices can now share the inference burden
(so that if one device is becoming too busy, another device can take more of the load)
Notice that with multi-device the application logic is left unchanged, so you don't need to explicitly load the network to every device, create and balance the inference requests and so on. From the application point of view, this is just another device that handles the actual machinery.
The only thing that is required to leverage performance is to provide the multi-device (and hence the underlying devices) with enough inference requests to crunch.
For example, if you were processing 4 cameras on the CPU (with 4 inference requests), you may now want to process more cameras (with more requests in flight) to keep CPU+GPU busy via multi-device.
Notice that with Multi-Device the application logic is left unchanged, so you don't need to explicitly load the network to every device, create and balance the inference requests and so on. From the application point of view, this is just another device that handles the actual machinery.
The only thing that is required to leverage performance is to provide the Multi-Device (and hence the underlying devices) with enough inference requests to crunch.
For example, if you were processing 4 cameras on the CPU (with 4 inference requests), you may now want to process more cameras (with more requests in flight) to keep CPU+GPU busy via Multi-Device.
The "setup" of Multi-Device can be described in three major steps:
* First is configuration of each device as usual (e.g. via conventional SetConfig method)
* Second is loading of a network to the Multi-Device plugin created on top of (prioritized) list of the configured devices. This is the only change that you need in your application.
* Finally, just like with any other ExecutableNetwork (resulted from LoadNetwork) you just create as many requests as needed to saturate the devices.
* First, configure of each device as usual (e.g., via conventional SetConfig method).
* Second, load a network to the Multi-Device plugin created on top of a (prioritized) list of the configured devices. This is the only change that you need in your application.
* Finally, just as with any other ExecutableNetwork (resulting from LoadNetwork), you create as many requests as needed to saturate the devices.
These steps are covered below in details.
## Defining and Configuring the Multi-Device plugin
@ -35,10 +42,10 @@ Notice that the priorities of the devices can be changed in real time for the ex
@snippet snippets/MULTI1.cpp part1
Finally, there is a way to specify number of requests that the multi-device will internally keep for each device. Suppose your original app was running 4 cameras with 4 inference requests. You would probably want to share these 4 requests between 2 devices used in the MULTI. The easiest way is to specify a number of requests for each device using parentheses: "MULTI:CPU(2),GPU(2)" and use the same 4 requests in your app. However, such an explicit configuration is not performance-portable and hence not recommended. Instead, the better way is to configure the individual devices and query the resulting number of requests to be used at the application level (see [Configuring the Individual Devices and Creating the Multi-Device On Top](#configuring-the-individual-devices-and-creating-the-multi-device-on-top)).
Finally, there is a way to specify number of requests that the Multi-Device will internally keep for each device. Suppose your original app was running 4 cameras with 4 inference requests. You would probably want to share these 4 requests between 2 devices used in MULTI. The easiest way is to specify a number of requests for each device using parentheses: "MULTI:CPU(2),GPU(2)" and use the same 4 requests in your app. However, such an explicit configuration is not performance-portable and hence not recommended. Instead, the better way is to configure the individual devices and query the resulting number of requests to be used at the application level (see [Configuring the Individual Devices and Creating the Multi-Device On Top](#configuring-the-individual-devices-and-creating-the-multi-device-on-top)).
## Enumerating Available Devices
Inference Engine now features a dedicated API to enumerate devices and their capabilities. See [Hello Query Device C++ Sample](../../../inference-engine/samples/hello_query_device/README.md). This is example output from the sample (truncated to the devices' names only):
Inference Engine now features a dedicated API to enumerate devices and their capabilities. See [Hello Query Device C++ Sample](../../../inference-engine/samples/hello_query_device/README.md). This is example output from the sample (truncated to the device names only):
```sh
./hello_query_device
@ -51,11 +58,11 @@ Available devices:
...
Device: HDDL
```
A simple programmatic way to enumerate the devices and use with the multi-device is as follows:
A simple programmatic way to enumerate the devices and use with the Multi-Device is as follows:
@snippet snippets/MULTI2.cpp part2
Beyond the trivial "CPU", "GPU", "HDDL" and so on, when multiple instances of a device are available the names are more qualified.
Beyond the trivial "CPU", "GPU" (alias for GPU.0), "HDDL" and so on, when multiple instances of a device are available, the names are more qualified.
For example, this is how two Intel® Movidius™ Myriad™ X sticks are listed with the hello_query_sample:
```
...
@ -76,30 +83,34 @@ As discussed in the first section, you shall configure each individual device as
Alternatively, you can combine all the individual device settings into single config and load that, allowing the Multi-Device plugin to parse and apply that to the right devices. See code example in the next section.
Notice that while the performance of accelerators combines really well with multi-device, the CPU+GPU execution poses some performance caveats, as these devices share the power, bandwidth and other resources. For example it is recommended to enable the GPU throttling hint (which save another CPU thread for the CPU inference).
See section of the [Using the multi-device with OpenVINO samples and benchmarking the performance](#using-the-multi-device-with-openvino-samples-and-benchmarking-the-performance) below.
Notice that while the performance of accelerators combines really well with Multi-Device, the CPU+GPU execution poses some performance caveats, as these devices share the power, bandwidth and other resources. For example it is recommended to enable the GPU throttling hint (which save another CPU thread for the CPU inference).
See the [Using the Multi-Device with OpenVINO samples and benchmarking the performance](#using-the-multi-device-with-openvino-samples-and-benchmarking-the-performance) section below.
## Querying the Optimal Number of Inference Requests
Notice that until R2 you had to calculate number of requests in your application for any device, e.g. you had to know that Intel® Vision Accelerator Design with Intel® Movidius™ VPUs required at least 32 inference requests to perform well. Now you can use the new GetMetric API to query the optimal number of requests. Similarly, when using the multi-device you don't need to sum over included devices yourself, you can query metric directly:
Notice that until R2 you had to calculate number of requests in your application for any device, e.g. you had to know that Intel® Vision Accelerator Design with Intel® Movidius™ VPUs required at least 32 inference requests to perform well. Now you can use the new GetMetric API to query the optimal number of requests. Similarly, when using the Multi-Device you don't need to sum over included devices yourself, you can query metric directly:
@snippet snippets/MULTI5.cpp part5
## Using the Multi-Device with OpenVINO Samples and Benchmarking the Performance
Notice that every OpenVINO sample that supports "-d" (which stands for "device") command-line option transparently accepts the multi-device.
The [Benchmark Application](../../../inference-engine/samples/benchmark_app/README.md) is the best reference to the optimal usage of the multi-device. As discussed multiple times earlier, you don't need to setup number of requests, CPU streams or threads as the application provides optimal out of the box performance.
Notice that every OpenVINO sample that supports "-d" (which stands for "device") command-line option transparently accepts the Multi-Device.
The [Benchmark Application](../../../inference-engine/samples/benchmark_app/README.md) is the best reference to the optimal usage of the Multi-Device. As discussed multiple times earlier, you don't need to setup number of requests, CPU streams or threads as the application provides optimal out of the box performance.
Below is example command-line to evaluate HDDL+GPU performance with that:
```sh
./benchmark_app –d MULTI:HDDL,GPU –m <model> -i <input> -niter 1000
```
Notice that you can use the FP16 IR to work with multi-device (as CPU automatically upconverts it to the fp32) and rest of devices support it naturally.
Also notice that no demos are (yet) fully optimized for the multi-device, by means of supporting the OPTIMAL_NUMBER_OF_INFER_REQUESTS metric, using the GPU streams/throttling, and so on.
Notice that you can use the FP16 IR to work with Multi-Device (as CPU automatically upconverts it to the fp32) and rest of devices support it naturally.
Also notice that no demos are (yet) fully optimized for the Multi-Device, by means of supporting the OPTIMAL_NUMBER_OF_INFER_REQUESTS metric, using the GPU streams/throttling, and so on.
## Video: MULTI Plugin
<iframe width="560" height="315" src="https://www.youtube.com/embed/xbORYFEmrqU" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
@sphinxdirective
.. raw:: html
<iframe width="560" height="315" src="https://www.youtube.com/embed/xbORYFEmrqU" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
@endsphinxdirective
## See Also
* [Supported Devices](Supported_Devices.md)

59
docs/IE_DG/supported_plugins/MYRIAD.md
View File

@ -4,63 +4,14 @@
The Inference Engine MYRIAD plugin has been developed for inference of neural networks on Intel&reg; Neural Compute Stick 2.
## Installation on Linux* OS
## Configuring the MYRIAD Plugin
For installation instructions, refer to the [Installation Guide for Linux*](../../install_guides/installing-openvino-linux.md).
## Installation on Windows* OS
For installation instructions, refer to the [Installation Guide for Windows*](../../install_guides/installing-openvino-windows.md).
## Supported networks
The Inference Engine MYRIAD plugin supports the following networks:
**Caffe\***:
* AlexNet
* CaffeNet
* GoogleNet (Inception) v1, v2, v4
* VGG family (VGG16, VGG19)
* SqueezeNet v1.0, v1.1
* ResNet v1 family (18\*\*\*, 50, 101, 152)
* MobileNet (mobilenet-v1-1.0-224, mobilenet-v2)
* Inception ResNet v2
* DenseNet family (121,161,169,201)
* SSD-300, SSD-512, SSD-MobileNet, SSD-GoogleNet, SSD-SqueezeNet
**TensorFlow\***:
* AlexNet
* Inception v1, v2, v3, v4
* Inception ResNet v2
* MobileNet v1, v2
* ResNet v1 family (50, 101, 152)
* ResNet v2 family (50, 101, 152)
* SqueezeNet v1.0, v1.1
* VGG family (VGG16, VGG19)
* Yolo family (yolo-v2, yolo-v3, tiny-yolo-v1, tiny-yolo-v2, tiny-yolo-v3)
* faster_rcnn_inception_v2, faster_rcnn_resnet101
* ssd_mobilenet_v1
* DeepLab-v3+
**MXNet\***:
* AlexNet and CaffeNet
* DenseNet family (121,161,169,201)
* SqueezeNet v1.1
* MobileNet v1, v2
* NiN
* ResNet v1 (101, 152)
* ResNet v2 (101)
* SqueezeNet v1.1
* VGG family (VGG16, VGG19)
* SSD-Inception-v3, SSD-MobileNet, SSD-ResNet-50, SSD-300
\*\*\* Network is tested on Intel&reg; Neural Compute Stick 2 with BatchNormalization fusion optimization disabled during Model Optimizer import
To configure your Intel® Vision Accelerator Design With Intel® Movidius™ on supported operating systemss, refer to the Steps for Intel® Vision Accelerator Design with Intel® Movidius™ VPUs section in the installation guides for [Linux](../../install_guides/installing-openvino-linux.md) or [Windows](../../install_guides/installing-openvino-windows.md).
## Supported Configuration Parameters
See VPU common configuration parameters for the [VPU Plugins](VPU.md).
When specifying key values as raw strings (that is, when using Python API), omit the `KEY_` prefix.
When specifying key values as raw strings (that is, when using the Python API), omit the `KEY_` prefix.
In addition to common parameters, the MYRIAD plugin accepts the following options:
@ -75,9 +26,9 @@ In addition to common parameters, the MYRIAD plugin accepts the following option
## Device allocation <a name="MYRIAD_DEVICE_ALLOC">&nbsp;</a>
Each `IExecutableNetwork` instance tries to allocate new device on `InferenceEngine::Core::LoadNetwork`, but if all available devices are already allocated it will use the one with the minimal number of uploaded networks.
The maximum number of networks single device can handle depends on device memory capacity and the size of the networks.
The maximum number of networks a single device can handle depends on device memory capacity and the size of the networks.
If `KEY_VPU_MYRIAD_FORCE_RESET` option is set to `YES` the plugin will reset all VPU devices in the system.
If the `KEY_VPU_MYRIAD_FORCE_RESET` option is set to `YES`, the plugin will reset all VPU devices in the system.
Single device cannot be shared across multiple processes.

5
docs/IE_DG/supported_plugins/Supported_Devices.md
View File

@ -1,7 +1,7 @@
Supported Devices {#openvino_docs_IE_DG_supported_plugins_Supported_Devices}
==================
The Inference Engine can infer models in different formats with various input and output formats. This section provides supported and optimal configurations per device.
The Inference Engine can infer models in different formats with various input and output formats. This section provides supported and optimal configurations per device. In OpenVINO™ documentation, "device" refers to an Intel® processors used for inference, which can be a supported CPU, GPU, VPU (vision processing unit), or GNA (Gaussian neural accelerator coprocessor), or a combination of those devices.
> **NOTE**: With OpenVINO™ 2020.4 release, Intel® Movidius™ Neural Compute Stick is no longer supported.
@ -28,7 +28,6 @@ This page shows supported and optimal configurations for each plugin.
| Acronym/Term | Description |
| :-----------------| :---------------------------------------------|
| DL | Deep Learning |
| FP32 format | Single-precision floating-point format |
| BF16 format | Brain floating-point format |
| FP16 format | Half-precision floating-point format |
@ -37,7 +36,7 @@ This page shows supported and optimal configurations for each plugin.
| U16 format | 2-byte unsigned integer format |
| U8 format | 1-byte unsigned integer format |
NHWC, NCHW, and NCDHW refer to the representation of batches of images.
NHWC, NCHW, and NCDHW refer to the data ordering in batches of images:
* NHWC and NCHW refer to image data layout.
* NCDHW refers to image sequence data layout.

82
docs/IE_DG/supported_plugins/VPU.md
View File

@ -1,20 +1,64 @@
# VPU Plugins {#openvino_docs_IE_DG_supported_plugins_VPU}
@sphinxdirective
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_IE_DG_supported_plugins_MYRIAD
openvino_docs_IE_DG_supported_plugins_HDDL
@endsphinxdirective
This chapter provides information on the Inference Engine plugins that enable inference of deep learning models on the supported VPU devices:
* Intel® Neural Compute Stick 2 powered by the Intel® Movidius™ Myriad™ X — Supported by the [MYRIAD Plugin](MYRIAD.md)
* Intel® Vision Accelerator Design with Intel® Movidius™ VPUs — Supported by the [HDDL Plugin](HDDL.md)
> **NOTE**: With OpenVINO™ 2020.4 release, Intel® Movidius™ Neural Compute Stick powered by the Intel® Movidius™ Myriad™ 2 is no longer supported.
> **NOTE**: With the OpenVINO™ 2020.4 release, Intel® Movidius™ Neural Compute Stick powered by the Intel® Movidius™ Myriad™ 2 is no longer supported.
## Known Layers Limitations
## Supported Networks
* `ScaleShift` layer is supported for zero value of `broadcast` attribute only.
* `CTCGreedyDecoder` layer works with `ctc_merge_repeated` attribute equal 1.
* `DetectionOutput` layer works with zero values of `interpolate_orientation` and `num_orient_classes` parameters only.
* `MVN` layer uses fixed value for `eps` parameters (1e-9).
* `Normalize` layer uses fixed value for `eps` parameters (1e-9) and is supported for zero value of `across_spatial` only.
* `Pad` layer works only with 4D tensors.
**Caffe\***:
* AlexNet
* CaffeNet
* GoogleNet (Inception) v1, v2, v4
* VGG family (VGG16, VGG19)
* SqueezeNet v1.0, v1.1
* ResNet v1 family (18\*\*\*, 50, 101, 152)
* MobileNet (mobilenet-v1-1.0-224, mobilenet-v2)
* Inception ResNet v2
* DenseNet family (121,161,169,201)
* SSD-300, SSD-512, SSD-MobileNet, SSD-GoogleNet, SSD-SqueezeNet
**TensorFlow\***:
* AlexNet
* Inception v1, v2, v3, v4
* Inception ResNet v2
* MobileNet v1, v2
* ResNet v1 family (50, 101, 152)
* ResNet v2 family (50, 101, 152)
* SqueezeNet v1.0, v1.1
* VGG family (VGG16, VGG19)
* Yolo family (yolo-v2, yolo-v3, tiny-yolo-v1, tiny-yolo-v2, tiny-yolo-v3)
* faster_rcnn_inception_v2, faster_rcnn_resnet101
* ssd_mobilenet_v1
* DeepLab-v3+
**MXNet\***:
* AlexNet and CaffeNet
* DenseNet family (121,161,169,201)
* SqueezeNet v1.1
* MobileNet v1, v2
* NiN
* ResNet v1 (101, 152)
* ResNet v2 (101)
* SqueezeNet v1.1
* VGG family (VGG16, VGG19)
* SSD-Inception-v3, SSD-MobileNet, SSD-ResNet-50, SSD-300
\*\*\* Network is tested on Intel&reg; Neural Compute Stick 2 with BatchNormalization fusion optimization disabled during Model Optimizer import
## Optimizations
@ -24,7 +68,7 @@ VPU plugins support layer fusion and decomposition.
#### Fusing Rules
Certain layers can be merged into Convolution, ReLU, and Eltwise layers according to the patterns below:
Certain layers can be merged into convolution, ReLU, and Eltwise layers according to the patterns below:
- Convolution
- Convolution + ReLU → Convolution
@ -46,6 +90,7 @@ Certain layers can be merged into Convolution, ReLU, and Eltwise layers accordin
> **NOTE**: Application of these rules depends on tensor sizes and resources available.
Layers can be joined only when the two conditions below are met:
- Layers are located on topologically independent branches.
- Layers can be executed simultaneously on the same hardware units.
@ -63,10 +108,9 @@ Layers can be joined only when the two conditions below are met:
> **NOTE**: VPU plugins can add extra layers like Copy.
## VPU Common Configuration Parameters
The VPU plugins supports the configuration parameters listed below.
VPU plugins support the configuration parameters listed below.
The parameters are passed as `std::map<std::string, std::string>` on `InferenceEngine::Core::LoadNetwork`
or `InferenceEngine::Core::SetConfig`.
When specifying key values as raw strings (that is, when using Python API), omit the `KEY_` prefix.
@ -83,20 +127,28 @@ When specifying key values as raw strings (that is, when using Python API), omit
## Data Transfer Pipelining <a name="VPU_DATA_TRANSFER_PIPELINING">&nbsp;</a>
MYRIAD plugin tries to pipeline data transfer to/from device with computations.
While one infer request is executed the data for next infer request can be uploaded to device in parallel.
Same applicable for result downloading.
While one infer request is executed, the data for next infer request can be uploaded to device in parallel.
The same applies to result downloading.
`KEY_VPU_PRINT_RECEIVE_TENSOR_TIME` configuration parameter can be used to check the efficiency of current pipelining.
The new record in performance counters will show the time that device spent waiting for input before starting the inference.
In perfect pipeline this time should be near to zero, which means that the data was already transferred when new inference started.
In a perfect pipeline this time should be near zero, which means that the data was already transferred when new inference started.
## Troubleshooting
**Get the following message when running inference with the VPU plugin: "[VPU] Cannot convert layer <layer_name> due to unsupported layer type <layer_type>"**
This means that your topology has a layer that is unsupported by your target VPU plugin. To resolve this issue, you can implement the custom layer for the target device using the [Inference Engine Extensibility mechanism](../Extensibility_DG/Intro.md). Or, to quickly get a working prototype, you can use the heterogeneous scenario with the default fallback policy (see the [HETERO Plugin](HETERO.md) section). Use the HETERO plugin with a fallback device that supports this layer, for example, CPU: `HETERO:MYRIAD,CPU`.
For a list of VPU supported layers, see the Supported Layers section of the [Supported Devices](Supported_Devices.md) topic.
For a list of VPU-supported layers, see the Supported Layers section of the [Supported Devices](Supported_Devices.md) page.
## Known Layers Limitations
* `ScaleShift` layer is supported for zero value of `broadcast` attribute only.
* `CTCGreedyDecoder` layer works with the `ctc_merge_repeated` attribute equal to 1.
* `DetectionOutput` layer works with zero values of `interpolate_orientation` and `num_orient_classes` parameters only.
* `MVN` layer uses fixed value for `eps` parameters (1e-9).
* `Normalize` layer uses fixed value for `eps` parameters (1e-9) and is supported for zero value of `across_spatial` only.
* `Pad` layer works only with 4D tensors.
## See Also

2
docs/IE_PLUGIN_DG/Intro.md
View File

@ -1,4 +1,4 @@
@mainpage Overview of Inference Engine Plugin Library
# Overview of Inference Engine Plugin Library {#openvino_docs_ie_plugin_dg_overview}
The plugin architecture of the Inference Engine allows to develop and plug independent inference
solutions dedicated to different devices. Physically, a plugin is represented as a dynamic library

696
docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md
View File

@ -1,5 +1,20 @@
# Model Optimizer Developer Guide {#openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide}
@sphinxdirective
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model
openvino_docs_MO_DG_Additional_Optimization_Use_Cases
openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Customize_Model_Optimizer
openvino_docs_MO_DG_prepare_model_Model_Optimizer_FAQ
openvino_docs_MO_DG_Known_Issues_Limitations
openvino_docs_MO_DG_Default_Model_Optimizer_Optimizations
@endsphinxdirective
## Introduction
Model Optimizer is a cross-platform command-line tool that facilitates the transition between the training and deployment environment, performs static model analysis, and adjusts deep learning models for optimal execution on end-point target devices.
@ -10,7 +25,7 @@ Model Optimizer process assumes you have a network model trained using supported
The scheme below illustrates the typical workflow for deploying a trained deep learning model:
![](img/workflow_steps.png)
![](img/BASIC_FLOW_MO_simplified.svg)
The IR is a pair of files describing the model:
@ -18,37 +33,660 @@ The IR is a pair of files describing the model:
* <code>.bin</code> - Contains the weights and biases binary data.
Below is a simple command running Model Optimizer to generate an IR for the input model:
> **TIP**: You also can work with the Model Optimizer inside the OpenVINO™ [Deep Learning Workbench](https://docs.openvinotoolkit.org/latest/workbench_docs_Workbench_DG_Introduction.html) (DL Workbench).
> [DL Workbench](https://docs.openvinotoolkit.org/latest/workbench_docs_Workbench_DG_Introduction.html) is a web-based graphical environment that enables you to optimize, fine-tune, analyze, visualize, and compare performance of deep learning models.
```sh
python3 mo.py --input_model INPUT_MODEL
```
To learn about all Model Optimizer parameters and conversion technics, see the [Converting a Model to IR](prepare_model/convert_model/Converting_Model.md) page.
## Install Model Optimizer Pre-Requisites
> **TIP**: You can quick start with the Model Optimizer inside the OpenVINO™ [Deep Learning Workbench](@ref
> openvino_docs_get_started_get_started_dl_workbench) (DL Workbench).
> [DL Workbench](@ref workbench_docs_Workbench_DG_Introduction) is the OpenVINO™ toolkit UI that enables you to
> import a model, analyze its performance and accuracy, visualize the outputs, optimize and prepare the model for
> deployment on various Intel® platforms.
Before running the Model Optimizer, you must install the Model Optimizer pre-requisites for the framework that was used to train the model.
@sphinxdirective
.. tab:: Using configuration scripts
.. tab:: Linux
.. tab:: All frameworks
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
./install_prerequisites.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
./install_prerequisites.shs
.. tab:: Caffe
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisitess
install_prerequisites_caffe.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_caffe.shs
.. tab:: Tensorflow 1.x
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_tf.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_tf.sh
.. tab:: Tensorflow 2.x
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_tf2.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_tf2.sh
.. tab:: MXNet
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_mxnet.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_mxnet.sh
.. tab:: ONNX
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_onnx.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_onnx.sh
.. tab:: Kaldi
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_kaldi.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_kaldi.sh
.. tab:: Windows
.. tab:: All frameworks
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites\
install_prerequisites.bat
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
install_prerequisites.bat
.. tab:: Caffe
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites\
install_prerequisites_caffe.bat
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
install_prerequisites_caffe.bat
.. tab:: Tensorflow 1.x
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites\
install_prerequisites_tf.bat
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
install_prerequisites_tf.bat
.. tab:: Tensorflow 2.x
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites\
install_prerequisites_tf2.bat
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
install_prerequisites_tf2.bat
.. tab:: MXNet
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites\
install_prerequisites_mxnet.bat
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
install_prerequisites_mxnet.bat
.. tab:: ONNX
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites\
install_prerequisites_onnx.bat
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
install_prerequisites_onnx.bat
.. tab:: Kaldi
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites\
install_prerequisites_kaldi.bat
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
install_prerequisites_kaldi.bat
.. tab:: macOS
.. tab:: All frameworks
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
./install_prerequisites.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
./install_prerequisites.shs
.. tab:: Caffe
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisitess
install_prerequisites_caffe.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_caffe.shs
.. tab:: Tensorflow 1.x
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_tf.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_tf.sh
.. tab:: Tensorflow 2.x
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_tf2.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_tf2.sh
.. tab:: MXNet
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_mxnet.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_mxnet.sh
.. tab:: ONNX
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_onnx.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_onnx.sh
.. tab:: Kaldi
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_kaldi.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_kaldi.sh
.. tab:: Using manual configuration process
.. tab:: Linux
.. tab:: All frameworks
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements.txt
.. tab:: Caffe
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_caffe.txt
.. tab:: Tensorflow 1.x
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_tf.txt
.. tab:: Tensorflow 2.x
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_tf2.txt
.. tab:: MXNet
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_mxnet.txt
.. tab:: ONNX
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_onnx.txt
.. tab:: Kaldi
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_kaldi.txt
.. tab:: Windows
.. tab:: All frameworks
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
pip install -r requirements.txt
.. tab:: Caffe
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
pip install -r requirements_caffe.txt
.. tab:: Tensorflow 1.x
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
pip install -r requirements_tf.txt
.. tab:: Tensorflow 2.x
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
pip install -r requirements_tf2.txt
.. tab:: MXNet
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
pip install -r requirements_mxnet.txt
.. tab:: ONNX
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
pip install -r requirements_onnx.txt
.. tab:: Kaldi
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
pip install -r requirements_kaldi.txt
.. tab:: macOS
.. tab:: All frameworks
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements.txt
.. tab:: Caffe
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_caffe.txt
.. tab:: Tensorflow 1.x
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_tf.txt
.. tab:: Tensorflow 2.x
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_tf2.txt
.. tab:: MXNet
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_mxnet.txt
.. tab:: ONNX
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_onnx.txt
.. tab:: Kaldi
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_kaldi.txt
@endsphinxdirective
## Run Model Optimizer
To convert the model to the Intermediate Representation (IR), run Model Optimizer using the command for your type of OpenVINO™ installation:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 <INSTALL_DIR>/deployment_tools/model_optimizer/mo.py --input_model INPUT_MODEL --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model INPUT_MODEL --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
You need to have have write permissions for an output directory.
> **NOTE**: Some models require using additional arguments to specify conversion parameters, such as `--input_shape`, `--scale`, `--scale_values`, `--mean_values`, `--mean_file`. To learn about when you need to use these parameters, refer to [Converting a Model to Intermediate Representation (IR)](Converting_Model.md).
To adjust the conversion process, you may use general parameters defined in the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) and
framework-specific parameters for:
* [Caffe](Convert_Model_From_Caffe.md)
* [TensorFlow](Convert_Model_From_TensorFlow.md)
* [MXNet](Convert_Model_From_MxNet.md)
* [ONNX](Convert_Model_From_ONNX.md)
* [Kaldi](Convert_Model_From_Kaldi.md)
## Videos
<table>
<tr>
<td>
<iframe width="220" src="https://www.youtube.com/embed/Kl1ptVb7aI8" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
</td>
<td>
<iframe width="220" src="https://www.youtube.com/embed/BBt1rseDcy0" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
</td>
<td>
<iframe width="220" src="https://www.youtube.com/embed/RF8ypHyiKrY" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
</td>
</tr>
<tr>
<td><strong>Model Optimizer Concept</strong>. <br>Duration: 3:56</td>
<td><strong>Model Optimizer Basic<br> Operation</strong>. <br>Duration: 2:57.</td>
<td><strong>Choosing the Right Precision</strong>. <br>Duration: 4:18.</td>
</tr>
</table>
@sphinxdirective
.. list-table::
* - .. raw:: html
<iframe width="220"
src="https://www.youtube.com/embed/Kl1ptVb7aI8">
</iframe>
- .. raw:: html
<iframe width="220"
src="https://www.youtube.com/embed/BBt1rseDcy0">
</iframe>
- .. raw:: html
<iframe width="220"
src="https://www.youtube.com/embed/RF8ypHyiKrY">
</iframe>
* - **Model Optimizer Concept.**
- **Model Optimizer Basic Operation.**
- **Choosing the Right Precision.**
* - Duration: 3:56
- Duration: 2:57
- Duration: 4:18
@endsphinxdirective

19
docs/MO_DG/img/BASIC_FLOW_MO_simplified.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 42 KiB

93
docs/MO_DG/img/THROUGHPUT.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 57 KiB

4
docs/MO_DG/img/workflow_steps.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5281f26cbaa468dc4cafa4ce2fde35d338fe0f658bbb796abaaf793e951939f6
size 13943
oid sha256:b89ec8a8231a4fa2c43ce8642d7a22d8f183497f5c99f084c4d1a92edb81df37
size 14332

15
docs/MO_DG/prepare_model/Additional_Optimizations.md Normal file
View File

@ -0,0 +1,15 @@
# Optimize Preprocessing Computation{#openvino_docs_MO_DG_Additional_Optimization_Use_Cases}
Model Optimizer performs preprocessing to a model. It is possible to optimize this step and improve first inference time, to do that, follow the tips bellow:
- **Image mean/scale parameters**<br>
Make sure to use the input image mean/scale parameters (`--scale` and `–mean_values`) with the Model Optimizer when you need pre-processing. It allows the tool to bake the pre-processing into the IR to get accelerated by the Inference Engine.
- **RGB vs. BGR inputs**<br>
If, for example, your network assumes the RGB inputs, the Model Optimizer can swap the channels in the first convolution using the `--reverse_input_channels` command line option, so you do not need to convert your inputs to RGB every time you get the BGR image, for example, from OpenCV*.
- **Larger batch size**<br>
Notice that the devices like GPU are doing better with larger batch size. While it is possible to set the batch size in the runtime using the Inference Engine [ShapeInference feature](../IE_DG/ShapeInference.md).
- **Resulting IR precision**<br>
The resulting IR precision, for instance, `FP16` or `FP32`, directly affects performance. As CPU now supports `FP16` (while internally upscaling to `FP32` anyway) and because this is the best precision for a GPU target, you may want to always convert models to `FP16`. Notice that this is the only precision that Intel&reg; Movidius&trade; Myriad&trade; 2 and Intel&reg; Myriad&trade; X VPUs support.

353
docs/MO_DG/prepare_model/Config_Model_Optimizer.md
View File

@ -1,5 +1,334 @@
# Installing Model Optimizer Pre-Requisites {#openvino_docs_MO_DG_prepare_model_Config_Model_Optimizer}
@sphinxdirective
.. tab:: Using configuration scripts
.. tab:: Linux
.. tab:: All frameworks
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
./install_prerequisites.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
./install_prerequisites.shs
.. tab:: Caffe
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisitess
install_prerequisites_caffe.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_caffe.shs
.. tab:: Tensorflow 1.x
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: Tensorflow 2.x
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: MXNet
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: ONNX
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: Kaldi
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: Windows
.. tab:: All frameworks
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: Caffe
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: Tensorflow 1.x
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: Tensorflow 2.x
.. tab:: Install globally
test
.. tab:: Install to virtualenv
tests
.. tab:: MXNet
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: ONNX
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: Kaldi
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: macOS
.. tab:: All frameworks
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: Caffe
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: Tensorflow 1.x
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: Tensorflow 2.x
.. tab:: Install globally
test
.. tab:: Install to virtualenv
tests
.. tab:: MXNet
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: ONNX
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: Kaldi
.. tab:: Install globally
test
.. tab:: Install to virtualenv
test
.. tab:: Using manual configuration process
.. tab:: Linux
.. tab:: All frameworks
test
.. tab:: Caffe
test
.. tab:: Tensorflow 1.x
test
.. tab:: Tensorflow 2.x
test
.. tab:: MXNet
test
.. tab:: ONNX
test
.. tab:: Kaldi
test
.. tab:: Windows
.. tab:: All frameworks
test
.. tab:: Caffe
test
.. tab:: Tensorflow 1.x
test
.. tab:: Tensorflow 2.x
test
.. tab:: MXNet
test
.. tab:: ONNX
test
.. tab:: Kaldi
test
.. tab:: macOS
.. tab:: All frameworks
test
.. tab:: Caffe
test
.. tab:: Tensorflow 1.x
test
.. tab:: Tensorflow 2.x
test
.. tab:: MXNet
test
.. tab:: ONNX
test
.. tab:: Kaldi
test
@endsphinxdirective
Before running the Model Optimizer, you must install the Model Optimizer pre-requisites for the framework that was used to train the model. This section tells you how to install the pre-requisites either through scripts or by using a manual process.
## Using Configuration Scripts
@ -152,8 +481,18 @@ pip3 install -r requirements_onnx.txt
```
## Using the protobuf Library in the Model Optimizer for Caffe\*
<details>
<summary>Click to expand</summary>
@sphinxdirective
.. raw:: html
<div class="collapsible-section" data-title="Click to expand">
@endsphinxdirective
@ -264,7 +603,15 @@ python3 -m easy_install dist/protobuf-3.6.1-py3.6-win-amd64.egg
set PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
```
</details>
@sphinxdirective
.. raw:: html
</div>
@endsphinxdirective
## See Also

11
docs/MO_DG/prepare_model/Default_Model_Optimizer_Optimizations.md Normal file
View File

@ -0,0 +1,11 @@
# Default Model Optimizer Optimizations {#openvino_docs_MO_DG_Default_Model_Optimizer_Optimizations}
Model Optimizer not only converts a model to IR format, but also performs a number of optimizations. For example, certain primitives like linear operations (BatchNorm and ScaleShift), are automatically fused into convolutions. Generally, these layers should not be manifested in the resulting IR:
![](../img/optimizations/resnet_269.png)
The picture above shows Caffe\* Resnet269\* topology. The left model is the original model, and the one on the right (after conversion) is the resulting model that the Model Optimizer produces, with BatchNorm and ScaleShift layers fused into the convolution weights rather than constituting separate layers.
If you still see these operations, inspect the Model Optimizer output carefully while searching for warnings, such as on the tool being unable to fuse. For example, non-linear operations (like activations) in between convolutions and linear operations might prevent the fusing. If performance is of concern, try to change (and potentially re-train) the topology. Refer to the [Model Optimizer Guide](../MO_DG/prepare_model/Model_Optimization_Techniques.md) for more optimizations.
Notice that the activation (`_relu`) is not touched by the Model Optimizer, and while it can be merged into convolution as well, this is rather a device-specific optimization, covered by Inference Engine during the model loading time. You are encouraged to inspect performance counters from plugins that should indicate that these particular layers are not executed (“Optimized out”). For more information, refer to <a href="#performance-counters">Internal Inference Performance Counters</a>.

133
docs/MO_DG/prepare_model/Getting_performance_numbers.md Normal file
View File

@ -0,0 +1,133 @@
# Getting Performance Numbers {#openvino_docs_MO_DG_Getting_Performance_Numbers}
## Tip 1. Measure the Proper Set of Operations
When evaluating performance of your model with the Inference Engine, you must measure the proper set of operations. To do so, consider the following tips:
- Avoid including one-time costs like model loading.
- Track separately the operations that happen outside the Inference Engine, like video decoding.
> **NOTE**: Some image pre-processing can be baked into the IR and accelerated. For more information, refer to [Model Optimizer Knobs Related to Performance](Additional_Optimizations.md)
## Tip 2. Getting Credible Performance Numbers
You need to build your performance conclusions on reproducible data. Do the performance measurements with a large number of invocations of the same routine. Since the first iteration is almost always significantly slower than the subsequent ones, you can use an aggregated value for the execution time for final projections:
- If the warm-up run does not help or execution time still varies, you can try running a large number of iterations and then average or find a mean of the results.
- For time values that range too much, use geomean.
Refer to the [Inference Engine Samples](../IE_DG/Samples_Overview.md) for code examples for the performance measurements. Almost every sample, except interactive demos, has a `-ni` option to specify the number of iterations.
## Getting performance numbers using OpenVINO tool
To get performance numbers use our Benchmark app.
[Benchmark App](../../inference-engine/samples/benchmark_app/README.md) sample is the best performance reference.
It has a lot of device-specific knobs, but the primary usage is as simple as:
```bash
$ ./benchmark_app –d GPU –m <model> -i <input>
```
to measure the performance of the model on the GPU.
Or
```bash
$ ./benchmark_app –d CPU –m <model> -i <input>
```
to execute on the CPU instead.
For example, for the CPU throughput mode from the previous section, you can play with number of streams (`-nstreams` command-line param).
Try different values of the `-nstreams` argument from `1` to a number of CPU cores and find one that provides the best performance. For example, on a 8-core CPU, compare the `-nstreams 1` (which is a latency-oriented scenario) to the `2`, `4` and `8` streams. Notice that `benchmark_app` automatically queries/creates/runs number of requests required to saturate the given number of streams.
Finally, notice that when you don't specify number of streams with `-nstreams`, "AUTO" value for the streams is used, e.g. for the CPU this is [CPU_THROUGHPUT_AUTO](supported_plugins/CPU.md). You can spot the actual value behind "AUTO" for your machine in the application output.
Notice that the "AUTO" number is not necessarily most optimal, so it is generally recommended to play either with the benchmark_app's "-nstreams" as described above, or via [new Workbench tool](@ref workbench_docs_Workbench_DG_Introduction).This allows you to simplify the app-logic, as you don't need to combine multiple inputs into a batch to achieve good CPU performance.
Instead, it is possible to keep a separate infer request per camera or another source of input and process the requests in parallel using Async API.
## Comparing Performance with Native/Framework Code
When comparing the Inference Engine performance with the framework or another reference code, make sure that both versions are as similar as possible:
- Wrap exactly the inference execution (refer to the [Inference Engine Samples](../IE_DG/Samples_Overview.md) for examples).
- Do not include model loading time.
- Ensure the inputs are identical for the Inference Engine and the framework. For example, Caffe\* allows to auto-populate the input with random values. Notice that it might give different performance than on real images.
- Similarly, for correct performance comparison, make sure the access pattern, for example, input layouts, is optimal for Inference Engine (currently, it is NCHW).
- Any user-side pre-processing should be tracked separately.
- Make sure to try the same environment settings that the framework developers recommend, for example, for TensorFlow*. In many cases, things that are more machine friendly, like respecting NUMA (see <a href="#cpu-checklist">CPU Checklist</a>), might work well for the Inference Engine as well.
- If applicable, use batching with the Inference Engine.
- If possible, demand the same accuracy. For example, TensorFlow allows `FP16` support, so when comparing to that, make sure to test the Inference Engine with the `FP16` as well.
## Using Tools <a name="using-tools"></a>
Whether you are tuning for the first time or doing advanced performance optimization, you need a a tool that provides accurate insights. Intel&reg; VTune&trade; Amplifier gives you the tool to mine it and interpret the profiling data.
Alternatively, you can gather the raw profiling data that samples report, the second chapter provides example of how to interpret these.
### Internal Inference Performance Counters <a name="performance-counters"></a>
Almost every sample (inspect command-line options for a specific sample with `-h`) supports a `-pc` command that outputs internal execution breakdown. Refer to the [samples code](../IE_DG/Samples_Overview.md) for the actual Inference Engine API behind that.
Below is example of CPU plugin output for a network (since the device is CPU, the layers wall clock `realTime` and the `cpu` time are the same):
```
conv1 EXECUTED layerType: Convolution realTime: 706 cpu: 706 execType: jit_avx2
conv2_1_x1 EXECUTED layerType: Convolution realTime: 137 cpu: 137 execType: jit_avx2_1x1
fc6 EXECUTED layerType: Convolution realTime: 233 cpu: 233 execType: jit_avx2_1x1
fc6_nChw8c_nchw EXECUTED layerType: Reorder realTime: 20 cpu: 20 execType: reorder
out_fc6 EXECUTED layerType: Output realTime: 3 cpu: 3 execType: unknown
relu5_9_x2 OPTIMIZED_OUT layerType: ReLU realTime: 0 cpu: 0 execType: undef
```
This contains layers name (as seen in IR), layers type and execution statistics. Notice the `OPTIMIZED_OUT`, which indicates that the particular activation was fused into adjacent convolution. Also, the `unknown` stays for the Inference Engine specific CPU (helper) primitives that are not part of the Intel MKL-DNN.
Notice that there are some helper layers in the CPU execution breakdown, which were not presented in the original topology. These are automatically added by the plugin. For example, the `Reorder` re-packs the Intel MKL-DNN internal (blocked) layout to the regular plain NCHW (that the user expects as the output). As explained in the <a href="#device-specific-tips">Few Device-Specific Tips</a>, if your custom kernels introduces a lot of outstanding/expensive Reorders, consider blocked implementation for the kernels.
Notice that in the heterogeneous cases, there will be additional information on which subgraph the statistics is about (the first subgraph is GPU, so its `cpu`/host time is really small compared to the actual `realTime`):
```
subgraph1: squeeze1x1 EXECUTED layerType: Convolution realTime: 227 cpu:3 execType: GPU
…
subgraph2: detection_out EXECUTED layerType: DetectionOutput realTime: 121 cpu:121 execType: unknown
…
```
As mentioned earlier, `unknown` here means CPU kernel with unknown (for example, not AVX2 or AVX512) acceleration path.
Since FPGA execution does not separate individual kernels, only bulk execution/data transfer statistics is available:
```
subgraph1: 1. input preprocessing (mean data/FPGA):EXECUTED layerType: preprocessing realTime: 129 cpu: 129
subgraph1: 2. input transfer to DDR:EXECUTED layerType: realTime: 201 cpu: 0
subgraph1: 3. FPGA execute time:EXECUTED layerType: realTime: 3808 cpu: 0 subgraph1: 4. output transfer from DDR:EXECUTED layerType: realTime: 55 cpu: 0
subgraph1: 5. FPGA output postprocessing:EXECUTED layerType: realTime: 7 cpu: 7
subgraph1: 6. softmax/copy: EXECUTED layerType: realTime: 2 cpu: 2
subgraph2: out_prob: NOT_RUN layerType: Output realTime: 0 cpu: 0
subgraph2: prob: EXECUTED layerType: SoftMax realTime: 10 cpu: 10
Total time: 4212 microseconds
```
The `softmax/copy` is a glue layer that connects the FPGA subgraph to the CPU subgraph (and copies the data).
### Intel&reg; VTune&trade; Examples <a name="vtune-examples"></a>
All major performance calls of the Inference Engine are instrumented with Instrumentation and Tracing Technology APIs. This allows viewing the Inference Engine calls on the Intel&reg; VTune&trade; timelines and aggregations plus correlating them to the underlying APIs, like OpenCL. In turn, this enables careful per-layer execution breakdown.
When choosing the Analysis type in Intel&reg; VTune&trade; Amplifier, make sure to select the **Analyze user tasks, events, and counters** option:
![](vtune_option.png)
See the [corresponding section in the Intel® VTune™ Amplifier User's Guide](https://software.intel.com/en-us/vtune-amplifier-help-task-analysis) for details.
Example of Inference Engine calls:
- On the Intel VTune Amplifier timeline.
Notice that `Task_runNOThrow` is an Async API wrapper and it is executed in a different thread and triggers the Intel MKL-DNN execution:
![](vtune_timeline.png)
- In the Intel VTune Amplifier **Top-down view**, grouped by the **Task Domain**.
Notice the `Task_runNoThrow` and `MKLDNN _INFER` that are bracketing the actual Intel MKL-DNN kernels execution:
![](vtune_topdown_view.jpg)
Similarly, you can use any GPU analysis in the Intel VTune Amplifier and get general correlation with Inference Engine API as well as the execution breakdown for OpenCL kernels.
Just like with regular native application, further drill down in the counters is possible, however, this is mostly useful for <a href="#optimizing-custom-kernels">optimizing custom kernels</a>. Finally, with the Intel VTune Amplifier, the profiling is not limited to your user-level code (see the [corresponding section in the Intel&reg; VTune&trade; Amplifier User's Guide](https://software.intel.com/en-us/vtune-amplifier-help-analyze-performance)).

68
docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md
View File

@ -348,18 +348,53 @@ You cannot specify the batch and the input shape at the same time. You should sp
The specified input shape cannot be parsed. Please, define it in one of the following ways:
*
```shell
python3 mo.py --input_model <INPUT_MODEL>.caffemodel --input_shape (1,3,227,227)
```
*
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model <INPUT_MODEL>.caffemodel --input_shape (1,3,227,227)
.. tab:: pip installation
.. code-block:: sh
mo --input_model <INPUT_MODEL>.caffemodel --input_shape (1,3,227,227)
@endsphinxdirective
*
```shell
python3 mo.py --input_model <INPUT_MODEL>.caffemodel --input_shape [1,3,227,227]
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model <INPUT_MODEL>.caffemodel --input_shape [1,3,227,227]
.. tab:: pip installation
.. code-block:: sh
mo --input_model <INPUT_MODEL>.caffemodel --input_shape [1,3,227,227]
@endsphinxdirective
* In case of multi input topology you should also specify inputs:
```shell
python3 mo.py --input_model /path-to/your-model.caffemodel --input data,rois --input_shape (1,3,227,227),(1,6,1,1)
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model /path-to/your-model.caffemodel --input data,rois --input_shape (1,3,227,227),(1,6,1,1)
.. tab:: pip installation
.. code-block:: sh
mo --input_model /path-to/your-model.caffemodel --input data,rois --input_shape (1,3,227,227),(1,6,1,1)
@endsphinxdirective
Keep in mind that there is no space between and inside the brackets for input shapes.
@ -616,19 +651,6 @@ You need to specify values for each input of the model. For more information, re
It means that you trying to convert the topology which contains '_contrib_box_nms' operation which is not supported directly. However the sub-graph of operations including the '_contrib_box_nms' could be replaced with DetectionOutput layer if your topology is one of the gluoncv topologies. Specify '--enable_ssd_gluoncv' command line parameter for the Model Optimizer to enable this transformation.
\htmlonly
<script>
window.addEventListener('load', function(){
var questionID = getURLParameter('question'); /* this function is defined in openvino-layout.js */
if (questionID) {
window.location = window.location.pathname + '#' + encodeURI(questionID);
}
});
</script>
\endhtmlonly
#### 103. What does the message "ModelOptimizer is not able to parse *.caffemodel" mean? <a name="question-103"></a>
If a '*.caffemodel' file exists and it is correct, the error possibly occured due to the use of Python protobuf implementation. In some cases, it shows error message during model parsing, for example: "'utf-8' codec can't decode byte 0xe0 in position 4: invalid continuation byte in field: mo_caffe.SpatialTransformerParameter.transform_type". You can either use Python 3.6/3.7 or build 'cpp' implementation of protobuf yourself for your version of Python. For the complete instructions about building `protobuf` from sources, see the appropriate section in [Converting a Model to Intermediate Representation](Config_Model_Optimizer.md).

79
docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Caffe.md
View File

@ -32,22 +32,31 @@ A summary of the steps for optimizing and deploying a model that was trained wit
* **Semantic segmentation models:**
* FCN8
> **NOTE:** It is necessary to specify mean and scale values for most of the Caffe\* models to convert them with the Model Optimizer. The exact values should be determined separately for each model. For example, for Caffe\* models trained on ImageNet, the mean values usually are `123.68`, `116.779`, `103.939` for blue, green and red channels respectively. The scale value is usually `127.5`. Refer to [Framework-agnostic parameters](Converting_Model_General.md) for the information on how to specify mean and scale values.
> **NOTE:** It is necessary to specify mean and scale values for most of the Caffe\* models to convert them with the Model Optimizer. The exact values should be determined separately for each model. For example, for Caffe\* models trained on ImageNet, the mean values usually are `123.68`, `116.779`, `103.939` for blue, green and red channels respectively. The scale value is usually `127.5`. Refer to the General Conversion Parameters section in [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) for the information on how to specify mean and scale values.
## Convert a Caffe* Model <a name="Convert_From_Caffe"></a>
To convert a Caffe\* model:
To convert a Caffe\* model, run Model Optimizer with the path to the input model `.caffemodel` file and the path to an output directory with write permissions:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
1. Go to the `$INTEL_OPENVINO_DIR/deployment_tools/model_optimizer` directory.
2. Use the `mo.py` script to simply convert a model, specifying the path to the input model `.caffemodel` file and the path to an output directory with write permissions:
```sh
python3 mo.py --input_model <INPUT_MODEL>.caffemodel --output_dir <OUTPUT_MODEL_DIR>
```
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
python3 mo.py --input_model <INPUT_MODEL>.caffemodel --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model <INPUT_MODEL>.caffemodel --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
Two groups of parameters are available to convert your model:
* [Framework-agnostic parameters](Converting_Model_General.md): These parameters are used to convert a model trained with any supported framework.
* [Caffe-specific parameters](#caffe_specific_conversion_params): Parameters used to convert only Caffe\* models
* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page.
* [Caffe-specific parameters](#caffe_specific_conversion_params) are used to convert only Caffe\* models.
### Using Caffe\*-Specific Conversion Parameters <a name="caffe_specific_conversion_params"></a>
@ -92,17 +101,38 @@ Caffe*-specific parameters:
#### Command-Line Interface (CLI) Examples Using Caffe\*-Specific Parameters
* Launching the Model Optimizer for the [bvlc_alexnet.caffemodel](https://github.com/BVLC/caffe/tree/master/models/bvlc_alexnet) with a specified `prototxt` file. This is needed when the name of the Caffe\* model and the `.prototxt` file are different or are placed in different directories. Otherwise, it is enough to provide only the path to the input `model.caffemodel` file. You must have write permissions for the output directory.
@sphinxdirective
.. tab:: Package, Docker, open-source installation
```sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --input_proto bvlc_alexnet.prototxt --output_dir <OUTPUT_MODEL_DIR>
```
.. code-block:: sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --input_proto bvlc_alexnet.prototxt --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model bvlc_alexnet.caffemodel --input_proto bvlc_alexnet.prototxt --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
* Launching the Model Optimizer for the [bvlc_alexnet.caffemodel](https://github.com/BVLC/caffe/tree/master/models/bvlc_alexnet) with a specified `CustomLayersMapping` file. This is the legacy method of quickly enabling model conversion if your model has custom layers. This requires the Caffe\* system on the computer. To read more about this, see [Legacy Mode for Caffe* Custom Layers](../customize_model_optimizer/Legacy_Mode_for_Caffe_Custom_Layers.md).
Optional parameters without default values and not specified by the user in the `.prototxt` file are removed from the Intermediate Representation, and nested parameters are flattened:
```sh
python3 mo.py --input_model bvlc_alexnet.caffemodel -k CustomLayersMapping.xml --disable_omitting_optional --enable_flattening_nested_params --output_dir <OUTPUT_MODEL_DIR>
```
This example shows a multi-input model with input layers: `data`, `rois`
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model bvlc_alexnet.caffemodel -k CustomLayersMapping.xml --disable_omitting_optional --enable_flattening_nested_params --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model bvlc_alexnet.caffemodel -k CustomLayersMapping.xml --disable_omitting_optional --enable_flattening_nested_params --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
This example shows a multi-input model with input layers: `data`, `rois`
```
layer {
name: "data"
@ -123,9 +153,20 @@ layer {
```
* Launching the Model Optimizer for a multi-input model with two inputs and providing a new shape for each input in the order they are passed to the Model Optimizer along with a writable output directory. In particular, for data, set the shape to `1,3,227,227`. For rois, set the shape to `1,6,1,1`:
```sh
python3 mo.py --input_model /path-to/your-model.caffemodel --input data,rois --input_shape (1,3,227,227),[1,6,1,1] --output_dir <OUTPUT_MODEL_DIR>
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model /path-to/your-model.caffemodel --input data,rois --input_shape (1,3,227,227),[1,6,1,1] --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model /path-to/your-model.caffemodel --input data,rois --input_shape (1,3,227,227),[1,6,1,1] --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
## Custom Layer Definition

89
docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md
View File

@ -1,5 +1,15 @@
# Converting a Kaldi* Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Kaldi}
@sphinxdirective
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_MO_DG_prepare_model_convert_model_kaldi_specific_Aspire_Tdnn_Model
@endsphinxdirective
A summary of the steps for optimizing and deploying a model that was trained with Kaldi\*:
1. [Configure the Model Optimizer](../Config_Model_Optimizer.md) for Kaldi\*.
@ -31,18 +41,27 @@ A summary of the steps for optimizing and deploying a model that was trained wit
## Convert a Kaldi* Model <a name="Convert_From_Kaldi"></a>
To convert a Kaldi\* model:
To convert a Kaldi\* model, run Model Optimizer with the path to the input model `.nnet` or `.mdl` file and to an output directory where you have write permissions:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
1. Go to the `<INSTALL_DIR>/deployment_tools/model_optimizer` directory.
2. Use the `mo.py` script to simply convert a model with the path to the input model `.nnet` or `.mdl` file and to an output directory where you have write permissions:
```sh
python3 mo.py --input_model <INPUT_MODEL>.nnet --output_dir <OUTPUT_MODEL_DIR>
```
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
python3 mo.py --input_model <INPUT_MODEL>.nnet --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model <INPUT_MODEL>.nnet --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
Two groups of parameters are available to convert your model:
* [Framework-agnostic parameters](Converting_Model_General.md): These parameters are used to convert any model trained in any supported framework.
* [Kaldi-specific parameters](#kaldi_specific_conversion_params): Parameters used to convert only Kaldi\* models.
* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page.
* [Kaldi-specific parameters](#kaldi_specific_conversion_params) are used to convert only Kaldi\* models.
### Using Kaldi\*-Specific Conversion Parameters <a name="kaldi_specific_conversion_params"></a>
@ -59,14 +78,36 @@ Kaldi-specific parameters:
### Examples of CLI Commands
* To launch the Model Optimizer for the wsj_dnn5b_smbr model with the specified `.nnet` file and an output directory where you have write permissions:
```sh
python3 mo.py --input_model wsj_dnn5b_smbr.nnet --output_dir <OUTPUT_MODEL_DIR>
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model wsj_dnn5b_smbr.nnet --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model wsj_dnn5b_smbr.nnet --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
* To launch the Model Optimizer for the wsj_dnn5b_smbr model with existing file that contains counts for the last layer with biases and a writable output directory:
```sh
python3 mo.py --input_model wsj_dnn5b_smbr.nnet --counts wsj_dnn5b_smbr.counts --output_dir <OUTPUT_MODEL_DIR>_
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model wsj_dnn5b_smbr.nnet --counts wsj_dnn5b_smbr.counts --output_dir <OUTPUT_MODEL_DIR>_
.. tab:: pip installation
.. code-block:: sh
mo --input_model wsj_dnn5b_smbr.nnet --counts wsj_dnn5b_smbr.counts --output_dir <OUTPUT_MODEL_DIR>_
@endsphinxdirective
* The Model Optimizer normalizes сounts in the following way:
\f[
S = \frac{1}{\sum_{j = 0}^{|C|}C_{j}}
@ -79,10 +120,22 @@ python3 mo.py --input_model wsj_dnn5b_smbr.nnet --counts wsj_dnn5b_smbr.counts -
* The normalized counts are subtracted from biases of the last or next to last layer (if last layer is SoftMax).
* If you want to remove the last SoftMax layer in the topology, launch the Model Optimizer with the
`--remove_output_softmax` flag.
```sh
python3 mo.py --input_model wsj_dnn5b_smbr.nnet --counts wsj_dnn5b_smbr.counts --remove_output_softmax --output_dir <OUTPUT_MODEL_DIR>_
```
`--remove_output_softmax` flag:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model wsj_dnn5b_smbr.nnet --counts wsj_dnn5b_smbr.counts --remove_output_softmax --output_dir <OUTPUT_MODEL_DIR>_
.. tab:: pip installation
.. code-block:: sh
mo --input_model wsj_dnn5b_smbr.nnet --counts wsj_dnn5b_smbr.counts --remove_output_softmax --output_dir <OUTPUT_MODEL_DIR>_
@endsphinxdirective
The Model Optimizer finds the last layer of the topology and removes this layer only if it is a SoftMax layer.
> **NOTE:** Model Optimizer can remove SoftMax layer only if the topology has one output.

38
docs/MO_DG/prepare_model/convert_model/Convert_Model_From_MxNet.md
View File

@ -1,5 +1,16 @@
# Converting an MXNet* Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_MxNet}
@sphinxdirective
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_Style_Transfer_From_MXNet
openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_GluonCV_Models
@endsphinxdirective
A summary of the steps for optimizing and deploying a model that was trained with the MXNet\* framework:
1. [Configure the Model Optimizer](../Config_Model_Optimizer.md) for MXNet* (MXNet was used to train your model)
@ -27,12 +38,14 @@ A summary of the steps for optimizing and deploying a model that was trained wit
|SSD-ResNet-50| [Repo](https://github.com/zhreshold/mxnet-ssd), [Symbol + Params](https://github.com/zhreshold/mxnet-ssd/releases/download/v0.6/resnet50_ssd_512_voc0712_trainval.zip)|
|SSD-VGG-16-300| [Repo](https://github.com/zhreshold/mxnet-ssd), [Symbol + Params](https://github.com/zhreshold/mxnet-ssd/releases/download/v0.5-beta/vgg16_ssd_300_voc0712_trainval.zip)|
|SSD-Inception v3| [Repo](https://github.com/zhreshold/mxnet-ssd), [Symbol + Params](https://github.com/zhreshold/mxnet-ssd/releases/download/v0.7-alpha/ssd_inceptionv3_512_voc0712trainval.zip)|
|FCN8 (Semantic Segmentation)| [Repo](https://github.com/apache/incubator-mxnet/tree/master/example/fcn-xs), [Symbol](https://www.dropbox.com/sh/578n5cxej7ofd6m/AAA9SFCBN8R_uL2CnAd3WQ5ia/FCN8s_VGG16-symbol.json?dl=0), [Params](https://www.dropbox.com/sh/578n5cxej7ofd6m/AABHWZHCtA2P6iR6LUflkxb_a/FCN8s_VGG16-0019-cpu.params?dl=0)|
|MTCNN part 1 (Face Detection)| [Repo](https://github.com/pangyupo/mxnet_mtcnn_face_detection), [Symbol](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det1-symbol.json), [Params](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det1-0001.params)|
|MTCNN part 2 (Face Detection)| [Repo](https://github.com/pangyupo/mxnet_mtcnn_face_detection), [Symbol](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det2-symbol.json), [Params](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det2-0001.params)|
|MTCNN part 3 (Face Detection)| [Repo](https://github.com/pangyupo/mxnet_mtcnn_face_detection), [Symbol](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det3-symbol.json), [Params](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det3-0001.params)|
|MTCNN part 4 (Face Detection)| [Repo](https://github.com/pangyupo/mxnet_mtcnn_face_detection), [Symbol](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det4-symbol.json), [Params](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det4-0001.params)|
|Lightened_moon| [Repo](https://github.com/tornadomeet/mxnet-face/tree/master/model/lightened_moon), [Symbol](https://github.com/tornadomeet/mxnet-face/blob/master/model/lightened_moon/lightened_moon_fuse-symbol.json), [Params](https://github.com/tornadomeet/mxnet-face/blob/master/model/lightened_moon/lightened_moon_fuse-0082.params)|
|RNN-Transducer| [Repo](https://github.com/HawkAaron/mxnet-transducer) |
|word_lm| [Repo](https://github.com/apache/incubator-mxnet/tree/master/example/rnn/word_lm) |
**Other supported topologies**
@ -41,18 +54,27 @@ A summary of the steps for optimizing and deploying a model that was trained wit
## Convert an MXNet* Model <a name="ConvertMxNet"></a>
To convert an MXNet\* model:
To convert an MXNet\* model, run Model Optimizer with a path to the input model `.params` file and to an output directory where you have write permissions:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
1. Go to the `<INSTALL_DIR>/deployment_tools/model_optimizer` directory.
2. To convert an MXNet\* model contained in a `model-file-symbol.json` and `model-file-0000.params`, run the Model Optimizer launch script `mo.py`, specifying a path to the input model file and a path to an output directory with write permissions:
```sh
python3 mo_mxnet.py --input_model model-file-0000.params --output_dir <OUTPUT_MODEL_DIR>
```
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
python3 mo.py --input_model model-file-0000.params --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model model-file-0000.params --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
Two groups of parameters are available to convert your model:
* [Framework-agnostic parameters](Converting_Model_General.md): These parameters are used to convert any model trained in any supported framework.
* [MXNet-specific parameters](#mxnet_specific_conversion_params): Parameters used to convert only MXNet models
* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page.
* [MXNet-specific parameters](#mxnet_specific_conversion_params) are used to convert only MXNet models.
### Using MXNet\*-Specific Conversion Parameters <a name="mxnet_specific_conversion_params"></a>

38
docs/MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md
View File

@ -1,5 +1,19 @@
# Converting a ONNX* Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_ONNX}
@sphinxdirective
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_Faster_RCNN
openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_Mask_RCNN
openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_GPT2
openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_DLRM
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_PyTorch
@endsphinxdirective
## Introduction to ONNX
[ONNX*](https://github.com/onnx/onnx) is a representation format for deep learning models. ONNX allows AI developers easily transfer models between different frameworks that helps to choose the best combination for them. Today, PyTorch\*, Caffe2\*, Apache MXNet\*, Microsoft Cognitive Toolkit\* and other tools are developing ONNX support.
@ -57,15 +71,25 @@ The list of supported topologies from the [models v1.5](https://github.com/Paddl
## Convert an ONNX* Model <a name="Convert_From_ONNX"></a>
The Model Optimizer process assumes you have an ONNX model that was directly downloaded from a public repository or converted from any framework that supports exporting to the ONNX format.
To convert an ONNX\* model:
To convert an ONNX\* model, run Model Optimizer with the path to the input model `.nnet` file and an output directory where you have write permissions:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
1. Go to the `<INSTALL_DIR>/deployment_tools/model_optimizer` directory.
2. Use the `mo.py` script to simply convert a model with the path to the input model `.nnet` file and an output directory where you have write permissions:
```sh
python3 mo.py --input_model <INPUT_MODEL>.onnx --output_dir <OUTPUT_MODEL_DIR>
```
.. code-block:: sh
There are no ONNX\* specific parameters, so only [framework-agnostic parameters](Converting_Model_General.md) are available to convert your model.
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
python3 mo.py --input_model <INPUT_MODEL>.onnx --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model <INPUT_MODEL>.onnx --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
There are no ONNX\* specific parameters, so only framework-agnostic parameters are available to convert your model. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page.
## Supported ONNX\* Layers
Refer to [Supported Framework Layers](../Supported_Frameworks_Layers.md) for the list of supported standard layers.

13
docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md
View File

@ -1,5 +1,18 @@
# Converting a PyTorch* Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_PyTorch}
@sphinxdirective
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_F3Net
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_QuartzNet
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_RNNT
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_YOLACT
@endsphinxdirective
## Supported Topologies
Here is the list of models that are tested and guaranteed to be supported. However, you can also use these instructions to convert PyTorch\* models that are not presented in the list.

38
docs/MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md
View File

@ -1,5 +1,27 @@
# Converting a TensorFlow* Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow}
@sphinxdirective
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_YOLO_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_FaceNet_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_NCF_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_DeepSpeech_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_lm_1b_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Object_Detection_API_Models
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Slim_Library_Models
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_CRNN_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_GNMT_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_BERT_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_XLNet_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_WideAndDeep_Family_Models
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_EfficientDet_Models
@endsphinxdirective
A summary of the steps for optimizing and deploying a model that was trained with the TensorFlow\* framework:
1. [Configure the Model Optimizer](../Config_Model_Optimizer.md) for TensorFlow\* (TensorFlow was used to train your model).
@ -37,7 +59,7 @@ Detailed information on how to convert models from the <a href="https://github.c
**Supported Pre-Trained Topologies from TensorFlow 1 Detection Model Zoo**
Detailed information on how to convert models from the <a href="https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf1_detection_zoo.md">TensorFlow 1 Object Detection Models Zoo</a> and <a href="https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md">TensorFlow 2 Object Detection Models Zoo</a> is available in the [Converting TensorFlow Object Detection API Models](tf_specific/Convert_Object_Detection_API_Models.md) chapter. The table below contains models from the Object Detection Models Zoo that are supported.
Detailed information on how to convert models from the <a href="https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf1_detection_zoo.md">TensorFlow 1 Detection Model Zoo</a> is available in the [Converting TensorFlow Object Detection API Models](tf_specific/Convert_Object_Detection_API_Models.md) chapter. The table below contains models from the Object Detection Models zoo that are supported.
| Model Name| TensorFlow 1 Object Detection API Models|
| :------------- | -----:|
@ -259,10 +281,10 @@ python3 mo_tf.py --input_model <INPUT_MODEL>.pb --output_dir <OUTPUT_MODEL_DIR>
Two groups of parameters are available to convert your model:
* [Framework-agnostic parameters](Converting_Model_General.md): These parameters are used to convert any model trained in any supported framework.
* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page.
* [TensorFlow-specific parameters](#tensorflow_specific_conversion_params): Parameters used to convert only TensorFlow models.
> **NOTE:** The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. For more information about the parameter, refer to **When to Reverse Input Channels** section of [Converting a Model Using General Conversion Parameters](Converting_Model_General.md).
> **NOTE:** The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. For more information about the parameter, refer to **When to Reverse Input Channels** section of [Converting a Model to Intermediate Representation (IR)](Converting_Model.md).
### Using TensorFlow\*-Specific Conversion Parameters <a name="tensorflow_specific_conversion_params"></a>
The following list provides the TensorFlow\*-specific parameters.
@ -406,7 +428,15 @@ The Model Optimizer provides explanatory messages if it is unable to run to comp
## Video: Converting a TensorFlow Model
<iframe width="560" height="315" src="https://www.youtube.com/embed/QW6532LtiTc" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
@sphinxdirective
.. raw:: html
<iframe width="560" height="315"
src="https://www.youtube.com/embed/QW6532LtiTc">
</iframe>
@endsphinxdirective
## Summary
In this document, you learned:

398
docs/MO_DG/prepare_model/convert_model/Converting_Model.md
View File

@ -1,14 +1,48 @@
# Converting a Model to Intermediate Representation (IR) {#openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model}
Use the <code>mo.py</code> script from the `<INSTALL_DIR>/deployment_tools/model_optimizer` directory to run the Model Optimizer and convert the model to the Intermediate Representation (IR):
```sh
python3 mo.py --input_model INPUT_MODEL --output_dir <OUTPUT_MODEL_DIR>
```
You need to have have write permissions for an output directory.
@sphinxdirective
> **NOTE**: Some models require using additional arguments to specify conversion parameters, such as `--input_shape`, `--scale`, `--scale_values`, `--mean_values`, `--mean_file`. To learn about when you need to use these parameters, refer to [Converting a Model Using General Conversion Parameters](Converting_Model_General.md).
.. toctree::
:maxdepth: 1
:hidden:
To adjust the conversion process, you may use general parameters defined in the [Converting a Model Using General Conversion Parameters](Converting_Model_General.md) and
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Caffe
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_MxNet
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Kaldi
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_ONNX
openvino_docs_MO_DG_prepare_model_Model_Optimization_Techniques
openvino_docs_MO_DG_prepare_model_convert_model_Cutting_Model
openvino_docs_MO_DG_prepare_model_Supported_Frameworks_Layers
openvino_docs_MO_DG_prepare_model_convert_model_IR_suitable_for_INT8_inference
openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Subgraph_Replacement_Model_Optimizer
openvino_docs_MO_DG_prepare_model_convert_model_Legacy_IR_Layers_Catalog_Spec
@endsphinxdirective
To convert the model to the Intermediate Representation (IR), run Model Optimizer using the command for your type of OpenVINO™ installation:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 <INSTALL_DIR>/deployment_tools/model_optimizer/mo.py \
--input_model INPUT_MODEL --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model INPUT_MODEL --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
The output directory must have write permissions, so you can run Model Optimizer from the output directory or specify an output path with the `--output_dir` option.
> **NOTE:** The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. For details, refer to [When to Reverse Input Channels](#when_to_reverse_input_channels).
To adjust the conversion process, you may use general parameters defined in the [General Conversion Parameters](#general_conversion_parameters) and
Framework-specific parameters for:
* [Caffe](Convert_Model_From_Caffe.md)
* [TensorFlow](Convert_Model_From_TensorFlow.md)
@ -17,6 +51,356 @@ Framework-specific parameters for:
* [Kaldi](Convert_Model_From_Kaldi.md)
## General Conversion Parameters
To adjust the conversion process, you can also use the general (framework-agnostic) parameters:
```sh
optional arguments:
-h, --help show this help message and exit
--framework {tf,caffe,mxnet,kaldi,onnx}
Name of the framework used to train the input model.
Framework-agnostic parameters:
--input_model INPUT_MODEL, -w INPUT_MODEL, -m INPUT_MODEL
Tensorflow*: a file with a pre-trained model (binary
or text .pb file after freezing). Caffe*: a model
proto file with model weights
--model_name MODEL_NAME, -n MODEL_NAME
Model_name parameter passed to the final create_ir
transform. This parameter is used to name a network in
a generated IR and output .xml/.bin files.
--output_dir OUTPUT_DIR, -o OUTPUT_DIR
Directory that stores the generated IR. By default, it
is the directory from where the Model Optimizer is
launched.
--input_shape INPUT_SHAPE
Input shape(s) that should be fed to an input node(s)
of the model. Shape is defined as a comma-separated
list of integer numbers enclosed in parentheses or
square brackets, for example [1,3,227,227] or
(1,227,227,3), where the order of dimensions depends
on the framework input layout of the model. For
example, [N,C,H,W] is used for Caffe* models and
[N,H,W,C] for TensorFlow* models. Model Optimizer
performs necessary transformations to convert the
shape to the layout required by Inference Engine
(N,C,H,W). The shape should not contain undefined
dimensions (? or -1) and should fit the dimensions
defined in the input operation of the graph. If there
are multiple inputs in the model, --input_shape should
contain definition of shape for each input separated
by a comma, for example: [1,3,227,227],[2,4] for a
model with two inputs with 4D and 2D shapes.
Alternatively, specify shapes with the --input
option.
--scale SCALE, -s SCALE
All input values coming from original network inputs
will be divided by this value. When a list of inputs
is overridden by the --input parameter, this scale is
not applied for any input that does not match with the
original input of the model.
--reverse_input_channels
Switch the input channels order from RGB to BGR (or
vice versa). Applied to original inputs of the model
if and only if a number of channels equals 3. Applied
after application of --mean_values and --scale_values
options, so numbers in --mean_values and
--scale_values go in the order of channels used in the
original model.
--log_level {CRITICAL,ERROR,WARN,WARNING,INFO,DEBUG,NOTSET}
Logger level
--input INPUT Quoted list of comma-separated input nodes names with
shapes, data types, and values for freezing. The shape
and value are specified as space-separated lists. The
data type of input node is specified in braces and can
have one of the values: f64 (float64), f32 (float32),
f16 (float16), i64 (int64), i32 (int32), u8 (uint8),
boolean. For example, use the following format to set
input port 0 of the node `node_name1` with the shape
[3 4] as an input node and freeze output port 1 of the
node `node_name2` with the value [20 15] of the int32
type and shape [2]: "0:node_name1[3
4],node_name2:1[2]{i32}->[20 15]".
--output OUTPUT The name of the output operation of the model. For
TensorFlow*, do not add :0 to this name.
--mean_values MEAN_VALUES, -ms MEAN_VALUES
Mean values to be used for the input image per
channel. Values to be provided in the (R,G,B) or
[R,G,B] format. Can be defined for desired input of
the model, for example: "--mean_values
data[255,255,255],info[255,255,255]". The exact
meaning and order of channels depend on how the
original model was trained.
--scale_values SCALE_VALUES
Scale values to be used for the input image per
channel. Values are provided in the (R,G,B) or [R,G,B]
format. Can be defined for desired input of the model,
for example: "--scale_values
data[255,255,255],info[255,255,255]". The exact
meaning and order of channels depend on how the
original model was trained.
--data_type {FP16,FP32,half,float}
Data type for all intermediate tensors and weights. If
original model is in FP32 and --data_type=FP16 is
specified, all model weights and biases are quantized
to FP16.
--disable_fusing Turn off fusing of linear operations to Convolution
--disable_resnet_optimization
Turn off resnet optimization
--finegrain_fusing FINEGRAIN_FUSING
Regex for layers/operations that won't be fused.
Example: --finegrain_fusing Convolution1,.*Scale.*
--disable_gfusing Turn off fusing of grouped convolutions
--enable_concat_optimization
Turn on Concat optimization.
--extensions EXTENSIONS
Directory or a comma separated list of directories
with extensions. To disable all extensions including
those that are placed at the default location, pass an
empty string.
--batch BATCH, -b BATCH
Input batch size
--version Version of Model Optimizer
--silent Prevent any output messages except those that
correspond to log level equals ERROR, that can be set
with the following option: --log_level. By default,
log level is already ERROR.
--freeze_placeholder_with_value FREEZE_PLACEHOLDER_WITH_VALUE
Replaces input layer with constant node with provided
value, for example: "node_name->True". It will be
DEPRECATED in future releases. Use --input option to
specify a value for freezing.
--static_shape Enables IR generation for fixed input shape (folding
`ShapeOf` operations and shape-calculating sub-graphs
to `Constant`). Changing model input shape using
the Inference Engine API in runtime may fail for such an IR.
--disable_weights_compression
Disable compression and store weights with original
precision.
--progress Enable model conversion progress display.
--stream_output Switch model conversion progress display to a
multiline mode.
--transformations_config TRANSFORMATIONS_CONFIG
Use the configuration file with transformations
description.
```
The sections below provide details on using particular parameters and examples of CLI commands.
## When to Specify Mean and Scale Values
Usually neural network models are trained with the normalized input data. This means that the input data values are converted to be in a specific range, for example, `[0, 1]` or `[-1, 1]`. Sometimes the mean values (mean images) are subtracted from the input data values as part of the pre-processing. There are two cases how the input data pre-processing is implemented.
* The input pre-processing operations are a part of a topology. In this case, the application that uses the framework to infer the topology does not pre-process the input.
* The input pre-processing operations are not a part of a topology and the pre-processing is performed within the application which feeds the model with an input data.
In the first case, the Model Optimizer generates the IR with required pre-processing layers and Inference Engine samples may be used to infer the model.
In the second case, information about mean/scale values should be provided to the Model Optimizer to embed it to the generated IR. Model Optimizer provides a number of command line parameters to specify them: `--scale`, `--scale_values`, `--mean_values`, `--mean_file`.
If both mean and scale values are specified, the mean is subtracted first and then scale is applied. Input values are *divided* by the scale value(s).
There is no a universal recipe for determining the mean/scale values for a particular model. The steps below could help to determine them:
* Read the model documentation. Usually the documentation describes mean/scale value if the pre-processing is required.
* Open the example script/application executing the model and track how the input data is read and passed to the framework.
* Open the model in a visualization tool and check for layers performing subtraction or multiplication (like `Sub`, `Mul`, `ScaleShift`, `Eltwise` etc) of the input data. If such layers exist, pre-processing is probably part of the model.
## When to Specify Input Shapes <a name="when_to_specify_input_shapes"></a>
There are situations when the input data shape for the model is not fixed, like for the fully-convolutional neural networks. In this case, for example, TensorFlow\* models contain `-1` values in the `shape` attribute of the `Placeholder` operation. Inference Engine does not support input layers with undefined size, so if the input shapes are not defined in the model, the Model Optimizer fails to convert the model. The solution is to provide the input shape(s) using the `--input` or `--input_shape` command line parameter for all input(s) of the model or provide the batch size using the `-b` command line parameter if the model contains just one input with undefined batch size only. In the latter case, the `Placeholder` shape for the TensorFlow\* model looks like this `[-1, 224, 224, 3]`.
## When to Reverse Input Channels <a name="when_to_reverse_input_channels"></a>
Input data for your application can be of RGB or BRG color input order. For example, Inference Engine samples load input images in the BGR channels order. However, the model may be trained on images loaded with the opposite order (for example, most TensorFlow\* models are trained with images in RGB order). In this case, inference results using the Inference Engine samples may be incorrect. The solution is to provide `--reverse_input_channels` command line parameter. Taking this parameter, the Model Optimizer performs first convolution or other channel dependent operation weights modification so these operations output will be like the image is passed with RGB channels order.
## When to Specify `--static_shape` Command Line Parameter
If the `--static_shape` command line parameter is specified the Model Optimizer evaluates shapes of all operations in the model (shape propagation) for a fixed input(s) shape(s). During the shape propagation the Model Optimizer evaluates operations *Shape* and removes them from the computation graph. With that approach, the initial model which can consume inputs of different shapes may be converted to IR working with the input of one fixed shape only. For example, consider the case when some blob is reshaped from 4D of a shape *[N, C, H, W]* to a shape *[N, C, H \* W]*. During the model conversion the Model Optimize calculates output shape as a constant 1D blob with values *[N, C, H \* W]*. So if the input shape changes to some other value *[N,C,H1,W1]* (it is possible scenario for a fully convolutional model) then the reshape layer becomes invalid.
Resulting Intermediate Representation will not be resizable with the help of Inference Engine.
## Examples of CLI Commands
Launch the Model Optimizer for the Caffe bvlc_alexnet model with debug log level:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --log_level DEBUG --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model bvlc_alexnet.caffemodel --log_level DEBUG --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
Launch the Model Optimizer for the Caffe bvlc_alexnet model with the output IR called `result.*` in the specified `output_dir`:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --model_name result --output_dir /../../models/
.. tab:: pip installation
.. code-block:: sh
mo --input_model bvlc_alexnet.caffemodel --model_name result --output_dir /../../models/
@endsphinxdirective
Launch the Model Optimizer for the Caffe bvlc_alexnet model with one input with scale values:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --scale_values [59,59,59] --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model bvlc_alexnet.caffemodel --scale_values [59,59,59] --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
Launch the Model Optimizer for the Caffe bvlc_alexnet model with multiple inputs with scale values:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --input data,rois --scale_values [59,59,59],[5,5,5] --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model bvlc_alexnet.caffemodel --input data,rois --scale_values [59,59,59],[5,5,5] --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
Launch the Model Optimizer for the Caffe bvlc_alexnet model with multiple inputs with scale and mean values specified for the particular nodes:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --input data,rois --mean_values data[59,59,59] --scale_values rois[5,5,5] --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model bvlc_alexnet.caffemodel --input data,rois --mean_values data[59,59,59] --scale_values rois[5,5,5] --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
Launch the Model Optimizer for the Caffe bvlc_alexnet model with specified input layer, overridden input shape, scale 5, batch 8 and specified name of an output operation:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --input "data[1 3 224 224]" --output pool5 -s 5 -b 8 --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model bvlc_alexnet.caffemodel --input "data[1 3 224 224]" --output pool5 -s 5 -b 8 --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
Launch the Model Optimizer for the Caffe bvlc_alexnet model with disabled fusing for linear operations to Convolution and grouped convolutions:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --disable_fusing --disable_gfusing --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model bvlc_alexnet.caffemodel --disable_fusing --disable_gfusing --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
Launch the Model Optimizer for the Caffe bvlc_alexnet model with reversed input channels order between RGB and BGR, specified mean values to be used for the input image per channel and specified data type for input tensor values:
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --reverse_input_channels --mean_values [255,255,255] --data_type FP16 --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model bvlc_alexnet.caffemodel --reverse_input_channels --mean_values [255,255,255] --data_type FP16 --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
Launch the Model Optimizer for the Caffe bvlc_alexnet model with extensions listed in specified directories, specified mean_images binaryproto.
file For more information about extensions, please refer to [this](../customize_model_optimizer/Extending_Model_Optimizer_with_New_Primitives.md) page.
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --extensions /home/,/some/other/path/ --mean_file /path/to/binaryproto --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model bvlc_alexnet.caffemodel --extensions /home/,/some/other/path/ --mean_file /path/to/binaryproto --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
Launch the Model Optimizer for TensorFlow* FaceNet* model with a placeholder freezing value.
It replaces the placeholder with a constant layer that contains the passed value.
For more information about FaceNet conversion, please refer to [this](tf_specific/Convert_FaceNet_From_Tensorflow.md) page.
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model FaceNet.pb --input "phase_train->False" --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model FaceNet.pb --input "phase_train->False" --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
Launch the Model Optimizer for any model with a placeholder freezing tensor of values.
It replaces the placeholder with a constant layer that contains the passed values.
Tensor here is represented in square brackets with each value separated from another by a whitespace.
If data type is set in the model, this tensor will be reshaped to a placeholder shape and casted to placeholder data type.
Otherwise, it will be casted to data type passed to `--data_type` parameter (by default, it is FP32).
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model FaceNet.pb --input "placeholder_layer_name->[0.1 1.2 2.3]" --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model FaceNet.pb --input "placeholder_layer_name->[0.1 1.2 2.3]" --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
## See Also
* [Configuring the Model Optimizer](../Config_Model_Optimizer.md)
* [IR Notation Reference](../../IR_and_opsets.md)

235
docs/MO_DG/prepare_model/convert_model/Converting_Model_General.md
View File

@ -1,235 +0,0 @@
# Converting a Model Using General Conversion Parameters {#openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General}
To simply convert a model trained by any supported framework, run the Model Optimizer launch script ``mo.py`` specifying a path to the input model file and an output directory where you have write permissions:
```sh
python3 mo.py --input_model INPUT_MODEL --output_dir <OUTPUT_MODEL_DIR>
```
The script is in `$INTEL_OPENVINO_DIR/deployment_tools/model_optimizer/`. The output directory must have write permissions, so you can run mo.py from the output directory or specify an output path with the `--output_dir` option.
> **NOTE:** The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. For details, refer to [When to Reverse Input Channels](#when_to_reverse_input_channels).
To adjust the conversion process, you can also use the general (framework-agnostic) parameters:
```sh
optional arguments:
-h, --help show this help message and exit
--framework {tf,caffe,mxnet,kaldi,onnx}
Name of the framework used to train the input model.
Framework-agnostic parameters:
--input_model INPUT_MODEL, -w INPUT_MODEL, -m INPUT_MODEL
Tensorflow*: a file with a pre-trained model (binary
or text .pb file after freezing). Caffe*: a model
proto file with model weights
--model_name MODEL_NAME, -n MODEL_NAME
Model_name parameter passed to the final create_ir
transform. This parameter is used to name a network in
a generated IR and output .xml/.bin files.
--output_dir OUTPUT_DIR, -o OUTPUT_DIR
Directory that stores the generated IR. By default, it
is the directory from where the Model Optimizer is
launched.
--input_shape INPUT_SHAPE
Input shape(s) that should be fed to an input node(s)
of the model. Shape is defined as a comma-separated
list of integer numbers enclosed in parentheses or
square brackets, for example [1,3,227,227] or
(1,227,227,3), where the order of dimensions depends
on the framework input layout of the model. For
example, [N,C,H,W] is used for Caffe* models and
[N,H,W,C] for TensorFlow* models. Model Optimizer
performs necessary transformations to convert the
shape to the layout required by Inference Engine
(N,C,H,W). The shape should not contain undefined
dimensions (? or -1) and should fit the dimensions
defined in the input operation of the graph. If there
are multiple inputs in the model, --input_shape should
contain definition of shape for each input separated
by a comma, for example: [1,3,227,227],[2,4] for a
model with two inputs with 4D and 2D shapes.
Alternatively, specify shapes with the --input
option.
--scale SCALE, -s SCALE
All input values coming from original network inputs
will be divided by this value. When a list of inputs
is overridden by the --input parameter, this scale is
not applied for any input that does not match with the
original input of the model.
--reverse_input_channels
Switch the input channels order from RGB to BGR (or
vice versa). Applied to original inputs of the model
if and only if a number of channels equals 3. Applied
after application of --mean_values and --scale_values
options, so numbers in --mean_values and
--scale_values go in the order of channels used in the
original model.
--log_level {CRITICAL,ERROR,WARN,WARNING,INFO,DEBUG,NOTSET}
Logger level
--input INPUT Quoted list of comma-separated input nodes names with
shapes, data types, and values for freezing. The shape
and value are specified as space-separated lists. The
data type of input node is specified in braces and can
have one of the values: f64 (float64), f32 (float32),
f16 (float16), i64 (int64), i32 (int32), u8 (uint8),
boolean. For example, use the following format to set
input port 0 of the node `node_name1` with the shape
[3 4] as an input node and freeze output port 1 of the
node `node_name2` with the value [20 15] of the int32
type and shape [2]: "0:node_name1[3
4],node_name2:1[2]{i32}->[20 15]".
--output OUTPUT The name of the output operation of the model. For
TensorFlow*, do not add :0 to this name.
--mean_values MEAN_VALUES, -ms MEAN_VALUES
Mean values to be used for the input image per
channel. Values to be provided in the (R,G,B) or
[R,G,B] format. Can be defined for desired input of
the model, for example: "--mean_values
data[255,255,255],info[255,255,255]". The exact
meaning and order of channels depend on how the
original model was trained.
--scale_values SCALE_VALUES
Scale values to be used for the input image per
channel. Values are provided in the (R,G,B) or [R,G,B]
format. Can be defined for desired input of the model,
for example: "--scale_values
data[255,255,255],info[255,255,255]". The exact
meaning and order of channels depend on how the
original model was trained.
--data_type {FP16,FP32,half,float}
Data type for all intermediate tensors and weights. If
original model is in FP32 and --data_type=FP16 is
specified, all model weights and biases are quantized
to FP16.
--disable_fusing Turn off fusing of linear operations to Convolution
--disable_resnet_optimization
Turn off resnet optimization
--finegrain_fusing FINEGRAIN_FUSING
Regex for layers/operations that won't be fused.
Example: --finegrain_fusing Convolution1,.*Scale.*
--disable_gfusing Turn off fusing of grouped convolutions
--enable_concat_optimization
Turn on Concat optimization.
--extensions EXTENSIONS
Directory or a comma separated list of directories
with extensions. To disable all extensions including
those that are placed at the default location, pass an
empty string.
--batch BATCH, -b BATCH
Input batch size
--version Version of Model Optimizer
--silent Prevent any output messages except those that
correspond to log level equals ERROR, that can be set
with the following option: --log_level. By default,
log level is already ERROR.
--freeze_placeholder_with_value FREEZE_PLACEHOLDER_WITH_VALUE
Replaces input layer with constant node with provided
value, for example: "node_name->True". It will be
DEPRECATED in future releases. Use --input option to
specify a value for freezing.
--static_shape Enables IR generation for fixed input shape (folding
`ShapeOf` operations and shape-calculating sub-graphs
to `Constant`). Changing model input shape using
the Inference Engine API in runtime may fail for such an IR.
--disable_weights_compression
Disable compression and store weights with original
precision.
--progress Enable model conversion progress display.
--stream_output Switch model conversion progress display to a
multiline mode.
--transformations_config TRANSFORMATIONS_CONFIG
Use the configuration file with transformations
description.
```
The sections below provide details on using particular parameters and examples of CLI commands.
## When to Specify Mean and Scale Values
Usually neural network models are trained with the normalized input data. This means that the input data values are converted to be in a specific range, for example, `[0, 1]` or `[-1, 1]`. Sometimes the mean values (mean images) are subtracted from the input data values as part of the pre-processing. There are two cases how the input data pre-processing is implemented.
* The input pre-processing operations are a part of a topology. In this case, the application that uses the framework to infer the topology does not pre-process the input.
* The input pre-processing operations are not a part of a topology and the pre-processing is performed within the application which feeds the model with an input data.
In the first case, the Model Optimizer generates the IR with required pre-processing layers and Inference Engine samples may be used to infer the model.
In the second case, information about mean/scale values should be provided to the Model Optimizer to embed it to the generated IR. Model Optimizer provides a number of command line parameters to specify them: `--scale`, `--scale_values`, `--mean_values`, `--mean_file`.
If both mean and scale values are specified, the mean is subtracted first and then scale is applied. Input values are *divided* by the scale value(s).
There is no a universal recipe for determining the mean/scale values for a particular model. The steps below could help to determine them:
* Read the model documentation. Usually the documentation describes mean/scale value if the pre-processing is required.
* Open the example script/application executing the model and track how the input data is read and passed to the framework.
* Open the model in a visualization tool and check for layers performing subtraction or multiplication (like `Sub`, `Mul`, `ScaleShift`, `Eltwise` etc) of the input data. If such layers exist, pre-processing is probably part of the model.
## When to Specify Input Shapes <a name="when_to_specify_input_shapes"></a>
There are situations when the input data shape for the model is not fixed, like for the fully-convolutional neural networks. In this case, for example, TensorFlow\* models contain `-1` values in the `shape` attribute of the `Placeholder` operation. Inference Engine does not support input layers with undefined size, so if the input shapes are not defined in the model, the Model Optimizer fails to convert the model. The solution is to provide the input shape(s) using the `--input` or `--input_shape` command line parameter for all input(s) of the model or provide the batch size using the `-b` command line parameter if the model contains just one input with undefined batch size only. In the latter case, the `Placeholder` shape for the TensorFlow\* model looks like this `[-1, 224, 224, 3]`.
## When to Reverse Input Channels <a name="when_to_reverse_input_channels"></a>
Input data for your application can be of RGB or BRG color input order. For example, Inference Engine samples load input images in the BGR channels order. However, the model may be trained on images loaded with the opposite order (for example, most TensorFlow\* models are trained with images in RGB order). In this case, inference results using the Inference Engine samples may be incorrect. The solution is to provide `--reverse_input_channels` command line parameter. Taking this parameter, the Model Optimizer performs first convolution or other channel dependent operation weights modification so these operations output will be like the image is passed with RGB channels order.
## When to Specify `--static_shape` Command Line Parameter
If the `--static_shape` command line parameter is specified the Model Optimizer evaluates shapes of all operations in the model (shape propagation) for a fixed input(s) shape(s). During the shape propagation the Model Optimizer evaluates operations *Shape* and removes them from the computation graph. With that approach, the initial model which can consume inputs of different shapes may be converted to IR working with the input of one fixed shape only. For example, consider the case when some blob is reshaped from 4D of a shape *[N, C, H, W]* to a shape *[N, C, H \* W]*. During the model conversion the Model Optimize calculates output shape as a constant 1D blob with values *[N, C, H \* W]*. So if the input shape changes to some other value *[N,C,H1,W1]* (it is possible scenario for a fully convolutional model) then the reshape layer becomes invalid.
Resulting Intermediate Representation will not be resizable with the help of Inference Engine.
## Examples of CLI Commands
Launch the Model Optimizer for the Caffe bvlc_alexnet model with debug log level:
```sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --log_level DEBUG --output_dir <OUTPUT_MODEL_DIR>
```
Launch the Model Optimizer for the Caffe bvlc_alexnet model with the output IR called `result.*` in the specified `output_dir`:
```sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --model_name result --output_dir /../../models/
```
Launch the Model Optimizer for the Caffe bvlc_alexnet model with one input with scale values:
```sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --scale_values [59,59,59] --output_dir <OUTPUT_MODEL_DIR>
```
Launch the Model Optimizer for the Caffe bvlc_alexnet model with multiple inputs with scale values:
```sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --input data,rois --scale_values [59,59,59],[5,5,5] --output_dir <OUTPUT_MODEL_DIR>
```
Launch the Model Optimizer for the Caffe bvlc_alexnet model with multiple inputs with scale and mean values specified for the particular nodes:
```sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --input data,rois --mean_values data[59,59,59] --scale_values rois[5,5,5] --output_dir <OUTPUT_MODEL_DIR>
```
Launch the Model Optimizer for the Caffe bvlc_alexnet model with specified input layer, overridden input shape, scale 5, batch 8 and specified name of an output operation:
```sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --input "data[1 3 224 224]" --output pool5 -s 5 -b 8 --output_dir <OUTPUT_MODEL_DIR>
```
Launch the Model Optimizer for the Caffe bvlc_alexnet model with disabled fusing for linear operations to Convolution and grouped convolutions:
```sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --disable_fusing --disable_gfusing --output_dir <OUTPUT_MODEL_DIR>
```
Launch the Model Optimizer for the Caffe bvlc_alexnet model with reversed input channels order between RGB and BGR, specified mean values to be used for the input image per channel and specified data type for input tensor values:
```sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --reverse_input_channels --mean_values [255,255,255] --data_type FP16 --output_dir <OUTPUT_MODEL_DIR>
```
Launch the Model Optimizer for the Caffe bvlc_alexnet model with extensions listed in specified directories, specified mean_images binaryproto file. For more information about extensions, please refer to [this](../customize_model_optimizer/Extending_Model_Optimizer_with_New_Primitives.md) page.
```sh
python3 mo.py --input_model bvlc_alexnet.caffemodel --extensions /home/,/some/other/path/ --mean_file /path/to/binaryproto --output_dir <OUTPUT_MODEL_DIR>
```
Launch the Model Optimizer for TensorFlow* FaceNet* model with a placeholder freezing value.
It replaces the placeholder with a constant layer that contains the passed value.
For more information about FaceNet conversion, please refer to [this](tf_specific/Convert_FaceNet_From_Tensorflow.md) page
```sh
python3 mo.py --input_model FaceNet.pb --input "phase_train->False" --output_dir <OUTPUT_MODEL_DIR>
```
Launch the Model Optimizer for any model with a placeholder freezing tensor of values.
It replaces the placeholder with a constant layer that contains the passed values.
Tensor here is represented in square brackets with each value separated from another by a whitespace.
If data type is set in the model, this tensor will be reshaped to a placeholder shape and casted to placeholder data type.
Otherwise, it will be casted to data type passed to `--data_type` parameter (by default, it is FP32).
```sh
python3 mo.py --input_model FaceNet.pb --input "placeholder_layer_name->[0.1 1.2 2.3]" --output_dir <OUTPUT_MODEL_DIR>
```

215
docs/MO_DG/prepare_model/convert_model/Cutting_Model.md
View File

@ -19,7 +19,7 @@ Model Optimizer provides command line options `--input` and `--output` to specif
* `--input` option accepts a comma-separated list of layer names of the input model that should be treated as new entry points to the model.
* `--output` option accepts a comma-separated list of layer names of the input model that should be treated as new exit points from the model.
The `--input` option is required for cases unrelated to model cutting. For example, when the model contains several inputs and `--input_shape` or `--mean_values` options are used, you should use the `--input` option to specify the order of input nodes for correct mapping between multiple items provided in `--input_shape` and `--mean_values` and the inputs in the model. Details on these options are out of scope for this document, which focuses on model cutting.
The `--input` option is required for cases unrelated to model cutting. For example, when the model contains several inputs and `--input_shape` or `--mean_values` options are used, you should use the `--input` option to specify the order of input nodes for correct mapping between multiple items provided in `--input_shape` and `--mean_values` and the inputs in the model. This is out of scope.
Model cutting is illustrated with Inception V1. This model is in `models/research/slim` repository. [This section](Converting_Model.md) describes pre-work to prepare the model for the Model Optimizer to be ready to proceed with this chapter.
@ -38,10 +38,22 @@ In the TensorBoard, it looks the following way together with some predecessors:
![TensorBoard with predecessors](../../img/inception_v1_std_output.png)
Convert this model and put the results in a writable output directory:
```sh
${INTEL_OPENVINO_DIR}/deployment_tools/model_optimizer
python3 mo.py --input_model inception_v1.pb -b 1 --output_dir <OUTPUT_MODEL_DIR>
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
python3 mo.py --input_model inception_v1.pb -b 1 --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model inception_v1.pb -b 1 --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
(The other examples on this page assume that you first cd to the `model_optimizer` directory and add the `--output_dir` argument with a directory where you have write permissions.)
The output `.xml` file with an Intermediate Representation contains the `Input` layer among other layers in the model:
@ -80,11 +92,24 @@ The last layer in the model is `InceptionV1/Logits/Predictions/Reshape_1`, which
</layer>
```
Due to automatic identification of inputs and outputs, you do not need to provide the `--input` and `--output` options to convert the whole model. The following commands are equivalent for the Inception V1 model:
```sh
python3 mo.py --input_model inception_v1.pb -b 1 --output_dir <OUTPUT_MODEL_DIR>
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model inception_v1.pb -b 1 --output_dir <OUTPUT_MODEL_DIR>
python3 mo.py --input_model inception_v1.pb -b 1 --input input --output InceptionV1/Logits/Predictions/Reshape_1 --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model inception_v1.pb -b 1 --output_dir <OUTPUT_MODEL_DIR>
mo --input_model inception_v1.pb -b 1 --input input --output InceptionV1/Logits/Predictions/Reshape_1 --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
python3 mo.py --input_model inception_v1.pb -b 1 --input input --output InceptionV1/Logits/Predictions/Reshape_1 --output_dir <OUTPUT_MODEL_DIR>
```
The Intermediate Representations are identical for both conversions. The same is true if the model has multiple inputs and/or outputs.
## Model Cutting
@ -98,9 +123,20 @@ Now consider how to cut some parts of the model off. This chapter uses the first
If you want to cut your model at the end, you have the following options:
1. The following command cuts off the rest of the model after the `InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu`, making this node the last in the model:
```sh
python3 mo.py --input_model inception_v1.pb -b 1 --output=InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model inception_v1.pb -b 1 --output=InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model inception_v1.pb -b 1 --output=InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
The resulting Intermediate Representation has three layers:
```xml
<?xml version="1.0" ?>
@ -142,9 +178,20 @@ python3 mo.py --input_model inception_v1.pb -b 1 --output=InceptionV1/InceptionV
As you can see in the TensorBoard picture, the original model has more nodes than Intermediate Representation. Model Optimizer has fused batch normalization `InceptionV1/InceptionV1/Conv2d_1a_7x7/BatchNorm` to the convolution `InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution`, and it is not present in the final Intermediate Representation. This is not an effect of the `--output` option, it is usual behavior of the Model Optimizer for batch normalizations and convolutions. The effect of the `--output` is that the `ReLU` layer becomes the last one in the converted model.
2. The following command cuts the edge that comes from 0 output port of the `InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu` and the rest of the model, making this node the last one in the model:
```sh
python3 mo.py --input_model inception_v1.pb -b 1 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu:0 --output_dir <OUTPUT_MODEL_DIR>
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model inception_v1.pb -b 1 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu:0 --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model inception_v1.pb -b 1 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu:0 --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
The resulting Intermediate Representation has three layers, which are the same as in the previous case:
```xml
<?xml version="1.0" ?>
@ -186,9 +233,20 @@ python3 mo.py --input_model inception_v1.pb -b 1 --output InceptionV1/InceptionV
This type of cutting is useful to cut edges in case of multiple output edges.
3. The following command cuts the edge that comes to 0 input port of the `InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu` and the rest of the model including `InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu`, deleting this node and making the previous node `InceptionV1/InceptionV1/Conv2d_1a_7x7/Conv2D` the last in the model:
```sh
python3 mo.py --input_model inception_v1.pb -b 1 --output=0:InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model inception_v1.pb -b 1 --output=0:InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model inception_v1.pb -b 1 --output=0:InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
The resulting Intermediate Representation has two layers, which are the same as the first two layers in the previous case:
```xml
<?xml version="1.0" ?>
@ -224,9 +282,20 @@ python3 mo.py --input_model inception_v1.pb -b 1 --output=0:InceptionV1/Inceptio
If you want to go further and cut the beginning of the model, leaving only the `ReLU` layer, you have the following options:
1. You can use the following command line, where `--input` and `--output` specify the same node in the graph:
```sh
python3 mo.py --input_model=inception_v1.pb -b 1 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --input InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model=inception_v1.pb -b 1 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --input InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model=inception_v1.pb -b 1 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --input InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
The resulting Intermediate Representation looks as follows:
```xml
<xml version="1.0">
@ -256,9 +325,20 @@ Even though `--input_shape` is not specified in the command line, the shapes for
2. You can cut edge incoming to layer by port number. To specify incoming port use notation `--input=port:input_node`.
So, to cut everything before `ReLU` layer, cut edge incoming in port 0 of `InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu` node:
```sh
python3 mo.py --input_model inception_v1.pb -b 1 --input 0:InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model inception_v1.pb -b 1 --input 0:InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model inception_v1.pb -b 1 --input 0:InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
The resulting Intermediate Representation looks as follows:
```xml
<xml version="1.0">
@ -288,9 +368,20 @@ Even though `--input_shape` is not specified in the command line, the shapes for
3. You can cut edge outcoming from layer by port number. To specify outcoming port use notation `--input=input_node:port`.
So, to cut everything before `ReLU` layer, cut edge from `InceptionV1/InceptionV1/Conv2d_1a_7x7/BatchNorm/batchnorm/add_1` node to `ReLU`:
```sh
python3 mo.py --input_model inception_v1.pb -b 1 --input InceptionV1/InceptionV1/Conv2d_1a_7x7/BatchNorm/batchnorm/add_1:0 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model inception_v1.pb -b 1 --input InceptionV1/InceptionV1/Conv2d_1a_7x7/BatchNorm/batchnorm/add_1:0 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model inception_v1.pb -b 1 --input_model inception_v1.pb -b 1 --input InceptionV1/InceptionV1/Conv2d_1a_7x7/BatchNorm/batchnorm/add_1:0 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
The resulting Intermediate Representation looks as follows:
```xml
<xml version="1.0">
@ -318,11 +409,21 @@ python3 mo.py --input_model inception_v1.pb -b 1 --input InceptionV1/InceptionV1
## Shape Override for New Inputs
The input shape can be overridden with `--input_shape`. In this case, the shape is applied to the node referenced in `--input`, not to the original `Placeholder` in the model. For example, this command line
```sh
python3 mo.py --input_model inception_v1.pb --input_shape=[1,5,10,20] --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --input InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
```
The input shape can be overridden with `--input_shape`. In this case, the shape is applied to the node referenced in `--input`, not to the original `Placeholder` in the model. For example, the command below :
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model inception_v1.pb --input_shape=[1,5,10,20] --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --input InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model inception_v1.pb --input_shape=[1,5,10,20] --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --input InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
gives the following shapes in the `Input` and `ReLU` layers:
```xml
@ -368,16 +469,39 @@ There are operations that contain more than one input ports. In the example cons
Following this behavior, the Model Optimizer creates an `Input` layer for port 0 only, leaving port 1 as a constant. So the result of:
```sh
python3 mo.py --input_model inception_v1.pb -b 1 --input InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --output_dir <OUTPUT_MODEL_DIR>
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model inception_v1.pb -b 1 --input InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model inception_v1.pb -b 1 --input InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
is identical to the result of conversion of the model as a whole, because this convolution is the first executable operation in Inception V1.
Different behavior occurs when `--input_shape` is also used as an attempt to override the input shape:
```sh
python3 mo.py --input_model inception_v1.pb--input=InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --input_shape [1,224,224,3] --output_dir <OUTPUT_MODEL_DIR>
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model inception_v1.pb--input=InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --input_shape [1,224,224,3] --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model inception_v1.pb--input=InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --input_shape [1,224,224,3] --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective
An error occurs (for more information, see <a href="MO_FAQ.html#FAQ30">FAQ #30</a>):
```sh
[ ERROR ] Node InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution has more than 1 input and input shapes were provided.
@ -387,6 +511,17 @@ For more information, see FAQ #30
In this case, when `--input_shape` is specified and the node contains multiple input ports, you need to specify an input port index together with an input node name. The input port index is specified in front of the node name with ':' as a separator (`PORT:NODE`). In the considered case, the port index 0 of the node `InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution` should be specified as `0:InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution`.
The correct command line is:
```sh
python3 mo.py --input_model inception_v1.pb --input 0:InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --input_shape=[1,224,224,3] --output_dir <OUTPUT_MODEL_DIR>
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model inception_v1.pb --input 0:InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --input_shape=[1,224,224,3] --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model inception_v1.pb --input 0:InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --input_shape=[1,224,224,3] --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective

4
docs/MO_DG/prepare_model/convert_model/IR_suitable_for_INT8_inference.md
View File

@ -9,7 +9,7 @@ Intermediate Representation (IR) should be specifically formed to be suitable fo
Such an IR is called a Low Precision IR and you can generate it in two ways:
- [Quantize regular IR with the Post-Training Optimization tool](@ref pot_README)
- Use the Model Optimizer for a model pretrained for Low Precision inference: TensorFlow\* pre-TFLite models (`.pb` model file with `FakeQuantize*` operations) and ONNX\* quantized models.
Both TensorFlow and ONNX quantized models could be prepared by [Neural Network Compression Framework](https://github.com/openvinotoolkit/nncf/blob/develop/README.md).
Both Tensorflow and ONNX quantized models could be prepared by [Neural Network Compression Framework](https://github.com/openvinotoolkit/nncf/blob/develop/README.md)
For an operation to be executed in INT8, it must have `FakeQuantize` operations as inputs.
See the [specification of `FakeQuantize` operation](../../../ops/quantization/FakeQuantize_1.md) for details.
@ -17,7 +17,7 @@ See the [specification of `FakeQuantize` operation](../../../ops/quantization/Fa
To execute the `Convolution` operation in INT8 on CPU, both data and weight inputs should have `FakeQuantize` as an input operation:
![](../../img/expanded_int8_Convolution_weights.png)
Low precision IR is also suitable for FP32 and FP16 inference if a chosen plugin supports all operations of the IR, because the only difference between a Low Precision IR and FP16 or FP32 IR is the existence of `FakeQuantize` in the Low Precision IR.
Low pecision IR is also suitable for FP32 and FP16 inference if a chosen plugin supports all operations of the IR, because the only difference between a Low Precision IR and FP16 or FP32 IR is the existence of `FakeQuantize` in the Low Precision IR.
Plugins with Low Precision Inference support recognize these sub-graphs and quantize them during the inference time.
Plugins without Low Precision support execute all operations, including `FakeQuantize`, as is in the FP32 or FP16 precision.

2
docs/MO_DG/prepare_model/convert_model/Legacy_IR_Layers_Catalog_Spec.md
View File

@ -1,4 +1,4 @@
# Intermediate Representation Notation Reference Catalog {#openvino_docs_MO_DG_prepare_model_convert_model_Legacy_IR_Layers_Catalog_Spec}
# [DEPRECATED] Intermediate Representation Notation Reference Catalog {#openvino_docs_MO_DG_prepare_model_convert_model_Legacy_IR_Layers_Catalog_Spec}
> **NOTE**: This IR Notation Reference is no longer supported since the new concept of operation sets is introduced in OpenVINO 2020.1 version. For a complete list of supported operations, see the [Intermediate Representation and Operation Sets](../../IR_and_opsets.md) topic.

20
docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_Style_Transfer_From_MXNet.md
View File

@ -90,8 +90,6 @@ Where the `models/13` string is composed of the following substrings:
* `models/`: path to the folder that contains .nd files with pre-trained styles weights
* `13`: prefix pointing to 13_decoder, which is the default decoder for the repository
>**NOTE**: If you get an error saying "No module named 'cPickle'", try running the script from this step in Python 2. Then return to Python 3 for the remaining steps.
You can choose any style from [collection of pre-trained weights](https://pan.baidu.com/s/1skMHqYp). (On the Chinese-language page, click the down arrow next to a size in megabytes. Then wait for an overlay box to appear, and click the blue button in it to download.) The `generate()` function generates `nst_vgg19-symbol.json` and `vgg19-symbol.json` files for the specified shape. In the code, it is [1024 x 768] for a 4:3 ratio, and you can specify another, for example, [224,224] for a square ratio.
#### 6. Run the Model Optimizer to generate an Intermediate Representation (IR):
@ -111,7 +109,19 @@ cp models/13_decoder_auxs.nd nst_model
> **NOTE**: Make sure that all the `.params` and `.json` files are in the same directory as the `.nd` files. Otherwise, the conversion process fails.
3. Run the Model Optimizer for MXNet. Use the `--nd_prefix_name` option to specify the decoder prefix and `--input_shape` to specify input shapes in [N,C,W,H] order. For example:<br>
```sh
python3 mo.py --input_symbol <path/to/nst_model>/nst_vgg19-symbol.json --framework mxnet --output_dir <path/to/output_dir> --input_shape [1,3,224,224] --nd_prefix_name 13_decoder --pretrained_model <path/to/nst_model>/vgg19-0000.params
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_symbol <path/to/nst_model>/nst_vgg19-symbol.json --framework mxnet --output_dir <path/to/output_dir> --input_shape [1,3,224,224] --nd_prefix_name 13_decoder --pretrained_model <path/to/nst_model>/vgg19-0000.params
.. tab:: pip installation
.. code-block:: sh
mo --input_symbol <path/to/nst_model>/nst_vgg19-symbol.json --framework mxnet --output_dir <path/to/output_dir> --input_shape [1,3,224,224] --nd_prefix_name 13_decoder --pretrained_model <path/to/nst_model>/vgg19-0000.params
@endsphinxdirective
4. The IR is generated (`.bin`, `.xml` and `.mapping` files) in the specified output directory and ready to be consumed by the Inference Engine.

17
docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_DLRM.md
View File

@ -10,9 +10,20 @@ These instructions are applicable only to the DLRM converted to the ONNX* file f
If you train the model using the [script provided in model repository](https://github.com/facebookresearch/dlrm/blob/master/dlrm_s_pytorch.py), just add the `--save-onnx` flag to the command line parameters and you'll get the `dlrm_s_pytorch.onnx` file containing the model serialized in ONNX* format.
**Step 2**. To generate the Intermediate Representation (IR) of the model, change your current working directory to the Model Optimizer installation directory and run the Model Optimizer with the following parameters:
```sh
python3 ./mo.py --input_model dlrm_s_pytorch.onnx
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model dlrm_s_pytorch.onnx
.. tab:: pip installation
.. code-block:: sh
mo --input_model dlrm_s_pytorch.onnx
@endsphinxdirective
Note that Pytorch model uses operation `torch.nn.EmbeddingBag`. This operation converts to onnx as custom `ATen` layer and not directly supported by OpenVINO*, but it is possible to convert this operation to:
* `Gather` if each "bag" consists of exactly one index. In this case `offsets` input becomes obsolete and not needed. They will be removed during conversion.

17
docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_GPT2.md
View File

@ -12,6 +12,17 @@ To download the model and sample test data, click **Download** on [https://githu
## Convert ONNX* GPT-2 Model to IR
To generate the Intermediate Representation (IR) of the model GPT-2, run the Model Optimizer with the following parameters:
```sh
python3 mo.py --input_model gpt2-10.onnx --input_shape [X,Y,Z] --output_dir <OUTPUT_MODEL_DIR>
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model gpt2-10.onnx --input_shape [X,Y,Z] --output_dir <OUTPUT_MODEL_DIR>
.. tab:: pip installation
.. code-block:: sh
mo --input_model gpt2-10.onnx --input_shape [X,Y,Z] --output_dir <OUTPUT_MODEL_DIR>
@endsphinxdirective

17
docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_Bert_ner.md
View File

@ -48,8 +48,19 @@ The script generates ONNX* model file `bert-ner.onnx`.
## Convert ONNX* BERT-NER model to IR
```bash
python mo.py --input_model bert-ner.onnx --input "input_mask[1 128],segment_ids[1 128],input_ids[1 128]"
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model bert-ner.onnx --input "input_mask[1 128],segment_ids[1 128],input_ids[1 128]"
.. tab:: pip installation
.. code-block:: sh
mo --input_model bert-ner.onnx --input "input_mask[1 128],segment_ids[1 128],input_ids[1 128]"
@endsphinxdirective
where `1` is `batch_size` and `128` is `sequence_length`.

28
docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_F3Net.md
View File

@ -2,17 +2,18 @@
[F3Net](https://github.com/weijun88/F3Net): Fusion, Feedback and Focus for Salient Object Detection
## Clone the F3Net Model Repository
## Clone the F3Net Repository
To clone the repository, run the following command:
```bash
git clone http://github.com/weijun88/F3Net.git
```sh
git clone http://github.com/weijun88/F3Net.git"
```
## Download and Convert the Model to ONNX*
To download the pre-trained model or train the model yourself, refer to the
[instruction](https://github.com/weijun88/F3Net/blob/master/README.md) in the F3Net model repository. First, convert the model to ONNX\* format. Create and run the script with the following content in the `src` directory of the model repository:
[instruction](https://github.com/weijun88/F3Net/blob/master/README.md) in the F3Net model repository. First, convert the model to ONNX\* format. Create and run the following Python script in the `src` directory of the model repository:
```python
import torch
from dataset import Config
@ -23,10 +24,21 @@ net = F3Net(cfg)
image = torch.zeros([1, 3, 352, 352])
torch.onnx.export(net, image, 'f3net.onnx', export_params=True, do_constant_folding=True, opset_version=11)
```
The script generates the ONNX\* model file `f3net.onnx`. This model conversion was tested with the repository hash commit `eecace3adf1e8946b571a4f4397681252f9dc1b8`.
The script generates the ONNX\* model file f3net.onnx. This model conversion was tested with the repository hash commit `eecace3adf1e8946b571a4f4397681252f9dc1b8`.
## Convert ONNX* F3Net Model to IR
```sh
./mo.py --input_model <MODEL_DIR>/f3net.onnx
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model <MODEL_DIR>/f3net.onnx
.. tab:: pip installation
.. code-block:: sh
mo --input_model <MODEL_DIR>/f3net.onnx
@endsphinxdirective

38
docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_QuartzNet.md
View File

@ -28,13 +28,37 @@ They are `decoder`, `encoder` and a combined `decoder(encoder(x))` models, respe
## Convert ONNX* QuartzNet model to IR
If using a combined model:
```sh
./mo.py --input_model <MODEL_DIR>/qt.onnx --input_shape [B,64,X]
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model <MODEL_DIR>/qt.onnx --input_shape [B,64,X]
.. tab:: pip installation
.. code-block:: sh
mo --input_model <MODEL_DIR>/qt.onnx --input_shape [B,64,X]
@endsphinxdirective
If using separate models:
```sh
./mo.py --input_model <MODEL_DIR>/encoder_qt.onnx --input_shape [B,64,X]
./mo.py --input_model <MODEL_DIR>/decoder_qt.onnx --input_shape [B,1024,Y]
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model <MODEL_DIR>/encoder_qt.onnx --input_shape [B,64,X]
python3 mo.py --input_model <MODEL_DIR>/decoder_qt.onnx --input_shape [B,1024,Y]
.. tab:: pip installation
.. code-block:: sh
mo ---input_model <MODEL_DIR>/encoder_qt.onnx --input_shape [B,64,X]
mo --input_model <MODEL_DIR>/decoder_qt.onnx --input_shape [B,1024,Y]
@endsphinxdirective
Where shape is determined by the audio file Mel-Spectrogram length: B - batch dimension, X - dimension based on the input length, Y - determined by encoder output, usually `X / 2`.

17
docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_RCAN.md
View File

@ -26,6 +26,17 @@ The script generates the ONNX\* model file RCAN.onnx. You can find more informat
## Convert ONNX* RCAN Model to IR
```sh
./mo.py --input_model RCAN.onnx
```
@sphinxdirective
.. tab:: Package, Docker, open-source installation
.. code-block:: sh
python3 mo.py --input_model RCAN.onnx
.. tab:: pip installation
.. code-block:: sh
mo --input_model RCAN.onnx
@endsphinxdirective

Some files were not shown because too many files have changed in this diff Show More