From 8d49595476f03775c5179f803b66ab74fa0e6154 Mon Sep 17 00:00:00 2001 From: Maciej Smyk Date: Wed, 28 Feb 2024 08:54:04 +0100 Subject: [PATCH] [DOCS] Update of hyperlinks to 2024 + new ov homepage diagram image for master (#23091) * Update of links in docs to 2024 in repo. * Replaced ov homepage diagram with a new version without Kalid, MXNet and Caffe --- README.md | 28 +- .../[legacy]-supported-model-formats.rst | 2 +- .../convert-tensorflow-retina-net.rst | 6 +- .../openvino-plugin-library.rst | 14 +- .../plugin-api-references.rst | 6 +- .../install-openvino-pip.rst | 8 +- .../openvino-samples/get-started-demos.rst | 2 +- .../hello-nv12-input-classification.rst | 2 +- .../running-inference/dynamic-shapes.rst | 4 +- ...tegrate-openvino-with-your-application.rst | 2 +- .../general-optimizations.rst | 12 +- .../integrate-save-preprocessing-use-case.rst | 39 ++- .../cmake_options_for_custom_compilation.md | 5 +- docs/dev/debug_capabilities.md | 2 +- docs/dev/pypi_publish/pypi-openvino-dev.md | 4 +- docs/dev/pypi_publish/pypi-openvino-rt.md | 10 +- docs/home.rst | 6 +- .../notebooks/001-hello-world-with-output.rst | 32 +- .../002-openvino-api-with-output.rst | 106 +++--- .../003-hello-segmentation-with-output.rst | 2 +- .../004-hello-detection-with-output.rst | 52 +-- ...classification-to-openvino-with-output.rst | 48 +-- ...2-pytorch-onnx-to-openvino-with-output.rst | 50 +-- .../102-pytorch-to-openvino-with-output.rst | 104 +++--- ...to-openvino-classification-with-output.rst | 58 ++-- ...105-language-quantize-bert-with-output.rst | 2 +- .../notebooks/106-auto-device-with-output.rst | 142 ++++---- ...tion-quantization-data2vec-with-output.rst | 4 +- ...tion-quantization-wav2vec2-with-output.rst | 4 +- docs/notebooks/108-gpu-device-with-output.rst | 198 +++++------ .../109-latency-tricks-with-output.rst | 6 +- .../109-throughput-tricks-with-output.rst | 134 ++++---- ...110-ct-scan-live-inference-with-output.rst | 4 +- ...segmentation-quantize-nncf-with-output.rst | 4 +- ...training-quantization-nncf-with-output.rst | 6 +- ...lassification-quantization-with-output.rst | 6 +- docs/notebooks/115-async-api-with-output.rst | 62 ++-- .../116-sparsity-optimization-with-output.rst | 50 +-- .../117-model-server-with-output.rst | 4 +- ...118-optimize-preprocessing-with-output.rst | 102 +++--- .../119-tflite-to-openvino-with-output.rst | 8 +- ...e-segmentation-to-openvino-with-output.rst | 6 +- ...ject-detection-to-openvino-with-output.rst | 10 +- .../121-convert-to-openvino-with-output.rst | 18 +- ...acy-mo-convert-to-openvino-with-output.rst | 322 +++++++++--------- ...tion-quantization-wav2vec2-with-output.rst | 2 +- ...tion-with-accuracy-control-with-output.rst | 2 +- .../124-hugging-face-hub-with-output.rst | 2 +- .../125-lraspp-segmentation-with-output.rst | 46 +-- .../201-vision-monodepth-with-output.rst | 76 ++--- ...sion-superresolution-image-with-output.rst | 130 +++---- ...sion-superresolution-video-with-output.rst | 66 ++-- .../203-meter-reader-with-output.rst | 2 +- ...nter-semantic-segmentation-with-output.rst | 82 ++--- ...-vision-background-removal-with-output.rst | 2 +- ...206-vision-paddlegan-anime-with-output.rst | 78 ++--- ...-paddlegan-superresolution-with-output.rst | 38 +-- ...ical-character-recognition-with-output.rst | 4 +- .../209-handwritten-ocr-with-output.rst | 4 +- .../215-image-inpainting-with-output.rst | 30 +- .../216-attention-center-with-output.rst | 44 +-- ...219-knowledge-graphs-conve-with-output.rst | 4 +- ...ss-lingual-books-alignment-with-output.rst | 18 +- .../223-text-prediction-with-output.rst | 2 +- ...-segmentation-point-clouds-with-output.rst | 52 +-- .../226-yolov7-optimization-with-output.rst | 4 +- ...228-clip-zero-shot-convert-with-output.rst | 2 +- ...rt-sequence-classification-with-output.rst | 34 +- ...lov8-instance-segmentation-with-output.rst | 2 +- ...-yolov8-keypoint-detection-with-output.rst | 2 +- ...30-yolov8-object-detection-with-output.rst | 4 +- ...ruct-pix2pix-image-editing-with-output.rst | 2 +- ...-diffusion-v2-optimum-demo-with-output.rst | 12 +- .../237-segment-anything-with-output.rst | 2 +- ...238-deep-floyd-if-optimize-with-output.rst | 2 +- .../239-image-bind-convert-with-output.rst | 56 +-- ...42-freevc-voice-conversion-with-output.rst | 2 +- ...tflite-selfie-segmentation-with-output.rst | 74 ++-- .../245-typo-detector-with-output.rst | 2 +- ...6-depth-estimation-videpth-with-output.rst | 2 +- .../247-code-language-id-with-output.rst | 4 +- .../250-music-generation-with-output.rst | 2 +- ...7-llava-multimodal-chatbot-with-output.rst | 4 +- ...eollava-multimodal-chatbot-with-output.rst | 4 +- .../260-pix2struct-docvqa-with-output.rst | 42 +-- ...sound-generation-audioldm2-with-output.rst | 114 +++---- .../274-efficient-sam-with-output.rst | 2 +- ...fusion-torchdynamo-backend-with-output.rst | 12 +- ...bilevlm-language-assistant-with-output.rst | 2 +- ...-shot-image-classification-with-output.rst | 2 +- ...low-training-openvino-nncf-with-output.rst | 10 +- ...uantization-aware-training-with-output.rst | 4 +- ...uantization-aware-training-with-output.rst | 116 +++---- .../401-object-detection-with-output.rst | 82 ++--- .../407-person-tracking-with-output.rst | 144 ++++---- docs/snippets/src/main.cpp | 6 +- .../_static/images/ov_homepage_diagram.png | 4 +- samples/c/hello_classification/README.md | 11 +- .../hello_nv12_input_classification/README.md | 9 +- .../cpp/benchmark/sync_benchmark/README.md | 8 +- .../benchmark/throughput_benchmark/README.md | 10 +- samples/cpp/benchmark_app/README.md | 10 +- .../cpp/classification_sample_async/README.md | 16 +- samples/cpp/hello_classification/README.md | 10 +- .../hello_nv12_input_classification/README.md | 9 +- samples/cpp/hello_query_device/README.md | 10 +- samples/cpp/hello_reshape_ssd/README.md | 10 +- samples/cpp/model_creation_sample/README.md | 10 +- .../python/benchmark/bert_benchmark/README.md | 4 +- .../python/benchmark/sync_benchmark/README.md | 14 +- .../benchmark/throughput_benchmark/README.md | 16 +- .../classification_sample_async/README.md | 19 +- samples/python/hello_classification/README.md | 38 +-- samples/python/hello_query_device/README.md | 14 +- samples/python/hello_reshape_ssd/README.md | 17 +- .../python/model_creation_sample/README.md | 34 +- src/README.md | 2 +- src/bindings/c/README.md | 8 +- .../how_to_wrap_openvino_interfaces_with_c.md | 6 +- .../how_to_wrap_openvino_objects_with_c.md | 2 +- src/bindings/c/docs/how_to_write_unit_test.md | 2 +- src/bindings/python/README.md | 6 +- .../python/src/openvino/preprocess/README.md | 6 +- src/core/README.md | 6 +- src/core/docs/api_details.md | 2 +- src/core/docs/debug_capabilities.md | 2 +- src/frontends/ir/README.md | 6 +- src/frontends/paddle/README.md | 2 +- src/frontends/pytorch/README.md | 10 +- src/frontends/tensorflow/README.md | 6 +- src/inference/docs/api_details.md | 6 +- src/plugins/auto/README.md | 4 +- src/plugins/auto/docs/architecture.md | 8 +- src/plugins/auto/docs/integration.md | 2 +- src/plugins/intel_cpu/docs/fake_quantize.md | 12 +- .../docs/internal_cpu_plugin_optimization.md | 12 +- .../docs/gpu_plugin_driver_troubleshooting.md | 12 +- .../intel_gpu/docs/source_code_structure.md | 10 +- src/plugins/proxy/README.md | 2 +- src/plugins/template/README.md | 4 +- tools/benchmark_tool/README.md | 12 +- 141 files changed, 1768 insertions(+), 1787 deletions(-) diff --git a/README.md b/README.md index 0df51e96894..4ac64c8a3fe 100644 --- a/README.md +++ b/README.md @@ -67,18 +67,18 @@ The OpenVINO™ Runtime can infer models on different hardware devices. This sec CPU - Intel CPU + Intel CPU openvino_intel_cpu_plugin Intel Xeon with Intel® Advanced Vector Extensions 2 (Intel® AVX2), Intel® Advanced Vector Extensions 512 (Intel® AVX-512), and AVX512_BF16, Intel Core Processors with Intel AVX2, Intel Atom Processors with Intel® Streaming SIMD Extensions (Intel® SSE), Intel® Advanced Matrix Extensions (Intel® AMX) - ARM CPU + ARM CPU openvino_arm_cpu_plugin Raspberry Pi™ 4 Model B, Apple® Mac mini with Apple silicon GPU - Intel GPU + Intel GPU openvino_intel_gpu_plugin Intel Processor Graphics, including Intel HD Graphics and Intel Iris Graphics @@ -96,22 +96,22 @@ OpenVINO™ Toolkit also contains several plugins which simplify loading models - Auto + Auto openvino_auto_plugin Auto plugin enables selecting Intel device for inference automatically - Auto Batch + Auto Batch openvino_auto_batch_plugin Auto batch plugin performs on-the-fly automatic batching (i.e. grouping inference requests together) to improve device utilization, with no programming effort from the user - Hetero + Hetero openvino_hetero_plugin Heterogeneous execution enables automatic inference splitting between several devices - Multi + Multi openvino_auto_plugin Multi plugin enables simultaneous inference of the same model on several devices in parallel @@ -160,9 +160,9 @@ You can also check out [Awesome OpenVINO](https://github.com/openvinotoolkit/awe ## System requirements The system requirements vary depending on platform and are available on dedicated pages: -- [Linux](https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_linux_header.html) -- [Windows](https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_windows_header.html) -- [macOS](https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_macos_header.html) +- [Linux](https://docs.openvino.ai/2024/get-started/install-openvino-overview/install-openvino-linux-header.html) +- [Windows](https://docs.openvino.ai/2024/get-started/install-openvino-overview/install-openvino-windows-header.html) +- [macOS](https://docs.openvino.ai/2024/get-started/install-openvino-overview/install-openvino-macos-header.html) ## How to build @@ -177,7 +177,7 @@ See [CONTRIBUTING](./CONTRIBUTING.md) for contribution details. Thank you! Visit [Intel DevHub Discord server](https://discord.gg/7pVRxUwdWG) if you need help or wish to talk to OpenVINO developers. You can go to the channel dedicated to Good First Issue support if you are working on a task. ## Take the issue -If you wish to be assigned to an issue please add a comment with `.take` command. +If you wish to be assigned to an issue please add a comment with `.take` command. ## Get support @@ -192,7 +192,7 @@ Report questions, issues and suggestions, using: * [OpenVINO Wiki](https://github.com/openvinotoolkit/openvino/wiki) * [OpenVINO Storage](https://storage.openvinotoolkit.org/) -* Additional OpenVINO™ toolkit modules: +* Additional OpenVINO™ toolkit modules: * [openvino_contrib](https://github.com/openvinotoolkit/openvino_contrib) * [Intel® Distribution of OpenVINO™ toolkit Product Page](https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit.html) * [Intel® Distribution of OpenVINO™ toolkit Release Notes](https://software.intel.com/en-us/articles/OpenVINO-RelNotes) @@ -206,6 +206,6 @@ Report questions, issues and suggestions, using: \* Other names and brands may be claimed as the property of others. [Open Model Zoo]:https://github.com/openvinotoolkit/open_model_zoo -[OpenVINO™ Runtime]:https://docs.openvino.ai/2023.3/openvino_docs_OV_UG_OV_Runtime_User_Guide.html -[OpenVINO Model Converter (OVC)]:https://docs.openvino.ai/2023.3/openvino_docs_model_processing_introduction.html#convert-a-model-in-cli-ovc +[OpenVINO™ Runtime]:https://docs.openvino.ai/2024/openvino-workflow/running-inference.html +[OpenVINO Model Converter (OVC)]:https://docs.openvino.ai/2024/openvino-workflow/model-preparation.html#convert-a-model-in-cli-ovc [Samples]:https://github.com/openvinotoolkit/openvino/tree/master/samples diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats.rst index bbc40b8b774..d012c2747dd 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats.rst @@ -555,7 +555,7 @@ to OpenVINO IR or ONNX before running inference should be considered the default OpenVINO versions of 2023 are mostly compatible with the old instructions, through a deprecated MO tool, installed with the deprecated OpenVINO Developer Tools package. - `OpenVINO 2023.0 `__ is the last + `OpenVINO 2023.0 `__ is the last release officially supporting the MO conversion process for the legacy formats. diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-retina-net.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-retina-net.rst index 153cd444347..18fcda5c99f 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-retina-net.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-retina-net.rst @@ -5,7 +5,7 @@ Converting a TensorFlow RetinaNet Model .. meta:: - :description: Learn how to convert a RetinaNet model + :description: Learn how to convert a RetinaNet model from TensorFlow to the OpenVINO Intermediate Representation. @@ -14,11 +14,11 @@ Converting a TensorFlow RetinaNet Model The code described here has been **deprecated!** Do not use it to avoid working with a legacy solution. It will be kept for some time to ensure backwards compatibility, but **you should not use** it in contemporary applications. This guide describes a deprecated conversion method. The guide on the new and recommended method can be found in the :doc:`Python tutorials `. - + This tutorial explains how to convert a RetinaNet model to the Intermediate Representation (IR). `Public RetinaNet model `__ does not contain pretrained TensorFlow weights. -To convert this model to the TensorFlow format, follow the `Reproduce Keras to TensorFlow Conversion tutorial `__. +To convert this model to the TensorFlow format, follow the `Reproduce Keras to TensorFlow Conversion tutorial `__. After converting the model to TensorFlow format, run the following command: diff --git a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library.rst b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library.rst index b0901c295ce..0160cf476fb 100644 --- a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library.rst +++ b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library.rst @@ -5,8 +5,8 @@ Overview of OpenVINO Plugin Library .. meta:: - :description: Develop and implement independent inference solutions for - different devices with the components of plugin architecture + :description: Develop and implement independent inference solutions for + different devices with the components of plugin architecture of OpenVINO. @@ -28,8 +28,8 @@ Overview of OpenVINO Plugin Library openvino_docs_ie_plugin_api_references -The plugin architecture of OpenVINO allows to develop and plug independent inference -solutions dedicated to different devices. Physically, a plugin is represented as a dynamic library +The plugin architecture of OpenVINO allows to develop and plug independent inference +solutions dedicated to different devices. Physically, a plugin is represented as a dynamic library exporting the single ``create_plugin_engine`` function that allows to create a new plugin instance. OpenVINO Plugin Library @@ -78,7 +78,7 @@ OpenVINO plugin dynamic library consists of several main components: * Provides the device specific remote tensor API and implementation. -.. note:: +.. note:: This documentation is written based on the ``Template`` plugin, which demonstrates plugin development details. Find the complete code of the ``Template``, which is fully compilable and up-to-date, at ``/src/plugins/template``. @@ -96,6 +96,6 @@ Detailed Guides API References ############## -* `OpenVINO Plugin API `__ -* `OpenVINO Transformation API `__ +* `OpenVINO Plugin API `__ +* `OpenVINO Transformation API `__ diff --git a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/plugin-api-references.rst b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/plugin-api-references.rst index c9eecf01bbd..d52cead0f83 100644 --- a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/plugin-api-references.rst +++ b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/plugin-api-references.rst @@ -5,7 +5,7 @@ Plugin API Reference .. meta:: - :description: Learn about extra API references required for the development of + :description: Learn about extra API references required for the development of plugins in OpenVINO. .. toctree:: @@ -17,6 +17,6 @@ Plugin API Reference The guides below provides extra API references needed for OpenVINO plugin development: -* `OpenVINO Plugin API `__ -* `OpenVINO Transformation API `__ +* `OpenVINO Plugin API `__ +* `OpenVINO Transformation API `__ diff --git a/docs/articles_en/get-started/install-openvino-overview/install-openvino-shared/install-openvino-pip.rst b/docs/articles_en/get-started/install-openvino-overview/install-openvino-shared/install-openvino-pip.rst index 1a80152624a..d8c9746c923 100644 --- a/docs/articles_en/get-started/install-openvino-overview/install-openvino-shared/install-openvino-pip.rst +++ b/docs/articles_en/get-started/install-openvino-overview/install-openvino-shared/install-openvino-pip.rst @@ -135,16 +135,16 @@ Now that you've installed OpenVINO Runtime, you're ready to run your own machine .. image:: https://user-images.githubusercontent.com/15709723/127752390-f6aa371f-31b5-4846-84b9-18dd4f662406.gif :width: 400 -Try the `Python Quick Start Example `__ to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook inside your web browser. +Try the `Python Quick Start Example `__ to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook inside your web browser. Get started with Python +++++++++++++++++++++++ Visit the :doc:`Tutorials ` page for more Jupyter Notebooks to get you started with OpenVINO, such as: -* `OpenVINO Python API Tutorial `__ -* `Basic image classification program with Hello Image Classification `__ -* `Convert a PyTorch model and use it for image background removal `__ +* `OpenVINO Python API Tutorial `__ +* `Basic image classification program with Hello Image Classification `__ +* `Convert a PyTorch model and use it for image background removal `__ diff --git a/docs/articles_en/learn-openvino/openvino-samples/get-started-demos.rst b/docs/articles_en/learn-openvino/openvino-samples/get-started-demos.rst index 0725be59611..143e3532114 100644 --- a/docs/articles_en/learn-openvino/openvino-samples/get-started-demos.rst +++ b/docs/articles_en/learn-openvino/openvino-samples/get-started-demos.rst @@ -264,7 +264,7 @@ You need a model that is specific for your inference task. You can get it from o Convert the Model -------------------- -If Your model requires conversion, check the `article `__ for information how to do it. +If Your model requires conversion, check the `article `__ for information how to do it. .. _download-media: diff --git a/docs/articles_en/learn-openvino/openvino-samples/hello-nv12-input-classification.rst b/docs/articles_en/learn-openvino/openvino-samples/hello-nv12-input-classification.rst index 183535feddd..e357328ff8b 100644 --- a/docs/articles_en/learn-openvino/openvino-samples/hello-nv12-input-classification.rst +++ b/docs/articles_en/learn-openvino/openvino-samples/hello-nv12-input-classification.rst @@ -213,6 +213,6 @@ Additional Resources - :doc:`Get Started with Samples ` - :doc:`Using OpenVINO Samples ` - :doc:`Convert a Model ` -- `API Reference `__ +- `API Reference `__ - `Hello NV12 Input Classification C++ Sample on Github `__ - `Hello NV12 Input Classification C Sample on Github `__ diff --git a/docs/articles_en/openvino-workflow/running-inference/dynamic-shapes.rst b/docs/articles_en/openvino-workflow/running-inference/dynamic-shapes.rst index 7e932380d12..74c55ad3116 100644 --- a/docs/articles_en/openvino-workflow/running-inference/dynamic-shapes.rst +++ b/docs/articles_en/openvino-workflow/running-inference/dynamic-shapes.rst @@ -64,7 +64,7 @@ Model input dimensions can be specified as dynamic using the model.reshape metho Some models may already have dynamic shapes out of the box and do not require additional configuration. This can either be because it was generated with dynamic shapes from the source framework, or because it was converted with Model Conversion API to use dynamic shapes. For more information, see the Dynamic Dimensions “Out of the Box” section. -The examples below show how to set dynamic dimensions with a model that has a static ``[1, 3, 224, 224]`` input shape (such as `mobilenet-v2 `__). The first example shows how to change the first dimension (batch size) to be dynamic. In the second example, the third and fourth dimensions (height and width) are set as dynamic. +The examples below show how to set dynamic dimensions with a model that has a static ``[1, 3, 224, 224]`` input shape. The first example shows how to change the first dimension (batch size) to be dynamic. In the second example, the third and fourth dimensions (height and width) are set as dynamic. .. tab-set:: @@ -177,7 +177,7 @@ The lower and/or upper bounds of a dynamic dimension can also be specified. They .. tab-item:: C :sync: c - The dimension bounds can be coded as arguments for `ov_dimension `__, as shown in these examples: + The dimension bounds can be coded as arguments for `ov_dimension `__, as shown in these examples: .. doxygensnippet:: docs/snippets/ov_dynamic_shapes.c :language: cpp diff --git a/docs/articles_en/openvino-workflow/running-inference/integrate-openvino-with-your-application.rst b/docs/articles_en/openvino-workflow/running-inference/integrate-openvino-with-your-application.rst index 7859f12d75d..d3ddcc38dee 100644 --- a/docs/articles_en/openvino-workflow/running-inference/integrate-openvino-with-your-application.rst +++ b/docs/articles_en/openvino-workflow/running-inference/integrate-openvino-with-your-application.rst @@ -440,7 +440,7 @@ To build your project using CMake with the default build tools currently availab Additional Resources #################### -* See the :doc:`OpenVINO Samples ` page or the `Open Model Zoo Demos `__ page for specific examples of how OpenVINO pipelines are implemented for applications like image classification, text prediction, and many others. +* See the :doc:`OpenVINO Samples ` page or the `Open Model Zoo Demos `__ page for specific examples of how OpenVINO pipelines are implemented for applications like image classification, text prediction, and many others. * :doc:`OpenVINO™ Runtime Preprocessing ` * :doc:`String Tensors ` * :doc:`Using Encrypted Models with OpenVINO ` diff --git a/docs/articles_en/openvino-workflow/running-inference/optimize-inference/general-optimizations.rst b/docs/articles_en/openvino-workflow/running-inference/optimize-inference/general-optimizations.rst index c96fa16e687..996438f05ed 100644 --- a/docs/articles_en/openvino-workflow/running-inference/optimize-inference/general-optimizations.rst +++ b/docs/articles_en/openvino-workflow/running-inference/optimize-inference/general-optimizations.rst @@ -5,12 +5,12 @@ General Optimizations .. meta:: - :description: General optimizations include application-level optimization - methods that improve data pipelining, pre-processing + :description: General optimizations include application-level optimization + methods that improve data pipelining, pre-processing acceleration and both latency and throughput. -This article covers application-level optimization techniques, such as asynchronous execution, to improve data pipelining, pre-processing acceleration and so on. +This article covers application-level optimization techniques, such as asynchronous execution, to improve data pipelining, pre-processing acceleration and so on. While the techniques (e.g. pre-processing) can be specific to end-user applications, the associated performance improvements are general and shall improve any target scenario -- both latency and throughput. .. _inputs_pre_processing: @@ -62,7 +62,7 @@ Below are example-codes for the regular and async-based approaches to compare: The technique can be generalized to any available parallel slack. For example, you can do inference and simultaneously encode the resulting or previous frames or run further inference, like emotion detection on top of the face detection results. -Refer to the `Object Detection C++ Demo `__ , `Object Detection Python Demo `__ (latency-oriented Async API showcase) and :doc:`Benchmark App Sample ` for complete examples of the Async API in action. +Refer to the `Object Detection C++ Demo `__ , `Object Detection Python Demo `__ (latency-oriented Async API showcase) and :doc:`Benchmark App Sample ` for complete examples of the Async API in action. .. note:: @@ -71,7 +71,7 @@ Refer to the `Object Detection C++ Demo ` +Previous sections covered the topic of the :doc:`preprocessing steps ` and the overview of :doc:`Layout ` API. -For many applications, it is also important to minimize read/load time of a model. -Therefore, performing integration of preprocessing steps every time on application -startup, after ``ov::runtime::Core::read_model``, may seem inconvenient. In such cases, -once pre and postprocessing steps have been added, it can be useful to store new execution +For many applications, it is also important to minimize read/load time of a model. +Therefore, performing integration of preprocessing steps every time on application +startup, after ``ov::runtime::Core::read_model``, may seem inconvenient. In such cases, +once pre and postprocessing steps have been added, it can be useful to store new execution model to OpenVINO Intermediate Representation (OpenVINO IR, `.xml` format). -Most available preprocessing steps can also be performed via command-line options, -using Model Optimizer. For details on such command-line options, refer to the +Most available preprocessing steps can also be performed via command-line options, +using Model Optimizer. For details on such command-line options, refer to the :doc:`Optimizing Preprocessing Computation `. Code example - Saving Model with Preprocessing to OpenVINO IR ############################################################# -When some preprocessing steps cannot be integrated into the execution graph using -Model Optimizer command-line options (for example, ``YUV``->``RGB`` color space conversion, +When some preprocessing steps cannot be integrated into the execution graph using +Model Optimizer command-line options (for example, ``YUV``->``RGB`` color space conversion, ``Resize``, etc.), it is possible to write a simple code which: * Reads the original model (OpenVINO IR, TensorFlow, TensorFlow Lite, ONNX, PaddlePaddle). * Adds the preprocessing/postprocessing steps. * Saves resulting model as IR (``.xml`` and ``.bin``). -Consider the example, where an original ONNX model takes one ``float32`` input with the -``{1, 3, 224, 224}`` shape, the ``RGB`` channel order, and mean/scale values applied. -In contrast, the application provides ``BGR`` image buffer with a non-fixed size and -input images as batches of two. Below is the model conversion code that can be applied +Consider the example, where an original ONNX model takes one ``float32`` input with the +``{1, 3, 224, 224}`` shape, the ``RGB`` channel order, and mean/scale values applied. +In contrast, the application provides ``BGR`` image buffer with a non-fixed size and +input images as batches of two. Below is the model conversion code that can be applied in the model preparation script for such a case. * Includes / Imports @@ -83,8 +83,8 @@ in the model preparation script for such a case. Application Code - Load Model to Target Device ############################################## -After this, the application code can load a saved file and stop preprocessing. In this case, enable -:doc:`model caching ` to minimize load +After this, the application code can load a saved file and stop preprocessing. In this case, enable +:doc:`model caching ` to minimize load time when the cached model is available. @@ -112,7 +112,6 @@ Additional Resources * :doc:`Layout API overview ` * :doc:`Model Optimizer - Optimize Preprocessing Computation ` * :doc:`Model Caching Overview ` -* The `ov::preprocess::PrePostProcessor `__ C++ class documentation -* The `ov::pass::Serialize `__ - pass to serialize model to XML/BIN -* The `ov::set_batch `__ - update batch dimension for a given model +* The `ov::preprocess::PrePostProcessor `__ C++ class documentation +* The `ov::pass::Serialize `__ - pass to serialize model to XML/BIN diff --git a/docs/dev/cmake_options_for_custom_compilation.md b/docs/dev/cmake_options_for_custom_compilation.md index feb9b7aa54a..a5184dbc460 100644 --- a/docs/dev/cmake_options_for_custom_compilation.md +++ b/docs/dev/cmake_options_for_custom_compilation.md @@ -1,6 +1,6 @@ # CMake Options for Custom Compilation -This document provides description and default values for CMake options that can be used to build a custom OpenVINO runtime using the open source version. For instructions on how to create a custom runtime from the prebuilt OpenVINO release package, refer to the [deployment manager] documentation. To understand all the dependencies when creating a custom runtime from the open source repository, refer to the [OpenVINO Runtime Introduction]. +This document provides description and default values for CMake options that can be used to build a custom OpenVINO runtime using the open source version. To understand all the dependencies when creating a custom runtime from the open source repository, refer to the [OpenVINO Runtime Introduction]. ## Table of contents: @@ -182,8 +182,7 @@ In this case OpenVINO CMake scripts take `TBBROOT` environment variable into acc [pugixml]:https://pugixml.org/ [ONNX]:https://onnx.ai/ [protobuf]:https://github.com/protocolbuffers/protobuf -[deployment manager]:https://docs.openvino.ai/2023.3/openvino_docs_install_guides_deployment_manager_tool.html -[OpenVINO Runtime Introduction]:https://docs.openvino.ai/2023.3/openvino_docs_OV_UG_Integrate_OV_with_your_application.html +[OpenVINO Runtime Introduction]:https://docs.openvino.ai/2024/openvino-workflow/running-inference/integrate-openvino-with-your-application.html [PDPD]:https://github.com/PaddlePaddle/Paddle [TensorFlow]:https://www.tensorflow.org/ [TensorFlow Lite]:https://www.tensorflow.org/lite diff --git a/docs/dev/debug_capabilities.md b/docs/dev/debug_capabilities.md index 78d9c381bfe..803b6b973f3 100644 --- a/docs/dev/debug_capabilities.md +++ b/docs/dev/debug_capabilities.md @@ -2,7 +2,7 @@ OpenVINO components provides different debug capabilities, to get more information please read: -* [OpenVINO Model Debug Capabilities](https://docs.openvino.ai/2023.3/openvino_docs_OV_UG_Model_Representation.html#model-debug-capabilities) +* [OpenVINO Model Debug Capabilities](https://docs.openvino.ai/2024/openvino_docs_OV_UG_Model_Representation.html#model-debugging-capabilities) * [OpenVINO Pass Manager Debug Capabilities](#todo) ## See also diff --git a/docs/dev/pypi_publish/pypi-openvino-dev.md b/docs/dev/pypi_publish/pypi-openvino-dev.md index 4c1790610a6..63eaedd7d75 100644 --- a/docs/dev/pypi_publish/pypi-openvino-dev.md +++ b/docs/dev/pypi_publish/pypi-openvino-dev.md @@ -3,7 +3,7 @@ > **NOTE**: This version is pre-release software and has not undergone full release validation or qualification. No support is offered on pre-release software and APIs/behavior are subject to change. It should NOT be incorporated into any production software/solution and instead should be used only for early testing and integration while awaiting a final release version of this software. -> **NOTE**: OpenVINO™ Development Tools package has been deprecated and will be discontinued with 2025.0 release. To learn more, refer to the [OpenVINO Legacy Features and Components page](https://docs.openvino.ai/2023.3/openvino_legacy_features.html). +> **NOTE**: OpenVINO™ Development Tools package has been deprecated and will be discontinued with 2025.0 release. To learn more, refer to the [OpenVINO Legacy Features and Components page](https://docs.openvino.ai/2024/documentation/legacy-features.html). Intel® Distribution of OpenVINO™ toolkit is an open-source toolkit for optimizing and deploying AI inference. It can be used to develop applications and solutions based on deep learning tasks, such as: emulation of human vision, automatic speech recognition, natural language processing, recommendation systems, etc. It provides high-performance and rich deployment options, from edge to cloud. @@ -126,7 +126,7 @@ For example, to install and configure the components for working with TensorFlow ## Troubleshooting -For general troubleshooting steps and issues, see [Troubleshooting Guide for OpenVINO Installation](https://docs.openvino.ai/2023.3/openvino_docs_get_started_guide_troubleshooting.html). The following sections also provide explanations to several error messages. +For general troubleshooting steps and issues, see [Troubleshooting Guide for OpenVINO Installation](https://docs.openvino.ai/2024/get-started/troubleshooting-install-config.html). The following sections also provide explanations to several error messages. ### Errors with Installing via PIP for Users in China diff --git a/docs/dev/pypi_publish/pypi-openvino-rt.md b/docs/dev/pypi_publish/pypi-openvino-rt.md index 832e67ba284..4e976dca74f 100644 --- a/docs/dev/pypi_publish/pypi-openvino-rt.md +++ b/docs/dev/pypi_publish/pypi-openvino-rt.md @@ -5,7 +5,7 @@ Intel® Distribution of OpenVINO™ toolkit is an open-source toolkit for optimizing and deploying AI inference. It can be used to develop applications and solutions based on deep learning tasks, such as: emulation of human vision, automatic speech recognition, natural language processing, recommendation systems, etc. It provides high-performance and rich deployment options, from edge to cloud. -If you have already finished developing your models and converting them to the OpenVINO model format, you can install OpenVINO Runtime to deploy your applications on various devices. The [OpenVINO™](https://docs.openvino.ai/2023.3/openvino_docs_OV_UG_OV_Runtime_User_Guide.html) Python package includes a set of libraries for an easy inference integration with your products. +If you have already finished developing your models and converting them to the OpenVINO model format, you can install OpenVINO Runtime to deploy your applications on various devices. The [OpenVINO™](https://docs.openvino.ai/2024/openvino-workflow/running-inference.html) Python package includes a set of libraries for an easy inference integration with your products. ## System Requirements @@ -75,13 +75,13 @@ If installation was successful, you will see the list of available devices. | Component | Content | Description | |------------------|---------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [OpenVINO Runtime](https://docs.openvino.ai/2023.3/openvino_docs_OV_UG_OV_Runtime_User_Guide.html) | `openvino package` |**OpenVINO Runtime** is a set of C++ libraries with C and Python bindings providing a common API to deliver inference solutions on the platform of your choice. Use the OpenVINO Runtime API to read PyTorch\*, TensorFlow\*, TensorFlow Lite\*, ONNX\*, and PaddlePaddle\* models and execute them on preferred devices. OpenVINO Runtime uses a plugin architecture and includes the following plugins: [CPU](https://docs.openvino.ai/2023.3/openvino_docs_OV_UG_supported_plugins_CPU.html), [GPU](https://docs.openvino.ai/2023.3/openvino_docs_OV_UG_supported_plugins_GPU.html), [Auto Batch](https://docs.openvino.ai/2023.3/openvino_docs_OV_UG_Automatic_Batching.html), [Auto](https://docs.openvino.ai/2023.3/openvino_docs_OV_UG_supported_plugins_AUTO.html), [Hetero](https://docs.openvino.ai/2023.3/openvino_docs_OV_UG_Hetero_execution.html). -| [OpenVINO Model Converter (OVC)](https://docs.openvino.ai/2023.3/openvino_docs_model_processing_introduction.html#convert-a-model-in-cli-ovc) | `ovc` |**OpenVINO Model Converter** converts models that were trained in popular frameworks to a format usable by OpenVINO components.
Supported frameworks include ONNX\*, TensorFlow\*, TensorFlow Lite\*, and PaddlePaddle\*. | -| [Benchmark Tool](https://docs.openvino.ai/2023.3/openvino_sample_benchmark_tool.html)| `benchmark_app` | **Benchmark Application** allows you to estimate deep learning inference performance on supported devices for synchronous and asynchronous modes. | +| [OpenVINO Runtime](https://docs.openvino.ai/2024/openvino-workflow/running-inference.html#) | `openvino package` |**OpenVINO Runtime** is a set of C++ libraries with C and Python bindings providing a common API to deliver inference solutions on the platform of your choice. Use the OpenVINO Runtime API to read PyTorch\*, TensorFlow\*, TensorFlow Lite\*, ONNX\*, and PaddlePaddle\* models and execute them on preferred devices. OpenVINO Runtime uses a plugin architecture and includes the following plugins: [CPU](https://docs.openvino.ai/2024/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device.html), [GPU](https://docs.openvino.ai/2024/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html), [Auto Batch](https://docs.openvino.ai/2024/openvino-workflow/running-inference/inference-devices-and-modes/automatic-batching.html), [Auto](https://docs.openvino.ai/2024/openvino-workflow/running-inference/inference-devices-and-modes/auto-device-selection.html), [Hetero](https://docs.openvino.ai/2024/openvino-workflow/running-inference/inference-devices-and-modes/hetero-execution.html). +| [OpenVINO Model Converter (OVC)](https://docs.openvino.ai/2024/openvino-workflow/model-preparation.html#convert-a-model-in-cli-ovc) | `ovc` |**OpenVINO Model Converter** converts models that were trained in popular frameworks to a format usable by OpenVINO components.
Supported frameworks include ONNX\*, TensorFlow\*, TensorFlow Lite\*, and PaddlePaddle\*. | +| [Benchmark Tool](https://docs.openvino.ai/2024/learn-openvino/openvino-samples/benchmark-tool.html)| `benchmark_app` | **Benchmark Application** allows you to estimate deep learning inference performance on supported devices for synchronous and asynchronous modes. | ## Troubleshooting -For general troubleshooting steps and issues, see [Troubleshooting Guide for OpenVINO Installation](https://docs.openvino.ai/2023.3/openvino_docs_get_started_guide_troubleshooting.html). The following sections also provide explanations to several error messages. +For general troubleshooting steps and issues, see [Troubleshooting Guide for OpenVINO Installation](https://docs.openvino.ai/2024/get-started/troubleshooting-install-config.html). The following sections also provide explanations to several error messages. ### Errors with Installing via PIP for Users in China diff --git a/docs/home.rst b/docs/home.rst index 8c6f814c609..51e54de7fdb 100644 --- a/docs/home.rst +++ b/docs/home.rst @@ -25,13 +25,13 @@ OpenVINO 2023.2
  • An open-source toolkit for optimizing and deploying deep learning models.
    Boost your AI deep-learning inference performance!
  • Use PyTorch models directly, without converting them first.
    - Learn more... + Learn more...
  • OpenVINO via PyTorch 2.0 torch.compile()
    Use OpenVINO directly in PyTorch-native applications!
    - Learn more... + Learn more...
  • Do you like Generative AI? You will love how it performs with OpenVINO!
    - Check out our new notebooks... + Check out our new notebooks... diff --git a/docs/notebooks/001-hello-world-with-output.rst b/docs/notebooks/001-hello-world-with-output.rst index e23c8da91e8..2806df1eaaa 100644 --- a/docs/notebooks/001-hello-world-with-output.rst +++ b/docs/notebooks/001-hello-world-with-output.rst @@ -5,7 +5,7 @@ This basic introduction to OpenVINO™ shows how to do inference with an image classification model. A pre-trained `MobileNetV3 -model `__ +model `__ from `Open Model Zoo `__ is used in this tutorial. For more information about how OpenVINO IR models are @@ -43,19 +43,19 @@ Imports .. code:: ipython3 from pathlib import Path - + import cv2 import matplotlib.pyplot as plt import numpy as np import openvino as ov - + # Fetch `notebook_utils` module import urllib.request urllib.request.urlretrieve( url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py', filename='notebook_utils.py' ) - + from notebook_utils import download_file Download the Model and data samples @@ -66,15 +66,15 @@ Download the Model and data samples .. code:: ipython3 base_artifacts_dir = Path('./artifacts').expanduser() - + model_name = "v3-small_224_1.0_float" model_xml_name = f'{model_name}.xml' model_bin_name = f'{model_name}.bin' - + model_xml_path = base_artifacts_dir / model_xml_name - + base_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/mobelinet-v3-tf/FP32/' - + if not model_xml_path.exists(): download_file(base_url + model_xml_name, model_xml_name, base_artifacts_dir) download_file(base_url + model_bin_name, model_bin_name, base_artifacts_dir) @@ -104,7 +104,7 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + core = ov.Core() device = widgets.Dropdown( options=core.available_devices + ["AUTO"], @@ -112,7 +112,7 @@ select device from dropdown list for running inference using OpenVINO description='Device:', disabled=False, ) - + device @@ -134,7 +134,7 @@ Load the Model core = ov.Core() model = core.read_model(model=model_xml_path) compiled_model = core.compile_model(model=model, device_name=device.value) - + output_layer = compiled_model.output(0) Load an Image @@ -149,13 +149,13 @@ Load an Image "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg", directory="data" ) - + # The MobileNet model expects images in RGB format. image = cv2.cvtColor(cv2.imread(filename=str(image_filename)), code=cv2.COLOR_BGR2RGB) - + # Resize to MobileNet image shape. input_image = cv2.resize(src=image, dsize=(224, 224)) - + # Reshape to model input shape. input_image = np.expand_dims(input_image, 0) plt.imshow(image); @@ -187,7 +187,7 @@ Do Inference "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_2012.txt", directory="data" ) - + imagenet_classes = imagenet_filename.read_text().splitlines() @@ -202,7 +202,7 @@ Do Inference # The model description states that for this model, class 0 is a background. # Therefore, a background must be added at the beginning of imagenet_classes. imagenet_classes = ['background'] + imagenet_classes - + imagenet_classes[result_index] diff --git a/docs/notebooks/002-openvino-api-with-output.rst b/docs/notebooks/002-openvino-api-with-output.rst index 0b210715fc4..26599ac60b8 100644 --- a/docs/notebooks/002-openvino-api-with-output.rst +++ b/docs/notebooks/002-openvino-api-with-output.rst @@ -41,16 +41,16 @@ Table of contents: .. code:: ipython3 # Required imports. Please execute this cell first. - %pip install -q "openvino>=2023.1.0" + %pip install -q "openvino>=2023.1.0" %pip install requests tqdm ipywidgets - + # Fetch `notebook_utils` module import urllib.request urllib.request.urlretrieve( url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py', filename='notebook_utils.py' ) - + from notebook_utils import download_file @@ -115,7 +115,7 @@ Initialize OpenVINO Runtime with ``ov.Core()`` .. code:: ipython3 import openvino as ov - + core = ov.Core() OpenVINO Runtime can load a network on a device. A device in this @@ -132,7 +132,7 @@ be faster. .. code:: ipython3 devices = core.available_devices - + for device in devices: device_name = core.get_property(device, "FULL_DEVICE_NAME") print(f"{device}: {device_name}") @@ -153,7 +153,7 @@ After initializing OpenVINO Runtime, first read the model file with ``compile_model()`` method. `OpenVINO™ supports several model -formats `__ +formats `__ and enables developers to convert them to its own OpenVINO IR format using a tool dedicated to this task. @@ -174,7 +174,7 @@ file has a different filename, it can be specified using the ``weights`` parameter in ``read_model()``. The OpenVINO `Model Conversion -API `__ +API `__ tool is used to convert models to OpenVINO IR format. Model conversion API reads the original model and creates an OpenVINO IR model (``.xml`` and ``.bin`` files) so inference can be performed without delays due to @@ -193,7 +193,7 @@ notebooks. ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/' ir_model_name_xml = 'classification.xml' ir_model_name_bin = 'classification.bin' - + download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model') download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model') @@ -221,10 +221,10 @@ notebooks. .. code:: ipython3 import openvino as ov - + core = ov.Core() classification_model_xml = "model/classification.xml" - + model = core.read_model(model=classification_model_xml) compiled_model = core.compile_model(model=model, device_name="CPU") @@ -249,7 +249,7 @@ points to the filename of an ONNX model. onnx_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/segmentation.onnx' onnx_model_name = 'segmentation.onnx' - + download_file(onnx_model_url, filename=onnx_model_name, directory='model') @@ -270,10 +270,10 @@ points to the filename of an ONNX model. .. code:: ipython3 import openvino as ov - + core = ov.Core() onnx_model_path = "model/segmentation.onnx" - + model_onnx = core.read_model(model=onnx_model_path) compiled_model_onnx = core.compile_model(model=model_onnx, device_name="CPU") @@ -298,7 +298,7 @@ without any conversion step. Pass the filename with extension to paddle_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/' paddle_model_name = 'inference.pdmodel' paddle_params_name = 'inference.pdiparams' - + download_file(paddle_model_url + paddle_model_name, filename=paddle_model_name, directory='model') download_file(paddle_model_url + paddle_params_name, filename=paddle_params_name, directory='model') @@ -326,10 +326,10 @@ without any conversion step. Pass the filename with extension to .. code:: ipython3 import openvino as ov - + core = ov.Core() paddle_model_path = 'model/inference.pdmodel' - + model_paddle = core.read_model(model=paddle_model_path) compiled_model_paddle = core.compile_model(model=model_paddle, device_name="CPU") @@ -349,7 +349,7 @@ TensorFlow models saved in frozen graph format can also be passed to pb_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/classification.pb' pb_model_name = 'classification.pb' - + download_file(pb_model_url, filename=pb_model_name, directory='model') @@ -370,10 +370,10 @@ TensorFlow models saved in frozen graph format can also be passed to .. code:: ipython3 import openvino as ov - + core = ov.Core() tf_model_path = "model/classification.pb" - + model_tf = core.read_model(model=tf_model_path) compiled_model_tf = core.compile_model(model=model_tf, device_name="CPU") @@ -398,10 +398,10 @@ It is pre-trained model optimized to work with TensorFlow Lite. .. code:: ipython3 from pathlib import Path - + tflite_model_url = 'https://www.kaggle.com/models/tensorflow/inception/frameworks/tfLite/variations/v4-quant/versions/1?lite-format=tflite' tflite_model_path = Path('model/classification.tflite') - + download_file(tflite_model_url, filename=tflite_model_path.name, directory=tflite_model_path.parent) @@ -422,9 +422,9 @@ It is pre-trained model optimized to work with TensorFlow Lite. .. code:: ipython3 import openvino as ov - + core = ov.Core() - + model_tflite = core.read_model(tflite_model_path) compiled_model_tflite = core.compile_model(model=model_tflite, device_name="CPU") @@ -453,15 +453,15 @@ model form torchvision library. After conversion model using import openvino as ov import torch from torchvision.models import resnet18, ResNet18_Weights - + core = ov.Core() - + pt_model = resnet18(weights=ResNet18_Weights.IMAGENET1K_V1) example_input = torch.zeros((1, 3, 224, 224)) ov_model_pytorch = ov.convert_model(pt_model, example_input=example_input) - + compiled_model_pytorch = core.compile_model(ov_model_pytorch, device_name="CPU") - + ov.save_model(ov_model_pytorch, "model/exported_pytorch_model.xml") Getting Information about a Model @@ -481,7 +481,7 @@ Information about the inputs and outputs of the model are in ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/' ir_model_name_xml = 'classification.xml' ir_model_name_bin = 'classification.bin' - + download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model') download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model') @@ -515,7 +515,7 @@ dictionary. .. code:: ipython3 import openvino as ov - + core = ov.Core() classification_model_xml = "model/classification.xml" model = core.read_model(model=classification_model_xml) @@ -587,7 +587,7 @@ Model Outputs .. code:: ipython3 import openvino as ov - + core = ov.Core() classification_model_xml = "model/classification.xml" model = core.read_model(model=classification_model_xml) @@ -691,7 +691,7 @@ produced data as values. ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/' ir_model_name_xml = 'classification.xml' ir_model_name_bin = 'classification.bin' - + download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model') download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model') @@ -717,7 +717,7 @@ produced data as values. .. code:: ipython3 import openvino as ov - + core = ov.Core() classification_model_xml = "model/classification.xml" model = core.read_model(model=classification_model_xml) @@ -734,7 +734,7 @@ the input layout of the network. .. code:: ipython3 import cv2 - + image_filename = download_file( "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_hollywood.jpg", directory="data" @@ -789,7 +789,7 @@ add the ``N`` dimension (where ``N``\ = 1) by calling the .. code:: ipython3 import numpy as np - + input_data = np.expand_dims(np.transpose(resized_image, (2, 0, 1)), 0).astype(np.float32) input_data.shape @@ -814,10 +814,10 @@ predicted result in ``np.array`` format. # for single input models only result = compiled_model(input_data)[output_layer] - + # for multiple inputs in a list result = compiled_model([input_data])[output_layer] - + # or using a dictionary, where the key is input tensor name or index result = compiled_model({input_layer.any_name: input_data})[output_layer] @@ -878,7 +878,7 @@ input shape. ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/' ir_model_name_xml = 'segmentation.xml' ir_model_name_bin = 'segmentation.bin' - + download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model') download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model') @@ -906,17 +906,17 @@ input shape. .. code:: ipython3 import openvino as ov - + core = ov.Core() segmentation_model_xml = "model/segmentation.xml" segmentation_model = core.read_model(model=segmentation_model_xml) segmentation_input_layer = segmentation_model.input(0) segmentation_output_layer = segmentation_model.output(0) - + print("~~~~ ORIGINAL MODEL ~~~~") print(f"input shape: {segmentation_input_layer.shape}") print(f"output shape: {segmentation_output_layer.shape}") - + new_shape = ov.PartialShape([1, 3, 544, 544]) segmentation_model.reshape({segmentation_input_layer.any_name: new_shape}) segmentation_compiled_model = core.compile_model(model=segmentation_model, device_name="CPU") @@ -966,7 +966,7 @@ set ``new_shape = (2,3,544,544)`` in the cell above. .. code:: ipython3 import openvino as ov - + segmentation_model_xml = "model/segmentation.xml" segmentation_model = core.read_model(model=segmentation_model_xml) segmentation_input_layer = segmentation_model.input(0) @@ -974,7 +974,7 @@ set ``new_shape = (2,3,544,544)`` in the cell above. new_shape = ov.PartialShape([2, 3, 544, 544]) segmentation_model.reshape({segmentation_input_layer.any_name: new_shape}) segmentation_compiled_model = core.compile_model(model=segmentation_model, device_name="CPU") - + print(f"input shape: {segmentation_input_layer.shape}") print(f"output shape: {segmentation_output_layer.shape}") @@ -993,7 +993,7 @@ input image through the network to see the result: import numpy as np import openvino as ov - + core = ov.Core() segmentation_model_xml = "model/segmentation.xml" segmentation_model = core.read_model(model=segmentation_model_xml) @@ -1003,9 +1003,9 @@ input image through the network to see the result: segmentation_model.reshape({segmentation_input_layer.any_name: new_shape}) segmentation_compiled_model = core.compile_model(model=segmentation_model, device_name="CPU") input_data = np.random.rand(2, 3, 544, 544) - + output = segmentation_compiled_model([input_data]) - + print(f"input data shape: {input_data.shape}") print(f"result data data shape: {segmentation_output_layer.shape}") @@ -1043,7 +1043,7 @@ the cache. ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/' ir_model_name_xml = 'classification.xml' ir_model_name_bin = 'classification.bin' - + download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model') download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model') @@ -1070,23 +1070,23 @@ the cache. import time from pathlib import Path - + import openvino as ov - + core = ov.Core() - - device_name = "GPU" - + + device_name = "GPU" + if device_name in core.available_devices: cache_path = Path("model/model_cache") cache_path.mkdir(exist_ok=True) # Enable caching for OpenVINO Runtime. To disable caching set enable_caching = False enable_caching = True config_dict = {"CACHE_DIR": str(cache_path)} if enable_caching else {} - + classification_model_xml = "model/classification.xml" model = core.read_model(model=classification_model_xml) - + start_time = time.perf_counter() compiled_model = core.compile_model(model=model, device_name=device_name, config=config_dict) end_time = time.perf_counter() diff --git a/docs/notebooks/003-hello-segmentation-with-output.rst b/docs/notebooks/003-hello-segmentation-with-output.rst index cc95396e99e..81b620b7ad3 100644 --- a/docs/notebooks/003-hello-segmentation-with-output.rst +++ b/docs/notebooks/003-hello-segmentation-with-output.rst @@ -4,7 +4,7 @@ Hello Image Segmentation A very basic introduction to using segmentation models with OpenVINO™. In this tutorial, a pre-trained -`road-segmentation-adas-0001 `__ +`road-segmentation-adas-0001 `__ model from the `Open Model Zoo `__ is used. ADAS stands for Advanced Driver Assistance Services. The model diff --git a/docs/notebooks/004-hello-detection-with-output.rst b/docs/notebooks/004-hello-detection-with-output.rst index 054333cbaf5..b7a25d706fd 100644 --- a/docs/notebooks/004-hello-detection-with-output.rst +++ b/docs/notebooks/004-hello-detection-with-output.rst @@ -5,7 +5,7 @@ A very basic introduction to using object detection models with OpenVINO™. The -`horizontal-text-detection-0001 `__ +`horizontal-text-detection-0001 `__ model from `Open Model Zoo `__ is used. It detects horizontal text in images and returns a blob of data in the @@ -50,14 +50,14 @@ Imports import numpy as np import openvino as ov from pathlib import Path - + # Fetch `notebook_utils` module import urllib.request urllib.request.urlretrieve( url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py', filename='notebook_utils.py' ) - + from notebook_utils import download_file Download model weights @@ -68,18 +68,18 @@ Download model weights .. code:: ipython3 base_model_dir = Path("./model").expanduser() - + model_name = "horizontal-text-detection-0001" model_xml_name = f'{model_name}.xml' model_bin_name = f'{model_name}.bin' - + model_xml_path = base_model_dir / model_xml_name model_bin_path = base_model_dir / model_bin_name - + if not model_xml_path.exists(): model_xml_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.xml" model_bin_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.bin" - + download_file(model_xml_url, model_xml_name, base_model_dir) download_file(model_bin_url, model_bin_name, base_model_dir) else: @@ -108,7 +108,7 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + core = ov.Core() device = widgets.Dropdown( options=core.available_devices + ["AUTO"], @@ -116,7 +116,7 @@ select device from dropdown list for running inference using OpenVINO description='Device:', disabled=False, ) - + device @@ -136,10 +136,10 @@ Load the Model .. code:: ipython3 core = ov.Core() - + model = core.read_model(model=model_xml_path) compiled_model = core.compile_model(model=model, device_name="CPU") - + input_layer_ir = compiled_model.input(0) output_layer_ir = compiled_model.output("boxes") @@ -155,19 +155,19 @@ Load an Image "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/intel_rnb.jpg", directory="data" ) - + # Text detection models expect an image in BGR format. image = cv2.imread(str(image_filename)) - + # N,C,H,W = batch size, number of channels, height, width. N, C, H, W = input_layer_ir.shape - + # Resize the image to meet network expected input sizes. resized_image = cv2.resize(image, (W, H)) - + # Reshape to the network input shape. input_image = np.expand_dims(resized_image.transpose(2, 0, 1), 0) - + plt.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)); @@ -190,7 +190,7 @@ Do Inference # Create an inference request. boxes = compiled_model([input_image])[output_layer_ir] - + # Remove zero only boxes. boxes = boxes[~np.all(boxes == 0, axis=1)] @@ -206,31 +206,31 @@ Visualize Results def convert_result_to_image(bgr_image, resized_image, boxes, threshold=0.3, conf_labels=True): # Define colors for boxes and descriptions. colors = {"red": (255, 0, 0), "green": (0, 255, 0)} - + # Fetch the image shapes to calculate a ratio. (real_y, real_x), (resized_y, resized_x) = bgr_image.shape[:2], resized_image.shape[:2] ratio_x, ratio_y = real_x / resized_x, real_y / resized_y - + # Convert the base image from BGR to RGB format. rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) - + # Iterate through non-zero boxes. for box in boxes: # Pick a confidence factor from the last place in an array. conf = box[-1] if conf > threshold: # Convert float to int and multiply corner position of each box by x and y ratio. - # If the bounding box is found at the top of the image, - # position the upper box bar little lower to make it visible on the image. + # If the bounding box is found at the top of the image, + # position the upper box bar little lower to make it visible on the image. (x_min, y_min, x_max, y_max) = [ - int(max(corner_position * ratio_y, 10)) if idx % 2 + int(max(corner_position * ratio_y, 10)) if idx % 2 else int(corner_position * ratio_x) for idx, corner_position in enumerate(box[:-1]) ] - + # Draw a box based on the position, parameters in rectangle function are: image, start_point, end_point, color, thickness. rgb_image = cv2.rectangle(rgb_image, (x_min, y_min), (x_max, y_max), colors["green"], 3) - + # Add text to the image based on position and confidence. # Parameters in text function are: image, text, bottom-left_corner_textfield, font, font_scale, color, thickness, line_type. if conf_labels: @@ -244,7 +244,7 @@ Visualize Results 1, cv2.LINE_AA, ) - + return rgb_image .. code:: ipython3 diff --git a/docs/notebooks/101-tensorflow-classification-to-openvino-with-output.rst b/docs/notebooks/101-tensorflow-classification-to-openvino-with-output.rst index 82a4441bbdd..1008434b092 100644 --- a/docs/notebooks/101-tensorflow-classification-to-openvino-with-output.rst +++ b/docs/notebooks/101-tensorflow-classification-to-openvino-with-output.rst @@ -2,11 +2,11 @@ Convert a TensorFlow Model to OpenVINO™ ======================================= This short tutorial shows how to convert a TensorFlow -`MobileNetV3 `__ +`MobileNetV3 `__ image classification model to OpenVINO `Intermediate -Representation `__ +Representation `__ (OpenVINO IR) format, using `Model Conversion -API `__. +API `__. After creating the OpenVINO IR, load the model in `OpenVINO Runtime `__ and do inference with a sample image. @@ -56,20 +56,20 @@ Imports import time from pathlib import Path - + import cv2 import matplotlib.pyplot as plt import numpy as np import openvino as ov import tensorflow as tf - + # Fetch `notebook_utils` module import urllib.request urllib.request.urlretrieve( url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py', filename='notebook_utils.py' ) - + from notebook_utils import download_file @@ -95,9 +95,9 @@ Settings # The paths of the source and converted models. model_dir = Path("model") model_dir.mkdir(exist_ok=True) - + model_path = Path("model/v3-small_224_1.0_float") - + ir_path = Path("model/v3-small_224_1.0_float.xml") Download model @@ -174,7 +174,7 @@ model directory and returns OpenVINO Model class instance which represents this model. Obtained model is ready to use and to be loaded on a device using ``ov.compile_model`` or can be saved on a disk using the ``ov.save_model`` function. See the -`tutorial `__ +`tutorial `__ for more information about using model conversion API with TensorFlow models. @@ -219,14 +219,14 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + device = widgets.Dropdown( options=core.available_devices + ["AUTO"], value='AUTO', description='Device:', disabled=False, ) - + device @@ -251,7 +251,7 @@ Get Model Information input_key = compiled_model.input(0) output_key = compiled_model.output(0) - network_input_shape = input_key.shape + network_input_shape = input_key.shape Load an Image ~~~~~~~~~~~~~ @@ -268,16 +268,16 @@ network. "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg", directory="data" ) - + # The MobileNet network expects images in RGB format. image = cv2.cvtColor(cv2.imread(filename=str(image_filename)), code=cv2.COLOR_BGR2RGB) - + # Resize the image to the network input shape. resized_image = cv2.resize(src=image, dsize=(224, 224)) - + # Transpose the image to the network input shape. input_image = np.expand_dims(resized_image, 0) - + plt.imshow(image); @@ -299,7 +299,7 @@ Do Inference .. code:: ipython3 result = compiled_model(input_image)[output_key] - + result_index = np.argmax(result) .. code:: ipython3 @@ -309,10 +309,10 @@ Do Inference "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_2012.txt", directory="data" ) - + # Convert the inference result to a class name. imagenet_classes = image_filename.read_text().splitlines() - + imagenet_classes[result_index] @@ -338,22 +338,22 @@ Timing Measure the time it takes to do inference on thousand images. This gives an indication of performance. For more accurate benchmarking, use the `Benchmark -Tool `__ +Tool `__ in OpenVINO. Note that many optimizations are possible to improve the performance. .. code:: ipython3 num_images = 1000 - + start = time.perf_counter() - + for _ in range(num_images): compiled_model([input_image]) - + end = time.perf_counter() time_ir = end - start - + print( f"IR model in OpenVINO Runtime/CPU: {time_ir/num_images:.4f} " f"seconds per image, FPS: {num_images/time_ir:.2f}" diff --git a/docs/notebooks/102-pytorch-onnx-to-openvino-with-output.rst b/docs/notebooks/102-pytorch-onnx-to-openvino-with-output.rst index 6b5c1d5c9c2..7953e74307b 100644 --- a/docs/notebooks/102-pytorch-onnx-to-openvino-with-output.rst +++ b/docs/notebooks/102-pytorch-onnx-to-openvino-with-output.rst @@ -92,20 +92,20 @@ Imports import time import warnings from pathlib import Path - + import cv2 import numpy as np import openvino as ov import torch from torchvision.models.segmentation import lraspp_mobilenet_v3_large, LRASPP_MobileNet_V3_Large_Weights - + # Fetch `notebook_utils` module import urllib.request urllib.request.urlretrieve( url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py', filename='notebook_utils.py' ) - + from notebook_utils import segmentation_map_to_image, viz_result_image, SegmentationMap, Label, download_file Settings @@ -125,7 +125,7 @@ transforms function, the model is pre-trained on images with a height of DIRECTORY_NAME = "model" BASE_MODEL_NAME = DIRECTORY_NAME + "/lraspp_mobilenet_v3_large" weights_path = Path(BASE_MODEL_NAME + ".pt") - + # Paths where ONNX and OpenVINO IR models will be stored. onnx_path = weights_path.with_suffix('.onnx') if not onnx_path.parent.exists(): @@ -156,7 +156,7 @@ have not downloaded the model before. .. code:: ipython3 - print("Downloading the LRASPP MobileNetV3 model (if it has not been downloaded already)...") + print("Downloading the LRASPP MobileNetV3 model (if it has not been downloaded already)...") download_file(LRASPP_MobileNet_V3_Large_Weights.COCO_WITH_VOC_LABELS_V1.url, filename=weights_path.name, directory=weights_path.parent) # create model object model = lraspp_mobilenet_v3_large() @@ -240,7 +240,7 @@ Convert ONNX Model to OpenVINO IR Format To convert the ONNX model to OpenVINO IR with ``FP16`` precision, use model conversion API. The models are saved inside the current directory. For more information on how to convert models, see this -`page `__. +`page `__. .. code:: ipython3 @@ -294,12 +294,12 @@ Images need to be normalized before propagating through the network. "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg", directory="data" ) - + image = cv2.cvtColor(cv2.imread(str(image_filename)), cv2.COLOR_BGR2RGB) - + resized_image = cv2.resize(image, (IMAGE_WIDTH, IMAGE_HEIGHT)) normalized_image = normalize(resized_image) - + # Convert the resized images to network input shape. input_image = np.expand_dims(np.transpose(resized_image, (2, 0, 1)), 0) normalized_input_image = np.expand_dims(np.transpose(normalized_image, (2, 0, 1)), 0) @@ -331,7 +331,7 @@ on an image. # Instantiate OpenVINO Core core = ov.Core() - + # Read model to OpenVINO Runtime model_onnx = core.read_model(model=onnx_path) @@ -345,14 +345,14 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + device = widgets.Dropdown( options=core.available_devices + ["AUTO"], value='AUTO', description='Device:', disabled=False, ) - + device @@ -368,7 +368,7 @@ select device from dropdown list for running inference using OpenVINO # Load model on device compiled_model_onnx = core.compile_model(model=model_onnx, device_name=device.value) - + # Run inference on the input image res_onnx = compiled_model_onnx([normalized_input_image])[0] @@ -403,7 +403,7 @@ be applied to each label for more convenient visualization. Label(index=20, color=(0, 64, 128), name="tv monitor") ] VOCLabels = SegmentationMap(voc_labels) - + # Convert the network result to a segmentation map and display the result. result_mask_onnx = np.squeeze(np.argmax(res_onnx, axis=1)).astype(np.uint8) viz_result_image( @@ -450,10 +450,10 @@ select device from dropdown list for running inference using OpenVINO core = ov.Core() model_ir = core.read_model(model=ir_path) compiled_model_ir = core.compile_model(model=model_ir, device_name=device.value) - + # Get input and output layers. output_layer_ir = compiled_model_ir.output(0) - + # Run inference on the input image. res_ir = compiled_model_ir([normalized_input_image])[output_layer_ir] @@ -486,7 +486,7 @@ looks the same as the output on the ONNX/OpenVINO IR models. model.eval() with torch.no_grad(): result_torch = model(torch.as_tensor(normalized_input_image).float()) - + result_mask_torch = torch.argmax(result_torch['out'], dim=1).squeeze(0).numpy().astype(np.uint8) viz_result_image( image, @@ -509,14 +509,14 @@ Performance Comparison Measure the time it takes to do inference on twenty images. This gives an indication of performance. For more accurate benchmarking, use the `Benchmark -Tool `__. +Tool `__. Keep in mind that many optimizations are possible to improve the performance. .. code:: ipython3 num_images = 100 - + with torch.no_grad(): start = time.perf_counter() for _ in range(num_images): @@ -527,7 +527,7 @@ performance. f"PyTorch model on CPU: {time_torch/num_images:.3f} seconds per image, " f"FPS: {num_images/time_torch:.2f}" ) - + compiled_model_onnx = core.compile_model(model=model_onnx, device_name="CPU") start = time.perf_counter() for _ in range(num_images): @@ -538,7 +538,7 @@ performance. f"ONNX model in OpenVINO Runtime/CPU: {time_onnx/num_images:.3f} " f"seconds per image, FPS: {num_images/time_onnx:.2f}" ) - + compiled_model_ir = core.compile_model(model=model_ir, device_name="CPU") start = time.perf_counter() for _ in range(num_images): @@ -549,7 +549,7 @@ performance. f"OpenVINO IR model in OpenVINO Runtime/CPU: {time_ir/num_images:.3f} " f"seconds per image, FPS: {num_images/time_ir:.2f}" ) - + if "GPU" in core.available_devices: compiled_model_onnx_gpu = core.compile_model(model=model_onnx, device_name="GPU") start = time.perf_counter() @@ -561,7 +561,7 @@ performance. f"ONNX model in OpenVINO/GPU: {time_onnx_gpu/num_images:.3f} " f"seconds per image, FPS: {num_images/time_onnx_gpu:.2f}" ) - + compiled_model_ir_gpu = core.compile_model(model=model_ir, device_name="GPU") start = time.perf_counter() for _ in range(num_images): @@ -616,6 +616,6 @@ References - `OpenVINO ONNX support `__ - `Model Conversion API - documentation `__ + documentation `__ - `Converting Pytorch - model `__ + model `__ diff --git a/docs/notebooks/102-pytorch-to-openvino-with-output.rst b/docs/notebooks/102-pytorch-to-openvino-with-output.rst index e2ed8dfafba..8d44159c790 100644 --- a/docs/notebooks/102-pytorch-to-openvino-with-output.rst +++ b/docs/notebooks/102-pytorch-to-openvino-with-output.rst @@ -99,23 +99,23 @@ Download input data and label map import requests from pathlib import Path from PIL import Image - + MODEL_DIR = Path("model") DATA_DIR = Path("data") - + MODEL_DIR.mkdir(exist_ok=True) DATA_DIR.mkdir(exist_ok=True) MODEL_NAME = "regnet_y_800mf" - + image = Image.open(requests.get("https://farm9.staticflickr.com/8225/8511402100_fea15da1c5_z.jpg", stream=True).raw) - + labels_file = DATA_DIR / "imagenet_2012.txt" - + if not labels_file.exists(): resp = requests.get("https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/data/dataset_classes/imagenet_2012.txt") with labels_file.open("wb") as f: f.write(resp.content) - + imagenet_classes = labels_file.open("r").read().splitlines() Load PyTorch Model @@ -141,14 +141,14 @@ enum ``RegNet_Y_800MF_Weights.DEFAULT``. .. code:: ipython3 import torchvision - + # get default weights using available weights Enum for model weights = torchvision.models.RegNet_Y_800MF_Weights.DEFAULT - + # create model topology and load weights model = torchvision.models.regnet_y_800mf(weights=weights) - - # switch model to inference mode + + # switch model to inference mode model.eval(); Prepare Input Data @@ -165,13 +165,13 @@ the first dimension. .. code:: ipython3 import torch - + # Initialize the Weight Transforms preprocess = weights.transforms() - + # Apply it to the input image img_transformed = preprocess(image) - + # Add batch dimension to image tensor input_tensor = img_transformed.unsqueeze(0) @@ -190,10 +190,10 @@ can be reused later. import numpy as np from scipy.special import softmax - + # Perform model inference on input tensor result = model(input_tensor) - + # Postprocessing function for getting results in the same way for both PyTorch model inference and OpenVINO def postprocess_result(output_tensor:np.ndarray, top_k:int = 5): """ @@ -209,10 +209,10 @@ can be reused later. topk_labels = np.argsort(softmaxed_scores)[-top_k:][::-1] topk_scores = softmaxed_scores[topk_labels] return topk_labels, topk_scores - + # Postprocess results top_labels, top_scores = postprocess_result(result.detach().numpy()) - + # Show results display(image) for idx, (label, score) in enumerate(zip(top_labels, top_scores)): @@ -241,7 +241,7 @@ Benchmark PyTorch Model Inference .. code:: ipython3 %%timeit - + # Run model inference model(input_tensor) @@ -260,7 +260,7 @@ Starting from the 2023.0 release OpenVINO supports direct PyTorch models conversion to OpenVINO Intermediate Representation (IR) format. OpenVINO model conversion API should be used for these purposes. More details regarding PyTorch model conversion can be found in OpenVINO -`documentation `__ +`documentation `__ The ``convert_model`` function accepts the PyTorch model object and returns the ``openvino.Model`` instance ready to load on a device using @@ -278,21 +278,21 @@ such as: and any other advanced options supported by model conversion Python API. More details can be found on this -`page `__ +`page `__ .. code:: ipython3 import openvino as ov - + # Create OpenVINO Core object instance core = ov.Core() - + # Convert model to openvino.runtime.Model object ov_model = ov.convert_model(model) - + # Save openvino.runtime.Model object on disk ov.save_model(ov_model, MODEL_DIR / f"{MODEL_NAME}_dynamic.xml") - + ov_model @@ -320,14 +320,14 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + device = widgets.Dropdown( options=core.available_devices + ["AUTO"], value='AUTO', description='Device:', disabled=False, ) - + device @@ -369,10 +369,10 @@ Run OpenVINO Model Inference # Run model inference result = compiled_model(input_tensor)[0] - + # Posptorcess results top_labels, top_scores = postprocess_result(result) - + # Show results display(image) for idx, (label, score) in enumerate(zip(top_labels, top_scores)): @@ -401,7 +401,7 @@ Benchmark OpenVINO Model Inference .. code:: ipython3 %%timeit - + compiled_model(input_tensor) @@ -499,10 +499,10 @@ Run OpenVINO Model Inference with Static Input Shape # Run model inference result = compiled_model(input_tensor)[0] - + # Posptorcess results top_labels, top_scores = postprocess_result(result) - + # Show results display(image) for idx, (label, score) in enumerate(zip(top_labels, top_scores)): @@ -531,7 +531,7 @@ Benchmark OpenVINO Model Inference with Static Input Shape .. code:: ipython3 %%timeit - + compiled_model(input_tensor) @@ -584,20 +584,20 @@ Reference =2.5.1,<2.6.0" else: @@ -73,7 +73,7 @@ Imports paddleclas 2.5.1 requires easydict, which is not installed. paddleclas 2.5.1 requires faiss-cpu==1.7.1.post2, but you have faiss-cpu 1.7.4 which is incompatible. paddleclas 2.5.1 requires gast==0.3.3, but you have gast 0.4.0 which is incompatible. - + .. parsed-literal:: @@ -97,13 +97,13 @@ Imports --2024-02-09 22:36:08-- http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb Resolving proxy-mu.intel.com (proxy-mu.intel.com)... 10.217.247.236 Connecting to proxy-mu.intel.com (proxy-mu.intel.com)|10.217.247.236|:911... connected. - Proxy request sent, awaiting response... + Proxy request sent, awaiting response... .. parsed-literal:: 404 Not Found 2024-02-09 22:36:08 ERROR 404: Not Found. - + .. parsed-literal:: @@ -116,20 +116,20 @@ Imports import time import tarfile from pathlib import Path - + import matplotlib.pyplot as plt import numpy as np import openvino as ov from paddleclas import PaddleClas from PIL import Image - + # Fetch `notebook_utils` module import urllib.request urllib.request.urlretrieve( url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py', filename='notebook_utils.py' ) - + from notebook_utils import download_file @@ -168,9 +168,9 @@ PaddleHub. This may take a while. "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_close.png", directory="data" ) - + IMAGE_FILENAME = img.as_posix() - + MODEL_NAME = "MobileNetV3_large_x1_0" MODEL_DIR = Path("model") if not MODEL_DIR.exists(): @@ -271,8 +271,8 @@ the same method. .. code:: ipython3 preprocess_ops = classifier.predictor.preprocess_ops - - + + def process_image(image): for op in preprocess_ops: image = op(image) @@ -341,7 +341,7 @@ accept path to PaddlePaddle model and returns OpenVINO Model class instance which represents this model. Obtained model is ready to use and loading on device using ``ov.compile_model`` or can be saved on disk using ``ov.save_model`` function. See the `Model Conversion -Guide `__ +Guide `__ for more information about the Model Conversion API. .. code:: ipython3 @@ -363,7 +363,7 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + core = ov.Core() device = widgets.Dropdown( options=core.available_devices + ["AUTO"], @@ -371,7 +371,7 @@ select device from dropdown list for running inference using OpenVINO description='Device:', disabled=False, ) - + device @@ -400,23 +400,23 @@ information. core = ov.Core() model = core.read_model(model_xml) compiled_model = core.compile_model(model=model, device_name="CPU") - + # Get model output output_layer = compiled_model.output(0) - + # Read, show, and preprocess input image # See the "Show Inference on PaddlePaddle Model" section for source of process_image image = Image.open(IMAGE_FILENAME) plt.imshow(image) input_image = process_image(np.array(image))[None,] - + # Do inference ov_result = compiled_model([input_image])[output_layer][0] - + # find the top three values top_indices = np.argsort(ov_result)[-3:][::-1] top_scores = ov_result[top_indices] - + # Convert the inference results to class names, using the same labels as the PaddlePaddle classifier for index, softmax_probability in zip(top_indices, top_scores): print(f"{class_id_map[index]}, {softmax_probability:.5f}") @@ -442,13 +442,13 @@ Measure the time it takes to do inference on fifty images and compare the result. The timing information gives an indication of performance. For a fair comparison, we include the time it takes to process the image. For more accurate benchmarking, use the `OpenVINO benchmark -tool `__. +tool `__. Note that many optimizations are possible to improve the performance. .. code:: ipython3 num_images = 50 - + image = Image.open(fp=IMAGE_FILENAME) .. code:: ipython3 @@ -456,7 +456,7 @@ Note that many optimizations are possible to improve the performance. # Show device information core = ov.Core() devices = core.available_devices - + for device_name in devices: device_full_name = core.get_property(device_name, "FULL_DEVICE_NAME") print(f"{device_name}: {device_full_name}") @@ -490,7 +490,7 @@ Note that many optimizations are possible to improve the performance. .. parsed-literal:: PaddlePaddle model on CPU: 0.0075 seconds per image, FPS: 133.16 - + PaddlePaddle result: Labrador retriever, 0.75138 German short-haired pointer, 0.02373 @@ -528,18 +528,18 @@ select device from dropdown list for running inference using OpenVINO # Show inference speed on OpenVINO IR model compiled_model = core.compile_model(model=model, device_name=device.value) output_layer = compiled_model.output(0) - - + + start = time.perf_counter() input_image = process_image(np.array(image))[None,] for _ in range(num_images): ie_result = compiled_model([input_image])[output_layer][0] top_indices = np.argsort(ie_result)[-5:][::-1] top_softmax = ie_result[top_indices] - + end = time.perf_counter() time_ir = end - start - + print( f"OpenVINO IR model in OpenVINO Runtime ({device.value}): {time_ir/num_images:.4f} " f"seconds per image, FPS: {num_images/time_ir:.2f}" @@ -554,7 +554,7 @@ select device from dropdown list for running inference using OpenVINO .. parsed-literal:: OpenVINO IR model in OpenVINO Runtime (AUTO): 0.0030 seconds per image, FPS: 328.87 - + OpenVINO result: Labrador retriever, 0.74909 German short-haired pointer, 0.02368 @@ -574,4 +574,4 @@ References - `PaddleClas `__ - `OpenVINO PaddlePaddle - support `__ + support `__ diff --git a/docs/notebooks/105-language-quantize-bert-with-output.rst b/docs/notebooks/105-language-quantize-bert-with-output.rst index f62318fe229..30715d34cee 100644 --- a/docs/notebooks/105-language-quantize-bert-with-output.rst +++ b/docs/notebooks/105-language-quantize-bert-with-output.rst @@ -572,7 +572,7 @@ Frames Per Second (FPS) for images. Finally, measure the inference performance of OpenVINO ``FP32`` and ``INT8`` models. For this purpose, use `Benchmark -Tool `__ +Tool `__ in OpenVINO. **NOTE**: The ``benchmark_app`` tool is able to measure the diff --git a/docs/notebooks/106-auto-device-with-output.rst b/docs/notebooks/106-auto-device-with-output.rst index b2b4bb85b78..15bd71f3f2e 100644 --- a/docs/notebooks/106-auto-device-with-output.rst +++ b/docs/notebooks/106-auto-device-with-output.rst @@ -2,19 +2,19 @@ Automatic Device Selection with OpenVINO™ ========================================= The `Auto -device `__ +device `__ (or AUTO in short) selects the most suitable device for inference by considering the model precision, power efficiency and processing capability of the available `compute -devices `__. +devices `__. The model precision (such as ``FP32``, ``FP16``, ``INT8``, etc.) is the first consideration to filter out the devices that cannot run the network efficiently. Next, if dedicated accelerators are available, these devices are preferred (for example, integrated and discrete -`GPU `__). -`CPU `__ +`GPU `__). +`CPU `__ is used as the default “fallback device”. Keep in mind that AUTO makes this selection only once, during the loading of a model. @@ -81,13 +81,13 @@ Import modules and create Core import time import sys - + import openvino as ov - + from IPython.display import Markdown, display - + core = ov.Core() - + if "GPU" not in core.available_devices: display(Markdown('
    Warning: A GPU device is not available. This notebook requires GPU device to have meaningful results.
    ')) @@ -121,17 +121,17 @@ with ``ov.compile_model`` or serialized for next usage with ``ov.save_model``. For more information about model conversion API, see this -`page `__. +`page `__. .. code:: ipython3 import torchvision from pathlib import Path - + base_model_dir = Path("./model") base_model_dir.mkdir(exist_ok=True) model_path = base_model_dir / "resnet50.xml" - + if not model_path.exists(): pt_model = torchvision.models.resnet50(weights="DEFAULT") ov_model = ov.convert_model(pt_model, input=[[1,3,224,224]]) @@ -164,12 +164,12 @@ By default, ``compile_model`` API will select **AUTO** as # Set LOG_LEVEL to LOG_INFO. core.set_property("AUTO", {"LOG_LEVEL":"LOG_INFO"}) - + # Load the model onto the target device. compiled_model = core.compile_model(ov_model) - + if isinstance(compiled_model, ov.CompiledModel): - print("Successfully compiled model without a device_name.") + print("Successfully compiled model without a device_name.") .. parsed-literal:: @@ -210,9 +210,9 @@ improve readability of your code. # Set LOG_LEVEL to LOG_NONE. core.set_property("AUTO", {"LOG_LEVEL":"LOG_NONE"}) - + compiled_model = core.compile_model(model=ov_model, device_name="AUTO") - + if isinstance(compiled_model, ov.CompiledModel): print("Successfully compiled model using AUTO.") @@ -271,16 +271,16 @@ function, we will reuse it for preparing input data. .. code:: ipython3 from PIL import Image - + # Download the image from the openvino_notebooks storage image_filename = download_file( "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg", directory="data" ) - + image = Image.open(str(image_filename)) input_transform = torchvision.models.ResNet50_Weights.DEFAULT.transforms() - + input_tensor = input_transform(image) input_tensor = input_tensor.unsqueeze(0).numpy() image @@ -307,14 +307,14 @@ Load the model to GPU device and perform inference if "GPU" not in core.available_devices: print(f"A GPU device is not available. Available devices are: {core.available_devices}") - else : + else : # Start time. gpu_load_start_time = time.perf_counter() compiled_model = core.compile_model(model=ov_model, device_name="GPU") # load to GPU - + # Execute the first inference. results = compiled_model(input_tensor)[0] - + # Measure time to the first inference. gpu_fil_end_time = time.perf_counter() gpu_fil_span = gpu_fil_end_time - gpu_load_start_time @@ -340,11 +340,11 @@ executed on CPU until GPU is ready. # Start time. auto_load_start_time = time.perf_counter() compiled_model = core.compile_model(model=ov_model) # The device_name is AUTO by default. - + # Execute the first inference. results = compiled_model(input_tensor)[0] - - + + # Measure time to the first inference. auto_fil_end_time = time.perf_counter() auto_fil_span = auto_fil_end_time - auto_load_start_time @@ -376,9 +376,9 @@ completely portable between devices – meaning AUTO can configure the performance hint on whichever device is being used. For more information, refer to the `Performance -Hints `__ +Hints `__ section of `Automatic Device -Selection `__ +Selection `__ article. Class and callback definition @@ -406,11 +406,11 @@ Class and callback definition """ self.fps = 0 self.latency = 0 - + self.start_time = time.perf_counter() self.latency_list = [] self.interval = interval - + def update(self, infer_request: ov.InferRequest) -> bool: """ Update the metrics if current ongoing @interval seconds duration is expired. Record the latency only if it is not expired. @@ -432,12 +432,12 @@ Class and callback definition return True else : return False - - + + class InferContext: """ Inference context. Record and update peforamnce metrics via @metrics, set @feed_inference to False once @remaining_update_num <=0 - :member: metrics: instance of class PerformanceMetrics + :member: metrics: instance of class PerformanceMetrics :member: remaining_update_num: the remaining times for peforamnce metrics updating. :member: feed_inference: if feed inference request is required or not. """ @@ -452,7 +452,7 @@ Class and callback definition self.metrics = PerformanceMetrics(update_interval) self.remaining_update_num = num self.feed_inference = True - + def update(self, infer_request: ov.InferRequest): """ Update the context. Set @feed_inference to False if the number of remaining performance metric updates (@remaining_update_num) reaches 0 @@ -461,13 +461,13 @@ Class and callback definition """ if self.remaining_update_num <= 0 : self.feed_inference = False - + if self.metrics.update(infer_request) : self.remaining_update_num = self.remaining_update_num - 1 if self.remaining_update_num <= 0 : self.feed_inference = False - - + + def completion_callback(infer_request: ov.InferRequest, context) -> None: """ callback for the inference request, pass the @infer_request to @context for updating @@ -476,8 +476,8 @@ Class and callback definition :returns: None """ context.update(infer_request) - - + + # Performance metrics update interval (seconds) and number of times. metrics_update_interval = 10 metrics_update_num = 6 @@ -493,29 +493,29 @@ Loop for inference and update the FPS/Latency every .. code:: ipython3 THROUGHPUT_hint_context = InferContext(metrics_update_interval, metrics_update_num) - + print("Compiling Model for AUTO device with THROUGHPUT hint") sys.stdout.flush() - + compiled_model = core.compile_model(model=ov_model, config={"PERFORMANCE_HINT":"THROUGHPUT"}) - + infer_queue = ov.AsyncInferQueue(compiled_model, 0) # Setting to 0 will query optimal number by default. infer_queue.set_callback(completion_callback) - + print(f"Start inference, {metrics_update_num: .0f} groups of FPS/latency will be measured over {metrics_update_interval: .0f}s intervals") sys.stdout.flush() - + while THROUGHPUT_hint_context.feed_inference: infer_queue.start_async(input_tensor, THROUGHPUT_hint_context) - + infer_queue.wait_all() - + # Take the FPS and latency of the latest period. THROUGHPUT_hint_fps = THROUGHPUT_hint_context.metrics.fps THROUGHPUT_hint_latency = THROUGHPUT_hint_context.metrics.latency - + print("Done") - + del compiled_model @@ -575,30 +575,30 @@ Loop for inference and update the FPS/Latency for each .. code:: ipython3 LATENCY_hint_context = InferContext(metrics_update_interval, metrics_update_num) - + print("Compiling Model for AUTO Device with LATENCY hint") sys.stdout.flush() - + compiled_model = core.compile_model(model=ov_model, config={"PERFORMANCE_HINT":"LATENCY"}) - + # Setting to 0 will query optimal number by default. infer_queue = ov.AsyncInferQueue(compiled_model, 0) infer_queue.set_callback(completion_callback) - + print(f"Start inference, {metrics_update_num: .0f} groups fps/latency will be out with {metrics_update_interval: .0f}s interval") sys.stdout.flush() - + while LATENCY_hint_context.feed_inference: infer_queue.start_async(input_tensor, LATENCY_hint_context) - + infer_queue.wait_all() - + # Take the FPS and latency of the latest period. LATENCY_hint_fps = LATENCY_hint_context.metrics.fps LATENCY_hint_latency = LATENCY_hint_context.metrics.latency - + print("Done") - + del compiled_model @@ -655,21 +655,21 @@ Difference in FPS and latency .. code:: ipython3 import matplotlib.pyplot as plt - + TPUT = 0 LAT = 1 labels = ["THROUGHPUT hint", "LATENCY hint"] - - fig1, ax1 = plt.subplots(1, 1) + + fig1, ax1 = plt.subplots(1, 1) fig1.patch.set_visible(False) - ax1.axis('tight') - ax1.axis('off') - + ax1.axis('tight') + ax1.axis('off') + cell_text = [] cell_text.append(['%.2f%s' % (THROUGHPUT_hint_fps," FPS"), '%.2f%s' % (THROUGHPUT_hint_latency, " ms")]) cell_text.append(['%.2f%s' % (LATENCY_hint_fps," FPS"), '%.2f%s' % (LATENCY_hint_latency, " ms")]) - - table = ax1.table(cellText=cell_text, colLabels=["FPS (Higher is better)", "Latency (Lower is better)"], rowLabels=labels, + + table = ax1.table(cellText=cell_text, colLabels=["FPS (Higher is better)", "Latency (Lower is better)"], rowLabels=labels, rowColours=["deepskyblue"] * 2, colColours=["deepskyblue"] * 2, cellLoc='center', loc='upper left') table.auto_set_font_size(False) @@ -677,7 +677,7 @@ Difference in FPS and latency table.auto_set_column_width(0) table.auto_set_column_width(1) table.scale(1, 3) - + fig1.tight_layout() plt.show() @@ -691,28 +691,28 @@ Difference in FPS and latency # Output the difference. width = 0.4 fontsize = 14 - + plt.rc('font', size=fontsize) fig, ax = plt.subplots(1,2, figsize=(10, 8)) - + rects1 = ax[0].bar([0], THROUGHPUT_hint_fps, width, label=labels[TPUT], color='#557f2d') rects2 = ax[0].bar([width], LATENCY_hint_fps, width, label=labels[LAT]) ax[0].set_ylabel("frames per second") - ax[0].set_xticks([width / 2]) + ax[0].set_xticks([width / 2]) ax[0].set_xticklabels(["FPS"]) ax[0].set_xlabel("Higher is better") - + rects1 = ax[1].bar([0], THROUGHPUT_hint_latency, width, label=labels[TPUT], color='#557f2d') rects2 = ax[1].bar([width], LATENCY_hint_latency, width, label=labels[LAT]) ax[1].set_ylabel("milliseconds") ax[1].set_xticks([width / 2]) ax[1].set_xticklabels(["Latency (ms)"]) ax[1].set_xlabel("Lower is better") - + fig.suptitle('Performance Hints') fig.legend(labels, fontsize=fontsize) fig.tight_layout() - + plt.show() diff --git a/docs/notebooks/107-speech-recognition-quantization-data2vec-with-output.rst b/docs/notebooks/107-speech-recognition-quantization-data2vec-with-output.rst index ca3683e05ef..7539979e60d 100644 --- a/docs/notebooks/107-speech-recognition-quantization-data2vec-with-output.rst +++ b/docs/notebooks/107-speech-recognition-quantization-data2vec-with-output.rst @@ -314,7 +314,7 @@ steps: accurate results, we should keep the operation in the postprocessing subgraph in floating point precision, using the ``ignored_scope`` parameter. For more information see `Tune quantization - parameters `__. + parameters `__. 3. Serialize OpenVINO IR model using ``ov.save_model`` function. .. code:: ipython3 @@ -490,7 +490,7 @@ Compare Performance of the Original and Quantized Models -------------------------------------------------------- `Benchmark -Tool `__ +Tool `__ is used to measure the inference performance of the ``FP16`` and ``INT8`` models. diff --git a/docs/notebooks/107-speech-recognition-quantization-wav2vec2-with-output.rst b/docs/notebooks/107-speech-recognition-quantization-wav2vec2-with-output.rst index 95e82386d07..8b83c82cf46 100644 --- a/docs/notebooks/107-speech-recognition-quantization-wav2vec2-with-output.rst +++ b/docs/notebooks/107-speech-recognition-quantization-wav2vec2-with-output.rst @@ -293,7 +293,7 @@ steps: accurate results, we should keep the operation in the postprocessing subgraph in floating point precision, using the ``ignored_scope`` parameter. For more information see `Tune quantization - parameters `__. + parameters `__. For this model, ignored scope was selected experimentally, based on result of quantization with accuracy control. For understanding how it works please check following @@ -625,7 +625,7 @@ Compare Performance of the Original and Quantized Models Finally, use `Benchmark -Tool `__ +Tool `__ to measure the inference performance of the ``FP16`` and ``INT8`` models. diff --git a/docs/notebooks/108-gpu-device-with-output.rst b/docs/notebooks/108-gpu-device-with-output.rst index dd7df6cf65c..0e436de4687 100644 --- a/docs/notebooks/108-gpu-device-with-output.rst +++ b/docs/notebooks/108-gpu-device-with-output.rst @@ -94,10 +94,10 @@ cards `__. To get started, first `install -OpenVINO `__ +OpenVINO `__ on a system equipped with one or more Intel GPUs. Follow the `GPU configuration -instructions `__ +instructions `__ to configure OpenVINO to work with your GPU. Then, read on to learn how to accelerate inference with GPUs in OpenVINO! @@ -110,7 +110,7 @@ Install required packages %pip install -q "openvino-dev>=2023.1.0" %pip install -q tensorflow - + # Fetch `notebook_utils` module import urllib.request urllib.request.urlretrieve( @@ -148,7 +148,7 @@ appear. .. code:: ipython3 import openvino as ov - + core = ov.Core() core.available_devices @@ -167,12 +167,12 @@ the system has a CPU, an integrated and discrete GPU, we should expect to see a list like this: ``['CPU', 'GPU.0', 'GPU.1']``. To simplify its use, the “GPU.0” can also be addressed with just “GPU”. For more details, see the `Device Naming -Convention `__ +Convention `__ section. If the GPUs are installed correctly on the system and still do not appear in the list, follow the steps described -`here `__ +`here `__ to configure your GPU drivers to work with OpenVINO. Once we have the GPUs working with OpenVINO, we can proceed with the next sections. @@ -192,7 +192,7 @@ To get the value of a property, such as the device name, we can use the .. code:: ipython3 device = "GPU" - + core.get_property(device, "FULL_DEVICE_NAME") @@ -216,7 +216,7 @@ for that property. print(f"{device} SUPPORTED_PROPERTIES:\n") supported_properties = core.get_property(device, "SUPPORTED_PROPERTIES") indent = len(max(supported_properties, key=len)) - + for property_key in supported_properties: if property_key not in ('SUPPORTED_METRICS', 'SUPPORTED_CONFIG_KEYS', 'SUPPORTED_PROPERTIES'): try: @@ -229,7 +229,7 @@ for that property. .. parsed-literal:: GPU SUPPORTED_PROPERTIES: - + AVAILABLE_DEVICES : ['0'] RANGE_FOR_ASYNC_INFER_REQUESTS: (1, 2, 1) RANGE_FOR_STREAMS : (1, 2) @@ -252,7 +252,7 @@ for that property. GPU_QUEUE_PRIORITY : Priority.MEDIUM GPU_QUEUE_THROTTLE : Priority.MEDIUM GPU_ENABLE_LOOP_UNROLLING : True - CACHE_DIR : + CACHE_DIR : PERFORMANCE_HINT : PerformanceMode.UNDEFINED COMPILATION_NUM_THREADS : 20 NUM_STREAMS : 1 @@ -288,7 +288,7 @@ the key properties are: speed up compilation time. To learn more about devices and properties, see the `Query Device -Properties `__ +Properties `__ page. Compiling a Model on GPU @@ -299,7 +299,7 @@ Compiling a Model on GPU Now, we know how to list the GPUs in the system and check their properties. We can easily use one for compiling and running models with OpenVINO `GPU -plugin `__. +plugin `__. Download and Convert a Model ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -328,23 +328,23 @@ package is already downloaded. import sys import tarfile from pathlib import Path - + sys.path.append("../utils") - + import notebook_utils as utils - + # A directory where the model will be downloaded. base_model_dir = Path("./model").expanduser() - + model_name = "ssdlite_mobilenet_v2" archive_name = Path(f"{model_name}_coco_2018_05_09.tar.gz") - + # Download the archive downloaded_model_path = base_model_dir / archive_name if not downloaded_model_path.exists(): model_url = f"http://download.tensorflow.org/models/object_detection/{archive_name}" utils.download_file(model_url, downloaded_model_path.name, downloaded_model_path.parent) - + # Unpack the model tf_model_path = base_model_dir / archive_name.with_suffix("").stem / "frozen_inference_graph.pb" if not tf_model_path.exists(): @@ -365,11 +365,11 @@ package is already downloaded. to the client in order to avoid crashing it. To change this limit, set the config variable `--NotebookApp.iopub_msg_rate_limit`. - + Current values: NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec) NotebookApp.rate_limit_window=3.0 (secs) - + Convert the Model to OpenVINO IR format @@ -380,20 +380,20 @@ Convert the Model to OpenVINO IR format To convert the model to OpenVINO IR with ``FP16`` precision, use model conversion API. The models are saved to the ``model/ir_model/`` directory. For more details about model conversion, see this -`page `__. +`page `__. .. code:: ipython3 from openvino.tools.mo.front import tf as ov_tf_front - + precision = 'FP16' - + # The output path for the conversion. model_path = base_model_dir / 'ir_model' / f'{model_name}_{precision.lower()}.xml' - + trans_config_path = Path(ov_tf_front.__file__).parent / "ssd_v2_support.json" pipeline_config = base_model_dir / archive_name.with_suffix("").stem / "pipeline.config" - + model = None if not model_path.exists(): model = ov.tools.mo.convert_model(input_model=tf_model_path, @@ -439,7 +439,7 @@ the ``available_devices`` method are valid device specifiers. You may also use “AUTO”, which will automatically select the best device for inference (which is often the GPU). To learn more about AUTO plugin, visit the `Automatic Device -Selection `__ +Selection `__ page as well as the `AUTO device tutorial `__. @@ -461,17 +461,17 @@ following: import time from pathlib import Path - + # Create cache folder cache_folder = Path("cache") cache_folder.mkdir(exist_ok=True) - + start = time.time() core = ov.Core() - + # Set cache folder core.set_property({'CACHE_DIR': cache_folder}) - + # Compile the model as before model = core.read_model(model=model_path) compiled_model = core.compile_model(model, device) @@ -494,7 +494,7 @@ compile times with caching enabled and disabled as follows: model = core.read_model(model=model_path) compiled_model = core.compile_model(model, device) print(f"Cache enabled - compile time: {time.time() - start}s") - + start = time.time() core = ov.Core() model = core.read_model(model=model_path) @@ -511,7 +511,7 @@ compile times with caching enabled and disabled as follows: The actual time improvements will depend on the environment as well as the model being used but it is definitely something to consider when optimizing an application. To read more about this, see the `Model -Caching `__ +Caching `__ docs. Throughput and Latency Performance Hints @@ -554,7 +554,7 @@ Using Multiple GPUs with Multi-Device and Cumulative Throughput The latency and throughput hints mentioned above are great and can make a difference when used adequately but they usually use just one device, either due to the `AUTO -plugin `__ +plugin `__ or by manual specification of the device name as above. When we have multiple devices, such as an integrated and discrete GPU, we may use both at the same time to improve the utilization of the resources. In @@ -586,7 +586,7 @@ manually specify devices to use. Below is an example showing how to use how to set up an asynchronous pipeline that takes advantage of parallelism to increase throughput.** To learn more, see `Asynchronous - Inferencing `__ + Inferencing `__ in OpenVINO as well as the `Asynchronous Inference notebook `__. @@ -612,7 +612,7 @@ Note that benchmark_app only requires the model path to run but both the device and hint arguments will be useful to us. For more advanced usages, the tool itself has other options that can be checked by running ``benchmark_app -h`` or reading the -`docs `__. +`docs `__. The following example shows how to benchmark a simple model, using a GPU with a latency focus: @@ -628,12 +628,12 @@ with a latency focus: [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] + [ INFO ] [ INFO ] Device info: [ INFO ] GPU [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] - [ INFO ] + [ INFO ] + [ INFO ] [Step 3/11] Setting device configuration [Step 4/11] Reading model files [ INFO ] Loading model files @@ -662,7 +662,7 @@ with a latency focus: [ INFO ] GPU_QUEUE_PRIORITY: Priority.MEDIUM [ INFO ] GPU_QUEUE_THROTTLE: Priority.MEDIUM [ INFO ] GPU_ENABLE_LOOP_UNROLLING: True - [ INFO ] CACHE_DIR: + [ INFO ] CACHE_DIR: [ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY [ INFO ] COMPILATION_NUM_THREADS: 20 [ INFO ] NUM_STREAMS: 1 @@ -671,7 +671,7 @@ with a latency focus: [ INFO ] DEVICE_ID: 0 [Step 9/11] Creating infer requests and preparing input tensors [ WARNING ] No input files were given for input 'image_tensor'!. This input will be filled with random values! - [ INFO ] Fill input 'image_tensor' with random values + [ INFO ] Fill input 'image_tensor' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 1 inference requests, limits: 60000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). [ INFO ] First inference took 6.17 ms @@ -709,12 +709,12 @@ CPU vs GPU with Latency Hint [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] + [ INFO ] [ INFO ] Device info: [ INFO ] CPU [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] - [ INFO ] + [ INFO ] + [ INFO ] [Step 3/11] Setting device configuration [Step 4/11] Reading model files [ INFO ] Loading model files @@ -746,7 +746,7 @@ CPU vs GPU with Latency Hint [ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0 [Step 9/11] Creating infer requests and preparing input tensors [ WARNING ] No input files were given for input 'image_tensor'!. This input will be filled with random values! - [ INFO ] Fill input 'image_tensor' with random values + [ INFO ] Fill input 'image_tensor' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 1 inference requests, limits: 60000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). [ INFO ] First inference took 4.42 ms @@ -773,12 +773,12 @@ CPU vs GPU with Latency Hint [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] + [ INFO ] [ INFO ] Device info: [ INFO ] GPU [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] - [ INFO ] + [ INFO ] + [ INFO ] [Step 3/11] Setting device configuration [Step 4/11] Reading model files [ INFO ] Loading model files @@ -807,7 +807,7 @@ CPU vs GPU with Latency Hint [ INFO ] GPU_QUEUE_PRIORITY: Priority.MEDIUM [ INFO ] GPU_QUEUE_THROTTLE: Priority.MEDIUM [ INFO ] GPU_ENABLE_LOOP_UNROLLING: True - [ INFO ] CACHE_DIR: + [ INFO ] CACHE_DIR: [ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY [ INFO ] COMPILATION_NUM_THREADS: 20 [ INFO ] NUM_STREAMS: 1 @@ -816,7 +816,7 @@ CPU vs GPU with Latency Hint [ INFO ] DEVICE_ID: 0 [Step 9/11] Creating infer requests and preparing input tensors [ WARNING ] No input files were given for input 'image_tensor'!. This input will be filled with random values! - [ INFO ] Fill input 'image_tensor' with random values + [ INFO ] Fill input 'image_tensor' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 1 inference requests, limits: 60000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). [ INFO ] First inference took 8.79 ms @@ -848,12 +848,12 @@ CPU vs GPU with Throughput Hint [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] + [ INFO ] [ INFO ] Device info: [ INFO ] CPU [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] - [ INFO ] + [ INFO ] + [ INFO ] [Step 3/11] Setting device configuration [Step 4/11] Reading model files [ INFO ] Loading model files @@ -885,7 +885,7 @@ CPU vs GPU with Throughput Hint [ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0 [Step 9/11] Creating infer requests and preparing input tensors [ WARNING ] No input files were given for input 'image_tensor'!. This input will be filled with random values! - [ INFO ] Fill input 'image_tensor' with random values + [ INFO ] Fill input 'image_tensor' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 5 inference requests, limits: 60000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). [ INFO ] First inference took 8.15 ms @@ -912,12 +912,12 @@ CPU vs GPU with Throughput Hint [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] + [ INFO ] [ INFO ] Device info: [ INFO ] GPU [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] - [ INFO ] + [ INFO ] + [ INFO ] [Step 3/11] Setting device configuration [Step 4/11] Reading model files [ INFO ] Loading model files @@ -946,7 +946,7 @@ CPU vs GPU with Throughput Hint [ INFO ] GPU_QUEUE_PRIORITY: Priority.MEDIUM [ INFO ] GPU_QUEUE_THROTTLE: Priority.MEDIUM [ INFO ] GPU_ENABLE_LOOP_UNROLLING: True - [ INFO ] CACHE_DIR: + [ INFO ] CACHE_DIR: [ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT [ INFO ] COMPILATION_NUM_THREADS: 20 [ INFO ] NUM_STREAMS: 2 @@ -955,7 +955,7 @@ CPU vs GPU with Throughput Hint [ INFO ] DEVICE_ID: 0 [Step 9/11] Creating infer requests and preparing input tensors [ WARNING ] No input files were given for input 'image_tensor'!. This input will be filled with random values! - [ INFO ] Fill input 'image_tensor' with random values + [ INFO ] Fill input 'image_tensor' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 4 inference requests, limits: 60000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). [ INFO ] First inference took 9.17 ms @@ -987,12 +987,12 @@ Single GPU vs Multiple GPUs [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] + [ INFO ] [ INFO ] Device info: [ INFO ] GPU [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] - [ INFO ] + [ INFO ] + [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Device GPU.1 does not support performance hint property(-hint). [ ERROR ] Config for device with 1 ID is not registered in GPU plugin @@ -1016,14 +1016,14 @@ Single GPU vs Multiple GPUs [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] + [ INFO ] [ INFO ] Device info: [ INFO ] AUTO [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 [ INFO ] GPU [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] - [ INFO ] + [ INFO ] + [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Device GPU.1 does not support performance hint property(-hint). [Step 4/11] Reading model files @@ -1063,14 +1063,14 @@ Single GPU vs Multiple GPUs [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] + [ INFO ] [ INFO ] Device info: [ INFO ] GPU [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 [ INFO ] MULTI [ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3 - [ INFO ] - [ INFO ] + [ INFO ] + [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Device GPU.1 does not support performance hint property(-hint). [Step 4/11] Reading model files @@ -1121,12 +1121,12 @@ Import Necessary Packages import time from pathlib import Path - + import cv2 import numpy as np from IPython.display import Video import openvino as ov - + # Instantiate OpenVINO Runtime core = ov.Core() core.available_devices @@ -1151,11 +1151,11 @@ Compile the Model model = core.read_model(model=model_path) device_name = "GPU" compiled_model = core.compile_model(model=model, device_name=device_name, config={"PERFORMANCE_HINT": "THROUGHPUT"}) - + # Get the input and output nodes input_layer = compiled_model.input(0) output_layer = compiled_model.output(0) - + # Get the input size num, height, width, channels = input_layer.shape print('Model input shape:', num, height, width, channels) @@ -1177,7 +1177,7 @@ Load and Preprocess Video Frames video_file = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/video/Coco%20Walking%20in%20Berkeley.mp4" video = cv2.VideoCapture(video_file) framebuf = [] - + # Go through every frame of video and resize it print('Loading video...') while video.isOpened(): @@ -1186,18 +1186,18 @@ Load and Preprocess Video Frames print('Video loaded!') video.release() break - + # Preprocess frames - convert them to shape expected by model input_frame = cv2.resize(src=frame, dsize=(width, height), interpolation=cv2.INTER_AREA) input_frame = np.expand_dims(input_frame, axis=0) - + # Append frame to framebuffer framebuf.append(input_frame) - - + + print('Frame shape: ', framebuf[0].shape) print('Number of frames: ', len(framebuf)) - + # Show original video file # If the video does not display correctly inside the notebook, please open it with your favorite media player Video(video_file) @@ -1251,10 +1251,10 @@ Callback Definition global frame_number stop_time = time.time() frame_number += 1 - + predictions = next(iter(infer_request.results.values())) results[frame_id] = predictions[:10] # Grab first 10 predictions for this frame - + total_time = stop_time - start_time frame_fps[frame_id] = frame_number / total_time @@ -1283,14 +1283,14 @@ Perform Inference start_time = time.time() for i, input_frame in enumerate(framebuf): infer_queue.start_async({0: input_frame}, i) - + infer_queue.wait_all() # Wait until all inference requests in the AsyncInferQueue are completed stop_time = time.time() - + # Calculate total inference time and FPS total_time = stop_time - start_time fps = len(framebuf) / total_time - time_per_frame = 1 / fps + time_per_frame = 1 / fps print(f'Total time to infer all frames: {total_time:.3f}s') print(f'Time per frame: {time_per_frame:.6f}s ({fps:.3f} FPS)') @@ -1310,20 +1310,20 @@ Process Results # Set minimum detection threshold min_thresh = .6 - + # Load video video = cv2.VideoCapture(video_file) - + # Get video parameters frame_width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)) frame_height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)) fps = int(video.get(cv2.CAP_PROP_FPS)) fourcc = int(video.get(cv2.CAP_PROP_FOURCC)) - + # Create folder and VideoWriter to save output video Path('./output').mkdir(exist_ok=True) output = cv2.VideoWriter('output/output.mp4', fourcc, fps, (frame_width, frame_height)) - + # Draw detection results on every frame of video and save as a new video file while video.isOpened(): current_frame = int(video.get(cv2.CAP_PROP_POS_FRAMES)) @@ -1333,12 +1333,12 @@ Process Results output.release() video.release() break - + # Draw info at the top left such as current fps, the devices and the performance hint being used cv2.putText(frame, f"fps {str(round(frame_fps[current_frame], 2))}", (5, 20), cv2.FONT_ITALIC, 0.6, (0, 0, 0), 1, cv2.LINE_AA) - cv2.putText(frame, f"device {device_name}", (5, 40), cv2.FONT_ITALIC, 0.6, (0, 0, 0), 1, cv2.LINE_AA) + cv2.putText(frame, f"device {device_name}", (5, 40), cv2.FONT_ITALIC, 0.6, (0, 0, 0), 1, cv2.LINE_AA) cv2.putText(frame, f"hint {compiled_model.get_property('PERFORMANCE_HINT')}", (5, 60), cv2.FONT_ITALIC, 0.6, (0, 0, 0), 1, cv2.LINE_AA) - + # prediction contains [image_id, label, conf, x_min, y_min, x_max, y_max] according to model for prediction in np.squeeze(results[current_frame]): if prediction[2] > min_thresh: @@ -1347,13 +1347,13 @@ Process Results x_max = int(prediction[5] * frame_width) y_max = int(prediction[6] * frame_height) label = classes[int(prediction[1])] - + # Draw a bounding box with its label above it cv2.rectangle(frame, (x_min, y_min), (x_max, y_max), (0, 255, 0), 1, cv2.LINE_AA) cv2.putText(frame, label, (x_min, y_min - 10), cv2.FONT_ITALIC, 1, (255, 0, 0), 1, cv2.LINE_AA) - + output.write(frame) - + # Show output video file # If the video does not display correctly inside the notebook, please open it with your favorite media player Video("output/output.mp4", width=800, embed=True) @@ -1390,18 +1390,18 @@ To read more about any of these topics, feel free to visit their corresponding documentation: - `GPU - Plugin `__ + Plugin `__ - `AUTO - Plugin `__ + Plugin `__ - `Model - Caching `__ + Caching `__ - `MULTI Device Mode `__ - `Query Device - Properties `__ + Properties `__ - `Configurations for GPUs with - OpenVINO `__ + OpenVINO `__ - `Benchmark Python - Tool `__ + Tool `__ - `Asynchronous - Inferencing `__ + Inferencing `__ diff --git a/docs/notebooks/109-latency-tricks-with-output.rst b/docs/notebooks/109-latency-tricks-with-output.rst index 09b0f4edeb2..62f7a218db5 100644 --- a/docs/notebooks/109-latency-tricks-with-output.rst +++ b/docs/notebooks/109-latency-tricks-with-output.rst @@ -604,7 +604,7 @@ OpenVINO IR model + more inference threads There is a possibility to add a config for any device (CPU in this case). We will increase the number of threads to an equal number of our cores. There are `more -options `__ +options `__ to be changed, so it’s worth playing with them to see what works best in our case. In some cases, this optimization may worsen the performance. If it is the case, don’t use it. @@ -642,7 +642,7 @@ OpenVINO IR model in latency mode OpenVINO offers a virtual device called -`AUTO `__, +`AUTO `__, which can select the best device for us based on a performance hint. There are three different hints: ``LATENCY``, ``THROUGHPUT``, and ``CUMULATIVE_THROUGHPUT``. As this notebook is focused on the latency @@ -773,6 +773,6 @@ object detection model. Even if you experience much better performance after running this notebook, please note this may not be valid for every hardware or every model. For the most accurate results, please use ``benchmark_app`` `command-line -tool `__. +tool `__. Note that ``benchmark_app`` cannot measure the impact of some tricks above, e.g., shared memory. diff --git a/docs/notebooks/109-throughput-tricks-with-output.rst b/docs/notebooks/109-throughput-tricks-with-output.rst index e6bdd1c78de..9e64293d339 100644 --- a/docs/notebooks/109-throughput-tricks-with-output.rst +++ b/docs/notebooks/109-throughput-tricks-with-output.rst @@ -93,7 +93,7 @@ Prerequisites import time from pathlib import Path from typing import Any, List, Tuple - + # Fetch `notebook_utils` module import urllib.request urllib.request.urlretrieve( @@ -116,24 +116,24 @@ object detection model. import numpy as np import cv2 - + FRAMES_NUMBER = 1024 - + IMAGE_WIDTH = 640 IMAGE_HEIGHT = 480 - + # load image image = utils.load_image("https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_bike.jpg") image = cv2.resize(image, dsize=(IMAGE_WIDTH, IMAGE_HEIGHT), interpolation=cv2.INTER_AREA) - + # preprocess it for YOLOv5 input_image = image / 255.0 input_image = np.transpose(input_image, axes=(2, 0, 1)) input_image = np.expand_dims(input_image, axis=0) - + # simulate video with many frames video_frames = np.tile(input_image, (FRAMES_NUMBER, 1, 1, 1, 1)) - + # show the image utils.show_array(image) @@ -164,13 +164,13 @@ PyTorch Hub and small enough to see the difference in performance. import torch from IPython.utils import io - + # directory for all models base_model_dir = Path("model") - + model_name = "yolov5n" model_path = base_model_dir / model_name - + # load YOLOv5n from PyTorch Hub pytorch_model = torch.hub.load("ultralytics/yolov5", "custom", path=model_path, device="cpu", skip_validation=True) # don't print full model architecture @@ -186,12 +186,12 @@ PyTorch Hub and small enough to see the difference in performance. .. parsed-literal:: YOLOv5 🚀 2023-4-21 Python-3.8.10 torch-2.1.0+cpu CPU - + .. parsed-literal:: - Fusing layers... + Fusing layers... .. parsed-literal:: @@ -201,7 +201,7 @@ PyTorch Hub and small enough to see the difference in performance. .. parsed-literal:: - Adding AutoShape... + Adding AutoShape... .. parsed-literal:: @@ -223,10 +223,10 @@ benchmarking process. .. code:: ipython3 import openvino as ov - + # initialize OpenVINO core = ov.Core() - + # print available devices for device in core.available_devices: device_name = core.get_property(device, "FULL_DEVICE_NAME") @@ -250,8 +250,8 @@ second (FPS). .. code:: ipython3 from openvino.runtime import AsyncInferQueue - - + + def benchmark_model(model: Any, frames: np.ndarray, async_queue: AsyncInferQueue = None, benchmark_name: str = "OpenVINO model", device_name: str = "CPU") -> float: """ Helper function for benchmarking the model. It measures the time and prints results. @@ -264,7 +264,7 @@ second (FPS). end = time.perf_counter() first_infer_time = end - start print(f"{benchmark_name} on {device_name}. First inference time: {first_infer_time :.4f} seconds") - + # benchmarking start = time.perf_counter() for batch in frames: @@ -273,15 +273,15 @@ second (FPS). if async_queue: async_queue.wait_all() end = time.perf_counter() - + # elapsed time infer_time = end - start - + # print second per image and FPS mean_infer_time = infer_time / FRAMES_NUMBER mean_fps = FRAMES_NUMBER / infer_time print(f"{benchmark_name} on {device_name}: {mean_infer_time :.4f} seconds per image ({mean_fps :.2f} FPS)") - + return mean_fps The following functions aim to post-process results and draw boxes on @@ -300,21 +300,21 @@ the image. "cell phone", "microwave", "oven", "oaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush" ] - + # Colors for the classes above (Rainbow Color Map). colors = cv2.applyColorMap( src=np.arange(0, 255, 255 / len(classes), dtype=np.float32).astype(np.uint8), colormap=cv2.COLORMAP_RAINBOW, ).squeeze() - - + + def postprocess(detections: np.ndarray) -> List[Tuple]: """ Postprocess the raw results from the model. """ # candidates - probability > 0.25 detections = detections[detections[..., 4] > 0.25] - + boxes = [] labels = [] scores = [] @@ -328,22 +328,22 @@ the image. ) labels.append(int(label)) scores.append(float(score)) - + # Apply non-maximum suppression to get rid of many overlapping entities. # See https://paperswithcode.com/method/non-maximum-suppression # This algorithm returns indices of objects to keep. indices = cv2.dnn.NMSBoxes( bboxes=boxes, scores=scores, score_threshold=0.25, nms_threshold=0.5 ) - + # If there are no boxes. if len(indices) == 0: return [] - + # Filter detected objects. return [(labels[idx], scores[idx], boxes[idx]) for idx in indices.flatten()] - - + + def draw_boxes(img: np.ndarray, boxes): """ Draw detected boxes on the image. @@ -355,7 +355,7 @@ the image. x2 = box[0] + box[2] y2 = box[1] + box[3] cv2.rectangle(img=img, pt1=box[:2], pt2=(x2, y2), color=color, thickness=2) - + # Draw a label name inside the box. cv2.putText( img=img, @@ -367,17 +367,17 @@ the image. thickness=1, lineType=cv2.LINE_AA, ) - - + + def show_result(results: np.ndarray): """ Postprocess the raw results, draw boxes and show the image. """ output_img = image.copy() - + detections = postprocess(results) draw_boxes(output_img, detections) - + utils.show_array(output_img) Optimizations @@ -400,7 +400,7 @@ optimizations applied. We will treat it as our baseline. .. code:: ipython3 import torch - + with torch.no_grad(): result = pytorch_model(torch.as_tensor(video_frames[0])).detach().numpy()[0] show_result(result) @@ -438,23 +438,23 @@ step in this notebook. .. code:: ipython3 onnx_path = base_model_dir / Path(f"{model_name}_{IMAGE_WIDTH}_{IMAGE_HEIGHT}").with_suffix(".onnx") - + # export PyTorch model to ONNX if it doesn't already exist if not onnx_path.exists(): dummy_input = torch.randn(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH) torch.onnx.export(pytorch_model, dummy_input, onnx_path) - + # convert ONNX model to IR, use FP16 ov_model = ov.convert_model(onnx_path) .. code:: ipython3 ov_cpu_model = core.compile_model(ov_model, device_name="CPU") - + result = ov_cpu_model(video_frames[0])[ov_cpu_model.output(0)][0] show_result(result) ov_cpu_fps = benchmark_model(model=ov_cpu_model, frames=video_frames, benchmark_name="OpenVINO model") - + del ov_cpu_model # release resources @@ -488,13 +488,13 @@ hardware and model. .. code:: ipython3 batch_size = 4 - + onnx_batch_path = base_model_dir / Path(f"{model_name}_{IMAGE_WIDTH}_{IMAGE_HEIGHT}_batch_{batch_size}").with_suffix(".onnx") - + if not onnx_batch_path.exists(): dummy_input = torch.randn(batch_size, 3, IMAGE_HEIGHT, IMAGE_WIDTH) torch.onnx.export(pytorch_model, dummy_input, onnx_batch_path) - + # export the model with the bigger batch size ov_batch_model = ov.convert_model(onnx_batch_path) @@ -510,13 +510,13 @@ hardware and model. .. code:: ipython3 ov_cpu_batch_model = core.compile_model(ov_batch_model, device_name="CPU") - + batched_video_frames = video_frames.reshape([-1, batch_size, 3, IMAGE_HEIGHT, IMAGE_WIDTH]) - + result = ov_cpu_batch_model(batched_video_frames[0])[ov_cpu_batch_model.output(0)][0] show_result(result) ov_cpu_batch_fps = benchmark_model(model=ov_cpu_batch_model, frames=batched_video_frames, benchmark_name="OpenVINO model + bigger batch") - + del ov_cpu_batch_model # release resources @@ -561,17 +561,17 @@ the pipeline. result = infer_request.get_output_tensor(0).data[0] show_result(result) pass - + infer_queue = ov.AsyncInferQueue(ov_model) infer_queue.set_callback(callback) # set callback to post-process (show) results - + infer_queue.start_async(video_frames[0]) infer_queue.wait_all() - + # don't show output for the remaining frames infer_queue.set_callback(lambda x, y: {}) fps = benchmark_model(model=infer_queue.start_async, frames=video_frames, async_queue=infer_queue, benchmark_name=benchmark_name, device_name=device_name) - + del infer_queue # release resources return fps @@ -586,15 +586,15 @@ configuration of the device. There are three different hints: notebook is focused on the throughput mode, we will use the latter two. The hints can be used with other devices as well. Throughput mode implicitly triggers using the `Automatic -Batching `__ +Batching `__ feature, which sets the batch size to the optimal level. .. code:: ipython3 ov_cpu_through_model = core.compile_model(ov_model, device_name="CPU", config={"PERFORMANCE_HINT": "THROUGHPUT"}) - + ov_cpu_through_fps = benchmark_async_mode(ov_cpu_through_model, benchmark_name="OpenVINO model", device_name="CPU (THROUGHPUT)") - + del ov_cpu_through_model # release resources @@ -633,9 +633,9 @@ execution. if "GPU" in core.available_devices: # compile for GPU ov_gpu_model = core.compile_model(ov_model, device_name="GPU", config={"PERFORMANCE_HINT": "THROUGHPUT"}) - + ov_gpu_fps = benchmark_async_mode(ov_gpu_model, benchmark_name="OpenVINO model", device_name="GPU (THROUGHPUT)") - + del ov_gpu_model # release resources OpenVINO IR model in throughput mode on AUTO @@ -644,16 +644,16 @@ OpenVINO IR model in throughput mode on AUTO OpenVINO offers a virtual device called -`AUTO `__, +`AUTO `__, which can select the best device for us based on the aforementioned performance hint. .. code:: ipython3 ov_auto_model = core.compile_model(ov_model, device_name="AUTO", config={"PERFORMANCE_HINT": "THROUGHPUT"}) - + ov_auto_fps = benchmark_async_mode(ov_auto_model, benchmark_name="OpenVINO model", device_name="AUTO (THROUGHPUT)") - + del ov_auto_model # release resources @@ -685,7 +685,7 @@ activate all devices. .. code:: ipython3 ov_auto_cumulative_model = core.compile_model(ov_model, device_name="AUTO", config={"PERFORMANCE_HINT": "CUMULATIVE_THROUGHPUT"}) - + ov_auto_cumulative_fps = benchmark_async_mode(ov_auto_cumulative_model, benchmark_name="OpenVINO model", device_name="AUTO (CUMULATIVE THROUGHPUT)") @@ -712,7 +712,7 @@ There are other tricks for performance improvement, such as advanced options, quantization and pre-post-processing or dedicated to latency mode. To get even more from your model, please visit `advanced throughput -options `__, +options `__, `109-latency-tricks <109-latency-tricks-with-output.html-with-output.html>`__, `111-detection-quantization <111-detection-quantization-with-output.html>`__, and `118-optimize-preprocessing <118-optimize-preprocessing-with-output.html>`__. @@ -733,20 +733,20 @@ steps, just skip them. .. code:: ipython3 from matplotlib import pyplot as plt - + labels = ["PyTorch model", "OpenVINO IR model", "OpenVINO IR model + bigger batch", "OpenVINO IR model in throughput mode", "OpenVINO IR model in throughput mode on GPU", "OpenVINO IR model in throughput mode on AUTO", "OpenVINO IR model in cumulative throughput mode on AUTO"] - + fps = [pytorch_fps, ov_cpu_fps, ov_cpu_batch_fps, ov_cpu_through_fps, ov_gpu_fps, ov_auto_fps, ov_auto_cumulative_fps] - + bar_colors = colors[::10] / 255.0 - + fig, ax = plt.subplots(figsize=(16, 8)) ax.bar(labels, fps, color=bar_colors) - + ax.set_ylabel("Throughput [FPS]") ax.set_title("Performance difference") - + plt.xticks(rotation='vertical') plt.show() @@ -765,6 +765,6 @@ object detection model. Even if you experience much better performance after running this notebook, please note this may not be valid for every hardware or every model. For the most accurate results, please use ``benchmark_app`` `command-line -tool `__. +tool `__. Note that ``benchmark_app`` cannot measure the impact of some tricks above. diff --git a/docs/notebooks/110-ct-scan-live-inference-with-output.rst b/docs/notebooks/110-ct-scan-live-inference-with-output.rst index 032eeb9b77d..b51fe0deffd 100644 --- a/docs/notebooks/110-ct-scan-live-inference-with-output.rst +++ b/docs/notebooks/110-ct-scan-live-inference-with-output.rst @@ -135,7 +135,7 @@ Benchmark Model Performance To measure the inference performance of the IR model, use `Benchmark -Tool `__ +Tool `__ - an inference performance measurement tool in OpenVINO. Benchmark tool is a command-line application that can be run in the notebook with ``! benchmark_app`` or ``%sx benchmark_app`` commands. @@ -335,7 +335,7 @@ Caching, refer to the `OpenVINO API tutorial <002-openvino-api-with-output.html>`__. We will use -`AsyncInferQueue `__ +`AsyncInferQueue `__ to perform asynchronous inference. It can be instantiated with compiled model and a number of jobs - parallel execution threads. If you don’t pass a number of jobs or pass ``0``, then OpenVINO will pick the optimal diff --git a/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output.rst b/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output.rst index bc414952359..76a1895a7e4 100644 --- a/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output.rst +++ b/docs/notebooks/110-ct-segmentation-quantize-nncf-with-output.rst @@ -14,7 +14,7 @@ scratch; the data is from This third tutorial in the series shows how to: - Convert an Original model to OpenVINO IR with `model conversion - API `__ + API `__ - Quantize a PyTorch model with NNCF - Evaluate the F1 score metric of the original model and the quantized model @@ -647,7 +647,7 @@ Compare Performance of the FP32 IR Model and Quantized Models To measure the inference performance of the ``FP32`` and ``INT8`` models, we use `Benchmark -Tool `__ +Tool `__ - OpenVINO’s inference performance measurement tool. Benchmark tool is a command line application, part of OpenVINO development tools, that can be run in the notebook with ``! benchmark_app`` or diff --git a/docs/notebooks/112-pytorch-post-training-quantization-nncf-with-output.rst b/docs/notebooks/112-pytorch-post-training-quantization-nncf-with-output.rst index 44efb5ed678..6aa5f62468f 100644 --- a/docs/notebooks/112-pytorch-post-training-quantization-nncf-with-output.rst +++ b/docs/notebooks/112-pytorch-post-training-quantization-nncf-with-output.rst @@ -527,7 +527,7 @@ layers. The framework is designed so that modifications to your original training code are minor. Quantization is the simplest scenario and requires a few modifications. For more information about NNCF Post Training Quantization (PTQ) API, refer to the `Basic Quantization Flow -Guide `__. +Guide `__. 1. Create a transformation function that accepts a sample from the dataset and returns data suitable for model inference. This enables @@ -695,7 +695,7 @@ Python API. The models will be saved to the ‘OUTPUT’ directory for later benchmarking. For more information about model conversion, refer to this -`page `__. +`page `__. .. code:: ipython3 @@ -883,7 +883,7 @@ IV. Compare performance of INT8 model and FP32 model in OpenVINO Finally, measure the inference performance of the ``FP32`` and ``INT8`` models, using `Benchmark -Tool `__ +Tool `__ - an inference performance measurement tool in OpenVINO. By default, Benchmark Tool runs inference for 60 seconds in asynchronous mode on CPU. It returns inference speed as latency (milliseconds per image) and diff --git a/docs/notebooks/113-image-classification-quantization-with-output.rst b/docs/notebooks/113-image-classification-quantization-with-output.rst index b684b407641..ee11d52fdb7 100644 --- a/docs/notebooks/113-image-classification-quantization-with-output.rst +++ b/docs/notebooks/113-image-classification-quantization-with-output.rst @@ -495,7 +495,7 @@ static shape. The converted model is ready to be loaded on a device for inference and can be saved on a disk for next usage via the ``save_model`` function. More details about model conversion Python API can be found on this -`page `__. +`page `__. .. code:: ipython3 @@ -2790,7 +2790,7 @@ dataset for performing basic quantization. Optionally, additional parameters like ``subset_size``, ``preset``, ``ignored_scope`` can be provided to improve quantization result if applicable. More details about supported parameters can be found on this -`page `__ +`page `__ .. code:: ipython3 @@ -2951,7 +2951,7 @@ Compare Performance of the Original and Quantized Models Finally, measure the inference performance of the ``FP32`` and ``INT8`` models, using `Benchmark -Tool `__ +Tool `__ - an inference performance measurement tool in OpenVINO. **NOTE**: For more accurate performance, it is recommended to run diff --git a/docs/notebooks/115-async-api-with-output.rst b/docs/notebooks/115-async-api-with-output.rst index 3c49ba2c288..e62bc0482a3 100644 --- a/docs/notebooks/115-async-api-with-output.rst +++ b/docs/notebooks/115-async-api-with-output.rst @@ -69,14 +69,14 @@ Imports import openvino as ov from IPython import display import matplotlib.pyplot as plt - + # Fetch the notebook utils script from the openvino_notebooks repo import urllib.request urllib.request.urlretrieve( url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py', filename='notebook_utils.py' ) - + import notebook_utils as utils Prepare model and data processing @@ -98,7 +98,7 @@ each frame of the video. # directory where model will be downloaded base_model_dir = "model" - + # model name as named in Open Model Zoo model_name = "person-detection-0202" precision = "FP16" @@ -116,7 +116,7 @@ each frame of the video. .. parsed-literal:: ################|| Downloading person-detection-0202 ||################ - + ========== Downloading model/intel/person-detection-0202/FP16/person-detection-0202.xml @@ -137,7 +137,7 @@ each frame of the video. ... 89%, 224 KB, 1678 KB/s, 0 seconds passed ... 100%, 248 KB, 1859 KB/s, 0 seconds passed - + ========== Downloading model/intel/person-detection-0202/FP16/person-detection-0202.bin @@ -315,7 +315,7 @@ each frame of the video. ... 99%, 3520 KB, 3392 KB/s, 1 seconds passed ... 100%, 3549 KB, 3419 KB/s, 1 seconds passed - + Load the model @@ -327,14 +327,14 @@ Load the model # initialize OpenVINO runtime core = ov.Core() - + # read the network and corresponding weights from file model = core.read_model(model=model_path) - + # compile the model for the CPU (you can choose manually CPU, GPU etc.) # or let the engine choose the best available device (AUTO) compiled_model = core.compile_model(model=model, device_name="CPU") - + # get input node input_layer_ir = model.input(0) N, C, H, W = input_layer_ir.shape @@ -350,7 +350,7 @@ Create functions for data processing def preprocess(image): """ Define the preprocess function for input data - + :param: image: the orignal input frame :returns: resized_image: the image processed @@ -360,12 +360,12 @@ Create functions for data processing resized_image = resized_image.transpose((2, 0, 1)) resized_image = np.expand_dims(resized_image, axis=0).astype(np.float32) return resized_image - - + + def postprocess(result, image, fps): """ Define the postprocess function for output data - + :param: result: the inference results image: the orignal input frame fps: average throughput calculated for each frame @@ -381,7 +381,7 @@ Create functions for data processing xmax = int(min((xmax * image.shape[1]), image.shape[1] - 10)) ymax = int(min((ymax * image.shape[0]), image.shape[0] - 10)) cv2.rectangle(image, (xmin, ymin), (xmax, ymax), (0, 255, 0), 2) - cv2.putText(image, str(round(fps, 2)) + " fps", (5, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 3) + cv2.putText(image, str(round(fps, 2)) + " fps", (5, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 3) return image Get the test video @@ -432,7 +432,7 @@ immediately processed: def sync_api(source, flip, fps, use_popup, skip_first_frames): """ Define the main function for video processing in sync mode - + :param: source: the video path or the ID of your webcam :returns: sync_fps: the inference throughput in sync mode @@ -456,13 +456,13 @@ immediately processed: break resized_frame = preprocess(frame) infer_request.set_tensor(input_layer_ir, ov.Tensor(resized_frame)) - # Start the inference request in synchronous mode + # Start the inference request in synchronous mode infer_request.infer() res = infer_request.get_output_tensor(0).data stop_time = time.time() total_time = stop_time - start_time frame_number = frame_number + 1 - sync_fps = frame_number / total_time + sync_fps = frame_number / total_time frame = postprocess(res, frame, sync_fps) # Display the results if use_popup: @@ -478,7 +478,7 @@ immediately processed: i = display.Image(data=encoded_img) # Display the image in this notebook display.clear_output(wait=True) - display.display(i) + display.display(i) # ctrl-c except KeyboardInterrupt: print("Interrupted") @@ -555,7 +555,7 @@ pipeline (decoding vs inference) and not by the sum of the stages. def async_api(source, flip, fps, use_popup, skip_first_frames): """ Define the main function for video processing in async mode - + :param: source: the video path or the ID of your webcam :returns: async_fps: the inference throughput in async mode @@ -597,7 +597,7 @@ pipeline (decoding vs inference) and not by the sum of the stages. stop_time = time.time() total_time = stop_time - start_time frame_number = frame_number + 1 - async_fps = frame_number / total_time + async_fps = frame_number / total_time frame = postprocess(res, frame, async_fps) # Display the results if use_popup: @@ -617,7 +617,7 @@ pipeline (decoding vs inference) and not by the sum of the stages. # Swap CURRENT and NEXT frames frame = next_frame # Swap CURRENT and NEXT infer requests - curr_request, next_request = next_request, curr_request + curr_request, next_request = next_request, curr_request # ctrl-c except KeyboardInterrupt: print("Interrupted") @@ -662,20 +662,20 @@ Compare the performance width = 0.4 fontsize = 14 - + plt.rc('font', size=fontsize) fig, ax = plt.subplots(1, 1, figsize=(10, 8)) - + rects1 = ax.bar([0], sync_fps, width, color='#557f2d') rects2 = ax.bar([width], async_fps, width) ax.set_ylabel("frames per second") - ax.set_xticks([0, width]) + ax.set_xticks([0, width]) ax.set_xticklabels(["Sync mode", "Async mode"]) ax.set_xlabel("Higher is better") - + fig.suptitle('Sync mode VS Async mode') fig.tight_layout() - + plt.show() @@ -689,7 +689,7 @@ Compare the performance Asynchronous mode pipelines can be supported with the -`AsyncInferQueue `__ +`AsyncInferQueue `__ wrapper class. This class automatically spawns the pool of ``InferRequest`` objects (also called “jobs”) and provides synchronization mechanisms to control the flow of the pipeline. It is a @@ -711,7 +711,7 @@ the possibility of passing runtime values. def callback(infer_request, info) -> None: """ Define the callback function for postprocessing - + :param: infer_request: the infer_request object info: a tuple includes original frame and starts time :returns: @@ -725,7 +725,7 @@ the possibility of passing runtime values. total_time = stop_time - start_time frame_number = frame_number + 1 inferqueue_fps = frame_number / total_time - + res = infer_request.get_output_tensor(0).data[0] frame = postprocess(res, frame, inferqueue_fps) # Encode numpy array to jpg @@ -741,7 +741,7 @@ the possibility of passing runtime values. def inferqueue(source, flip, fps, skip_first_frames) -> None: """ Define the main function for video processing with async infer queue - + :param: source: the video path or the ID of your webcam :retuns: None @@ -763,7 +763,7 @@ the possibility of passing runtime values. print("Source ended") break resized_frame = preprocess(frame) - # Start the inference request with async infer queue + # Start the inference request with async infer queue infer_queue.start_async({input_layer_ir.any_name: resized_frame}, (frame, start_time)) except KeyboardInterrupt: print("Interrupted") diff --git a/docs/notebooks/116-sparsity-optimization-with-output.rst b/docs/notebooks/116-sparsity-optimization-with-output.rst index 4a4df59ee93..4732777289c 100644 --- a/docs/notebooks/116-sparsity-optimization-with-output.rst +++ b/docs/notebooks/116-sparsity-optimization-with-output.rst @@ -12,7 +12,7 @@ datasets `__ using `Optimum-Intel `__. It demonstrates the inference performance advantage on 4th Gen Intel® Xeon® Scalable Processors by running it with `Sparse Weight -Decompression `__, +Decompression `__, a runtime option that seizes model sparsity for efficiency. The notebook consists of the following steps: @@ -67,7 +67,7 @@ Imports import shutil from pathlib import Path - + from optimum.intel.openvino import OVModelForSequenceClassification from transformers import AutoTokenizer, pipeline from huggingface_hub import hf_hub_download @@ -112,17 +112,17 @@ model card on Hugging Face. # The following model has been quantized, sparsified using Optimum-Intel 1.7 which is enabled by OpenVINO and NNCF # for reproducibility, refer https://huggingface.co/OpenVINO/bert-base-uncased-sst2-int8-unstructured80 model_id = "OpenVINO/bert-base-uncased-sst2-int8-unstructured80" - + # The following two steps will set up the model and download them to HF Cache folder ov_model = OVModelForSequenceClassification.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) - + # Let's take the model for a spin! sentiment_classifier = pipeline("text-classification", model=ov_model, tokenizer=tokenizer) - + text = "He's a dreadful magician." outputs = sentiment_classifier(text) - + print(outputs) @@ -149,14 +149,14 @@ the IRs into a single folder. # create a folder quantized_sparse_dir = Path("bert_80pc_sparse_quantized_ir") quantized_sparse_dir.mkdir(parents=True, exist_ok=True) - - # following return path to specified filename in cache folder (which we've with the + + # following return path to specified filename in cache folder (which we've with the ov_ir_xml_path = hf_hub_download(repo_id=model_id, filename="openvino_model.xml") ov_ir_bin_path = hf_hub_download(repo_id=model_id, filename="openvino_model.bin") - + # copy IRs to the folder shutil.copy(ov_ir_xml_path, quantized_sparse_dir) - shutil.copy(ov_ir_bin_path, quantized_sparse_dir) + shutil.copy(ov_ir_bin_path, quantized_sparse_dir) @@ -210,7 +210,7 @@ as an example. It is recommended to tune based on your applications. [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: [ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3 - [ INFO ] + [ INFO ] [ INFO ] Device info: @@ -218,8 +218,8 @@ as an example. It is recommended to tune based on your applications. [ INFO ] CPU [ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3 - [ INFO ] - [ INFO ] + [ INFO ] + [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files @@ -275,9 +275,9 @@ as an example. It is recommended to tune based on your applications. [ WARNING ] No input files were given for input 'input_ids'!. This input will be filled with random values! [ WARNING ] No input files were given for input 'attention_mask'!. This input will be filled with random values! [ WARNING ] No input files were given for input 'token_type_ids'!. This input will be filled with random values! - [ INFO ] Fill input 'input_ids' with random values - [ INFO ] Fill input 'attention_mask' with random values - [ INFO ] Fill input 'token_type_ids' with random values + [ INFO ] Fill input 'input_ids' with random values + [ INFO ] Fill input 'attention_mask' with random values + [ INFO ] Fill input 'token_type_ids' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 4 inference requests, limits: 60000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). @@ -314,7 +314,7 @@ for which a layer will be enabled. .. code:: ipython3 # Dump benchmarking config for dense inference - # "CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE" controls minimum sparsity rate for weights to consider + # "CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE" controls minimum sparsity rate for weights to consider # for sparse optimization at the runtime. with (quantized_sparse_dir / "perf_config_sparse.json").open("w") as outfile: outfile.write( @@ -345,7 +345,7 @@ for which a layer will be enabled. [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: [ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3 - [ INFO ] + [ INFO ] [ INFO ] Device info: @@ -353,8 +353,8 @@ for which a layer will be enabled. [ INFO ] CPU [ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3 - [ INFO ] - [ INFO ] + [ INFO ] + [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files @@ -390,7 +390,7 @@ for which a layer will be enabled. [ ERROR ] Exception from src/inference/src/core.cpp:99: [ GENERAL_ERROR ] Exception from src/plugins/intel_cpu/src/config.cpp:158: Wrong value for property key CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE. Expected only float numbers - + Traceback (most recent call last): File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 408, in main compiled_model = benchmark.core.compile_model(model, benchmark.device, device_config) @@ -399,8 +399,8 @@ for which a layer will be enabled. RuntimeError: Exception from src/inference/src/core.cpp:99: [ GENERAL_ERROR ] Exception from src/plugins/intel_cpu/src/config.cpp:158: Wrong value for property key CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE. Expected only float numbers - - + + When this might be helpful @@ -417,6 +417,6 @@ For more details about asynchronous inference with OpenVINO, refer to the following documentation: - `Deployment Optimization - Guide `__ + Guide `__ - `Inference Request - API `__ + API `__ diff --git a/docs/notebooks/117-model-server-with-output.rst b/docs/notebooks/117-model-server-with-output.rst index f14db107155..a2610f7393a 100644 --- a/docs/notebooks/117-model-server-with-output.rst +++ b/docs/notebooks/117-model-server-with-output.rst @@ -269,7 +269,7 @@ Check whether the OVMS container is running normally: The required Model Server parameters are listed below. For additional configuration options, see the `Model Server Parameters -section `__. +section `__. .. raw:: html @@ -928,6 +928,6 @@ References 1. `OpenVINO™ Model Server - documentation `__ + documentation `__ 2. `OpenVINO™ Model Server GitHub repository `__ diff --git a/docs/notebooks/118-optimize-preprocessing-with-output.rst b/docs/notebooks/118-optimize-preprocessing-with-output.rst index 1b5ed13154d..7c0488ae99f 100644 --- a/docs/notebooks/118-optimize-preprocessing-with-output.rst +++ b/docs/notebooks/118-optimize-preprocessing-with-output.rst @@ -9,9 +9,9 @@ instrument, that enables integration of preprocessing steps into an execution graph and performing it on a selected device, which can improve device utilization. For more information about Preprocessing API, see this -`overview `__ +`overview `__ and -`details `__ +`details `__ This tutorial include following steps: @@ -88,13 +88,13 @@ Imports import time from pathlib import Path - + import cv2 import matplotlib.pyplot as plt import numpy as np import openvino as ov import tensorflow as tf - + # Fetch `notebook_utils` module import urllib.request urllib.request.urlretrieve( @@ -140,7 +140,7 @@ Setup image and device .. code:: ipython3 import ipywidgets as widgets - + core = ov.Core() device = widgets.Dropdown( options=core.available_devices + ["AUTO"], @@ -148,7 +148,7 @@ Setup image and device description='Device:', disabled=False, ) - + device @@ -182,12 +182,12 @@ and save it to the disk. .. code:: ipython3 model_name = "InceptionResNetV2" - + model_dir = Path("model") model_dir.mkdir(exist_ok=True) - + model_path = model_dir / model_name - + model = tf.keras.applications.InceptionV3() model.save(model_path) @@ -273,7 +273,7 @@ Graph modifications of a model shall be performed after the model is read from a drive and before it is loaded on the actual device. Pre-processing support following operations (please, see more details -`here `__) +`here `__) - Mean/Scale Normalization - Converting Precision @@ -292,13 +292,13 @@ The options for preprocessing are not required. .. code:: ipython3 ir_path = model_dir / "ir_model" / f"{model_name}.xml" - + ppp_model = None - + if ir_path.exists(): ppp_model = core.read_model(model=ir_path) print(f"Model in OpenVINO format already exists: {ir_path}") - else: + else: ppp_model = ov.convert_model(model_path, input=[1,299,299,3]) ov.save_model(ppp_model, str(ir_path)) @@ -309,14 +309,14 @@ Create ``PrePostProcessor`` Object The -`PrePostProcessor() `__ +`PrePostProcessor() `__ class enables specifying the preprocessing and postprocessing steps for a model. .. code:: ipython3 from openvino.preprocess import PrePostProcessor - + ppp = PrePostProcessor(ppp_model) Declare User’s Data Format @@ -334,7 +334,7 @@ about user’s input tensor will be initialized to same data (type/shape/etc) as model’s input parameter. User application can override particular parameters according to application’s data. Refer to the following -`page `__ +`page `__ for more information about parameters for overriding. Below is all the specified input information: @@ -372,13 +372,13 @@ Declaring Model Layout Model input already has information about precision and shape. Preprocessing API is not intended to modify this. The only thing that may be specified is input data -`layout `__. +`layout `__. .. code:: ipython3 input_layer_ir = next(iter(ppp_model.inputs)) print(f"The input shape of the model is {input_layer_ir.shape}") - + ppp.input().model().set_layout(ov.Layout('NHWC')) @@ -402,7 +402,7 @@ Preprocessing Steps Now, the sequence of preprocessing steps can be defined. For more information about preprocessing steps, see -`here `__. +`here `__. Perform the following: @@ -411,7 +411,7 @@ Perform the following: dynamic size, for example, ``{?, 3, ?, ?}`` resize will not know how to resize the picture. Therefore, in this case, target height/ width should be specified. For more details, see also the - `PreProcessSteps.resize() `__. + `PreProcessSteps.resize() `__. - Subtract mean from each channel. - Divide each pixel data to appropriate scale value. @@ -421,7 +421,7 @@ then such conversion will be added explicitly. .. code:: ipython3 from openvino.preprocess import ResizeAlgorithm - + ppp.input().preprocess().convert_element_type(ov.Type.f32) \ .resize(ResizeAlgorithm.RESIZE_LINEAR)\ .mean([127.5,127.5,127.5])\ @@ -461,7 +461,7 @@ configuration for debugging purposes. resize to model width/height: ([1,?,?,3], [N,H,W,C], f32) -> ([1,299,299,3], [N,H,W,C], f32) mean (127.5,127.5,127.5): ([1,299,299,3], [N,H,W,C], f32) -> ([1,299,299,3], [N,H,W,C], f32) scale (127.5,127.5,127.5): ([1,299,299,3], [N,H,W,C], f32) -> ([1,299,299,3], [N,H,W,C], f32) - + Load model and perform inference @@ -475,12 +475,12 @@ Load model and perform inference image = cv2.imread(image_path) input_tensor = np.expand_dims(image, 0) return input_tensor - - + + compiled_model_with_preprocess_api = core.compile_model(model=ppp_model, device_name=device.value) - + ppp_output_layer = compiled_model_with_preprocess_api.output(0) - + ppp_input_tensor = prepare_image_api_preprocess(image_path) results = compiled_model_with_preprocess_api(ppp_input_tensor)[ppp_output_layer][0] @@ -508,22 +508,22 @@ Load image and fit it to model input def manual_image_preprocessing(path_to_image, compiled_model): input_layer_ir = next(iter(compiled_model.inputs)) - + # N, H, W, C = batch size, height, width, number of channels N, H, W, C = input_layer_ir.shape - + # load image, image will be resized to model input size and converted to RGB img = tf.keras.preprocessing.image.load_img(image_path, target_size=(H, W), color_mode='rgb') - + x = tf.keras.preprocessing.image.img_to_array(img) x = np.expand_dims(x, axis=0) - + # will scale input pixels between -1 and 1 input_tensor = tf.keras.applications.inception_resnet_v2.preprocess_input(x) - + return input_tensor - - + + input_tensor = manual_image_preprocessing(image_path, compiled_model) print(f"The shape of the image is {input_tensor.shape}") print(f"The data type of the image is {input_tensor.dtype}") @@ -543,7 +543,7 @@ Perform inference .. code:: ipython3 output_layer = compiled_model.output(0) - + result = compiled_model(input_tensor)[output_layer] Compare results @@ -560,18 +560,18 @@ Compare results on one image def check_results(input_tensor, compiled_model, imagenet_classes): output_layer = compiled_model.output(0) - + results = compiled_model(input_tensor)[output_layer][0] - + top_indices = np.argsort(results)[-5:][::-1] top_softmax = results[top_indices] - + for index, softmax_probability in zip(top_indices, top_softmax): print(f"{imagenet_classes[index]}, {softmax_probability:.5f}") - + return top_indices, top_softmax - - + + # Convert the inference result to a class name. imagenet_filename = download_file( "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_2012.txt", @@ -579,13 +579,13 @@ Compare results on one image ) imagenet_classes = imagenet_filename.read_text().splitlines() imagenet_classes = ['background'] + imagenet_classes - + # get result for inference with preprocessing api print("Result of inference with Preprocessing API:") res = check_results(ppp_input_tensor, compiled_model_with_preprocess_api, imagenet_classes) - + print("\n") - + # get result for inference with the manual preparing of the image print("Result of inference with manual image setup:") res = check_results(input_tensor, compiled_model, imagenet_classes) @@ -605,8 +605,8 @@ Compare results on one image n02108915 French bulldog, 0.01915 n02111129 Leonberg, 0.00825 n02097047 miniature schnauzer, 0.00294 - - + + Result of inference with manual image setup: n02098413 Lhasa, Lhasa apso, 0.76843 n02099601 golden retriever, 0.19322 @@ -624,24 +624,24 @@ Compare performance def check_performance(compiled_model, preprocessing_function=None): num_images = 1000 - + start = time.perf_counter() - + for _ in range(num_images): input_tensor = preprocessing_function(image_path, compiled_model) compiled_model(input_tensor) - + end = time.perf_counter() time_ir = end - start - + return time_ir, num_images - + time_ir, num_images = check_performance(compiled_model, manual_image_preprocessing) print( f"IR model in OpenVINO Runtime/CPU with manual image preprocessing: {time_ir/num_images:.4f} " f"seconds per image, FPS: {num_images/time_ir:.2f}" ) - + time_ir, num_images = check_performance(compiled_model_with_preprocess_api, prepare_image_api_preprocess) print( f"IR model in OpenVINO Runtime/CPU with preprocessing API: {time_ir/num_images:.4f} " diff --git a/docs/notebooks/119-tflite-to-openvino-with-output.rst b/docs/notebooks/119-tflite-to-openvino-with-output.rst index 1e8d7f1382d..25819684d09 100644 --- a/docs/notebooks/119-tflite-to-openvino-with-output.rst +++ b/docs/notebooks/119-tflite-to-openvino-with-output.rst @@ -8,7 +8,7 @@ machine learning models to edge devices. This short tutorial shows how to convert a TensorFlow Lite `EfficientNet-Lite-B0 `__ image classification model to OpenVINO `Intermediate -Representation `__ +Representation `__ (OpenVINO IR) format, using Model Converter. After creating the OpenVINO IR, load the model in `OpenVINO Runtime `__ @@ -124,9 +124,9 @@ using ``ov.save_model`` function, reducing loading time for next running. By default, model weights are compressed to FP16 during serialization by ``ov.save_model``. For more information about model conversion, see this -`page `__. +`page `__. For TensorFlow Lite models support, refer to this -`tutorial `__. +`tutorial `__. .. code:: ipython3 @@ -239,7 +239,7 @@ Estimate Model Performance -------------------------- `Benchmark -Tool `__ +Tool `__ is used to measure the inference performance of the model on CPU and GPU. diff --git a/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output.rst b/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output.rst index 71fadfc91f3..7576df3b77e 100644 --- a/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output.rst +++ b/docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output.rst @@ -17,9 +17,9 @@ This tutorial shows how to convert a TensorFlow `Mask R-CNN with Inception ResNet V2 `__ instance segmentation model to OpenVINO `Intermediate -Representation `__ +Representation `__ (OpenVINO IR) format, using `Model Conversion -API `__. +API `__. After creating the OpenVINO IR, load the model in `OpenVINO Runtime `__ and do inference with a sample image. @@ -698,4 +698,4 @@ utilization. For more information, refer to the `Optimize Preprocessing tutorial <118-optimize-preprocessing-with-output.html>`__ and to the overview of `Preprocessing -API `__. +API `__. diff --git a/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output.rst b/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output.rst index 6bda545e1f6..568cb9b67a1 100644 --- a/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output.rst +++ b/docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output.rst @@ -17,10 +17,10 @@ This tutorial shows how to convert a TensorFlow `Faster R-CNN with Resnet-50 V1 `__ object detection model to OpenVINO `Intermediate -Representation `__ +Representation `__ (OpenVINO IR) format, using Model Converter. After creating the OpenVINO IR, load the model in `OpenVINO -Runtime `__ +Runtime `__ and do inference with a sample image. Table of contents: @@ -189,9 +189,9 @@ or saved on disk using the ``save_model`` function to reduce loading time when the model is run in the future. See the `Model Preparation -Guide `__ +Guide `__ for more information about model conversion and TensorFlow `models -support `__. +support `__. .. code:: ipython3 @@ -709,4 +709,4 @@ utilization. For more information, refer to the `Optimize Preprocessing tutorial <118-optimize-preprocessing-with-output.html>`__ and to the overview of `Preprocessing -API `__. +API `__. diff --git a/docs/notebooks/121-convert-to-openvino-with-output.rst b/docs/notebooks/121-convert-to-openvino-with-output.rst index 504bb7a9474..6227c048667 100644 --- a/docs/notebooks/121-convert-to-openvino-with-output.rst +++ b/docs/notebooks/121-convert-to-openvino-with-output.rst @@ -54,7 +54,7 @@ OpenVINO IR format OpenVINO `Intermediate Representation -(IR) `__ is the +(IR) `__ is the proprietary model format of OpenVINO. It is produced after converting a model with model conversion API. Model conversion API translates the frequently used deep learning operations to their respective similar @@ -73,7 +73,7 @@ These model formats can be read, compiled, and converted to OpenVINO IR, either automatically or explicitly. For more details, refer to `Model -Preparation `__ +Preparation `__ documentation. .. code:: ipython3 @@ -504,7 +504,7 @@ inputs. Doing so at the model preparation stage, not at runtime, can be beneficial in terms of performance and memory consumption. For more information refer to `Setting Input -Shapes `__ +Shapes `__ documentation. .. code:: ipython3 @@ -666,9 +666,9 @@ Convert Models from memory Model conversion API supports passing original framework Python object directly. More details can be found in -`PyTorch `__, -`TensorFlow `__, -`PaddlePaddle `__ +`PyTorch `__, +`TensorFlow `__, +`PaddlePaddle `__ frameworks conversion guides. .. code:: ipython3 @@ -722,7 +722,7 @@ OVC or can be replaced with functionality from ``ov.PrePostProcessor`` class. Refer to `Optimize Preprocessing notebook `__ for more information about `Preprocessing -API `__. +API `__. Here is the migration guide from legacy model preprocessing to Preprocessing API. @@ -736,7 +736,7 @@ for both inputs and outputs. Some preprocessing requires to set input layouts, for example, setting a batch, applying mean or scales, and reversing input channels (BGR<->RGB). For the layout syntax, check the `Layout API -overview `__. +overview `__. To specify the layout, you can use the layout option followed by the layout value. @@ -885,6 +885,6 @@ the new conversion API. Instead, we recommend performing the cut in the original framework. Examples of model cutting of TensorFlow protobuf, TensorFlow SavedModel, and ONNX formats with tools provided by the Tensorflow and ONNX frameworks can be found in `documentation -guide `__. +guide `__. For PyTorch, TensorFlow 2 Keras, and PaddlePaddle, we recommend changing the original model code to perform the model cut. diff --git a/docs/notebooks/121-legacy-mo-convert-to-openvino-with-output.rst b/docs/notebooks/121-legacy-mo-convert-to-openvino-with-output.rst index 078b5066438..8fc65572387 100644 --- a/docs/notebooks/121-legacy-mo-convert-to-openvino-with-output.rst +++ b/docs/notebooks/121-legacy-mo-convert-to-openvino-with-output.rst @@ -49,7 +49,7 @@ OpenVINO IR format OpenVINO `Intermediate Representation -(IR) `__ is the +(IR) `__ is the proprietary model format of OpenVINO. It is produced after converting a model with model conversion API. Model conversion API translates the frequently used deep learning operations to their respective similar @@ -69,25 +69,25 @@ tool. You can choose one of them based on whichever is most convenient for you. There should not be any differences in the results of model conversion if the same set of parameters is used. For more details, refer to `Model -Preparation `__ +Preparation `__ documentation. .. code:: ipython3 # Model Optimizer CLI tool parameters description - + ! mo --help .. parsed-literal:: usage: main.py [options] - + optional arguments: -h, --help show this help message and exit --framework FRAMEWORK Name of the framework used to train the input model. - + Framework-agnostic parameters: --model_name MODEL_NAME, -n MODEL_NAME Model_name parameter passed to the final create_ir @@ -296,7 +296,7 @@ documentation. provided only for ONNX models that do not require fallback to the legacy ONNX frontend for the conversion. - + TensorFlow*-specific parameters: --input_model_is_text [INPUT_MODEL_IS_TEXT] TensorFlow*: treat the input model file as a text @@ -327,7 +327,7 @@ documentation. --tensorflow_custom_layer_libraries TENSORFLOW_CUSTOM_LAYER_LIBRARIES TensorFlow*: comma separated list of shared libraries with TensorFlow* custom operations implementation. - + Caffe*-specific parameters: --input_proto INPUT_PROTO, -d INPUT_PROTO Deploy-ready prototxt file that contains a topology @@ -349,7 +349,7 @@ documentation. attributes of a custom layer to IR with flattened nested parameters. Default behavior is to transfer the attributes without flattening nested parameters. - + MXNet-specific parameters: --input_symbol INPUT_SYMBOL Symbol file (for example, model-symbol.json) that @@ -369,7 +369,7 @@ documentation. --enable_ssd_gluoncv [ENABLE_SSD_GLUONCV] Enable pattern matchers replacers for converting gluoncv ssd topologies. - + Kaldi-specific parameters: --counts COUNTS Path to the counts file --remove_output_softmax [REMOVE_OUTPUT_SOFTMAX] @@ -383,52 +383,52 @@ documentation. # Python conversion API parameters description from openvino.tools import mo - - + + mo.convert_model(help=True) .. parsed-literal:: Optional parameters: - --help + --help Print available parameters. - --framework + --framework Name of the framework used to train the input model. - + Framework-agnostic parameters: - --input_model + --input_model Model object in original framework (PyTorch, Tensorflow) or path to model file. Tensorflow*: a file with a pre-trained model (binary or text .pb file after freezing). Caffe*: a model proto file with model weights - + Supported formats of input model: - + PaddlePaddle paddle.hapi.model.Model paddle.fluid.dygraph.layers.Layer paddle.fluid.executor.Executor - + PyTorch torch.nn.Module torch.jit.ScriptModule torch.jit.ScriptFunction - + TF tf.compat.v1.Graph tf.compat.v1.GraphDef tf.compat.v1.wrap_function tf.compat.v1.session - + TF2 / Keras tf.keras.Model tf.keras.layers.Layer tf.function tf.Module tf.train.checkpoint - --input + --input Input can be set by passing a list of InputCutInfo objects or by a list of tuples. Each tuple can contain optionally input name, input type or input shape. Example: input=("op_name", PartialShape([-1, @@ -453,12 +453,12 @@ documentation. `node_name1` with the shape [3,4] as an input node and freeze output port 1 of the node `node_name2` with the value [20,15] of the int32 type and shape [2]: "0:node_name1[3,4],node_name2:1[2]{i32}->[20,15]". - - --output + + --output The name of the output operation of the model or list of names. For TensorFlow*, do not add :0 to this name.The order of outputs in converted model is the same as order of specified operation names. - --input_shape + --input_shape Input shape(s) that should be fed to an input node(s) of the model. Input shapes can be defined by passing a list of objects of type PartialShape, Shape, [Dimension, ...] or [int, ...] or by a string of the following @@ -475,19 +475,19 @@ documentation. for each input separated by a comma, for example: [1,3,227,227],[2,4] for a model with two inputs with 4D and 2D shapes. Alternatively, specify shapes with the --input option. - --example_input + --example_input Sample of model input in original framework. For PyTorch it can be torch.Tensor. For Tensorflow it can be tf.Tensor or numpy.ndarray. For PaddlePaddle it can be Paddle Variable. - --batch + --batch Set batch size. It applies to 1D or higher dimension inputs. The default dimension index for the batch is zero. Use a label 'n' in --layout or --source_layout option to set the batch dimension. For example, "x(hwnc)" defines the third dimension to be the batch. - - --mean_values + + --mean_values Mean values to be used for the input image per channel. Mean values can be set by passing a dictionary, where key is input name and value is mean value. For example mean_values={'data':[255,255,255],'info':[255,255,255]}. @@ -496,7 +496,7 @@ documentation. input of the model, for example: "--mean_values data[255,255,255],info[255,255,255]". The exact meaning and order of channels depend on how the original model was trained. - --scale_values + --scale_values Scale values to be used for the input image per channel. Scale values can be set by passing a dictionary, where key is input name and value is scale value. For example scale_values={'data':[255,255,255],'info':[255,255,255]}. @@ -507,14 +507,14 @@ documentation. was trained. If both --mean_values and --scale_values are specified, the mean is subtracted first and then scale is applied regardless of the order of options in command line. - --scale + --scale All input values coming from original network inputs will be divided by this value. When a list of inputs is overridden by the --input parameter, this scale is not applied for any input that does not match with the original input of the model. If both --mean_values and --scale are specified, the mean is subtracted first and then scale is applied regardless of the order of options in command line. - --reverse_input_channels + --reverse_input_channels Switch the input channels order from RGB to BGR (or vice versa). Applied to original inputs of the model if and only if a number of channels equals 3. When --mean_values/--scale_values are also specified, reversing @@ -523,7 +523,7 @@ documentation. in the original model. In other words, if both options are specified, then the data flow in the model looks as following: Parameter -> ReverseInputChannels -> Mean apply-> Scale apply -> the original body of the model. - --source_layout + --source_layout Layout of the input or output of the model in the framework. Layout can be set by passing a dictionary, where key is input name and value is LayoutMap object. Or layout can be set by string of the following format. Layout @@ -532,11 +532,11 @@ documentation. Layout can be partially defined, "?" can be used to specify undefined layout for one dimension, "..." can be used to specify undefined layout for multiple dimensions, for example "?c??", "nc...", "n...c", etc. - - --target_layout + + --target_layout Same as --source_layout, but specifies target layout that will be in the model after processing by ModelOptimizer. - --layout + --layout Combination of --source_layout and --target_layout. Can't be used with either of them. If model has one input it is sufficient to specify layout of this input, for example --layout nhwc. To specify layouts @@ -545,20 +545,20 @@ documentation. --layout "name1(nhwc->nchw),name2(cn->nc)". Also "*" in long layout form can be used to fuse dimensions, for example "[n,c,...]->[n*c,...]". - --compress_to_fp16 + --compress_to_fp16 If the original model has FP32 weights or biases, they are compressed to FP16. All intermediate data is kept in original precision. Option can be specified alone as "--compress_to_fp16", or explicit True/False values can be set, for example: "--compress_to_fp16=False", or "--compress_to_fp16=True" - - --extensions + + --extensions Paths to libraries (.so or .dll) with extensions, comma-separated list of paths, objects derived from BaseExtension class or lists of objects. For the legacy MO path (if `--use_legacy_frontend` is used), a directory or a comma-separated list of directories with extensions are supported. To disable all extensions including those that are placed at the default location, pass an empty string. - --transform + --transform Apply additional transformations. 'transform' can be set by a list of tuples, where the first element is transform name and the second element is transform parameters. For example: [('LowLatency2', {{'use_const_initializer': @@ -570,114 +570,114 @@ documentation. "--transform "MakeStateful[param_res_names= {'input_name_1':'output_name_1','input_name_2':'output_name_2'}]"" Available transformations: "LowLatency2", "MakeStateful", "Pruning" - - --transformations_config + + --transformations_config Use the configuration file with transformations description or pass object derived from BaseExtension class. Transformations file can be specified as relative path from the current directory, as absolute path or as relative path from the mo root directory. - --silent + --silent Prevent any output messages except those that correspond to log level equals ERROR, that can be set with the following option: --log_level. By default, log level is already ERROR. - --log_level + --log_level Logger level of logging massages from MO. Expected one of ['CRITICAL', 'ERROR', 'WARN', 'WARNING', 'INFO', 'DEBUG', 'NOTSET']. - --version + --version Version of Model Optimizer - --progress + --progress Enable model conversion progress display. - --stream_output + --stream_output Switch model conversion progress display to a multiline mode. - --share_weights + --share_weights Map memory of weights instead reading files or share memory from input model. Currently, mapping feature is provided only for ONNX models that do not require fallback to the legacy ONNX frontend for the conversion. - - + + PaddlePaddle-specific parameters: - --example_output + --example_output Sample of model output in original framework. For PaddlePaddle it can be Paddle Variable. - + TensorFlow*-specific parameters: - --input_model_is_text + --input_model_is_text TensorFlow*: treat the input model file as a text protobuf format. If not specified, the Model Optimizer treats it as a binary file by default. - - --input_checkpoint + + --input_checkpoint TensorFlow*: variables file to load. - --input_meta_graph + --input_meta_graph Tensorflow*: a file with a meta-graph of the model before freezing - --saved_model_dir + --saved_model_dir TensorFlow*: directory with a model in SavedModel format of TensorFlow 1.x or 2.x version. - --saved_model_tags + --saved_model_tags Group of tag(s) of the MetaGraphDef to load, in string format, separated by ','. For tag-set contains multiple tags, all tags must be passed in. - - --tensorflow_custom_operations_config_update + + --tensorflow_custom_operations_config_update TensorFlow*: update the configuration file with node name patterns with input/output nodes information. - --tensorflow_object_detection_api_pipeline_config + --tensorflow_object_detection_api_pipeline_config TensorFlow*: path to the pipeline configuration file used to generate model created with help of Object Detection API. - --tensorboard_logdir + --tensorboard_logdir TensorFlow*: dump the input graph to a given directory that should be used with TensorBoard. - --tensorflow_custom_layer_libraries + --tensorflow_custom_layer_libraries TensorFlow*: comma separated list of shared libraries with TensorFlow* custom operations implementation. - + MXNet-specific parameters: - --input_symbol + --input_symbol Symbol file (for example, model-symbol.json) that contains a topology structure and layer attributes - --nd_prefix_name + --nd_prefix_name Prefix name for args.nd and argx.nd files. - --pretrained_model_name + --pretrained_model_name Name of a pretrained MXNet model without extension and epoch number. This model will be merged with args.nd and argx.nd files - --save_params_from_nd + --save_params_from_nd Enable saving built parameters file from .nd files - --legacy_mxnet_model + --legacy_mxnet_model Enable MXNet loader to make a model compatible with the latest MXNet version. Use only if your model was trained with MXNet version lower than 1.0.0 - --enable_ssd_gluoncv + --enable_ssd_gluoncv Enable pattern matchers replacers for converting gluoncv ssd topologies. - - + + Caffe*-specific parameters: - --input_proto + --input_proto Deploy-ready prototxt file that contains a topology structure and layer attributes - --caffe_parser_path + --caffe_parser_path Path to Python Caffe* parser generated from caffe.proto - --k + --k Path to CustomLayersMapping.xml to register custom layers - --disable_omitting_optional + --disable_omitting_optional Disable omitting optional attributes to be used for custom layers. Use this option if you want to transfer all attributes of a custom layer to IR. Default behavior is to transfer the attributes with default values and the attributes defined by the user to IR. - --enable_flattening_nested_params + --enable_flattening_nested_params Enable flattening optional params to be used for custom layers. Use this option if you want to transfer attributes of a custom layer to IR with flattened nested parameters. Default behavior is to transfer the attributes without flattening nested parameters. - + Kaldi-specific parameters: - --counts + --counts Path to the counts file - --remove_output_softmax + --remove_output_softmax Removes the SoftMax layer that is the output layer - --remove_memory + --remove_memory Removes the Memory layer and use additional inputs outputs instead - - + + Fetching example models @@ -695,7 +695,7 @@ This notebook uses two models for conversion examples: .. code:: ipython3 from pathlib import Path - + # create a directory for models files MODEL_DIRECTORY_PATH = Path("model") MODEL_DIRECTORY_PATH.mkdir(exist_ok=True) @@ -708,10 +708,10 @@ NLP model from Hugging Face and export it in ONNX format: from transformers import AutoModelForSequenceClassification, AutoTokenizer from transformers.onnx import export, FeaturesManager - - + + ONNX_NLP_MODEL_PATH = MODEL_DIRECTORY_PATH / "distilbert.onnx" - + # download model hf_model = AutoModelForSequenceClassification.from_pretrained( "distilbert-base-uncased-finetuned-sst-2-english" @@ -720,14 +720,14 @@ NLP model from Hugging Face and export it in ONNX format: tokenizer = AutoTokenizer.from_pretrained( "distilbert-base-uncased-finetuned-sst-2-english" ) - + # get model onnx config function for output feature format sequence-classification model_kind, model_onnx_config = FeaturesManager.check_supported_model_or_raise( hf_model, feature="sequence-classification" ) # fill onnx config based on pytorch model config onnx_config = model_onnx_config(hf_model.config) - + # export to onnx format export( preprocessor=tokenizer, @@ -771,8 +771,8 @@ CV classification model from torchvision: .. code:: ipython3 from torchvision.models import resnet50, ResNet50_Weights - - + + # create model object pytorch_model = resnet50(weights=ResNet50_Weights.DEFAULT) # switch model from training to inference mode @@ -968,10 +968,10 @@ Convert PyTorch model to ONNX format: import torch import warnings - - + + ONNX_CV_MODEL_PATH = MODEL_DIRECTORY_PATH / "resnet.onnx" - + if ONNX_CV_MODEL_PATH.exists(): print(f"ONNX model {ONNX_CV_MODEL_PATH} already exists.") else: @@ -998,7 +998,7 @@ To convert a model to OpenVINO IR, use the following command: .. code:: ipython3 # Model Optimizer CLI - + ! mo --input_model model/distilbert.onnx --output_dir model @@ -1016,7 +1016,7 @@ To convert a model to OpenVINO IR, use the following command: Find more information about compression to FP16 at https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_FP16_Compression.html [ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11. Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/2023.0/openvino_2_0_transition_guide.html - [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html @@ -1031,13 +1031,13 @@ To convert a model to OpenVINO IR, use the following command: # Python conversion API from openvino.tools import mo - + # mo.convert_model returns an openvino.runtime.Model object ov_model = mo.convert_model(ONNX_NLP_MODEL_PATH) - + # then model can be serialized to *.xml & *.bin files from openvino.runtime import serialize - + serialize(ov_model, xml_path=MODEL_DIRECTORY_PATH / "distilbert.xml") @@ -1058,20 +1058,20 @@ Both Python conversion API and Model Optimizer command-line tool provide the following capabilities: \* overriding original input shapes for model conversion with ``input`` and ``input_shape`` parameters. `Setting Input Shapes -guide `__. +guide `__. \* cutting off unwanted parts of a model (such as unsupported operations and training sub-graphs) using the ``input`` and ``output`` parameters to define new inputs and outputs of the converted model. `Cutting Off Parts of a Model -guide `__. +guide `__. \* inserting additional input pre-processing sub-graphs into the converted model by using the ``mean_values``, ``scales_values``, ``layout``, and other parameters. `Embedding Preprocessing Computation -article `__. +article `__. \* compressing the model weights (for example, weights for convolutions and matrix multiplications) to FP16 data type using ``compress_to_fp16`` compression parameter. `Compression of a Model to FP16 -guide `__. +guide `__. If the out-of-the-box conversion (only the ``input_model`` parameter is specified) is not successful, it may be required to use the parameters @@ -1092,14 +1092,14 @@ up static shapes, model conversion API provides the ``input`` and ``input_shape`` parameters. For more information refer to `Setting Input Shapes -guide `__. +guide `__. .. code:: ipython3 # Model Optimizer CLI - + ! mo --input_model model/distilbert.onnx --input input_ids,attention_mask --input_shape [1,128],[1,128] --output_dir model - + # alternatively ! mo --input_model model/distilbert.onnx --input input_ids[1,128],attention_mask[1,128] --output_dir model @@ -1118,7 +1118,7 @@ guide `__. +guide `__. .. code:: ipython3 # Model Optimizer CLI - + # cut at the end ! mo --input_model model/distilbert.onnx --output /classifier/Gemm --output_dir model - - + + # cut from the beginning ! mo --input_model model/distilbert.onnx --input /distilbert/embeddings/LayerNorm/Add_1,attention_mask --output_dir model @@ -1325,7 +1325,7 @@ guide `__. +article `__. Specifying Layout ^^^^^^^^^^^^^^^^^ @@ -1404,7 +1404,7 @@ for both inputs and outputs. Some preprocessing requires to set input layouts, for example, setting a batch, applying mean or scales, and reversing input channels (BGR<->RGB). For the layout syntax, check the `Layout API -overview `__. +overview `__. To specify the layout, you can use the layout option followed by the layout value. @@ -1414,7 +1414,7 @@ Resnet50 model that was exported to the ONNX format: .. code:: ipython3 # Model Optimizer CLI - + ! mo --input_model model/resnet.onnx --layout nchw --output_dir model @@ -1432,7 +1432,7 @@ Resnet50 model that was exported to the ONNX format: Find more information about compression to FP16 at https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_FP16_Compression.html [ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11. Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/2023.0/openvino_2_0_transition_guide.html - [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html [ SUCCESS ] Generated IR version 11 model. [ SUCCESS ] XML file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/121-convert-to-openvino/model/resnet.xml @@ -1443,8 +1443,8 @@ Resnet50 model that was exported to the ONNX format: # Python conversion API from openvino.tools import mo - - + + ov_model = mo.convert_model(ONNX_CV_MODEL_PATH, layout="nchw") Changing Model Layout @@ -1459,9 +1459,9 @@ presented by input data. Use either ``layout`` or ``source_layout`` with .. code:: ipython3 # Model Optimizer CLI - + ! mo --input_model model/resnet.onnx --layout "nchw->nhwc" --output_dir model - + # alternatively use source_layout and target_layout parameters ! mo --input_model model/resnet.onnx --source_layout nchw --target_layout nhwc --output_dir model @@ -1480,7 +1480,7 @@ presented by input data. Use either ``layout`` or ``source_layout`` with Find more information about compression to FP16 at https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_FP16_Compression.html [ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11. Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/2023.0/openvino_2_0_transition_guide.html - [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html @@ -1505,7 +1505,7 @@ presented by input data. Use either ``layout`` or ``source_layout`` with Find more information about compression to FP16 at https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_FP16_Compression.html [ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11. Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/2023.0/openvino_2_0_transition_guide.html - [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html @@ -1520,10 +1520,10 @@ presented by input data. Use either ``layout`` or ``source_layout`` with # Python conversion API from openvino.tools import mo - - + + ov_model = mo.convert_model(ONNX_CV_MODEL_PATH, layout="nchw->nhwc") - + # alternatively use source_layout and target_layout parameters ov_model = mo.convert_model( ONNX_CV_MODEL_PATH, source_layout="nchw", target_layout="nhwc" @@ -1543,9 +1543,9 @@ that the preprocessing takes negligible time for inference. .. code:: ipython3 # Model Optimizer CLI - + ! mo --input_model model/resnet.onnx --mean_values [123,117,104] --scale 255 --output_dir model - + ! mo --input_model model/resnet.onnx --mean_values [123,117,104] --scale_values [255,255,255] --output_dir model @@ -1563,7 +1563,7 @@ that the preprocessing takes negligible time for inference. Find more information about compression to FP16 at https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_FP16_Compression.html [ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11. Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/2023.0/openvino_2_0_transition_guide.html - [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html @@ -1588,7 +1588,7 @@ that the preprocessing takes negligible time for inference. Find more information about compression to FP16 at https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_FP16_Compression.html [ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11. Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/2023.0/openvino_2_0_transition_guide.html - [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html @@ -1603,10 +1603,10 @@ that the preprocessing takes negligible time for inference. # Python conversion API from openvino.tools import mo - - + + ov_model = mo.convert_model(ONNX_CV_MODEL_PATH, mean_values=[123, 117, 104], scale=255) - + ov_model = mo.convert_model( ONNX_CV_MODEL_PATH, mean_values=[123, 117, 104], scale_values=[255, 255, 255] ) @@ -1625,7 +1625,7 @@ the color channels before inference. .. code:: ipython3 # Model Optimizer CLI - + ! mo --input_model model/resnet.onnx --reverse_input_channels --output_dir model @@ -1643,7 +1643,7 @@ the color channels before inference. Find more information about compression to FP16 at https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_FP16_Compression.html [ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11. Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/2023.0/openvino_2_0_transition_guide.html - [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html @@ -1658,8 +1658,8 @@ the color channels before inference. # Python conversion API from openvino.tools import mo - - + + ov_model = mo.convert_model(ONNX_CV_MODEL_PATH, reverse_input_channels=True) Compressing a Model to FP16 @@ -1676,7 +1676,7 @@ models, this decrease is negligible. .. code:: ipython3 # Model Optimizer CLI - + ! mo --input_model model/resnet.onnx --compress_to_fp16=True --output_dir model @@ -1694,7 +1694,7 @@ models, this decrease is negligible. Find more information about compression to FP16 at https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_FP16_Compression.html [ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11. Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/2023.0/openvino_2_0_transition_guide.html - [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html @@ -1709,8 +1709,8 @@ models, this decrease is negligible. # Python conversion API from openvino.tools import mo - - + + ov_model = mo.convert_model(ONNX_CV_MODEL_PATH, compress_to_fp16=True) Convert Models Represented as Python Objects @@ -1727,8 +1727,8 @@ training scripts). # Python conversion API from openvino.tools import mo - - + + ov_model = mo.convert_model(pytorch_model) @@ -1745,17 +1745,17 @@ string analogs, similar to the command-line tool. # Python conversion API from openvino.tools import mo - - + + ov_model = mo.convert_model( pytorch_model, input_shape=[1, 3, 100, 100], mean_values=[127, 127, 127], layout="nchw", ) - + ov_model = mo.convert_model(pytorch_model, source_layout="nchw", target_layout="nhwc") - + ov_model = mo.convert_model( pytorch_model, compress_to_fp16=True, reverse_input_channels=True ) diff --git a/docs/notebooks/122-speech-recognition-quantization-wav2vec2-with-output.rst b/docs/notebooks/122-speech-recognition-quantization-wav2vec2-with-output.rst index d33e967ce19..8270d1105cd 100644 --- a/docs/notebooks/122-speech-recognition-quantization-wav2vec2-with-output.rst +++ b/docs/notebooks/122-speech-recognition-quantization-wav2vec2-with-output.rst @@ -24,7 +24,7 @@ The advanced quantization flow allows to apply 8-bit quantization to the model with control of accuracy metric. This is achieved by keeping the most impactful operations within the model in the original precision. The flow is based on the `Basic 8-bit -quantization `__ +quantization `__ and has the following differences: - Besides the calibration dataset, a validation dataset is required to diff --git a/docs/notebooks/122-yolov8-quantization-with-accuracy-control-with-output.rst b/docs/notebooks/122-yolov8-quantization-with-accuracy-control-with-output.rst index 207ad779fa4..b3247c870fd 100644 --- a/docs/notebooks/122-yolov8-quantization-with-accuracy-control-with-output.rst +++ b/docs/notebooks/122-yolov8-quantization-with-accuracy-control-with-output.rst @@ -14,7 +14,7 @@ quantization flow allows to apply 8-bit quantization to the model with control of accuracy metric. This is achieved by keeping the most impactful operations within the model in the original precision. The flow is based on the `Basic 8-bit -quantization `__ +quantization `__ and has the following differences: - Besides the calibration dataset, a validation dataset is required to diff --git a/docs/notebooks/124-hugging-face-hub-with-output.rst b/docs/notebooks/124-hugging-face-hub-with-output.rst index eb4d32901b6..102b001cb7c 100644 --- a/docs/notebooks/124-hugging-face-hub-with-output.rst +++ b/docs/notebooks/124-hugging-face-hub-with-output.rst @@ -177,7 +177,7 @@ Converting the Model to OpenVINO IR format We use the OpenVINO `Model conversion -API `__ +API `__ to convert the model (this one is implemented in PyTorch) to OpenVINO Intermediate Representation (IR). diff --git a/docs/notebooks/125-lraspp-segmentation-with-output.rst b/docs/notebooks/125-lraspp-segmentation-with-output.rst index 4c47502d7ff..dde28680272 100644 --- a/docs/notebooks/125-lraspp-segmentation-with-output.rst +++ b/docs/notebooks/125-lraspp-segmentation-with-output.rst @@ -70,7 +70,7 @@ Prerequisites .. code:: ipython3 from pathlib import Path - + import openvino as ov import torch @@ -84,11 +84,11 @@ First of all lets get a test image from an open dataset. .. code:: ipython3 import urllib.request - + from torchvision.io import read_image import torchvision.transforms as transforms - - + + img_path = 'cats_image.jpeg' urllib.request.urlretrieve( url='https://huggingface.co/datasets/huggingface/cats-image/resolve/main/cats_image.jpeg', @@ -122,12 +122,12 @@ models `__. +`page `__. .. code:: ipython3 ov_model_xml_path = Path('models/ov_lraspp_model.xml') - - + + if not ov_model_xml_path.exists(): ov_model_xml_path.parent.mkdir(parents=True, exist_ok=True) dummy_input = torch.randn(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH) @@ -219,7 +219,7 @@ Select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + core = ov.Core() device = widgets.Dropdown( options=core.available_devices + ["AUTO"], @@ -227,7 +227,7 @@ Select device from dropdown list for running inference using OpenVINO description='Device:', disabled=False, ) - + device @@ -266,13 +266,13 @@ visualize the image with a ``cat`` mask for the PyTorch model. import torch import matplotlib.pyplot as plt - + import torchvision.transforms.functional as F - - + + plt.rcParams["savefig.bbox"] = 'tight' - - + + def show(imgs): if not isinstance(imgs, list): imgs = [imgs] @@ -293,11 +293,11 @@ Prepare and display a cat mask. 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor' ] sem_class_to_idx = {cls: idx for (idx, cls) in enumerate(sem_classes)} - + normalized_mask = torch.nn.functional.softmax(result_torch, dim=1) - + cat_mask = normalized_mask[0, sem_class_to_idx['cat']] - + show(cat_mask) @@ -322,7 +322,7 @@ And now we can plot a boolean mask on top of the original image. .. code:: ipython3 from torchvision.utils import draw_segmentation_masks - + show(draw_segmentation_masks(image, masks=boolean_cat_mask, alpha=0.7, colors='yellow')) diff --git a/docs/notebooks/201-vision-monodepth-with-output.rst b/docs/notebooks/201-vision-monodepth-with-output.rst index 233548081cb..55c0c67a9fc 100644 --- a/docs/notebooks/201-vision-monodepth-with-output.rst +++ b/docs/notebooks/201-vision-monodepth-with-output.rst @@ -3,7 +3,7 @@ Monodepth Estimation with OpenVINO This tutorial demonstrates Monocular Depth Estimation with MidasNet in OpenVINO. Model information can be found -`here `__. +`here `__. .. figure:: https://user-images.githubusercontent.com/36741649/127173017-a0bbcf75-db24-4d2c-81b9-616e04ab7cd9.gif :alt: monodepth @@ -69,7 +69,7 @@ Install requirements %pip install -q "openvino>=2023.1.0" %pip install -q matplotlib opencv-python requests tqdm - + # Fetch `notebook_utils` module import urllib.request urllib.request.urlretrieve( @@ -105,7 +105,7 @@ Imports import time from pathlib import Path - + import cv2 import matplotlib.cm import matplotlib.pyplot as plt @@ -120,7 +120,7 @@ Imports display, ) import openvino as ov - + from notebook_utils import download_file, load_image Download the model @@ -134,14 +134,14 @@ The model is in the `OpenVINO Intermediate Representation .. code:: ipython3 model_folder = Path('model') - + ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/depth-estimation-midas/FP32/' ir_model_name_xml = 'MiDaS_small.xml' ir_model_name_bin = 'MiDaS_small.bin' - + download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory=model_folder) download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory=model_folder) - + model_xml_path = model_folder / ir_model_name_xml @@ -167,13 +167,13 @@ Functions def normalize_minmax(data): """Normalizes the values in `data` between 0 and 1""" return (data - data.min()) / (data.max() - data.min()) - - + + def convert_result_to_image(result, colormap="viridis"): """ Convert network result of floating point numbers to an RGB image with integer values from 0-255 by applying a colormap. - + `result` is expected to be a single network result in 1,H,W shape `colormap` is a matplotlib colormap. See https://matplotlib.org/stable/tutorials/colors/colormaps.html @@ -184,8 +184,8 @@ Functions result = cmap(result)[:, :, :3] * 255 result = result.astype(np.uint8) return result - - + + def to_rgb(image_data) -> np.ndarray: """ Convert image_data from BGR to RGB @@ -202,7 +202,7 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + core = ov.Core() device = widgets.Dropdown( options=core.available_devices + ["AUTO"], @@ -210,7 +210,7 @@ select device from dropdown list for running inference using OpenVINO description='Device:', disabled=False, ) - + device @@ -237,10 +237,10 @@ output keys and the expected input shape for the model. core.set_property({'CACHE_DIR': '../cache'}) model = core.read_model(model_xml_path) compiled_model = core.compile_model(model=model, device_name=device.value) - + input_key = compiled_model.input(0) output_key = compiled_model.output(0) - + network_input_shape = list(input_key.shape) network_image_height, network_image_width = network_input_shape[2:] @@ -262,10 +262,10 @@ H=height, W=width). IMAGE_FILE = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_bike.jpg" image = load_image(path=IMAGE_FILE) - + # Resize to input shape for network. resized_image = cv2.resize(src=image, dsize=(network_image_height, network_image_width)) - + # Reshape the image to network input shape NCHW. input_image = np.expand_dims(np.transpose(resized_image, (2, 0, 1)), 0) @@ -280,11 +280,11 @@ original image shape. .. code:: ipython3 result = compiled_model([input_image])[output_key] - + # Convert the network result of disparity map to an image that shows # distance as colors. result_image = convert_result_to_image(result=result) - + # Resize back to original image shape. The `cv2.resize` function expects shape # in (width, height), [::-1] reverses the (height, width) shape to match this. result_image = cv2.resize(result_image, image.shape[:2][::-1]) @@ -346,7 +346,7 @@ Video Settings # Try the `THEO` encoding if you have FFMPEG installed. # FOURCC = cv2.VideoWriter_fourcc(*"THEO") FOURCC = cv2.VideoWriter_fourcc(*"vp09") - + # Create Path objects for the input video and the result video. output_directory = Path("output") output_directory.mkdir(exist_ok=True) @@ -369,11 +369,11 @@ compute values for these properties for the monodepth video. raise ValueError(f"The video at {VIDEO_FILE} cannot be read.") input_fps = cap.get(cv2.CAP_PROP_FPS) input_video_frame_height, input_video_frame_width = image.shape[:2] - + target_fps = input_fps / ADVANCE_FRAMES target_frame_height = int(input_video_frame_height * SCALE_OUTPUT) target_frame_width = int(input_video_frame_width * SCALE_OUTPUT) - + cap.release() print( f"The input video has a frame width of {input_video_frame_width}, " @@ -403,10 +403,10 @@ Do Inference on a Video and Create Monodepth Video input_video_frame_nr = 0 start_time = time.perf_counter() total_inference_duration = 0 - + # Open the input video cap = cv2.VideoCapture(str(VIDEO_FILE)) - + # Create a result video. out_video = cv2.VideoWriter( str(result_video_path), @@ -414,36 +414,36 @@ Do Inference on a Video and Create Monodepth Video target_fps, (target_frame_width * 2, target_frame_height), ) - + num_frames = int(NUM_SECONDS * input_fps) total_frames = cap.get(cv2.CAP_PROP_FRAME_COUNT) if num_frames == 0 else num_frames progress_bar = ProgressBar(total=total_frames) progress_bar.display() - + try: while cap.isOpened(): ret, image = cap.read() if not ret: cap.release() break - + if input_video_frame_nr >= total_frames: break - + # Only process every second frame. # Prepare a frame for inference. # Resize to the input shape for network. resized_image = cv2.resize(src=image, dsize=(network_image_height, network_image_width)) # Reshape the image to network input shape NCHW. input_image = np.expand_dims(np.transpose(resized_image, (2, 0, 1)), 0) - + # Do inference. inference_start_time = time.perf_counter() result = compiled_model([input_image])[output_key] inference_stop_time = time.perf_counter() inference_duration = inference_stop_time - inference_start_time total_inference_duration += inference_duration - + if input_video_frame_nr % (10 * ADVANCE_FRAMES) == 0: clear_output(wait=True) progress_bar.display() @@ -457,7 +457,7 @@ Do Inference on a Video and Create Monodepth Video f"({1/inference_duration:.2f} FPS)" ) ) - + # Transform the network result to a RGB image. result_frame = to_rgb(convert_result_to_image(result)) # Resize the image and the result to a target frame shape. @@ -467,13 +467,13 @@ Do Inference on a Video and Create Monodepth Video stacked_frame = np.hstack((image, result_frame)) # Save a frame to the video. out_video.write(stacked_frame) - + input_video_frame_nr = input_video_frame_nr + ADVANCE_FRAMES cap.set(1, input_video_frame_nr) - + progress_bar.progress = input_video_frame_nr progress_bar.update() - + except KeyboardInterrupt: print("Processing interrupted.") finally: @@ -483,7 +483,7 @@ Do Inference on a Video and Create Monodepth Video cap.release() end_time = time.perf_counter() duration = end_time - start_time - + print( f"Processed {processed_frames} frames in {duration:.2f} seconds. " f"Total FPS (including video processing): {processed_frames/duration:.2f}." @@ -494,7 +494,7 @@ Do Inference on a Video and Create Monodepth Video .. parsed-literal:: - Processed 60 frames in 37.50 seconds. Total FPS (including video processing): 1.60.Inference FPS: 43.40 + Processed 60 frames in 37.50 seconds. Total FPS (including video processing): 1.60.Inference FPS: 43.40 Monodepth Video saved to 'output/Coco%20Walking%20in%20Berkeley_monodepth.mp4'. @@ -525,7 +525,7 @@ Display Monodepth Video Showing monodepth video saved at /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/201-vision-monodepth/output/Coco%20Walking%20in%20Berkeley_monodepth.mp4 - If you cannot see the video in your browser, please click on the following link to download the video + If you cannot see the video in your browser, please click on the following link to download the video diff --git a/docs/notebooks/202-vision-superresolution-image-with-output.rst b/docs/notebooks/202-vision-superresolution-image-with-output.rst index 1b2abe7d296..ebaf082b668 100644 --- a/docs/notebooks/202-vision-superresolution-image-with-output.rst +++ b/docs/notebooks/202-vision-superresolution-image-with-output.rst @@ -5,7 +5,7 @@ Super Resolution is the process of enhancing the quality of an image by increasing the pixel count using deep learning. This notebook shows the Single Image Super Resolution (SISR) which takes just one low resolution image. A model called -`single-image-super-resolution-1032 `__, +`single-image-super-resolution-1032 `__, which is available in Open Model Zoo, is used in this tutorial. It is based on the research paper cited below. @@ -95,7 +95,7 @@ Imports import os import time from pathlib import Path - + import cv2 import matplotlib.pyplot as plt import numpy as np @@ -129,7 +129,7 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + core = ov.Core() device = widgets.Dropdown( options=core.available_devices + ["AUTO"], @@ -137,7 +137,7 @@ select device from dropdown list for running inference using OpenVINO description='Device:', disabled=False, ) - + device @@ -153,20 +153,20 @@ select device from dropdown list for running inference using OpenVINO # 1032: 4x superresolution, 1033: 3x superresolution model_name = 'single-image-super-resolution-1032' - + base_model_dir = Path("./model").expanduser() - + model_xml_name = f'{model_name}.xml' model_bin_name = f'{model_name}.bin' - + model_xml_path = base_model_dir / model_xml_name model_bin_path = base_model_dir / model_bin_name - + if not model_xml_path.exists(): base_url = f'https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/{model_name}/FP16/' model_xml_url = base_url + model_xml_name model_bin_url = base_url + model_bin_name - + download_file(model_xml_url, model_xml_path) download_file(model_bin_url, model_bin_path) else: @@ -183,7 +183,7 @@ Functions """ Write the specified text in the top left corner of the image as white text with a black border. - + :param image: image as numpy arry with HWC shape, RGB or BGR :param text: text to write :return: image with written text, as numpy array @@ -196,11 +196,11 @@ Functions font_thickness = 2 text_color_bg = (0, 0, 0) x, y = org - + image = cv2.UMat(image) (text_w, text_h), _ = cv2.getTextSize(text, font, font_scale, font_thickness) result_im = cv2.rectangle(image, org, (x + text_w, y + text_h), text_color_bg, -1) - + textim = cv2.putText( result_im, text, @@ -212,13 +212,13 @@ Functions line_type, ) return textim.get() - - + + def convert_result_to_image(result) -> np.ndarray: """ Convert network result of floating point numbers to image with integer values from 0-255. Values outside this range are clipped to 0 and 255. - + :param result: a single superresolution network result in N,C,H,W shape """ result = result.squeeze(0).transpose(1, 2, 0) @@ -227,8 +227,8 @@ Functions result[result > 255] = 255 result = result.astype(np.uint8) return result - - + + def to_rgb(image_data) -> np.ndarray: """ Convert image_data from BGR to RGB @@ -254,23 +254,23 @@ information about the network inputs and outputs. core = ov.Core() model = core.read_model(model=model_xml_path) compiled_model = core.compile_model(model=model, device_name=device.value) - + # Network inputs and outputs are dictionaries. Get the keys for the # dictionaries. original_image_key, bicubic_image_key = compiled_model.inputs output_key = compiled_model.output(0) - + # Get the expected input and target shape. The `.dims[2:]` returns the height # and width. The `resize` function of OpenCV expects the shape as (width, height), # so reverse the shape with `[::-1]` and convert it to a tuple. input_height, input_width = list(original_image_key.shape)[2:] target_height, target_width = list(bicubic_image_key.shape)[2:] - + upsample_factor = int(target_height / input_height) - + print(f"The network expects inputs with a width of {input_width}, " f"height of {input_height}") print(f"The network returns images with a width of {target_width}, " f"height of {target_height}") - + print( f"The image sides are upsampled by a factor of {upsample_factor}. " f"The new image is {upsample_factor**2} times as large as the " @@ -298,17 +298,17 @@ Load and Show the Input Image IMAGE_PATH = Path("./data/tower.jpg") OUTPUT_PATH = Path("output/") - + os.makedirs(str(OUTPUT_PATH), exist_ok=True) - + download_file('https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/tower.jpg', IMAGE_PATH) full_image = cv2.imread(str(IMAGE_PATH)) - + # Uncomment these lines to load a raw image as BGR. # import rawpy # with rawpy.imread(IMAGE_PATH) as raw: # full_image = raw.postprocess()[:,:,(2,1,0)] - + plt.imshow(to_rgb(full_image)) print(f"Showing full image with width {full_image.shape[1]} " f"and height {full_image.shape[0]}") @@ -353,17 +353,17 @@ as the crop size. # Set it to 1 to crop the image with the exact input size. CROP_FACTOR = 2 adjusted_upsample_factor = upsample_factor // CROP_FACTOR - + image_id = "flag" # A tag to recognize the saved images. starty = 3200 startx = 0 - + # Perform the crop. image_crop = full_image[ starty : starty + input_height * CROP_FACTOR, startx : startx + input_width * CROP_FACTOR, ] - + # Show the cropped image. print(f"Showing image crop with width {image_crop.shape[1]} and " f"height {image_crop.shape[0]}.") plt.imshow(to_rgb(image_crop)); @@ -394,11 +394,11 @@ interpolation. This bicubic image is the second input to the network. bicubic_image = cv2.resize( src=image_crop, dsize=(target_width, target_height), interpolation=cv2.INTER_CUBIC ) - + # If required, resize the image to the input image shape. if CROP_FACTOR > 1: image_crop = cv2.resize(src=image_crop, dsize=(input_width, input_height)) - + # Reshape the images from (H,W,C) to (N,C,H,W). input_image_original = np.expand_dims(image_crop.transpose(2, 0, 1), axis=0) input_image_bicubic = np.expand_dims(bicubic_image.transpose(2, 0, 1), axis=0) @@ -418,7 +418,7 @@ Do inference and convert the inference result to an ``RGB`` image. bicubic_image_key.any_name: input_image_bicubic, } )[output_key] - + # Get inference result as numpy array and reshape to image shape and data type result_image = convert_result_to_image(result) @@ -460,7 +460,7 @@ Save Superresolution and Bicubic Image Crop # Add a text with "SUPER" or "BICUBIC" to the superresolution or bicubic image. image_super = write_text_on_image(image=result_image, text="SUPER") image_bicubic = write_text_on_image(image=bicubic_image, text="BICUBIC") - + # Store the image and the results. crop_image_path = Path(f"{OUTPUT_PATH.stem}/{image_id}_{adjusted_upsample_factor}x_crop.png") superres_image_path = Path( @@ -493,11 +493,11 @@ Write Animated GIF with Bicubic/Superresolution Comparison print(image_bicubic.shape) print(image_super.shape) - + result_pil = Image.fromarray(to_rgb(image_super)) bicubic_pil = Image.fromarray(to_rgb(image_bicubic)) gif_image_path = Path(f"{OUTPUT_PATH.stem}/{image_id}_comparison_{adjusted_upsample_factor}x.gif") - + result_pil.save( fp=str(gif_image_path), format="GIF", @@ -506,7 +506,7 @@ Write Animated GIF with Bicubic/Superresolution Comparison duration=1000, loop=0, ) - + # The `DisplayImage(str(gif_image_path))` function does not work in Colab. DisplayImage(data=open(gif_image_path, "rb").read(), width=1920 // 2) @@ -540,7 +540,7 @@ the ``Files`` tool. .. code:: ipython3 FOURCC = cv2.VideoWriter_fourcc(*"MJPG") - + result_video_path = Path( f"{OUTPUT_PATH.stem}/{image_id}_crop_comparison_{adjusted_upsample_factor}x.avi" ) @@ -548,22 +548,22 @@ the ``Files`` tool. result_image.shape[0] // 2, result_image.shape[1] // 2, ) - + out_video = cv2.VideoWriter( filename=str(result_video_path), fourcc=FOURCC, fps=90, frameSize=(video_target_width, video_target_height), ) - + resized_result_image = cv2.resize(src=result_image, dsize=(video_target_width, video_target_height)) resized_bicubic_image = cv2.resize( src=bicubic_image, dsize=(video_target_width, video_target_height) ) - + progress_bar = ProgressBar(total=video_target_width) progress_bar.display() - + for i in range(video_target_width): # Create a frame where the left part (until i pixels width) contains the # superresolution image, and the right part (from i pixels width) contains @@ -582,7 +582,7 @@ the ``Files`` tool. progress_bar.update() out_video.release() clear_output() - + video_link = FileLink(result_video_path) video_link.html_link_str = "%s" display(HTML(f"The video has been saved to {video_link._repr_html_()}")) @@ -619,9 +619,9 @@ Compute patches CROPLINES = 10 # See Superresolution on one crop of the image for description of `CROP_FACTOR`. CROP_FACTOR = 2 - + full_image_height, full_image_width = full_image.shape[:2] - + # Compute x and y coordinates of left top of image tiles. x_coords = list(range(0, full_image_width, input_width * CROP_FACTOR - CROPLINES * 2)) while full_image_width - x_coords[-1] < input_width * CROP_FACTOR: @@ -629,12 +629,12 @@ Compute patches y_coords = list(range(0, full_image_height, input_height * CROP_FACTOR - CROPLINES * 2)) while full_image_height - y_coords[-1] < input_height * CROP_FACTOR: y_coords.pop(-1) - + # Compute the width and height to crop the full image. The full image is # cropped at the border to tiles of the input size. crop_width = x_coords[-1] + input_width * CROP_FACTOR crop_height = y_coords[-1] + input_height * CROP_FACTOR - + # Compute the width and height of the target superresolution image. new_width = ( x_coords[-1] * (upsample_factor // CROP_FACTOR) @@ -677,62 +677,62 @@ as total time to process each patch. num_patches = len(x_coords) * len(y_coords) progress_bar = ProgressBar(total=num_patches) progress_bar.display() - + # Crop image to fit tiles of the input size. full_image_crop = full_image.copy()[:crop_height, :crop_width, :] - + # Create an empty array of the target size. full_superresolution_image = np.empty((new_height, new_width, 3), dtype=np.uint8) - + # Create a bicubic upsampled image of the target size for comparison. full_bicubic_image = cv2.resize( src=full_image_crop[CROPLINES:-CROPLINES, CROPLINES:-CROPLINES, :], dsize=(new_width, new_height), interpolation=cv2.INTER_CUBIC, ) - + total_inference_duration = 0 for y in y_coords: for x in x_coords: patch_nr += 1 - + # Crop the input image. image_crop = full_image_crop[ y : y + input_height * CROP_FACTOR, x : x + input_width * CROP_FACTOR, ] - + # Resize the images to the target shape with bicubic interpolation bicubic_image = cv2.resize( src=image_crop, dsize=(target_width, target_height), interpolation=cv2.INTER_CUBIC, ) - + if CROP_FACTOR > 1: image_crop = cv2.resize(src=image_crop, dsize=(input_width, input_height)) - + input_image_original = np.expand_dims(image_crop.transpose(2, 0, 1), axis=0) - + input_image_bicubic = np.expand_dims(bicubic_image.transpose(2, 0, 1), axis=0) - + # Do inference. inference_start_time = time.perf_counter() - + result = compiled_model( { original_image_key.any_name: input_image_original, bicubic_image_key.any_name: input_image_bicubic, } )[output_key] - + inference_stop_time = time.perf_counter() inference_duration = inference_stop_time - inference_start_time total_inference_duration += inference_duration - + # Reshape an inference result to the image shape and the data type. result_image = convert_result_to_image(result) - + # Add the inference result of this patch to the full superresolution # image. adjusted_upsample_factor = upsample_factor // CROP_FACTOR @@ -746,10 +746,10 @@ as total time to process each patch. CROPLINES * adjusted_upsample_factor : -CROPLINES * adjusted_upsample_factor, :, ] - + progress_bar.progress = patch_nr progress_bar.update() - + if patch_nr % 10 == 0: clear_output(wait=True) progress_bar.display() @@ -760,7 +760,7 @@ as total time to process each patch. f"({1/inference_duration:.2f} FPS)" ) ) - + end_time = time.perf_counter() duration = end_time - start_time clear_output(wait=True) @@ -775,7 +775,7 @@ as total time to process each patch. .. parsed-literal:: Processed 42 patches in 4.64 seconds. Total patches per second (including processing): 9.05. - Inference patches per second: 17.92 + Inference patches per second: 17.92 Save superresolution image and the bicubic image @@ -789,7 +789,7 @@ Save superresolution image and the bicubic image f"{OUTPUT_PATH.stem}/full_superres_{adjusted_upsample_factor}x.jpg" ) full_bicubic_image_path = Path(f"{OUTPUT_PATH.stem}/full_bicubic_{adjusted_upsample_factor}x.jpg") - + cv2.imwrite(str(full_superresolution_image_path), full_superresolution_image) cv2.imwrite(str(full_bicubic_image_path), full_bicubic_image); diff --git a/docs/notebooks/202-vision-superresolution-video-with-output.rst b/docs/notebooks/202-vision-superresolution-video-with-output.rst index 465022c097d..31d16bf0fb0 100644 --- a/docs/notebooks/202-vision-superresolution-video-with-output.rst +++ b/docs/notebooks/202-vision-superresolution-video-with-output.rst @@ -5,7 +5,7 @@ Super Resolution is the process of enhancing the quality of an image by increasing the pixel count using deep learning. This notebook applies Single Image Super Resolution (SISR) to frames in a 360p (480×360) video in 360p resolution. A model called -`single-image-super-resolution-1032 `__, +`single-image-super-resolution-1032 `__, which is available in Open Model Zoo, is used in this tutorial. It is based on the research paper cited below. @@ -81,7 +81,7 @@ Imports import time from pathlib import Path - + import cv2 import numpy as np from IPython.display import ( @@ -120,7 +120,7 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + core = ov.Core() device = widgets.Dropdown( options=core.available_devices + ["AUTO"], @@ -128,7 +128,7 @@ select device from dropdown list for running inference using OpenVINO description='Device:', disabled=False, ) - + device @@ -144,20 +144,20 @@ select device from dropdown list for running inference using OpenVINO # 1032: 4x superresolution, 1033: 3x superresolution model_name = 'single-image-super-resolution-1032' - + base_model_dir = Path('./model').expanduser() - + model_xml_name = f'{model_name}.xml' model_bin_name = f'{model_name}.bin' - + model_xml_path = base_model_dir / model_xml_name model_bin_path = base_model_dir / model_bin_name - + if not model_xml_path.exists(): base_url = f'https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/{model_name}/FP16/' model_xml_url = base_url + model_xml_name model_bin_url = base_url + model_bin_name - + download_file(model_xml_url, model_xml_path) download_file(model_bin_url, model_bin_path) else: @@ -180,7 +180,7 @@ Functions """ Convert network result of floating point numbers to image with integer values from 0-255. Values outside this range are clipped to 0 and 255. - + :param result: a single superresolution network result in N,C,H,W shape """ result = result.squeeze(0).transpose(1, 2, 0) @@ -215,18 +215,18 @@ resolution version of the image in 1920x1080. # dictionaries. original_image_key, bicubic_image_key = compiled_model.inputs output_key = compiled_model.output(0) - + # Get the expected input and target shape. The `.dims[2:]` function returns the height # and width.The `resize` function of OpenCV expects the shape as (width, height), # so reverse the shape with `[::-1]` and convert it to a tuple. input_height, input_width = list(original_image_key.shape)[2:] target_height, target_width = list(bicubic_image_key.shape)[2:] - + upsample_factor = int(target_height / input_height) - + print(f"The network expects inputs with a width of {input_width}, " f"height of {input_height}") print(f"The network returns images with a width of {target_width}, " f"height of {target_height}") - + print( f"The image sides are upsampled by a factor of {upsample_factor}. " f"The new image is {upsample_factor**2} times as large as the " @@ -264,7 +264,7 @@ Settings .. code:: ipython3 OUTPUT_DIR = "output" - + Path(OUTPUT_DIR).mkdir(exist_ok=True) # Maximum number of frames to read from the input video. Set to 0 to read all frames. NUM_FRAMES = 100 @@ -290,10 +290,10 @@ Download and Prepare Video filename = Path(stream.default_filename.encode("ascii", "ignore").decode("ascii")).stem stream.download(output_path=OUTPUT_DIR, filename=filename) print(f"Video {filename} downloaded to {OUTPUT_DIR}") - + # Create Path objects for the input video and the resulting videos. video_path = Path(stream.get_file_path(filename, OUTPUT_DIR)) - + # Path names for the result videos. superres_video_path = Path(f"{OUTPUT_DIR}/{video_path.stem}_superres.mp4") bicubic_video_path = Path(f"{OUTPUT_DIR}/{video_path.stem}_bicubic.mp4") @@ -314,14 +314,14 @@ Download and Prepare Video raise ValueError(f"The video at '{video_path}' cannot be read.") fps = cap.get(cv2.CAP_PROP_FPS) frame_count = cap.get(cv2.CAP_PROP_FRAME_COUNT) - + if NUM_FRAMES == 0: total_frames = frame_count else: total_frames = min(frame_count, NUM_FRAMES) - + original_frame_height, original_frame_width = image.shape[:2] - + cap.release() print( f"The input video has a frame width of {original_frame_width}, " @@ -389,10 +389,10 @@ video. start_time = time.perf_counter() frame_nr = 0 total_inference_duration = 0 - + progress_bar = ProgressBar(total=total_frames) progress_bar.display() - + cap = cv2.VideoCapture(filename=str(video_path)) try: while cap.isOpened(): @@ -400,22 +400,22 @@ video. if not ret: cap.release() break - + if frame_nr >= total_frames: break - + # Resize the input image to the network shape and convert it from (H,W,C) to # (N,C,H,W). resized_image = cv2.resize(src=image, dsize=(input_width, input_height)) input_image_original = np.expand_dims(resized_image.transpose(2, 0, 1), axis=0) - + # Resize and reshape the image to the target shape with bicubic # interpolation. bicubic_image = cv2.resize( src=image, dsize=(target_width, target_height), interpolation=cv2.INTER_CUBIC ) input_image_bicubic = np.expand_dims(bicubic_image.transpose(2, 0, 1), axis=0) - + # Do inference. inference_start_time = time.perf_counter() result = compiled_model( @@ -427,19 +427,19 @@ video. inference_stop_time = time.perf_counter() inference_duration = inference_stop_time - inference_start_time total_inference_duration += inference_duration - + # Transform the inference result into an image. result_frame = convert_result_to_image(result=result) - + # Write the result image and the bicubic image to a video file. superres_video.write(image=result_frame) bicubic_video.write(image=bicubic_image) - + stacked_frame = np.hstack((bicubic_image, result_frame)) comparison_video.write(image=stacked_frame) - + frame_nr = frame_nr + 1 - + # Update the progress bar and the status message. progress_bar.progress = frame_nr progress_bar.update() @@ -453,8 +453,8 @@ video. f"({1/inference_duration:.2f} FPS)" ) ) - - + + except KeyboardInterrupt: print("Processing interrupted.") finally: diff --git a/docs/notebooks/203-meter-reader-with-output.rst b/docs/notebooks/203-meter-reader-with-output.rst index 771bc9c07c3..f590dd5fd90 100644 --- a/docs/notebooks/203-meter-reader-with-output.rst +++ b/docs/notebooks/203-meter-reader-with-output.rst @@ -590,7 +590,7 @@ select device from dropdown list for running inference using OpenVINO The number of detected meter from detection network can be arbitrary in some scenarios, which means the batch size of segmentation network input is a `dynamic -dimension `__, +dimension `__, and it should be specified as ``-1`` or the ``ov::Dimension()`` instead of a positive number used for static dimensions. In this case, for memory consumption optimization, we can specify the lower and/or upper diff --git a/docs/notebooks/204-segmenter-semantic-segmentation-with-output.rst b/docs/notebooks/204-segmenter-semantic-segmentation-with-output.rst index 9447dd7c2e0..9baee251d05 100644 --- a/docs/notebooks/204-segmenter-semantic-segmentation-with-output.rst +++ b/docs/notebooks/204-segmenter-semantic-segmentation-with-output.rst @@ -90,13 +90,13 @@ Prerequisites import sys from pathlib import Path - + # clone Segmenter repo if not Path("segmenter").exists(): !git clone https://github.com/rstrudel/segmenter else: print("Segmenter repo already cloned") - + # include path to Segmenter repo to use its functions sys.path.append("./segmenter") @@ -432,7 +432,7 @@ Resolving deltas: 100% (117/117), done. import numpy as np import yaml - + # Fetch the notebook utils script from the openvino_notebooks repo import urllib.request urllib.request.urlretrieve( @@ -453,13 +453,13 @@ config for our model. # here we use tiny model, there are also better but larger models available in repository WEIGHTS_LINK = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/segmenter/checkpoints/ade20k/seg_tiny_mask/checkpoint.pth" CONFIG_LINK = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/segmenter/checkpoints/ade20k/seg_tiny_mask/variant.yml" - + MODEL_DIR = Path("model/") MODEL_DIR.mkdir(exist_ok=True) - + download_file(WEIGHTS_LINK, directory=MODEL_DIR, show_progress=True) download_file(CONFIG_LINK, directory=MODEL_DIR, show_progress=True) - + WEIGHT_PATH = MODEL_DIR / "checkpoint.pth" CONFIG_PATH = MODEL_DIR / "variant.yaml" @@ -497,7 +497,7 @@ initialize the model. .. code:: ipython3 from segmenter.segm.model.factory import load_model - + pytorch_model, config = load_model(WEIGHT_PATH) # put model into eval mode, to set it for inference pytorch_model.eval() @@ -560,12 +560,12 @@ normalized with given mean and standard deviation provided in from PIL import Image import torch import torchvision.transforms.functional as F - - + + def preprocess(im: Image, normalization: dict) -> torch.Tensor: """ Preprocess image: scale, normalize and unsqueeze - + :param im: input image :param normalization: dictionary containing normalization data from config file :return: @@ -577,7 +577,7 @@ normalized with given mean and standard deviation provided in im = F.normalize(im, normalization["mean"], normalization["std"]) # change dim from [C, H, W] to [1, C, H, W] im = im.unsqueeze(0) - + return im Visualization @@ -601,29 +601,29 @@ corresponding to the inferred labels. from segmenter.segm.data.utils import dataset_cat_description, seg_to_rgb from segmenter.segm.data.ade20k import ADE20K_CATS_PATH - - + + def apply_segmentation_mask(pil_im: Image, results: torch.Tensor) -> Image: """ Combine segmentation masks with the image - + :param pil_im: original input image :param results: tensor containing segmentation masks for each pixel :return: pil_blend: image with colored segmentation masks overlay """ cat_names, cat_colors = dataset_cat_description(ADE20K_CATS_PATH) - + # 3D array, where each pixel has values for all classes, take index of max as label seg_map = results.argmax(0, keepdim=True) # transform label id to colors seg_rgb = seg_to_rgb(seg_map, cat_colors) seg_rgb = (255 * seg_rgb.cpu().numpy()).astype(np.uint8) pil_seg = Image.fromarray(seg_rgb[0]) - + # overlay segmentation mask over original image pil_blend = Image.blend(pil_im, pil_seg, 0.5).convert("RGB") - + return pil_blend Validation of inference of original model @@ -637,15 +637,15 @@ example image ``coco_hollywood.jpg``. .. code:: ipython3 from segmenter.segm.model.utils import inference - + # load image with PIL image = load_image("https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_hollywood.jpg") # load_image reads the image in BGR format, [:,:,::-1] reshape transfroms it to RGB pil_image = Image.fromarray(image[:,:,::-1]) - + # preprocess image with normalization params loaded in previous steps image = preprocess(pil_image, normalization) - + # inference function needs some meta parameters, where we specify that we don't flip images in inference mode im_meta = dict(flip=False) # perform inference with function from repository @@ -665,7 +665,7 @@ previous steps. # combine segmentation mask with image blended_image = apply_segmentation_mask(pil_image, original_results) - + # show image with segmentation mask overlay blended_image @@ -712,15 +712,15 @@ they are not a problem. .. code:: ipython3 import openvino as ov - + # get input sizes from config file batch_size = 2 channels = 3 image_size = config["dataset_kwargs"]["image_size"] - + # make dummy input with correct shapes obtained from config file dummy_input = torch.randn(batch_size, channels, image_size, image_size) - + model = ov.convert_model(pytorch_model, example_input=dummy_input, input=([batch_size, channels, image_size, image_size], )) # serialize model for saving IR ov.save_model(model, MODEL_DIR / "segmenter.xml") @@ -763,7 +763,7 @@ any additional custom code required to process input. class SegmenterOV: """ Class containing OpenVINO model with all attributes required to work with inference function. - + :param model: compiled OpenVINO model :type model: CompiledModel :param output_blob: output blob used in inference @@ -774,14 +774,14 @@ any additional custom code required to process input. :type n_cls: int :param normalization: :type normalization: dict - + """ - + def __init__(self, model_path: Path, device:str = "CPU"): """ Constructor method. Initializes OpenVINO model and sets all required attributes - + :param model_path: path to model's .xml file, also containing variant.yml :param device: device string for selecting inference device """ @@ -791,23 +791,23 @@ any additional custom code required to process input. model_xml = core.read_model(model_path) self.model = core.compile_model(model_xml, device) self.output_blob = self.model.output(0) - + # load model configs variant_path = Path(model_path).parent / "variant.yml" with open(variant_path, "r") as f: self.config = yaml.load(f, Loader=yaml.FullLoader) - + # load normalization specs from config normalization_name = self.config["dataset_kwargs"]["normalization"] self.normalization = STATS[normalization_name] - + # load number of classes from config self.n_cls = self.config["net_kwargs"]["n_cls"] - + def forward(self, data: torch.Tensor) -> torch.Tensor: """ Perform inference on data and return the result in Tensor format - + :param data: input data to model :return: data inferred by model """ @@ -826,7 +826,7 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + core = ov.Core() device = widgets.Dropdown( options=core.available_devices + ["AUTO"], @@ -834,7 +834,7 @@ select device from dropdown list for running inference using OpenVINO description='Device:', disabled=False, ) - + device @@ -866,7 +866,7 @@ select device from dropdown list for running inference using OpenVINO # combine segmentation mask with image converted_blend = apply_segmentation_mask(pil_image, results) - + # show image with segmentation mask overlay converted_blend @@ -885,7 +885,7 @@ Benchmarking performance of converted model Finally, use the OpenVINO `Benchmark -Tool `__ +Tool `__ to measure the inference performance of the model. NOTE: For more accurate performance, it is recommended to run @@ -928,12 +928,12 @@ to measure the inference performance of the model. [ WARNING ] Default duration 120 seconds is used for unknown device AUTO [ INFO ] OpenVINO: [ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3 - [ INFO ] + [ INFO ] [ INFO ] Device info: [ INFO ] AUTO [ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3 - [ INFO ] - [ INFO ] + [ INFO ] + [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files @@ -985,7 +985,7 @@ to measure the inference performance of the model. [ INFO ] LOADED_FROM_CACHE: False [Step 9/11] Creating infer requests and preparing input tensors [ WARNING ] No input files were given for input 'im'!. This input will be filled with random values! - [ INFO ] Fill input 'im' with random values + [ INFO ] Fill input 'im' with random values .. parsed-literal:: diff --git a/docs/notebooks/205-vision-background-removal-with-output.rst b/docs/notebooks/205-vision-background-removal-with-output.rst index 5a3e1a62dbf..59fd2ae3df9 100644 --- a/docs/notebooks/205-vision-background-removal-with-output.rst +++ b/docs/notebooks/205-vision-background-removal-with-output.rst @@ -492,7 +492,7 @@ References - `PIP install openvino-dev `__ - `Model Conversion - API `__ + API `__ - `U^2-Net `__ - U^2-Net research paper: `U^2-Net: Going Deeper with Nested U-Structure for Salient Object diff --git a/docs/notebooks/206-vision-paddlegan-anime-with-output.rst b/docs/notebooks/206-vision-paddlegan-anime-with-output.rst index cfd0f16e9b4..ccf0abca8b4 100644 --- a/docs/notebooks/206-vision-paddlegan-anime-with-output.rst +++ b/docs/notebooks/206-vision-paddlegan-anime-with-output.rst @@ -61,10 +61,10 @@ Install requirements .. code:: ipython3 %pip install -q "openvino>=2023.1.0" - + %pip install -q "paddlepaddle>=2.5.1" "paddle2onnx>=0.6" %pip install -q "git+https://github.com/PaddlePaddle/PaddleGAN.git" --no-deps - + %pip install -q opencv-python matplotlib scikit-learn scikit-image %pip install -q "imageio==2.9.0" "imageio-ffmpeg" "numba>=0.53.1" easydict munch natsort @@ -97,7 +97,7 @@ Install requirements ppgan 2.1.0 requires librosa==0.8.1, but you have librosa 0.10.1 which is incompatible. ppgan 2.1.0 requires opencv-python<=4.6.0.66, but you have opencv-python 4.9.0.80 which is incompatible. scikit-image 0.21.0 requires imageio>=2.27, but you have imageio 2.9.0 which is incompatible. - + .. parsed-literal:: @@ -116,13 +116,13 @@ Imports import os from pathlib import Path import urllib.request - + import cv2 import matplotlib.pyplot as plt import numpy as np import openvino as ov from IPython.display import HTML, display - + # PaddlePaddle requires a C++ compiler. If importing the paddle packages fails, # install C++. try: @@ -159,9 +159,9 @@ Settings MODEL_DIR = "model" MODEL_NAME = "paddlegan_anime" - + os.makedirs(MODEL_DIR, exist_ok=True) - + # Create filenames of the models that will be converted in this notebook. model_path = Path(f"{MODEL_DIR}/{MODEL_NAME}") ir_path = model_path.with_suffix(".xml") @@ -234,7 +234,7 @@ cell. PADDLEGAN_INFERENCE = True OUTPUT_DIR = "output" - + os.makedirs(OUTPUT_DIR, exist_ok=True) # Step 1. Load the image and convert to RGB. image_path = Path("./data/coco_bricks.png") @@ -244,33 +244,33 @@ cell. "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_bricks.png", image_path ) - + image = cv2.cvtColor(cv2.imread(str(image_path), flags=cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB) - + ## Inference takes a long time on large images. Resize to a max width of 600. image = resize_to_max_width(image, 600) - + # Step 2. Transform the image. transformed_image = predictor.transform(image) input_tensor = paddle.to_tensor(transformed_image[None, ::]) - + if PADDLEGAN_INFERENCE: - # Step 3. Do inference. + # Step 3. Do inference. predictor.generator.eval() with paddle.no_grad(): result = predictor.generator(input_tensor) - + # Step 4. Convert the inference result to an image, following the same steps as # PaddleGAN's predictor.run() function. result_image_pg = (result * 0.5 + 0.5)[0].numpy() * 255 result_image_pg = result_image_pg.transpose((1, 2, 0)) - + # Step 5. Resize the result image. result_image_pg = cv2.resize(result_image_pg, image.shape[:2][::-1]) - + # Step 6. Adjust the brightness. result_image_pg = predictor.adjust_brightness(result_image_pg, image) - + # Step 7. Save the result image. anime_image_path_pg = Path(f"{OUTPUT_DIR}/{image_path.stem}_anime_pg").with_suffix(".jpg") if cv2.imwrite(str(anime_image_path_pg), result_image_pg[:, :, (2, 1, 0)]): @@ -409,17 +409,17 @@ feeding them to the converted model. Now we use model conversion API and convert the model to OpenVINO IR. **Convert ONNX Model to OpenVINO IR with**\ `Model Conversion Python -API `__ +API `__ .. code:: ipython3 print("Exporting ONNX model to OpenVINO IR... This may take a few minutes.") - + model = ov.convert_model( onnx_path, input=[1, 3, target_height, target_width], ) - + # Serialize model in IR format ov.save_model(model, str(ir_path)) @@ -469,17 +469,17 @@ OpenVINO IR model .. code:: ipython3 # Copyright (c) 2020 PaddlePaddle Authors. Licensed under the Apache License, Version 2.0 - - + + def calc_avg_brightness(img): R = img[..., 0].mean() G = img[..., 1].mean() B = img[..., 2].mean() - + brightness = 0.299 * R + 0.587 * G + 0.114 * B return brightness, B, G, R - - + + def adjust_brightness(dst, src): brightness1, B1, G1, R1 = AnimeGANPredictor.calc_avg_brightness(src) brightness2, B2, G2, R2 = AnimeGANPredictor.calc_avg_brightness(dst) @@ -513,7 +513,7 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + core = ov.Core() device = widgets.Dropdown( options=core.available_devices + ["AUTO"], @@ -521,7 +521,7 @@ select device from dropdown list for running inference using OpenVINO description='Device:', disabled=False, ) - + device @@ -537,7 +537,7 @@ select device from dropdown list for running inference using OpenVINO # Load and prepare the IR model. core = ov.Core() - + model = core.read_model(model=ir_path) compiled_model = core.compile_model(model=model, device_name=device.value) input_key = compiled_model.input(0) @@ -548,7 +548,7 @@ select device from dropdown list for running inference using OpenVINO # Step 1. Load an image and convert it to RGB. image_path = Path("./data/coco_bricks.png") image = cv2.cvtColor(cv2.imread(str(image_path), flags=cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB) - + # Step 2. Do preprocess transformations. # Resize the image resized_image = cv2.resize(image, (target_width, target_height)) @@ -557,21 +557,21 @@ select device from dropdown list for running inference using OpenVINO input_mean = np.array([127.5,127.5,127.5]).reshape(1, 3, 1, 1) input_scale = np.array([127.5,127.5,127.5]).reshape(1, 3, 1, 1) input_image = (input_image - input_mean) / input_scale - + # Step 3. Do inference. result_ir = compiled_model([input_image])[output_key] - + # Step 4. Convert the inference result to an image, following the same steps as # PaddleGAN's predictor.run() function. result_image_ir = (result_ir * 0.5 + 0.5)[0] * 255 result_image_ir = result_image_ir.transpose((1, 2, 0)) - + # Step 5. Resize the result image. result_image_ir = cv2.resize(result_image_ir, image.shape[:2][::-1]) - + # Step 6. Adjust the brightness. result_image_ir = adjust_brightness(result_image_ir, image) - + # Step 7. Save the result image. anime_fn_ir = Path(f"{OUTPUT_DIR}/{image_path.stem}_anime_ir").with_suffix(".jpg") if cv2.imwrite(str(anime_fn_ir), result_image_ir[:, :, (2, 1, 0)]): @@ -621,13 +621,13 @@ measure inference on one image. For more accurate benchmarking, use f"OpenVINO IR model in OpenVINO Runtime/CPU: {time_ir/NUM_IMAGES:.3f} " f"seconds per image, FPS: {NUM_IMAGES/time_ir:.2f}" ) - + ## `PADDLEGAN_INFERENCE` is defined in the "Inference on PaddleGAN model" section above. ## Uncomment the next line to enable a performance comparison with the PaddleGAN model - ## if you disabled it earlier. - + ## if you disabled it earlier. + # PADDLEGAN_INFERENCE = True - + if PADDLEGAN_INFERENCE: with paddle.no_grad(): start = time.perf_counter() @@ -661,7 +661,7 @@ References - `OpenVINO ONNX support `__ - `Model Conversion - API `__ + API `__ The PaddleGAN code that is shown in this notebook is written by PaddlePaddle Authors and licensed under the Apache 2.0 license. The diff --git a/docs/notebooks/207-vision-paddlegan-superresolution-with-output.rst b/docs/notebooks/207-vision-paddlegan-superresolution-with-output.rst index 7236dc119c2..6bf7350261e 100644 --- a/docs/notebooks/207-vision-paddlegan-superresolution-with-output.rst +++ b/docs/notebooks/207-vision-paddlegan-superresolution-with-output.rst @@ -52,9 +52,9 @@ Imports .. code:: ipython3 %pip install -q "openvino>=2023.1.0" - + %pip install -q "paddlepaddle>=2.5.1" "paddle2onnx>=0.6" - + %pip install -q "imageio==2.9.0" "imageio-ffmpeg" "numba>=0.53.1" "easydict" "munch" "natsort" %pip install -q "git+https://github.com/PaddlePaddle/PaddleGAN.git" --no-deps %pip install -q scikit-image @@ -86,7 +86,7 @@ Imports ppgan 2.1.0 requires imageio==2.9.0, but you have imageio 2.33.1 which is incompatible. ppgan 2.1.0 requires librosa==0.8.1, but you have librosa 0.10.1 which is incompatible. ppgan 2.1.0 requires opencv-python<=4.6.0.66, but you have opencv-python 4.9.0.80 which is incompatible. - + .. parsed-literal:: @@ -98,7 +98,7 @@ Imports import time import warnings from pathlib import Path - + import cv2 import matplotlib.pyplot as plt import numpy as np @@ -109,7 +109,7 @@ Imports from PIL import Image from paddle.static import InputSpec from ppgan.apps import RealSRPredictor - + # Fetch `notebook_utils` module import urllib.request urllib.request.urlretrieve( @@ -130,7 +130,7 @@ Settings MODEL_DIR = Path("model") OUTPUT_DIR = Path("output") OUTPUT_DIR.mkdir(exist_ok=True) - + model_path = MODEL_DIR / MODEL_NAME ir_path = model_path.with_suffix(".xml") onnx_path = model_path.with_suffix(".onnx") @@ -312,7 +312,7 @@ Convert PaddlePaddle Model to ONNX 2024-02-09 23:42:19 [INFO] ONNX model saved in model/paddlegan_sr.onnx. -Convert ONNX Model to OpenVINO IR with `Model Conversion Python API `__ +Convert ONNX Model to OpenVINO IR with `Model Conversion Python API `__ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -320,12 +320,12 @@ Convert ONNX Model to OpenVINO IR with `Model Conversion Python API `__ +`horizontal-text-detection-0001 `__ and -`text-recognition-resnet `__ +`text-recognition-resnet `__ models are used together for text detection and then text recognition. In this tutorial, Open Model Zoo tools including Model Downloader, Model diff --git a/docs/notebooks/209-handwritten-ocr-with-output.rst b/docs/notebooks/209-handwritten-ocr-with-output.rst index 5812496245a..135b910fc66 100644 --- a/docs/notebooks/209-handwritten-ocr-with-output.rst +++ b/docs/notebooks/209-handwritten-ocr-with-output.rst @@ -8,9 +8,9 @@ Latin alphabet is available in `notebook This model is capable of processing only one line of symbols at a time. The models used in this notebook are -`handwritten-japanese-recognition-0001 `__ +`handwritten-japanese-recognition-0001 `__ and -`handwritten-simplified-chinese-0001 `__. +`handwritten-simplified-chinese-0001 `__. To decode model outputs as readable text `kondate_nakayosi `__ and diff --git a/docs/notebooks/215-image-inpainting-with-output.rst b/docs/notebooks/215-image-inpainting-with-output.rst index f6ce5d5d06c..fba49c6bf49 100644 --- a/docs/notebooks/215-image-inpainting-with-output.rst +++ b/docs/notebooks/215-image-inpainting-with-output.rst @@ -34,7 +34,7 @@ Table of contents: .. parsed-literal:: - + [notice] A new release of pip is available: 23.2.1 -> 23.3.1 [notice] To update, run: pip install --upgrade pip Note: you may need to restart the kernel to use updated packages. @@ -44,13 +44,13 @@ Table of contents: import sys from pathlib import Path - + import cv2 import matplotlib.pyplot as plt import numpy as np from zipfile import ZipFile import openvino as ov - + sys.path.append("../utils") import notebook_utils as utils @@ -63,7 +63,7 @@ Download ``gmcnn-places2-tf``\ model (this step will be skipped if the model is already downloaded) and then unzip it. Downloaded model stored in TensorFlow frozen graph format. The steps how this frozen graph can be obtained from original model checkpoint can be found in this -`instruction `__ +`instruction `__ .. code:: ipython3 @@ -71,14 +71,14 @@ be obtained from original model checkpoint can be found in this base_model_dir = "model" # The name of the model from Open Model Zoo. model_name = "gmcnn-places2-tf" - + model_path = Path(f"{base_model_dir}/public/{model_name}/frozen_model.pb") if not model_path.exists(): model_url = f"https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2022.1/gmcnn-places2-tf/{model_name}.zip" utils.download_file(model_url, model_name, base_model_dir) else: print("Already downloaded") - + with ZipFile(f'{base_model_dir}/{model_name}' + '', "r") as zip_ref: zip_ref.extractall(path=Path(base_model_dir, 'public', )) @@ -96,14 +96,14 @@ Convert Tensorflow model to OpenVINO IR format The pre-trained model is in TensorFlow format. To use it with OpenVINO, convert it to OpenVINO IR format with model conversion API. For more information about model conversion, see this -`page `__. +`page `__. This step is also skipped if the model is already converted. .. code:: ipython3 model_dir = Path(base_model_dir, 'public', 'ir') ir_path = Path(f"{model_dir}/frozen_model.xml") - + # Run model conversion API to convert model to OpenVINO IR FP32 format, if the IR file does not exist. if not ir_path.exists(): ov_model = ov.convert_model(model_path, input=[[1,512,680,3],[1,512,680,1]]) @@ -135,21 +135,21 @@ Only a few lines of code are required to run the model: .. code:: ipython3 core = ov.Core() - + # Read the model.xml and weights file model = core.read_model(model=ir_path) .. code:: ipython3 import ipywidgets as widgets - + device = widgets.Dropdown( options=core.available_devices + ["AUTO"], value='AUTO', description='Device:', disabled=False, ) - + device @@ -194,14 +194,14 @@ original image. def create_mask(image_width, image_height, size_x=30, size_y=30, number=1): """ Create a square mask of defined size on a random location. - + :param: image_width: width of the image :param: image_height: height of the image :param: size: size in pixels of one side :returns: mask: grayscale float32 mask of size shaped [image_height, image_width, 1] """ - + mask = np.zeros((image_height, image_width, 1), dtype=np.float32) for _ in range(number): start_x = np.random.randint(image_width - size_x) @@ -237,14 +237,14 @@ you like. Just change the URL below. .. code:: ipython3 img_path = Path("data/laptop.png") - + if not img_path.exists(): # Download an image. url = "https://user-images.githubusercontent.com/29454499/281372079-fa8d84c4-8bf9-4a82-a1b9-5a74ad42ce47.png" image_file = utils.download_file( url, filename="laptop.png", directory="data", show_progress=False, silent=True, timeout=30 ) - + # Read the image. image = cv2.imread(str(img_path)) # Resize the image to meet network expected input sizes. diff --git a/docs/notebooks/216-attention-center-with-output.rst b/docs/notebooks/216-attention-center-with-output.rst index 6ef7808432b..cf97979a890 100644 --- a/docs/notebooks/216-attention-center-with-output.rst +++ b/docs/notebooks/216-attention-center-with-output.rst @@ -81,7 +81,7 @@ Table of contents: .. parsed-literal:: DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 - + .. parsed-literal:: @@ -96,12 +96,12 @@ Imports .. code:: ipython3 import cv2 - + import numpy as np import tensorflow as tf from pathlib import Path import matplotlib.pyplot as plt - + import openvino as ov @@ -565,7 +565,7 @@ The attention-center model is pre-trained model in TensorFlow Lite format. In this Notebook the model will be converted to OpenVINO IR format with model conversion API. For more information about model conversion, see this -`page `__. +`page `__. This step is also skipped if the model is already converted. Also TFLite models format is supported in OpenVINO by TFLite frontend, @@ -576,11 +576,11 @@ find example in .. code:: ipython3 tflite_model_path = Path("./attention-center/model/center.tflite") - + ir_model_path = Path("./model/ir_center_model.xml") - + core = ov.Core() - + if not ir_model_path.exists(): model = ov.convert_model(tflite_model_path, input=[('image:0', [1,480,640,3], ov.Type.f32)]) ov.save_model(model, ir_model_path) @@ -605,14 +605,14 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + device = widgets.Dropdown( options=core.available_devices + ["AUTO"], value='AUTO', description='Device:', disabled=False, ) - + device @@ -645,7 +645,7 @@ input. self.model_input_image_shape = model_input_image_shape self.image = None self.real_input_image_shape = None - + if image_path is not None: self.image = cv2.imread(str(image_path)) self.real_input_image_shape = self.image.shape @@ -654,20 +654,20 @@ input. self.real_input_image_shape = self.image.shape else: raise Exception("Sorry, image can't be found, please, specify image_path or image") - + def prepare_image_tensor(self): rgb_image = cv2.cvtColor(self.image, cv2.COLOR_BGR2RGB) resized_image = cv2.resize(rgb_image, (self.model_input_image_shape[1], self.model_input_image_shape[0])) - + image_tensor = tf.constant(np.expand_dims(resized_image, axis=0), dtype=tf.float32) return image_tensor - + def scalt_center_to_real_image_shape(self, predicted_center): new_center_y = round(predicted_center[0] * self.real_input_image_shape[1] / self.model_input_image_shape[1]) new_center_x = round(predicted_center[1] * self.real_input_image_shape[0] / self.model_input_image_shape[0]) return (int(new_center_y), int(new_center_x)) - + def draw_attention_center_point(self, predicted_center): image_with_circle = cv2.circle(self.image, predicted_center, @@ -675,12 +675,12 @@ input. color=(3, 3, 255), thickness=-1) return image_with_circle - + def print_image(self, predicted_center=None): image_to_print = self.image if predicted_center is not None: image_to_print = self.draw_attention_center_point(predicted_center) - + plt.imshow(cv2.cvtColor(image_to_print, cv2.COLOR_BGR2RGB)) Load input image @@ -693,11 +693,11 @@ Upload input image using file loading button .. code:: ipython3 import ipywidgets as widgets - + load_file_widget = widgets.FileUpload( accept="image/*", multiple=False, description="Image file", ) - + load_file_widget @@ -714,18 +714,18 @@ Upload input image using file loading button import io import PIL from urllib.request import urlretrieve - + img_path = Path("data/coco.jpg") img_path.parent.mkdir(parents=True, exist_ok=True) urlretrieve( "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg", img_path, ) - + # read uploaded image image = PIL.Image.open(io.BytesIO(list(load_file_widget.value.values())[-1]['content'])) if load_file_widget.value else PIL.Image.open(img_path) image.convert("RGB") - + input_image = Image((480, 640), image=(np.ascontiguousarray(image)[:, :, ::-1]).astype(np.uint8)) image_tensor = input_image.prepare_image_tensor() input_image.print_image() @@ -753,7 +753,7 @@ Get result with OpenVINO IR model .. code:: ipython3 output_layer = compiled_model.output(0) - + # make inference, get result in input image resolution res = compiled_model([image_tensor])[output_layer] # scale point to original image resulution diff --git a/docs/notebooks/219-knowledge-graphs-conve-with-output.rst b/docs/notebooks/219-knowledge-graphs-conve-with-output.rst index 00226d02f23..09d6caa4b27 100644 --- a/docs/notebooks/219-knowledge-graphs-conve-with-output.rst +++ b/docs/notebooks/219-knowledge-graphs-conve-with-output.rst @@ -561,7 +561,7 @@ Benchmark the converted OpenVINO model using benchmark app The OpenVINO toolkit provides a benchmarking application to gauge the platform specific runtime performance that can be obtained under optimal configuration parameters for a given model. For more details refer to: -https://docs.openvino.ai/2023.3/openvino_sample_benchmark_tool.html +https://docs.openvino.ai/2024/learn-openvino/openvino-samples/benchmark-tool.html Here, we use the benchmark application to obtain performance estimates under optimal configuration for the knowledge graph model inference. We @@ -674,7 +674,7 @@ evaluation on the knowledge graph. Then, we determine the platform specific speedup in runtime performance that can be obtained through OpenVINO graph optimizations. To learn more about the OpenVINO performance optimizations, refer to: -https://docs.openvino.ai/2023.3/openvino_docs_deployment_optimization_guide_dldt_optimization_guide.html +https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference.html References ~~~~~~~~~~ diff --git a/docs/notebooks/220-cross-lingual-books-alignment-with-output.rst b/docs/notebooks/220-cross-lingual-books-alignment-with-output.rst index eb00d5b5ca6..7ecbbd81698 100644 --- a/docs/notebooks/220-cross-lingual-books-alignment-with-output.rst +++ b/docs/notebooks/220-cross-lingual-books-alignment-with-output.rst @@ -476,7 +476,7 @@ Optimize the Model with OpenVINO The LaBSE model is quite large and can be slow to infer on some hardware, so let’s optimize it with OpenVINO. `Model Conversion -API `__ +API `__ accepts the PyTorch/Transformers model object and additional information about model inputs. An ``example_input`` is needed to trace the model execution graph, as PyTorch constructs it dynamically during inference. @@ -856,7 +856,7 @@ the pipeline - getting embeddings. You might wonder why, when using OpenVINO, you need to compile the model after reading it. There are two main reasons for this: 1. Compatibility with different devices. The model can be compiled to run on a `specific -device `__, +device `__, like CPU, GPU or GNA. Each device may work with different data types, support different features, and gain performance by changing the neural network for a specific computing model. With OpenVINO, you do not need @@ -865,13 +865,13 @@ hardware. A universal OpenVINO model representation is enough. 1. Optimization for different scenarios. For example, one scenario prioritizes minimizing the *time between starting and finishing model inference* (`latency-oriented -optimization `__). +optimization `__). In our case, it is more important *how many texts per second the model can process* (`throughput-oriented -optimization `__). +optimization `__). To get a throughput-optimized model, pass a `performance -hint `__ +hint `__ as a configuration during compilation. Then OpenVINO selects the optimal parameters for execution on the available hardware. @@ -935,7 +935,7 @@ advance and fill it in as the inference requests are executed. Let’s compare the models and plot the results. **NOTE**: To get a more accurate benchmark, use the `Benchmark Python - Tool `__ + Tool `__ .. code:: ipython3 @@ -1044,8 +1044,8 @@ boost. Here are useful links with information about the techniques used in this notebook: - `OpenVINO performance -hints `__ +hints `__ - `OpenVINO Async -API `__ +API `__ - `Throughput -Optimizations `__ +Optimizations `__ diff --git a/docs/notebooks/223-text-prediction-with-output.rst b/docs/notebooks/223-text-prediction-with-output.rst index cbfcc1655da..8fc602e1355 100644 --- a/docs/notebooks/223-text-prediction-with-output.rst +++ b/docs/notebooks/223-text-prediction-with-output.rst @@ -197,7 +197,7 @@ converted to OpenVINO Intermediate Representation (IR) format. HuggingFace provides a GPT-Neo model in PyTorch format, which is supported in OpenVINO via Model Conversion API. The ``ov.convert_model`` Python function of `model conversion -API `__ +API `__ can be used for converting the model. The function returns instance of OpenVINO Model class, which is ready to use in Python interface. The Model can also be save on device in OpenVINO IR format for future diff --git a/docs/notebooks/224-3D-segmentation-point-clouds-with-output.rst b/docs/notebooks/224-3D-segmentation-point-clouds-with-output.rst index d0299a5b08a..9fabecab330 100644 --- a/docs/notebooks/224-3D-segmentation-point-clouds-with-output.rst +++ b/docs/notebooks/224-3D-segmentation-point-clouds-with-output.rst @@ -41,7 +41,7 @@ Table of contents: .. parsed-literal:: DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 - + .. parsed-literal:: @@ -60,14 +60,14 @@ Imports import numpy as np import matplotlib.pyplot as plt import openvino as ov - + # Fetch `notebook_utils` module import urllib.request urllib.request.urlretrieve( url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py', filename='notebook_utils.py' ) - + from notebook_utils import download_file Prepare the Model @@ -97,14 +97,14 @@ function returns an OpenVINO model ready to load on a device and start making predictions. We can save it on a disk for next usage with ``ov.save_model``. For more information about model conversion Python API, see this -`page `__. +`page `__. .. code:: ipython3 ir_model_xml = onnx_model_path.with_suffix(".xml") - + core = ov.Core() - + if not ir_model_xml.exists(): # Convert model to OpenVINO Model model = ov.convert_model(onnx_model_path) @@ -113,7 +113,7 @@ API, see this else: # Read model model = core.read_model(model=ir_model_xml) - + Data Processing Module ---------------------- @@ -125,37 +125,37 @@ Data Processing Module def load_data(point_file: Union[str, Path]): """ Load the point cloud data and convert it to ndarray - + Parameters: point_file: string, path of .pts data Returns: point_set: point clound represented in np.array format """ - + point_set = np.loadtxt(point_file).astype(np.float32) - + # normailization point_set = point_set - np.expand_dims(np.mean(point_set, axis=0), 0) # center dist = np.max(np.sqrt(np.sum(point_set ** 2, axis=1)), 0) point_set = point_set / dist # scale - + return point_set - - + + def visualize(point_set:np.ndarray): """ Create a 3D view for data visualization - + Parameters: point_set: np.ndarray, the coordinate data in X Y Z format """ - + fig = plt.figure(dpi=192, figsize=(4, 4)) ax = fig.add_subplot(111, projection='3d') X = point_set[:, 0] Y = point_set[:, 2] Z = point_set[:, 1] - + # Scale the view of each axis to adapt to the coordinate data distribution max_range = np.array([X.max() - X.min(), Y.max() - Y.min(), Z.max() - Z.min()]).max() * 0.5 mid_x = (X.max() + X.min()) * 0.5 @@ -164,12 +164,12 @@ Data Processing Module ax.set_xlim(mid_x - max_range, mid_x + max_range) ax.set_ylim(mid_y - max_range, mid_y + max_range) ax.set_zlim(mid_z - max_range, mid_z + max_range) - + plt.tick_params(labelsize=5) ax.set_xlabel('X', fontsize=10) ax.set_ylabel('Y', fontsize=10) ax.set_zlabel('Z', fontsize=10) - + return ax Visualize the original 3D data @@ -189,7 +189,7 @@ chair for example. "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/pts/chair.pts", directory="data" ) - + points = load_data(str(point_data)) X = points[:, 0] Y = points[:, 2] @@ -232,11 +232,11 @@ each input point. # Parts of a chair classes = ['back', 'seat', 'leg', 'arm'] - + # Preprocess the input data point = points.transpose(1, 0) point = np.expand_dims(point, axis=0) - + # Print info about model input and output shape print(f"input shape: {model.input(0).partial_shape}") print(f"output shape: {model.output(0).partial_shape}") @@ -258,14 +258,14 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + device = widgets.Dropdown( options=core.available_devices + ["AUTO"], value='AUTO', description='Device:', disabled=False, ) - + device @@ -283,7 +283,7 @@ select device from dropdown list for running inference using OpenVINO compiled_model = core.compile_model(model=model, device_name=device.value) output_layer = compiled_model.output(0) result = compiled_model([point])[output_layer] - + # Find the label map for all points of chair with highest confidence pred = np.argmax(result[0], axis=1) ax = visualize(point) @@ -299,10 +299,10 @@ select device from dropdown list for running inference using OpenVINO XCur = np.array(XCur) YCur = np.array(YCur) ZCur = np.array(ZCur) - + # add current point of the part ax.scatter(XCur, YCur, ZCur, s=5, cmap="jet", marker="o", label=classes[i]) - + ax.set_title('3D Segmentation Visualization') plt.legend(loc='upper right', fontsize=8) plt.show() diff --git a/docs/notebooks/226-yolov7-optimization-with-output.rst b/docs/notebooks/226-yolov7-optimization-with-output.rst index 1bae731eaf8..f16088414c2 100644 --- a/docs/notebooks/226-yolov7-optimization-with-output.rst +++ b/docs/notebooks/226-yolov7-optimization-with-output.rst @@ -660,7 +660,7 @@ While ONNX models are directly supported by OpenVINO runtime, it can be useful to convert them to IR format to take the advantage of OpenVINO model conversion API features. The ``ov.convert_model`` python function of `model conversion -API `__ +API `__ can be used for converting the model. The function returns instance of OpenVINO Model class, which is ready to use in Python interface. However, it can also be save on device in OpenVINO IR format using @@ -1439,7 +1439,7 @@ Compare Performance of the Original and Quantized Models Finally, use the OpenVINO `Benchmark -Tool `__ +Tool `__ to measure the inference performance of the ``FP32`` and ``INT8`` models. diff --git a/docs/notebooks/228-clip-zero-shot-convert-with-output.rst b/docs/notebooks/228-clip-zero-shot-convert-with-output.rst index fc21a7025b8..de2f3db78a9 100644 --- a/docs/notebooks/228-clip-zero-shot-convert-with-output.rst +++ b/docs/notebooks/228-clip-zero-shot-convert-with-output.rst @@ -165,7 +165,7 @@ For best results with OpenVINO, it is recommended to convert the model to OpenVINO IR format. OpenVINO supports PyTorch via Model conversion API. To convert the PyTorch model to OpenVINO IR format we will use ``ov.convert_model`` of `model conversion -API `__. +API `__. The ``ov.convert_model`` Python function returns an OpenVINO Model object ready to load on the device and start making predictions. We can save it on disk for the next usage with ``ov.save_model``. diff --git a/docs/notebooks/229-distilbert-sequence-classification-with-output.rst b/docs/notebooks/229-distilbert-sequence-classification-with-output.rst index 62dbb164e01..d2e92206fbb 100644 --- a/docs/notebooks/229-distilbert-sequence-classification-with-output.rst +++ b/docs/notebooks/229-distilbert-sequence-classification-with-output.rst @@ -76,7 +76,7 @@ Imports .. parsed-literal:: DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 - + .. parsed-literal:: @@ -136,7 +136,7 @@ Convert Model to OpenVINO Intermediate Representation format `Model conversion -API `__ +API `__ facilitates the transition between training and deployment environments, performs static model analysis, and adjusts deep learning models for optimal execution on end-point target devices. @@ -144,11 +144,11 @@ optimal execution on end-point target devices. .. code:: ipython3 import torch - + ir_xml_name = checkpoint + ".xml" MODEL_DIR = "model/" ir_xml_path = Path(MODEL_DIR) / ir_xml_name - + MAX_SEQ_LENGTH = 128 input_info = [(ov.PartialShape([1, -1]), ov.Type.i64), (ov.PartialShape([1, -1]), ov.Type.i64)] default_input = torch.ones(1, MAX_SEQ_LENGTH, dtype=torch.int64) @@ -156,7 +156,7 @@ optimal execution on end-point target devices. "input_ids": default_input, "attention_mask": default_input, } - + ov_model = ov.convert_model(model, input=input_info, example_input=inputs) ov.save_model(ov_model, ir_xml_path) @@ -168,14 +168,14 @@ optimal execution on end-point target devices. OpenVINO™ Runtime uses the `Infer -Request `__ +Request `__ mechanism which enables running models on different devices in asynchronous or synchronous manners. The model graph is sent as an argument to the OpenVINO API and an inference request is created. The default inference mode is AUTO but it can be changed according to requirements and hardware available. You can explore the different inference modes and their usage `in -documentation. `__ +documentation. `__ .. code:: ipython3 @@ -191,14 +191,14 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + device = widgets.Dropdown( options=core.available_devices + ["AUTO"], value='AUTO', description='Device:', disabled=False, ) - + device @@ -225,7 +225,7 @@ select device from dropdown list for running inference using OpenVINO Parameters: Logits array Returns: Probabilities """ - + e_x = np.exp(x - np.max(x)) return e_x / e_x.sum() @@ -244,7 +244,7 @@ Inference Parameters: Text to be processed Returns: Label: Positive or Negative. """ - + input_text = tokenizer( input_text, truncation=True, @@ -293,7 +293,7 @@ Read from a text file filename='notebook_utils.py' ) from notebook_utils import download_file - + # Download the text from the openvino_notebooks storage vocab_file_path = download_file( "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/text/food_reviews.txt", @@ -324,11 +324,11 @@ Read from a text file .. parsed-literal:: User Input: The food was horrible. - - Label: NEGATIVE - + + Label: NEGATIVE + User Input: We went because the restaurant had good reviews. - Label: POSITIVE - + Label: POSITIVE + Total Time: 0.03 seconds diff --git a/docs/notebooks/230-yolov8-instance-segmentation-with-output.rst b/docs/notebooks/230-yolov8-instance-segmentation-with-output.rst index a1ae017a36e..a010c6ecaa5 100644 --- a/docs/notebooks/230-yolov8-instance-segmentation-with-output.rst +++ b/docs/notebooks/230-yolov8-instance-segmentation-with-output.rst @@ -989,7 +989,7 @@ Compare performance of the Original and Quantized Models Finally, use the OpenVINO `Benchmark -Tool `__ +Tool `__ to measure the inference performance of the ``FP32`` and ``INT8`` models. diff --git a/docs/notebooks/230-yolov8-keypoint-detection-with-output.rst b/docs/notebooks/230-yolov8-keypoint-detection-with-output.rst index 8728bc40966..b1729485993 100644 --- a/docs/notebooks/230-yolov8-keypoint-detection-with-output.rst +++ b/docs/notebooks/230-yolov8-keypoint-detection-with-output.rst @@ -981,7 +981,7 @@ Compare performance of the Original and Quantized Models Finally, use the OpenVINO `Benchmark -Tool `__ +Tool `__ to measure the inference performance of the ``FP32`` and ``INT8`` models. diff --git a/docs/notebooks/230-yolov8-object-detection-with-output.rst b/docs/notebooks/230-yolov8-object-detection-with-output.rst index 9558abba942..66c18d24214 100644 --- a/docs/notebooks/230-yolov8-object-detection-with-output.rst +++ b/docs/notebooks/230-yolov8-object-detection-with-output.rst @@ -951,7 +951,7 @@ Compare performance object detection models Finally, use the OpenVINO `Benchmark -Tool `__ +Tool `__ to measure the inference performance of the ``FP32`` and ``INT8`` models. @@ -1223,7 +1223,7 @@ CPU as part of an application. This will improve selected device utilization. For more information, refer to the overview of `Preprocessing -API `__. +API `__. For example, we can integrate converting input data layout and normalization defined in ``image_to_tensor`` function. diff --git a/docs/notebooks/231-instruct-pix2pix-image-editing-with-output.rst b/docs/notebooks/231-instruct-pix2pix-image-editing-with-output.rst index 1021ebf2773..bcb1b315561 100644 --- a/docs/notebooks/231-instruct-pix2pix-image-editing-with-output.rst +++ b/docs/notebooks/231-instruct-pix2pix-image-editing-with-output.rst @@ -100,7 +100,7 @@ Convert Models to OpenVINO IR OpenVINO supports PyTorch models using `Model Conversion -API `__ +API `__ to convert the model to IR format. ``ov.convert_model`` function accepts PyTorch model object and example input and then converts it to ``ov.Model`` class instance that ready to use for loading on device or diff --git a/docs/notebooks/236-stable-diffusion-v2-optimum-demo-with-output.rst b/docs/notebooks/236-stable-diffusion-v2-optimum-demo-with-output.rst index 2609ecc5d0f..56f28a07cbc 100644 --- a/docs/notebooks/236-stable-diffusion-v2-optimum-demo-with-output.rst +++ b/docs/notebooks/236-stable-diffusion-v2-optimum-demo-with-output.rst @@ -53,7 +53,7 @@ in this notebook is `helenai/stabilityai-stable-diffusion-2-1-base-ov `__. Let’s download the pre-converted model Stable Diffusion 2.1 `Intermediate Representation Format -(IR) `__ +(IR) `__ Showing Info Available Devices ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -77,10 +77,10 @@ this .. code:: ipython3 from openvino.runtime import Core - + ie = Core() devices = ie.available_devices - + for device in devices: device_name = ie.get_property(device, "FULL_DEVICE_NAME") print(f"{device}: {device_name}") @@ -103,7 +103,7 @@ Download Pre-Converted Stable Diffusion 2.1 IR from optimum.intel.openvino import OVStableDiffusionPipeline # download the pre-converted SD v2.1 model from Hugging Face Hub name = "helenai/stabilityai-stable-diffusion-2-1-base-ov" - + pipe = OVStableDiffusionPipeline.from_pretrained(name, compile=False) pipe.reshape(batch_size=1, height=512, width=512, num_images_per_prompt=1) @@ -216,8 +216,8 @@ Be creative, add the prompt and enjoy the result .. code:: ipython3 import gc - - # Generate an image. + + # Generate an image. prompt = "red car in snowy forest, epic vista, beautiful landscape, 4k, 8k" output = pipe(prompt, num_inference_steps=17, output_type="pil").images[0] output.save("image.png") diff --git a/docs/notebooks/237-segment-anything-with-output.rst b/docs/notebooks/237-segment-anything-with-output.rst index bcfc646f5c9..b8c679e3fa4 100644 --- a/docs/notebooks/237-segment-anything-with-output.rst +++ b/docs/notebooks/237-segment-anything-with-output.rst @@ -1529,7 +1529,7 @@ Compare Performance of the Original and Quantized Models Finally, use the OpenVINO `Benchmark -Tool `__ +Tool `__ to measure the inference performance of the ``FP32`` and ``INT8`` models. diff --git a/docs/notebooks/238-deep-floyd-if-optimize-with-output.rst b/docs/notebooks/238-deep-floyd-if-optimize-with-output.rst index 4a44147d750..13845873dde 100644 --- a/docs/notebooks/238-deep-floyd-if-optimize-with-output.rst +++ b/docs/notebooks/238-deep-floyd-if-optimize-with-output.rst @@ -749,7 +749,7 @@ Compare performance time of the converted and optimized models To measure the inference performance of OpenVINO FP16 and INT8 models, use `Benchmark -Tool `__. +Tool `__. **NOTE**: For more accurate performance, run ``benchmark_app`` in a terminal/command prompt after closing other applications. Run diff --git a/docs/notebooks/239-image-bind-convert-with-output.rst b/docs/notebooks/239-image-bind-convert-with-output.rst index d2974d5fa10..045601f71e8 100644 --- a/docs/notebooks/239-image-bind-convert-with-output.rst +++ b/docs/notebooks/239-image-bind-convert-with-output.rst @@ -95,14 +95,14 @@ Prerequisites .. code:: ipython3 import sys - + %pip install -q soundfile pytorchvideo ftfy "timm>=0.6.7" einops fvcore "openvino>=2023.1.0" numpy scipy matplotlib --extra-index-url https://download.pytorch.org/whl/cpu - + if sys.version_info.minor < 8: %pip install -q "decord" else: %pip install -q "eva-decord" - + if sys.platform != "linux": %pip install -q "torch>=2.0.1" "torchvision>=0.15.2,<0.17.0" "torchaudio>=2.0.2" else: @@ -111,12 +111,12 @@ Prerequisites .. code:: ipython3 from pathlib import Path - + repo_dir = Path("ImageBind") - + if not repo_dir.exists(): !git clone https://github.com/facebookresearch/ImageBind.git - + %cd {repo_dir} @@ -147,7 +147,7 @@ card `__. import torch from imagebind.models import imagebind_model from imagebind.models.imagebind_model import ModalityType - + # Instantiate model model = imagebind_model.imagebind_huge(pretrained=True) model.eval(); @@ -186,11 +186,11 @@ data reading and preprocessing for each modality. .. code:: ipython3 # Prepare inputs - + text_list = ["A car", "A bird", "A dog"] image_paths = [".assets/dog_image.jpg", ".assets/car_image.jpg", ".assets/bird_image.jpg"] audio_paths = [".assets/dog_audio.wav", ".assets/bird_audio.wav", ".assets/car_audio.wav"] - + inputs = { ModalityType.TEXT: data.load_and_transform_text(text_list, "cpu"), ModalityType.VISION: data.load_and_transform_vision_data(image_paths, "cpu"), @@ -204,7 +204,7 @@ Convert Model to OpenVINO Intermediate Representation (IR) format OpenVINO supports PyTorch through Model Conversion API. You will use `model conversion Python -API `__ +API `__ to convert model to IR format. The ``ov.convert_model`` function returns OpenVINO Model class instance ready to load on a device or save on a disk for next loading using ``ov.save_model``. @@ -223,14 +223,14 @@ embeddings. super().__init__() self.model = model self.modality = modality - + def forward(self, data): return self.model({self.modality: data}) .. code:: ipython3 import openvino as ov - + core = ov.Core() Select inference device @@ -243,14 +243,14 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + device = widgets.Dropdown( options=core.available_devices + ["AUTO"], value='AUTO', description='Device:', disabled=False, ) - + device @@ -265,7 +265,7 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 ov_modality_models = {} - + modalities = [ModalityType.TEXT, ModalityType.VISION, ModalityType.AUDIO] for modality in modalities: export_dir = Path(f"image-bind-{modality}") @@ -347,17 +347,17 @@ they represent the same object. import matplotlib.pyplot as plt import numpy as np from scipy.special import softmax - - + + def visualize_prob_matrix(matrix, x_label, y_label): fig, ax = plt.subplots() ax.matshow(matrix, cmap='winter') - + for (i, j), z in np.ndenumerate(matrix): ax.text(j, i, '{:0.3f}'.format(z), ha='center', va='center') ax.set_xticks(range(len(x_label)), x_label) ax.set_yticks(range(len(y_label)), y_label) - + image_list = [img.split('/')[-1] for img in image_paths] audio_list = [audio.split('/')[-1] for audio in audio_paths] @@ -369,7 +369,7 @@ Text-Image classification .. code:: ipython3 text_vision_scores = softmax(embeddings[ModalityType.VISION] @ embeddings[ModalityType.TEXT].T, axis=-1) - + visualize_prob_matrix(text_vision_scores, text_list, image_list) @@ -385,7 +385,7 @@ Text-Audio classification .. code:: ipython3 text_audio_scores = softmax(embeddings[ModalityType.AUDIO] @ embeddings[ModalityType.TEXT].T, axis=-1) - + visualize_prob_matrix(text_audio_scores, text_list, audio_list) @@ -401,7 +401,7 @@ Image-Audio classification .. code:: ipython3 audio_vision_scores = softmax(embeddings[ModalityType.VISION] @ embeddings[ModalityType.AUDIO].T, axis=-1) - + visualize_prob_matrix(audio_vision_scores, image_list, audio_list) @@ -424,7 +424,7 @@ Putting all together, we can match text, image, and sound for our data. .. parsed-literal:: - Predicted label: A car + Predicted label: A car probability for image - 1.000 probability for audio - 1.000 @@ -437,7 +437,7 @@ Putting all together, we can match text, image, and sound for our data. .. raw:: html - +