natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[DOCS] Updating Interactive Tutorials (#23466) 

An update of the `Interactive Tutorials` section.
This commit is contained in:
Sebastian Golebiewski 2024-03-15 11:39:54 +01:00 committed by GitHub
parent 6f8b70f245
commit d5890513f8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
350 changed files with 57961 additions and 70031 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/nbdoc/consts.py
View File

@ -6,7 +6,7 @@ repo_directory = "notebooks"
repo_owner = "openvinotoolkit"
repo_name = "openvino_notebooks"
repo_branch = "tree/main"
artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20240209220807/dist/rst_files/"
artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20240312220809/dist/rst_files/"
blacklisted_extensions = ['.xml', '.bin']
notebooks_repo = "https://github.com/openvinotoolkit/openvino_notebooks/blob/main/"
notebooks_binder = "https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath="

30
docs/notebooks/001-hello-world-with-output.rst
View File

@ -43,19 +43,19 @@ Imports
.. code:: ipython3
from pathlib import Path
import cv2
import matplotlib.pyplot as plt
import numpy as np
import openvino as ov
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py',
filename='notebook_utils.py'
)
from notebook_utils import download_file
Download the Model and data samples
@ -66,15 +66,15 @@ Download the Model and data samples
.. code:: ipython3
base_artifacts_dir = Path('./artifacts').expanduser()
model_name = "v3-small_224_1.0_float"
model_xml_name = f'{model_name}.xml'
model_bin_name = f'{model_name}.bin'
model_xml_path = base_artifacts_dir / model_xml_name
base_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/mobelinet-v3-tf/FP32/'
if not model_xml_path.exists():
download_file(base_url + model_xml_name, model_xml_name, base_artifacts_dir)
download_file(base_url + model_bin_name, model_bin_name, base_artifacts_dir)
@ -104,7 +104,7 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
@ -112,7 +112,7 @@ select device from dropdown list for running inference using OpenVINO
description='Device:',
disabled=False,
)
device
@ -134,7 +134,7 @@ Load the Model
core = ov.Core()
model = core.read_model(model=model_xml_path)
compiled_model = core.compile_model(model=model, device_name=device.value)
output_layer = compiled_model.output(0)
Load an Image
@ -149,13 +149,13 @@ Load an Image
"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg",
directory="data"
)
# The MobileNet model expects images in RGB format.
image = cv2.cvtColor(cv2.imread(filename=str(image_filename)), code=cv2.COLOR_BGR2RGB)
# Resize to MobileNet image shape.
input_image = cv2.resize(src=image, dsize=(224, 224))
# Reshape to model input shape.
input_image = np.expand_dims(input_image, 0)
plt.imshow(image);
@ -187,7 +187,7 @@ Do Inference
"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_2012.txt",
directory="data"
)
imagenet_classes = imagenet_filename.read_text().splitlines()
@ -202,7 +202,7 @@ Do Inference
# The model description states that for this model, class 0 is a background.
# Therefore, a background must be added at the beginning of imagenet_classes.
imagenet_classes = ['background'] + imagenet_classes
imagenet_classes[result_index]

2
docs/notebooks/001-hello-world-with-output_files/001-hello-world-with-output_11_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2dd4338c6c163e7693885ce544e8c9cd2aecedf3b136fa295e22877f37b5634c
oid sha256:5712bd24e962ae0e0267607554ebe1f2869c223b108876ce10e5d20fe6285126
size 387941

254
docs/notebooks/002-openvino-api-with-output.rst
View File

@ -41,16 +41,16 @@ Table of contents:
.. code:: ipython3
# Required imports. Please execute this cell first.
%pip install -q "openvino>=2023.1.0"
%pip install requests tqdm ipywidgets
%pip install -q "openvino>=2023.1.0"
%pip install -q requests tqdm ipywidgets
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py',
filename='notebook_utils.py'
)
from notebook_utils import download_file
@ -59,47 +59,6 @@ Table of contents:
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.31.0)
Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.66.1)
Requirement already satisfied: ipywidgets in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (8.1.2)
Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (3.3.2)
Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (3.6)
Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (2.2.0)
Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (2024.2.2)
Requirement already satisfied: comm>=0.1.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets) (0.2.1)
Requirement already satisfied: ipython>=6.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets) (8.12.3)
Requirement already satisfied: traitlets>=4.3.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets) (5.14.1)
Requirement already satisfied: widgetsnbextension~=4.0.10 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets) (4.0.10)
Requirement already satisfied: jupyterlab-widgets~=3.0.10 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets) (3.0.10)
.. parsed-literal::
Requirement already satisfied: backcall in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (0.2.0)
Requirement already satisfied: decorator in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1)
Requirement already satisfied: jedi>=0.16 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (0.19.1)
Requirement already satisfied: matplotlib-inline in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (0.1.6)
Requirement already satisfied: pickleshare in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (0.7.5)
Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.43)
Requirement already satisfied: pygments>=2.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (2.17.2)
Requirement already satisfied: stack-data in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3)
Requirement already satisfied: typing-extensions in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (4.9.0)
Requirement already satisfied: pexpect>4.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets) (4.9.0)
.. parsed-literal::
Requirement already satisfied: parso<0.9.0,>=0.8.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.3)
Requirement already satisfied: ptyprocess>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)
Requirement already satisfied: wcwidth in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets) (0.2.13)
Requirement already satisfied: executing>=1.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.0.1)
Requirement already satisfied: asttokens>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.4.1)
Requirement already satisfied: pure-eval in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (0.2.2)
Requirement already satisfied: six>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0)
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
@ -115,7 +74,7 @@ Initialize OpenVINO Runtime with ``ov.Core()``
.. code:: ipython3
import openvino as ov
core = ov.Core()
OpenVINO Runtime can load a network on a device. A device in this
@ -124,15 +83,10 @@ context means a CPU, an Intel GPU, a Neural Compute Stick 2, etc. The
system. The “FULL_DEVICE_NAME” option to ``core.get_property()`` shows
the name of the device.
In this notebook, the CPU device is used. To use an integrated GPU, use
``device_name="GPU"`` instead. Be aware that loading a network on GPU
will be slower than loading a network on CPU, but inference will likely
be faster.
.. code:: ipython3
devices = core.available_devices
for device in devices:
device_name = core.get_property(device, "FULL_DEVICE_NAME")
print(f"{device}: {device_name}")
@ -143,6 +97,34 @@ be faster.
CPU: Intel(R) Core(TM) i9-10920X CPU @ 3.50GHz
Select device for inference
~~~~~~~~~~~~~~~~~~~~~~~~~~~
You can specify which device from available devices will be used for
inference using this widget
.. code:: ipython3
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices,
value=core.available_devices[0],
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', options=('CPU',), value='CPU')
Loading a Model
---------------
@ -153,7 +135,7 @@ After initializing OpenVINO Runtime, first read the model file with
``compile_model()`` method.
`OpenVINO™ supports several model
formats <https://docs.openvino.ai/2024/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/%5Blegacy%5D-supported-model-formats.html>`__
formats <https://docs.openvino.ai/2024/openvino-workflow/model-preparation/convert-model-to-ir.html>`__
and enables developers to convert them to its own OpenVINO IR format
using a tool dedicated to this task.
@ -193,7 +175,7 @@ notebooks.
ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/'
ir_model_name_xml = 'classification.xml'
ir_model_name_bin = 'classification.bin'
download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model')
download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model')
@ -214,19 +196,19 @@ notebooks.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
.. code:: ipython3
import openvino as ov
core = ov.Core()
classification_model_xml = "model/classification.xml"
model = core.read_model(model=classification_model_xml)
compiled_model = core.compile_model(model=model, device_name="CPU")
compiled_model = core.compile_model(model=model, device_name=device.value)
ONNX Model
~~~~~~~~~~
@ -249,7 +231,7 @@ points to the filename of an ONNX model.
onnx_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/segmentation.onnx'
onnx_model_name = 'segmentation.onnx'
download_file(onnx_model_url, filename=onnx_model_name, directory='model')
@ -263,19 +245,19 @@ points to the filename of an ONNX model.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.onnx')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.onnx')
.. code:: ipython3
import openvino as ov
core = ov.Core()
onnx_model_path = "model/segmentation.onnx"
model_onnx = core.read_model(model=onnx_model_path)
compiled_model_onnx = core.compile_model(model=model_onnx, device_name="CPU")
compiled_model_onnx = core.compile_model(model=model_onnx, device_name=device.value)
The ONNX model can be exported to OpenVINO IR with ``save_model()``:
@ -298,7 +280,7 @@ without any conversion step. Pass the filename with extension to
paddle_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/'
paddle_model_name = 'inference.pdmodel'
paddle_params_name = 'inference.pdiparams'
download_file(paddle_model_url + paddle_model_name, filename=paddle_model_name, directory='model')
download_file(paddle_model_url + paddle_params_name, filename=paddle_params_name, directory='model')
@ -319,19 +301,19 @@ without any conversion step. Pass the filename with extension to
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/inference.pdiparams')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/inference.pdiparams')
.. code:: ipython3
import openvino as ov
core = ov.Core()
paddle_model_path = 'model/inference.pdmodel'
model_paddle = core.read_model(model=paddle_model_path)
compiled_model_paddle = core.compile_model(model=model_paddle, device_name="CPU")
compiled_model_paddle = core.compile_model(model=model_paddle, device_name=device.value)
.. code:: ipython3
@ -349,7 +331,7 @@ TensorFlow models saved in frozen graph format can also be passed to
pb_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/classification.pb'
pb_model_name = 'classification.pb'
download_file(pb_model_url, filename=pb_model_name, directory='model')
@ -363,19 +345,19 @@ TensorFlow models saved in frozen graph format can also be passed to
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.pb')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.pb')
.. code:: ipython3
import openvino as ov
core = ov.Core()
tf_model_path = "model/classification.pb"
model_tf = core.read_model(model=tf_model_path)
compiled_model_tf = core.compile_model(model=model_tf, device_name="CPU")
compiled_model_tf = core.compile_model(model=model_tf, device_name=device.value)
.. code:: ipython3
@ -398,10 +380,10 @@ It is pre-trained model optimized to work with TensorFlow Lite.
.. code:: ipython3
from pathlib import Path
tflite_model_url = 'https://www.kaggle.com/models/tensorflow/inception/frameworks/tfLite/variations/v4-quant/versions/1?lite-format=tflite'
tflite_model_path = Path('model/classification.tflite')
download_file(tflite_model_url, filename=tflite_model_path.name, directory=tflite_model_path.parent)
@ -415,18 +397,18 @@ It is pre-trained model optimized to work with TensorFlow Lite.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.tflite')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.tflite')
.. code:: ipython3
import openvino as ov
core = ov.Core()
model_tflite = core.read_model(tflite_model_path)
compiled_model_tflite = core.compile_model(model=model_tflite, device_name="CPU")
compiled_model_tflite = core.compile_model(model=model_tflite, device_name=device.value)
.. code:: ipython3
@ -453,15 +435,15 @@ model form torchvision library. After conversion model using
import openvino as ov
import torch
from torchvision.models import resnet18, ResNet18_Weights
core = ov.Core()
pt_model = resnet18(weights=ResNet18_Weights.IMAGENET1K_V1)
example_input = torch.zeros((1, 3, 224, 224))
ov_model_pytorch = ov.convert_model(pt_model, example_input=example_input)
compiled_model_pytorch = core.compile_model(ov_model_pytorch, device_name="CPU")
compiled_model_pytorch = core.compile_model(ov_model_pytorch, device_name=device.value)
ov.save_model(ov_model_pytorch, "model/exported_pytorch_model.xml")
Getting Information about a Model
@ -481,7 +463,7 @@ Information about the inputs and outputs of the model are in
ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/'
ir_model_name_xml = 'classification.xml'
ir_model_name_bin = 'classification.bin'
download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model')
download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model')
@ -500,7 +482,7 @@ Information about the inputs and outputs of the model are in
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
@ -515,7 +497,7 @@ dictionary.
.. code:: ipython3
import openvino as ov
core = ov.Core()
classification_model_xml = "model/classification.xml"
model = core.read_model(model=classification_model_xml)
@ -587,7 +569,7 @@ Model Outputs
.. code:: ipython3
import openvino as ov
core = ov.Core()
classification_model_xml = "model/classification.xml"
model = core.read_model(model=classification_model_xml)
@ -691,7 +673,7 @@ produced data as values.
ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/'
ir_model_name_xml = 'classification.xml'
ir_model_name_bin = 'classification.bin'
download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model')
download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model')
@ -710,18 +692,18 @@ produced data as values.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
.. code:: ipython3
import openvino as ov
core = ov.Core()
classification_model_xml = "model/classification.xml"
model = core.read_model(model=classification_model_xml)
compiled_model = core.compile_model(model=model, device_name="CPU")
compiled_model = core.compile_model(model=model, device_name=device.value)
input_layer = compiled_model.input(0)
output_layer = compiled_model.output(0)
@ -734,7 +716,7 @@ the input layout of the network.
.. code:: ipython3
import cv2
image_filename = download_file(
"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_hollywood.jpg",
directory="data"
@ -789,7 +771,7 @@ add the ``N`` dimension (where ``N``\ = 1) by calling the
.. code:: ipython3
import numpy as np
input_data = np.expand_dims(np.transpose(resized_image, (2, 0, 1)), 0).astype(np.float32)
input_data.shape
@ -814,10 +796,10 @@ predicted result in ``np.array`` format.
# for single input models only
result = compiled_model(input_data)[output_layer]
# for multiple inputs in a list
result = compiled_model([input_data])[output_layer]
# or using a dictionary, where the key is input tensor name or index
result = compiled_model({input_layer.any_name: input_data})[output_layer]
@ -878,7 +860,7 @@ input shape.
ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/'
ir_model_name_xml = 'segmentation.xml'
ir_model_name_bin = 'segmentation.bin'
download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model')
download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model')
@ -899,27 +881,27 @@ input shape.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.bin')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.bin')
.. code:: ipython3
import openvino as ov
core = ov.Core()
segmentation_model_xml = "model/segmentation.xml"
segmentation_model = core.read_model(model=segmentation_model_xml)
segmentation_input_layer = segmentation_model.input(0)
segmentation_output_layer = segmentation_model.output(0)
print("~~~~ ORIGINAL MODEL ~~~~")
print(f"input shape: {segmentation_input_layer.shape}")
print(f"output shape: {segmentation_output_layer.shape}")
new_shape = ov.PartialShape([1, 3, 544, 544])
segmentation_model.reshape({segmentation_input_layer.any_name: new_shape})
segmentation_compiled_model = core.compile_model(model=segmentation_model, device_name="CPU")
segmentation_compiled_model = core.compile_model(model=segmentation_model, device_name=device.value)
# help(segmentation_compiled_model)
print("~~~~ RESHAPED MODEL ~~~~")
print(f"model input shape: {segmentation_input_layer.shape}")
@ -966,15 +948,15 @@ set ``new_shape = (2,3,544,544)`` in the cell above.
.. code:: ipython3
import openvino as ov
segmentation_model_xml = "model/segmentation.xml"
segmentation_model = core.read_model(model=segmentation_model_xml)
segmentation_input_layer = segmentation_model.input(0)
segmentation_output_layer = segmentation_model.output(0)
new_shape = ov.PartialShape([2, 3, 544, 544])
segmentation_model.reshape({segmentation_input_layer.any_name: new_shape})
segmentation_compiled_model = core.compile_model(model=segmentation_model, device_name="CPU")
segmentation_compiled_model = core.compile_model(model=segmentation_model, device_name=device.value)
print(f"input shape: {segmentation_input_layer.shape}")
print(f"output shape: {segmentation_output_layer.shape}")
@ -993,7 +975,7 @@ input image through the network to see the result:
import numpy as np
import openvino as ov
core = ov.Core()
segmentation_model_xml = "model/segmentation.xml"
segmentation_model = core.read_model(model=segmentation_model_xml)
@ -1001,11 +983,11 @@ input image through the network to see the result:
segmentation_output_layer = segmentation_model.output(0)
new_shape = ov.PartialShape([2, 3, 544, 544])
segmentation_model.reshape({segmentation_input_layer.any_name: new_shape})
segmentation_compiled_model = core.compile_model(model=segmentation_model, device_name="CPU")
segmentation_compiled_model = core.compile_model(model=segmentation_model, device_name=device.value)
input_data = np.random.rand(2, 3, 544, 544)
output = segmentation_compiled_model([input_data])
print(f"input data shape: {input_data.shape}")
print(f"result data data shape: {segmentation_output_layer.shape}")
@ -1043,7 +1025,7 @@ the cache.
ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/'
ir_model_name_xml = 'classification.xml'
ir_model_name_bin = 'classification.bin'
download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model')
download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model')
@ -1062,7 +1044,7 @@ the cache.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
@ -1070,27 +1052,30 @@ the cache.
import time
from pathlib import Path
import openvino as ov
core = ov.Core()
cache_path = Path("model/model_cache")
cache_path.mkdir(exist_ok=True)
# Enable caching for OpenVINO Runtime. To disable caching set enable_caching = False
enable_caching = True
config_dict = {"CACHE_DIR": str(cache_path)} if enable_caching else {}
classification_model_xml = "model/classification.xml"
model = core.read_model(model=classification_model_xml)
start_time = time.perf_counter()
compiled_model = core.compile_model(model=model, device_name=device.value, config=config_dict)
end_time = time.perf_counter()
print(f"Loading the network to the {device.value} device took {end_time-start_time:.2f} seconds.")
device_name = "GPU"
if device_name in core.available_devices:
cache_path = Path("model/model_cache")
cache_path.mkdir(exist_ok=True)
# Enable caching for OpenVINO Runtime. To disable caching set enable_caching = False
enable_caching = True
config_dict = {"CACHE_DIR": str(cache_path)} if enable_caching else {}
.. parsed-literal::
classification_model_xml = "model/classification.xml"
model = core.read_model(model=classification_model_xml)
Loading the network to the CPU device took 0.16 seconds.
start_time = time.perf_counter()
compiled_model = core.compile_model(model=model, device_name=device_name, config=config_dict)
end_time = time.perf_counter()
print(f"Loading the network to the {device_name} device took {end_time-start_time:.2f} seconds.")
After running the previous cell, we know the model exists in the cache
directory. Then, we delete the compiled model and load it again. Now, we
@ -1098,9 +1083,14 @@ measure the time it takes now.
.. code:: ipython3
if device_name in core.available_devices:
del compiled_model
start_time = time.perf_counter()
compiled_model = core.compile_model(model=model, device_name=device_name, config=config_dict)
end_time = time.perf_counter()
print(f"Loading the network to the {device_name} device took {end_time-start_time:.2f} seconds.")
del compiled_model
start_time = time.perf_counter()
compiled_model = core.compile_model(model=model, device_name=device.value, config=config_dict)
end_time = time.perf_counter()
print(f"Loading the network to the {device.value} device took {end_time-start_time:.2f} seconds.")
.. parsed-literal::
Loading the network to the CPU device took 0.09 seconds.

52
docs/notebooks/003-hello-segmentation-with-output.rst
View File

@ -44,14 +44,14 @@ Imports
import matplotlib.pyplot as plt
import numpy as np
import openvino as ov
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py',
filename='notebook_utils.py'
)
from notebook_utils import segmentation_map_to_image, download_file
Download model weights
@ -62,19 +62,19 @@ Download model weights
.. code:: ipython3
from pathlib import Path
base_model_dir = Path("./model").expanduser()
model_name = "road-segmentation-adas-0001"
model_xml_name = f'{model_name}.xml'
model_bin_name = f'{model_name}.bin'
model_xml_path = base_model_dir / model_xml_name
if not model_xml_path.exists():
model_xml_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/road-segmentation-adas-0001/FP32/road-segmentation-adas-0001.xml"
model_bin_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/road-segmentation-adas-0001/FP32/road-segmentation-adas-0001.bin"
download_file(model_xml_url, model_xml_name, base_model_dir)
download_file(model_bin_url, model_bin_name, base_model_dir)
else:
@ -103,7 +103,7 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
@ -111,7 +111,7 @@ select device from dropdown list for running inference using OpenVINO
description='Device:',
disabled=False,
)
device
@ -131,10 +131,10 @@ Load the Model
.. code:: ipython3
core = ov.Core()
model = core.read_model(model=model_xml_path)
compiled_model = core.compile_model(model=model, device_name=device.value)
input_layer_ir = compiled_model.input(0)
output_layer_ir = compiled_model.output(0)
@ -152,23 +152,23 @@ is provided.
"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/empty_road_mapillary.jpg",
directory="data"
)
# The segmentation network expects images in BGR format.
image = cv2.imread(str(image_filename))
rgb_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
image_h, image_w, _ = image.shape
# N,C,H,W = batch size, number of channels, height, width.
N, C, H, W = input_layer_ir.shape
# OpenCV resize expects the destination size as (width, height).
resized_image = cv2.resize(image, (W, H))
# Reshape to the network input shape.
input_image = np.expand_dims(
resized_image.transpose(2, 0, 1), 0
)
)
plt.imshow(rgb_image)
@ -182,7 +182,7 @@ is provided.
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f11c8142580>
<matplotlib.image.AxesImage at 0x7fe68e54a130>
@ -199,7 +199,7 @@ Do Inference
# Run the inference.
result = compiled_model([input_image])[output_layer_ir]
# Prepare data for visualization.
segmentation_mask = np.argmax(result, axis=1)
plt.imshow(segmentation_mask.transpose(1, 2, 0))
@ -209,7 +209,7 @@ Do Inference
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f11c8051eb0>
<matplotlib.image.AxesImage at 0x7fe65848a160>
@ -226,14 +226,14 @@ Prepare Data for Visualization
# Define colormap, each color represents a class.
colormap = np.array([[68, 1, 84], [48, 103, 141], [53, 183, 120], [199, 216, 52]])
# Define the transparency of the segmentation mask on the photo.
alpha = 0.3
# Use function from notebook_utils.py to transform mask to an RGB image.
mask = segmentation_map_to_image(segmentation_mask, colormap)
resized_mask = cv2.resize(mask, (image_w, image_h))
# Create an image with mask.
image_with_mask = cv2.addWeighted(resized_mask, alpha, rgb_image, 1 - alpha, 0)
@ -246,16 +246,16 @@ Visualize data
# Define titles with images.
data = {"Base Photo": rgb_image, "Segmentation": mask, "Masked Photo": image_with_mask}
# Create a subplot to visualize images.
fig, axs = plt.subplots(1, len(data.items()), figsize=(15, 10))
# Fill the subplot.
for ax, (name, image) in zip(axs, data.items()):
ax.axis('off')
ax.set_title(name)
ax.imshow(image)
# Display an image.
plt.show(fig)

2
docs/notebooks/003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_11_2.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:90fac1140a8f375892b9637d611b5b4d7e51f78a45f3af3ab3b33e74b97baa18
oid sha256:96f0eb3a9535d57b8784be4b717dc9f280e4bf107e5b61d7cf51b36e142e4c7a
size 249032

2
docs/notebooks/003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_13_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:57f846855a237159135e3491021459d4d94b97ddb0458c74b7bc35cca5ff0d5a
oid sha256:caef59a6c15a5a1d512f4dd22395b12fbd754bba264ea5f0deae323ff8edee39
size 20550

2
docs/notebooks/003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_17_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:655bbfae2c0894ca52bb1cdaf7de64bbad747265984e968afc86f0198bc6600d
oid sha256:6a3137d9359a44fb19e1900e6b808f9e7e7ded0ba209abe8c4bd90fcf37b1c6a
size 260045

52
docs/notebooks/004-hello-detection-with-output.rst
View File

@ -50,14 +50,14 @@ Imports
import numpy as np
import openvino as ov
from pathlib import Path
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py',
filename='notebook_utils.py'
)
from notebook_utils import download_file
Download model weights
@ -68,18 +68,18 @@ Download model weights
.. code:: ipython3
base_model_dir = Path("./model").expanduser()
model_name = "horizontal-text-detection-0001"
model_xml_name = f'{model_name}.xml'
model_bin_name = f'{model_name}.bin'
model_xml_path = base_model_dir / model_xml_name
model_bin_path = base_model_dir / model_bin_name
if not model_xml_path.exists():
model_xml_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.xml"
model_bin_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.bin"
download_file(model_xml_url, model_xml_name, base_model_dir)
download_file(model_bin_url, model_bin_name, base_model_dir)
else:
@ -108,7 +108,7 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
@ -116,7 +116,7 @@ select device from dropdown list for running inference using OpenVINO
description='Device:',
disabled=False,
)
device
@ -136,10 +136,10 @@ Load the Model
.. code:: ipython3
core = ov.Core()
model = core.read_model(model=model_xml_path)
compiled_model = core.compile_model(model=model, device_name="CPU")
compiled_model = core.compile_model(model=model, device_name=device.value)
input_layer_ir = compiled_model.input(0)
output_layer_ir = compiled_model.output("boxes")
@ -155,19 +155,19 @@ Load an Image
"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/intel_rnb.jpg",
directory="data"
)
# Text detection models expect an image in BGR format.
image = cv2.imread(str(image_filename))
# N,C,H,W = batch size, number of channels, height, width.
N, C, H, W = input_layer_ir.shape
# Resize the image to meet network expected input sizes.
resized_image = cv2.resize(image, (W, H))
# Reshape to the network input shape.
input_image = np.expand_dims(resized_image.transpose(2, 0, 1), 0)
plt.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB));
@ -190,7 +190,7 @@ Do Inference
# Create an inference request.
boxes = compiled_model([input_image])[output_layer_ir]
# Remove zero only boxes.
boxes = boxes[~np.all(boxes == 0, axis=1)]
@ -206,31 +206,31 @@ Visualize Results
def convert_result_to_image(bgr_image, resized_image, boxes, threshold=0.3, conf_labels=True):
# Define colors for boxes and descriptions.
colors = {"red": (255, 0, 0), "green": (0, 255, 0)}
# Fetch the image shapes to calculate a ratio.
(real_y, real_x), (resized_y, resized_x) = bgr_image.shape[:2], resized_image.shape[:2]
ratio_x, ratio_y = real_x / resized_x, real_y / resized_y
# Convert the base image from BGR to RGB format.
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
# Iterate through non-zero boxes.
for box in boxes:
# Pick a confidence factor from the last place in an array.
conf = box[-1]
if conf > threshold:
# Convert float to int and multiply corner position of each box by x and y ratio.
# If the bounding box is found at the top of the image,
# position the upper box bar little lower to make it visible on the image.
# If the bounding box is found at the top of the image,
# position the upper box bar little lower to make it visible on the image.
(x_min, y_min, x_max, y_max) = [
int(max(corner_position * ratio_y, 10)) if idx % 2
int(max(corner_position * ratio_y, 10)) if idx % 2
else int(corner_position * ratio_x)
for idx, corner_position in enumerate(box[:-1])
]
# Draw a box based on the position, parameters in rectangle function are: image, start_point, end_point, color, thickness.
rgb_image = cv2.rectangle(rgb_image, (x_min, y_min), (x_max, y_max), colors["green"], 3)
# Add text to the image based on position and confidence.
# Parameters in text function are: image, text, bottom-left_corner_textfield, font, font_scale, color, thickness, line_type.
if conf_labels:
@ -244,7 +244,7 @@ Visualize Results
1,
cv2.LINE_AA,
)
return rgb_image
.. code:: ipython3

2
docs/notebooks/004-hello-detection-with-output_files/004-hello-detection-with-output_11_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:538f477cad7f1fad28669553d8cac16e28f403c0eab9e95a2546b071dd2fbfa1
oid sha256:c7a830fedc5653fd506c656144decc048cad5a7651c8e498024f0eb0ab8c8e96
size 305482

2
docs/notebooks/004-hello-detection-with-output_files/004-hello-detection-with-output_16_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:18a797607c8b3a7b14e6949e480c9ccd6a9629fa2026cdedfe5c4b6ab891edbd
oid sha256:edb00cb4f0e2c42cd9e0f90939afbd6352ca40c90866821898f2c42c1fd9df64
size 457214

64
docs/notebooks/101-tensorflow-classification-to-openvino-with-output.rst
View File

@ -8,7 +8,7 @@ Representation <https://docs.openvino.ai/2024/documentation/openvino-ir-format/o
(OpenVINO IR) format, using `Model Conversion
API <https://docs.openvino.ai/2024/openvino-workflow/model-preparation.html>`__.
After creating the OpenVINO IR, load the model in `OpenVINO
Runtime <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_OV_Runtime_User_Guide.html>`__
Runtime <https://docs.openvino.ai/2024/openvino-workflow/running-inference.html>`__
and do inference with a sample image.
Table of contents:
@ -56,33 +56,33 @@ Imports
import time
from pathlib import Path
import cv2
import matplotlib.pyplot as plt
import numpy as np
import openvino as ov
import tensorflow as tf
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py',
filename='notebook_utils.py'
)
from notebook_utils import download_file
.. parsed-literal::
2024-02-09 22:34:07.759850: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-02-09 22:34:07.794264: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2024-03-12 22:18:47.211125: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-03-12 22:18:47.245088: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
.. parsed-literal::
2024-02-09 22:34:08.310440: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2024-03-12 22:18:47.761261: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
Settings
@ -95,9 +95,9 @@ Settings
# The paths of the source and converted models.
model_dir = Path("model")
model_dir.mkdir(exist_ok=True)
model_path = Path("model/v3-small_224_1.0_float")
ir_path = Path("model/v3-small_224_1.0_float.xml")
Download model
@ -122,12 +122,12 @@ and save it to the disk.
.. parsed-literal::
2024-02-09 22:34:11.190073: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW
2024-02-09 22:34:11.190106: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:168] retrieving CUDA diagnostic information for host: iotg-dev-workstation-07
2024-02-09 22:34:11.190111: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:175] hostname: iotg-dev-workstation-07
2024-02-09 22:34:11.190249: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:199] libcuda reported version is: 470.223.2
2024-02-09 22:34:11.190264: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:203] kernel reported version is: 470.182.3
2024-02-09 22:34:11.190268: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration
2024-03-12 22:18:50.501463: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW
2024-03-12 22:18:50.501497: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:168] retrieving CUDA diagnostic information for host: iotg-dev-workstation-07
2024-03-12 22:18:50.501501: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:175] hostname: iotg-dev-workstation-07
2024-03-12 22:18:50.501645: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:199] libcuda reported version is: 470.223.2
2024-03-12 22:18:50.501661: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:203] kernel reported version is: 470.182.3
2024-03-12 22:18:50.501665: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration
.. parsed-literal::
@ -137,13 +137,13 @@ and save it to the disk.
.. parsed-literal::
2024-02-09 22:34:15.411337: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
2024-03-12 22:18:54.762256: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
[[{{node inputs}}]]
.. parsed-literal::
2024-02-09 22:34:18.568762: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
2024-03-12 22:18:57.938498: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
[[{{node inputs}}]]
WARNING:absl:Found untraced functions such as _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op while saving (showing 5 of 54). These functions will not be directly callable after loading.
@ -219,14 +219,14 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
@ -251,7 +251,7 @@ Get Model Information
input_key = compiled_model.input(0)
output_key = compiled_model.output(0)
network_input_shape = input_key.shape
network_input_shape = input_key.shape
Load an Image
~~~~~~~~~~~~~
@ -268,16 +268,16 @@ network.
"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg",
directory="data"
)
# The MobileNet network expects images in RGB format.
image = cv2.cvtColor(cv2.imread(filename=str(image_filename)), code=cv2.COLOR_BGR2RGB)
# Resize the image to the network input shape.
resized_image = cv2.resize(src=image, dsize=(224, 224))
# Transpose the image to the network input shape.
input_image = np.expand_dims(resized_image, 0)
plt.imshow(image);
@ -299,7 +299,7 @@ Do Inference
.. code:: ipython3
result = compiled_model(input_image)[output_key]
result_index = np.argmax(result)
.. code:: ipython3
@ -309,10 +309,10 @@ Do Inference
"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_2012.txt",
directory="data"
)
# Convert the inference result to a class name.
imagenet_classes = image_filename.read_text().splitlines()
imagenet_classes[result_index]
@ -345,15 +345,15 @@ performance.
.. code:: ipython3
num_images = 1000
start = time.perf_counter()
for _ in range(num_images):
compiled_model([input_image])
end = time.perf_counter()
time_ir = end - start
print(
f"IR model in OpenVINO Runtime/CPU: {time_ir/num_images:.4f} "
f"seconds per image, FPS: {num_images/time_ir:.2f}"
@ -362,5 +362,5 @@ performance.
.. parsed-literal::
IR model in OpenVINO Runtime/CPU: 0.0011 seconds per image, FPS: 926.34
IR model in OpenVINO Runtime/CPU: 0.0011 seconds per image, FPS: 946.40

2
docs/notebooks/101-tensorflow-classification-to-openvino-with-output_files/101-tensorflow-classification-to-openvino-with-output_19_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2dd4338c6c163e7693885ce544e8c9cd2aecedf3b136fa295e22877f37b5634c
oid sha256:5712bd24e962ae0e0267607554ebe1f2869c223b108876ce10e5d20fe6285126
size 387941

74
docs/notebooks/102-pytorch-onnx-to-openvino-with-output.rst
View File

@ -92,20 +92,20 @@ Imports
import time
import warnings
from pathlib import Path
import cv2
import numpy as np
import openvino as ov
import torch
from torchvision.models.segmentation import lraspp_mobilenet_v3_large, LRASPP_MobileNet_V3_Large_Weights
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py',
filename='notebook_utils.py'
)
from notebook_utils import segmentation_map_to_image, viz_result_image, SegmentationMap, Label, download_file
Settings
@ -125,7 +125,7 @@ transforms function, the model is pre-trained on images with a height of
DIRECTORY_NAME = "model"
BASE_MODEL_NAME = DIRECTORY_NAME + "/lraspp_mobilenet_v3_large"
weights_path = Path(BASE_MODEL_NAME + ".pt")
# Paths where ONNX and OpenVINO IR models will be stored.
onnx_path = weights_path.with_suffix('.onnx')
if not onnx_path.parent.exists():
@ -156,7 +156,7 @@ have not downloaded the model before.
.. code:: ipython3
print("Downloading the LRASPP MobileNetV3 model (if it has not been downloaded already)...")
print("Downloading the LRASPP MobileNetV3 model (if it has not been downloaded already)...")
download_file(LRASPP_MobileNet_V3_Large_Weights.COCO_WITH_VOC_LABELS_V1.url, filename=weights_path.name, directory=weights_path.parent)
# create model object
model = lraspp_mobilenet_v3_large()
@ -294,12 +294,12 @@ Images need to be normalized before propagating through the network.
"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg",
directory="data"
)
image = cv2.cvtColor(cv2.imread(str(image_filename)), cv2.COLOR_BGR2RGB)
resized_image = cv2.resize(image, (IMAGE_WIDTH, IMAGE_HEIGHT))
normalized_image = normalize(resized_image)
# Convert the resized images to network input shape.
input_image = np.expand_dims(np.transpose(resized_image, (2, 0, 1)), 0)
normalized_input_image = np.expand_dims(np.transpose(normalized_image, (2, 0, 1)), 0)
@ -331,7 +331,7 @@ on an image.
# Instantiate OpenVINO Core
core = ov.Core()
# Read model to OpenVINO Runtime
model_onnx = core.read_model(model=onnx_path)
@ -345,14 +345,14 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
@ -368,7 +368,7 @@ select device from dropdown list for running inference using OpenVINO
# Load model on device
compiled_model_onnx = core.compile_model(model=model_onnx, device_name=device.value)
# Run inference on the input image
res_onnx = compiled_model_onnx([normalized_input_image])[0]
@ -403,7 +403,7 @@ be applied to each label for more convenient visualization.
Label(index=20, color=(0, 64, 128), name="tv monitor")
]
VOCLabels = SegmentationMap(voc_labels)
# Convert the network result to a segmentation map and display the result.
result_mask_onnx = np.squeeze(np.argmax(res_onnx, axis=1)).astype(np.uint8)
viz_result_image(
@ -450,10 +450,10 @@ select device from dropdown list for running inference using OpenVINO
core = ov.Core()
model_ir = core.read_model(model=ir_path)
compiled_model_ir = core.compile_model(model=model_ir, device_name=device.value)
# Get input and output layers.
output_layer_ir = compiled_model_ir.output(0)
# Run inference on the input image.
res_ir = compiled_model_ir([normalized_input_image])[output_layer_ir]
@ -486,7 +486,7 @@ looks the same as the output on the ONNX/OpenVINO IR models.
model.eval()
with torch.no_grad():
result_torch = model(torch.as_tensor(normalized_input_image).float())
result_mask_torch = torch.argmax(result_torch['out'], dim=1).squeeze(0).numpy().astype(np.uint8)
viz_result_image(
image,
@ -516,7 +516,7 @@ performance.
.. code:: ipython3
num_images = 100
with torch.no_grad():
start = time.perf_counter()
for _ in range(num_images):
@ -527,66 +527,44 @@ performance.
f"PyTorch model on CPU: {time_torch/num_images:.3f} seconds per image, "
f"FPS: {num_images/time_torch:.2f}"
)
compiled_model_onnx = core.compile_model(model=model_onnx, device_name="CPU")
compiled_model_onnx = core.compile_model(model=model_onnx, device_name=device.value)
start = time.perf_counter()
for _ in range(num_images):
compiled_model_onnx([normalized_input_image])
end = time.perf_counter()
time_onnx = end - start
print(
f"ONNX model in OpenVINO Runtime/CPU: {time_onnx/num_images:.3f} "
f"ONNX model in OpenVINO Runtime/{device.value}: {time_onnx/num_images:.3f} "
f"seconds per image, FPS: {num_images/time_onnx:.2f}"
)
compiled_model_ir = core.compile_model(model=model_ir, device_name="CPU")
compiled_model_ir = core.compile_model(model=model_ir, device_name=device.value)
start = time.perf_counter()
for _ in range(num_images):
compiled_model_ir([input_image])
end = time.perf_counter()
time_ir = end - start
print(
f"OpenVINO IR model in OpenVINO Runtime/CPU: {time_ir/num_images:.3f} "
f"OpenVINO IR model in OpenVINO Runtime/{device.value}: {time_ir/num_images:.3f} "
f"seconds per image, FPS: {num_images/time_ir:.2f}"
)
if "GPU" in core.available_devices:
compiled_model_onnx_gpu = core.compile_model(model=model_onnx, device_name="GPU")
start = time.perf_counter()
for _ in range(num_images):
compiled_model_onnx_gpu([input_image])
end = time.perf_counter()
time_onnx_gpu = end - start
print(
f"ONNX model in OpenVINO/GPU: {time_onnx_gpu/num_images:.3f} "
f"seconds per image, FPS: {num_images/time_onnx_gpu:.2f}"
)
compiled_model_ir_gpu = core.compile_model(model=model_ir, device_name="GPU")
start = time.perf_counter()
for _ in range(num_images):
compiled_model_ir_gpu([input_image])
end = time.perf_counter()
time_ir_gpu = end - start
print(
f"IR model in OpenVINO/GPU: {time_ir_gpu/num_images:.3f} "
f"seconds per image, FPS: {num_images/time_ir_gpu:.2f}"
)
.. parsed-literal::
PyTorch model on CPU: 0.040 seconds per image, FPS: 24.69
PyTorch model on CPU: 0.039 seconds per image, FPS: 25.55
.. parsed-literal::
ONNX model in OpenVINO Runtime/CPU: 0.018 seconds per image, FPS: 56.48
ONNX model in OpenVINO Runtime/AUTO: 0.018 seconds per image, FPS: 55.42
.. parsed-literal::
OpenVINO IR model in OpenVINO Runtime/CPU: 0.018 seconds per image, FPS: 55.11
OpenVINO IR model in OpenVINO Runtime/AUTO: 0.019 seconds per image, FPS: 52.62
**Show Device Information**

2
docs/notebooks/102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_22_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:bdc636d912f3e7146a2aacdaf744a83a90ad66778eacac5924a2499d8acb351f
oid sha256:085fc2e0ffdf71311eed99aebb0c6efafffe33ab3f01ca34c44f99feba46a565
size 465692

2
docs/notebooks/102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_27_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:1481c5d43590bbae0644ab25ea265919fb1d8ecd7e4f628e25e2ef69e822c4f0
oid sha256:fe706f0dda105330b25a3a8d1679019ef40d88c86b28a5aa2ca77449800ec939
size 465695

2
docs/notebooks/102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_29_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:bdc636d912f3e7146a2aacdaf744a83a90ad66778eacac5924a2499d8acb351f
oid sha256:085fc2e0ffdf71311eed99aebb0c6efafffe33ab3f01ca34c44f99feba46a565
size 465692

118
docs/notebooks/102-pytorch-to-openvino-with-output.rst
View File

@ -99,23 +99,23 @@ Download input data and label map
import requests
from pathlib import Path
from PIL import Image
MODEL_DIR = Path("model")
DATA_DIR = Path("data")
MODEL_DIR.mkdir(exist_ok=True)
DATA_DIR.mkdir(exist_ok=True)
MODEL_NAME = "regnet_y_800mf"
image = Image.open(requests.get("https://farm9.staticflickr.com/8225/8511402100_fea15da1c5_z.jpg", stream=True).raw)
labels_file = DATA_DIR / "imagenet_2012.txt"
if not labels_file.exists():
resp = requests.get("https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/data/dataset_classes/imagenet_2012.txt")
with labels_file.open("wb") as f:
f.write(resp.content)
imagenet_classes = labels_file.open("r").read().splitlines()
Load PyTorch Model
@ -141,14 +141,14 @@ enum ``RegNet_Y_800MF_Weights.DEFAULT``.
.. code:: ipython3
import torchvision
# get default weights using available weights Enum for model
weights = torchvision.models.RegNet_Y_800MF_Weights.DEFAULT
# create model topology and load weights
model = torchvision.models.regnet_y_800mf(weights=weights)
# switch model to inference mode
# switch model to inference mode
model.eval();
Prepare Input Data
@ -165,13 +165,13 @@ the first dimension.
.. code:: ipython3
import torch
# Initialize the Weight Transforms
preprocess = weights.transforms()
# Apply it to the input image
img_transformed = preprocess(image)
# Add batch dimension to image tensor
input_tensor = img_transformed.unsqueeze(0)
@ -190,10 +190,10 @@ can be reused later.
import numpy as np
from scipy.special import softmax
# Perform model inference on input tensor
result = model(input_tensor)
# Postprocessing function for getting results in the same way for both PyTorch model inference and OpenVINO
def postprocess_result(output_tensor:np.ndarray, top_k:int = 5):
"""
@ -209,10 +209,10 @@ can be reused later.
topk_labels = np.argsort(softmaxed_scores)[-top_k:][::-1]
topk_scores = softmaxed_scores[topk_labels]
return topk_labels, topk_scores
# Postprocess results
top_labels, top_scores = postprocess_result(result.detach().numpy())
# Show results
display(image)
for idx, (label, score) in enumerate(zip(top_labels, top_scores)):
@ -241,14 +241,14 @@ Benchmark PyTorch Model Inference
.. code:: ipython3
%%timeit
# Run model inference
model(input_tensor)
.. parsed-literal::
17.6 ms ± 52.8 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
16.1 ms ± 389 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
Convert PyTorch Model to OpenVINO Intermediate Representation
@ -278,21 +278,21 @@ such as:
and any other advanced options supported by model conversion Python API.
More details can be found on this
`page <https://docs.openvino.ai/2024/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api.html>`__
`page <https://docs.openvino.ai/2024/openvino-workflow/model-preparation/conversion-parameters.html>`__
.. code:: ipython3
import openvino as ov
# Create OpenVINO Core object instance
core = ov.Core()
# Convert model to openvino.runtime.Model object
ov_model = ov.convert_model(model)
# Save openvino.runtime.Model object on disk
ov.save_model(ov_model, MODEL_DIR / f"{MODEL_NAME}_dynamic.xml")
ov_model
@ -320,14 +320,14 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
@ -369,10 +369,10 @@ Run OpenVINO Model Inference
# Run model inference
result = compiled_model(input_tensor)[0]
# Posptorcess results
top_labels, top_scores = postprocess_result(result)
# Show results
display(image)
for idx, (label, score) in enumerate(zip(top_labels, top_scores)):
@ -401,13 +401,13 @@ Benchmark OpenVINO Model Inference
.. code:: ipython3
%%timeit
compiled_model(input_tensor)
.. parsed-literal::
3.42 ms ± 7.33 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
3.13 ms ± 17.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
Convert PyTorch Model with Static Input Shape
@ -435,7 +435,7 @@ reshaping example please check the following
.. parsed-literal::
<Model: 'Model66'
<Model: 'Model65'
inputs[
<ConstOutput: names[x] shape[1,3,224,224] type: f32>
]
@ -499,10 +499,10 @@ Run OpenVINO Model Inference with Static Input Shape
# Run model inference
result = compiled_model(input_tensor)[0]
# Posptorcess results
top_labels, top_scores = postprocess_result(result)
# Show results
display(image)
for idx, (label, score) in enumerate(zip(top_labels, top_scores)):
@ -531,13 +531,13 @@ Benchmark OpenVINO Model Inference with Static Input Shape
.. code:: ipython3
%%timeit
compiled_model(input_tensor)
.. parsed-literal::
2.9 ms ± 17.7 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
2.87 ms ± 22 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
Convert TorchScript Model to OpenVINO Intermediate Representation
@ -584,20 +584,20 @@ Reference <https://pytorch.org/docs/stable/jit_language_reference.html#language-
# Get model path
scripted_model_path = MODEL_DIR / f"{MODEL_NAME}_scripted.pth"
# Compile and save model if it has not been compiled before or load compiled model
if not scripted_model_path.exists():
scripted_model = torch.jit.script(model)
torch.jit.save(scripted_model, scripted_model_path)
else:
scripted_model = torch.jit.load(scripted_model_path)
# Run scripted model inference
result = scripted_model(input_tensor)
# Postprocess results
top_labels, top_scores = postprocess_result(result.detach().numpy())
# Show results
display(image)
for idx, (label, score) in enumerate(zip(top_labels, top_scores)):
@ -626,13 +626,13 @@ Benchmark Scripted Model Inference
.. code:: ipython3
%%timeit
scripted_model(input_tensor)
.. parsed-literal::
13 ms ± 50.6 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
13.1 ms ± 68.6 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
Convert PyTorch Scripted Model to OpenVINO Intermediate Representation
@ -647,16 +647,16 @@ the original PyTorch model.
# Convert model to openvino.runtime.Model object
ov_model = ov.convert_model(scripted_model)
# Load OpenVINO model on device
compiled_model = core.compile_model(ov_model, device.value)
# Run OpenVINO model inference
result = compiled_model(input_tensor, device.value)[0]
# Postprocess results
top_labels, top_scores = postprocess_result(result)
# Show results
display(image)
for idx, (label, score) in enumerate(zip(top_labels, top_scores)):
@ -685,13 +685,13 @@ Benchmark OpenVINO Model Inference Converted From Scripted Model
.. code:: ipython3
%%timeit
compiled_model(input_tensor)
.. parsed-literal::
3.42 ms ± 6.53 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
3.18 ms ± 9.04 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
Traced Model
@ -719,20 +719,20 @@ original PyTorch model code definitions.
# Get model path
traced_model_path = MODEL_DIR / f"{MODEL_NAME}_traced.pth"
# Trace and save model if it has not been traced before or load traced model
if not traced_model_path.exists():
traced_model = torch.jit.trace(model, example_inputs=input_tensor)
torch.jit.save(traced_model, traced_model_path)
else:
traced_model = torch.jit.load(traced_model_path)
# Run traced model inference
result = traced_model(input_tensor)
# Postprocess results
top_labels, top_scores = postprocess_result(result.detach().numpy())
# Show results
display(image)
for idx, (label, score) in enumerate(zip(top_labels, top_scores)):
@ -761,13 +761,13 @@ Benchmark Traced Model Inference
.. code:: ipython3
%%timeit
traced_model(input_tensor)
.. parsed-literal::
13.4 ms ± 4.67 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
13.6 ms ± 336 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
Convert PyTorch Traced Model to OpenVINO Intermediate Representation
@ -782,16 +782,16 @@ original PyTorch model.
# Convert model to openvino.runtime.Model object
ov_model = ov.convert_model(traced_model)
# Load OpenVINO model on device
compiled_model = core.compile_model(ov_model, device.value)
# Run OpenVINO model inference
result = compiled_model(input_tensor)[0]
# Postprocess results
top_labels, top_scores = postprocess_result(result)
# Show results
display(image)
for idx, (label, score) in enumerate(zip(top_labels, top_scores)):
@ -820,11 +820,11 @@ Benchmark OpenVINO Model Inference Converted From Traced Model
.. code:: ipython3
%%timeit
compiled_model(input_tensor)[0]
.. parsed-literal::
3.47 ms ± 10.3 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
3.21 ms ± 21.9 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

80
docs/notebooks/103-paddle-to-openvino-classification-with-output.rst
View File

@ -6,7 +6,7 @@ This notebook shows how to convert a MobileNetV3 model from
on the `ImageNet <https://www.image-net.org>`__ dataset, to OpenVINO IR.
It also shows how to perform classification inference on a sample image,
using `OpenVINO
Runtime <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_OV_Runtime_User_Guide.html>`__
Runtime <https://docs.openvino.ai/2024/openvino-workflow/running-inference.html>`__
and compares the results of the
`PaddlePaddle <https://github.com/PaddlePaddle/Paddle>`__ model with the
IR model.
@ -46,7 +46,7 @@ Imports
.. code:: ipython3
import platform
if platform.system() == "Windows":
%pip install -q "paddlepaddle>=2.5.1,<2.6.0"
else:
@ -71,9 +71,9 @@ Imports
ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
paddleclas 2.5.1 requires easydict, which is not installed.
paddleclas 2.5.1 requires faiss-cpu==1.7.1.post2, but you have faiss-cpu 1.7.4 which is incompatible.
paddleclas 2.5.1 requires faiss-cpu==1.7.1.post2, but you have faiss-cpu 1.8.0 which is incompatible.
paddleclas 2.5.1 requires gast==0.3.3, but you have gast 0.4.0 which is incompatible.
.. parsed-literal::
@ -94,16 +94,16 @@ Imports
.. parsed-literal::
--2024-02-09 22:36:08-- http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb
Resolving proxy-mu.intel.com (proxy-mu.intel.com)... 10.217.247.236
Connecting to proxy-mu.intel.com (proxy-mu.intel.com)|10.217.247.236|:911... connected.
Proxy request sent, awaiting response...
--2024-03-12 22:20:41-- http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb
Resolving proxy-dmz.intel.com (proxy-dmz.intel.com)... 10.241.208.166
Connecting to proxy-dmz.intel.com (proxy-dmz.intel.com)|10.241.208.166|:911... connected.
Proxy request sent, awaiting response...
.. parsed-literal::
404 Not Found
2024-02-09 22:36:08 ERROR 404: Not Found.
2024-03-12 22:20:41 ERROR 404: Not Found.
.. parsed-literal::
@ -116,31 +116,31 @@ Imports
import time
import tarfile
from pathlib import Path
import matplotlib.pyplot as plt
import numpy as np
import openvino as ov
from paddleclas import PaddleClas
from PIL import Image
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py',
filename='notebook_utils.py'
)
from notebook_utils import download_file
.. parsed-literal::
2024-02-09 22:36:10 INFO: Loading faiss with AVX2 support.
2024-03-12 22:20:43 INFO: Loading faiss with AVX512 support.
.. parsed-literal::
2024-02-09 22:36:10 INFO: Successfully loaded faiss with AVX2 support.
2024-03-12 22:20:43 INFO: Successfully loaded faiss with AVX512 support.
Settings
@ -168,9 +168,9 @@ PaddleHub. This may take a while.
"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_close.png",
directory="data"
)
IMAGE_FILENAME = img.as_posix()
MODEL_NAME = "MobileNetV3_large_x1_0"
MODEL_DIR = Path("model")
if not MODEL_DIR.exists():
@ -224,7 +224,7 @@ inference on that image, and then show the top three prediction results.
.. parsed-literal::
[2024/02/09 22:36:38] ppcls WARNING: The current running environment does not support the use of GPU. CPU has been used instead.
[2024/03/12 22:21:11] ppcls WARNING: The current running environment does not support the use of GPU. CPU has been used instead.
.. parsed-literal::
@ -271,8 +271,8 @@ the same method.
.. code:: ipython3
preprocess_ops = classifier.predictor.preprocess_ops
def process_image(image):
for op in preprocess_ops:
image = op(image)
@ -294,7 +294,7 @@ clipping values.
.. parsed-literal::
2024-02-09 22:36:39 WARNING: Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
2024-03-12 22:21:12 WARNING: Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
.. parsed-literal::
@ -306,7 +306,7 @@ clipping values.
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f9b4c389670>
<matplotlib.image.AxesImage at 0x7f1d380bd2b0>
@ -363,7 +363,7 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
@ -371,7 +371,7 @@ select device from dropdown list for running inference using OpenVINO
description='Device:',
disabled=False,
)
device
@ -399,24 +399,24 @@ information.
# Load OpenVINO Runtime and OpenVINO IR model
core = ov.Core()
model = core.read_model(model_xml)
compiled_model = core.compile_model(model=model, device_name="CPU")
compiled_model = core.compile_model(model=model, device_name=device.value)
# Get model output
output_layer = compiled_model.output(0)
# Read, show, and preprocess input image
# See the "Show Inference on PaddlePaddle Model" section for source of process_image
image = Image.open(IMAGE_FILENAME)
plt.imshow(image)
input_image = process_image(np.array(image))[None,]
# Do inference
ov_result = compiled_model([input_image])[output_layer][0]
# find the top three values
top_indices = np.argsort(ov_result)[-3:][::-1]
top_scores = ov_result[top_indices]
# Convert the inference results to class names, using the same labels as the PaddlePaddle classifier
for index, softmax_probability in zip(top_indices, top_scores):
print(f"{class_id_map[index]}, {softmax_probability:.5f}")
@ -448,7 +448,7 @@ Note that many optimizations are possible to improve the performance.
.. code:: ipython3
num_images = 50
image = Image.open(fp=IMAGE_FILENAME)
.. code:: ipython3
@ -456,7 +456,7 @@ Note that many optimizations are possible to improve the performance.
# Show device information
core = ov.Core()
devices = core.available_devices
for device_name in devices:
device_full_name = core.get_property(device_name, "FULL_DEVICE_NAME")
print(f"{device_name}: {device_full_name}")
@ -489,8 +489,8 @@ Note that many optimizations are possible to improve the performance.
.. parsed-literal::
PaddlePaddle model on CPU: 0.0075 seconds per image, FPS: 133.16
PaddlePaddle model on CPU: 0.0074 seconds per image, FPS: 134.43
PaddlePaddle result:
Labrador retriever, 0.75138
German short-haired pointer, 0.02373
@ -528,18 +528,18 @@ select device from dropdown list for running inference using OpenVINO
# Show inference speed on OpenVINO IR model
compiled_model = core.compile_model(model=model, device_name=device.value)
output_layer = compiled_model.output(0)
start = time.perf_counter()
input_image = process_image(np.array(image))[None,]
for _ in range(num_images):
ie_result = compiled_model([input_image])[output_layer][0]
top_indices = np.argsort(ie_result)[-5:][::-1]
top_softmax = ie_result[top_indices]
end = time.perf_counter()
time_ir = end - start
print(
f"OpenVINO IR model in OpenVINO Runtime ({device.value}): {time_ir/num_images:.4f} "
f"seconds per image, FPS: {num_images/time_ir:.2f}"
@ -553,8 +553,8 @@ select device from dropdown list for running inference using OpenVINO
.. parsed-literal::
OpenVINO IR model in OpenVINO Runtime (AUTO): 0.0030 seconds per image, FPS: 328.87
OpenVINO IR model in OpenVINO Runtime (AUTO): 0.0028 seconds per image, FPS: 353.76
OpenVINO result:
Labrador retriever, 0.74909
German short-haired pointer, 0.02368

2
docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_15_3.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:37fe65815997e6f67c6a635a98caf7ad3d5066aee57709f650fe8ef4c8bdfe11
oid sha256:99b8398ef76f2959d210e2d30bb44420f8d34a885a4480bc26e2af6627ba7119
size 120883

2
docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_23_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:22866fa0a0a063c7772d4a884812ca79fb0737c1eb2281bc825ee18eded729c5
oid sha256:1381e5922057c6bc70eb4ba9a04f3164382ad01191d320c1acbc819e7261f8c1
size 224886

2
docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_27_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:22866fa0a0a063c7772d4a884812ca79fb0737c1eb2281bc825ee18eded729c5
oid sha256:1381e5922057c6bc70eb4ba9a04f3164382ad01191d320c1acbc819e7261f8c1
size 224886

2
docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_30_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:22866fa0a0a063c7772d4a884812ca79fb0737c1eb2281bc825ee18eded729c5
oid sha256:1381e5922057c6bc70eb4ba9a04f3164382ad01191d320c1acbc819e7261f8c1
size 224886

2
docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_8_2.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:22866fa0a0a063c7772d4a884812ca79fb0737c1eb2281bc825ee18eded729c5
oid sha256:1381e5922057c6bc70eb4ba9a04f3164382ad01191d320c1acbc819e7261f8c1
size 224886

1241
docs/notebooks/104-model-tools-with-output.rst
View File

File diff suppressed because it is too large Load Diff

130
docs/notebooks/105-language-quantize-bert-with-output.rst
View File

@ -5,7 +5,7 @@ This tutorial demonstrates how to apply ``INT8`` quantization to the
Natural Language Processing model known as
`BERT <https://en.wikipedia.org/wiki/BERT_(language_model)>`__, using
the `Post-Training Quantization
API <https://docs.openvino.ai/nightly/basic_quantization_flow.html>`__
API <https://docs.openvino.ai/2024/openvino-workflow/model-optimization-guide/quantizing-models-post-training/basic-quantization-flow.html>`__
(NNCF library). A fine-tuned `HuggingFace
BERT <https://huggingface.co/transformers/model_doc/bert.html>`__
`PyTorch <https://pytorch.org/>`__ model, trained on the `Microsoft
@ -42,7 +42,7 @@ Table of contents:
.. code:: ipython3
%pip install -q "nncf>=2.5.0"
%pip install -q "nncf>=2.5.0"
%pip install -q transformers datasets evaluate --extra-index-url https://download.pytorch.org/whl/cpu
%pip install -q "openvino>=2023.1.0"
@ -75,7 +75,7 @@ Imports
from zipfile import ZipFile
from typing import Iterable
from typing import Any
import datasets
import evaluate
import numpy as np
@ -84,7 +84,7 @@ Imports
import openvino as ov
import torch
from transformers import BertForSequenceClassification, BertTokenizer
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
@ -96,14 +96,14 @@ Imports
.. parsed-literal::
2024-02-09 22:38:10.763464: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-02-09 22:38:10.797722: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2024-03-12 22:22:23.157910: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-03-12 22:22:23.191787: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
.. parsed-literal::
2024-02-09 22:38:11.441310: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2024-03-12 22:22:23.830567: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
@ -124,7 +124,7 @@ Settings
MODEL_LINK = "https://download.pytorch.org/tutorial/MRPC.zip"
FILE_NAME = MODEL_LINK.split("/")[-1]
PRETRAINED_MODEL_DIR = os.path.join(MODEL_DIR, "MRPC")
os.makedirs(DATA_DIR, exist_ok=True)
os.makedirs(MODEL_DIR, exist_ok=True)
@ -169,10 +169,10 @@ PyTorch model formats are supported:
input_shape = ov.PartialShape([1, -1])
ir_model_xml = Path(MODEL_DIR) / "bert_mrpc.xml"
core = ov.Core()
torch_model = BertForSequenceClassification.from_pretrained(PRETRAINED_MODEL_DIR)
torch_model.eval
input_info = [("input_ids", input_shape, np.int64),("attention_mask", input_shape, np.int64),("token_type_ids", input_shape, np.int64)]
default_input = torch.ones(1, MAX_SEQ_LENGTH, dtype=torch.int64)
inputs = {
@ -180,7 +180,7 @@ PyTorch model formats are supported:
"attention_mask": default_input,
"token_type_ids": default_input,
}
# Convert the PyTorch model to OpenVINO IR FP32.
if not ir_model_xml.exists():
model = ov.convert_model(torch_model, example_input=inputs, input=input_info)
@ -191,7 +191,7 @@ PyTorch model formats are supported:
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly. To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly. To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
return self.fget.__get__(instance, owner)()
@ -215,6 +215,12 @@ PyTorch model formats are supported:
No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda'
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4193: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead
warnings.warn(
Prepare the Dataset
-------------------
@ -230,16 +236,16 @@ tokenizer from HuggingFace.
def create_data_source():
raw_dataset = datasets.load_dataset('glue', 'mrpc', split='validation')
tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_DIR)
def _preprocess_fn(examples):
texts = (examples['sentence1'], examples['sentence2'])
result = tokenizer(*texts, padding='max_length', max_length=MAX_SEQ_LENGTH, truncation=True)
result['labels'] = examples['label']
return result
processed_dataset = raw_dataset.map(_preprocess_fn, batched=True, batch_size=1)
return processed_dataset
data_source = create_data_source()
Optimize model using NNCF Post-training Quantization API
@ -261,7 +267,7 @@ The optimization process contains the following steps:
.. code:: ipython3
INPUT_NAMES = [key for key in inputs.keys()]
def transform_fn(data_item):
"""
Extract the model's input from the data item.
@ -272,7 +278,7 @@ The optimization process contains the following steps:
name: np.asarray([data_item[name]], dtype=np.int64) for name in INPUT_NAMES
}
return inputs
calibration_dataset = nncf.Dataset(data_source, transform_fn)
# Quantize the model. By specifying model_type, we specify additional transformer patterns in the model.
quantized_model = nncf.quantize(model, calibration_dataset,
@ -286,7 +292,9 @@ The optimization process contains the following steps:
.. raw:: html
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"></pre>
@ -305,7 +313,9 @@ The optimization process contains the following steps:
.. raw:: html
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"></pre>
@ -334,7 +344,9 @@ The optimization process contains the following steps:
.. raw:: html
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"></pre>
@ -353,7 +365,9 @@ The optimization process contains the following steps:
.. raw:: html
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"></pre>
@ -392,14 +406,14 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
@ -427,10 +441,10 @@ changing ``sample_idx`` to another value (from 0 to 407).
sample_idx = 5
sample = data_source[sample_idx]
inputs = {k: torch.unsqueeze(torch.tensor(sample[k]), 0) for k in ['input_ids', 'token_type_ids', 'attention_mask']}
result = compiled_quantized_model(inputs)[output_layer]
result = np.argmax(result)
print(f"Text 1: {sample['sentence1']}")
print(f"Text 2: {sample['sentence2']}")
print(f"The same meaning: {'yes' if result == 1 else 'no'}")
@ -452,12 +466,12 @@ Compare F1-score of FP32 and INT8 models
def validate(model: ov.Model, dataset: Iterable[Any]) -> float:
"""
Evaluate the model on GLUE dataset.
Evaluate the model on GLUE dataset.
Returns F1 score metric.
"""
compiled_model = core.compile_model(model, device_name=device.value)
output_layer = compiled_model.output(0)
metric = evaluate.load('glue', 'mrpc')
for batch in dataset:
inputs = [
@ -468,14 +482,14 @@ Compare F1-score of FP32 and INT8 models
metric.add_batch(predictions=[predictions], references=[batch['labels']])
metrics = metric.compute()
f1_score = metrics['f1']
return f1_score
print('Checking the accuracy of the original model:')
metric = validate(model, data_source)
print(f'F1 score: {metric:.4f}')
print('Checking the accuracy of the quantized model:')
metric = validate(quantized_model, data_source)
print(f'F1 score: {metric:.4f}')
@ -517,7 +531,7 @@ Frames Per Second (FPS) for images.
num_samples = 50
sample = data_source[0]
inputs = {k: torch.unsqueeze(torch.tensor(sample[k]), 0) for k in ['input_ids', 'token_type_ids', 'attention_mask']}
with torch.no_grad():
start = time.perf_counter()
for _ in range(num_samples):
@ -528,7 +542,7 @@ Frames Per Second (FPS) for images.
f"PyTorch model on CPU: {time_torch / num_samples:.3f} seconds per sentence, "
f"SPS: {num_samples / time_torch:.2f}"
)
start = time.perf_counter()
for _ in range(num_samples):
compiled_model(inputs)
@ -538,7 +552,7 @@ Frames Per Second (FPS) for images.
f"IR FP32 model in OpenVINO Runtime/{device.value}: {time_ir / num_samples:.3f} "
f"seconds per sentence, SPS: {num_samples / time_ir:.2f}"
)
start = time.perf_counter()
for _ in range(num_samples):
compiled_quantized_model(inputs)
@ -557,17 +571,17 @@ Frames Per Second (FPS) for images.
.. parsed-literal::
PyTorch model on CPU: 0.073 seconds per sentence, SPS: 13.77
PyTorch model on CPU: 0.073 seconds per sentence, SPS: 13.63
.. parsed-literal::
IR FP32 model in OpenVINO Runtime/AUTO: 0.021 seconds per sentence, SPS: 47.89
IR FP32 model in OpenVINO Runtime/AUTO: 0.020 seconds per sentence, SPS: 48.91
.. parsed-literal::
OpenVINO IR INT8 model in OpenVINO Runtime/AUTO: 0.009 seconds per sentence, SPS: 109.72
OpenVINO IR INT8 model in OpenVINO Runtime/AUTO: 0.009 seconds per sentence, SPS: 112.60
Finally, measure the inference performance of OpenVINO ``FP32`` and
@ -575,7 +589,7 @@ Finally, measure the inference performance of OpenVINO ``FP32`` and
Tool <https://docs.openvino.ai/2024/learn-openvino/openvino-samples/benchmark-tool.html>`__
in OpenVINO.
**NOTE**: The ``benchmark_app`` tool is able to measure the
**Note**: The ``benchmark_app`` tool is able to measure the
performance of the OpenVINO Intermediate Representation (OpenVINO IR)
models only. For more accurate performance, run ``benchmark_app`` in
a terminal/command prompt after closing other applications. Run
@ -597,24 +611,24 @@ in OpenVINO.
[Step 2/11] Loading OpenVINO Runtime
[ WARNING ] Default duration 120 seconds is used for unknown device device.value
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ] Device info:
[ INFO ]
[ INFO ]
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ ERROR ] Exception from src/inference/src/core.cpp:228:
Exception from src/inference/src/dev/core_impl.cpp:560:
[ ERROR ] Exception from src/inference/src/cpp/core.cpp:216:
Exception from src/inference/src/dev/core_impl.cpp:556:
Device with "device" name is not registered in the OpenVINO Runtime
Traceback (most recent call last):
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 166, in main
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 166, in main
supported_properties = benchmark.core.get_property(device, properties.supported_properties())
RuntimeError: Exception from src/inference/src/core.cpp:228:
Exception from src/inference/src/dev/core_impl.cpp:560:
RuntimeError: Exception from src/inference/src/cpp/core.cpp:216:
Exception from src/inference/src/dev/core_impl.cpp:556:
Device with "device" name is not registered in the OpenVINO Runtime
.. code:: ipython3
@ -630,22 +644,22 @@ in OpenVINO.
[Step 2/11] Loading OpenVINO Runtime
[ WARNING ] Default duration 120 seconds is used for unknown device device.value
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ] Device info:
[ INFO ]
[ INFO ]
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ ERROR ] Exception from src/inference/src/core.cpp:228:
Exception from src/inference/src/dev/core_impl.cpp:560:
[ ERROR ] Exception from src/inference/src/cpp/core.cpp:216:
Exception from src/inference/src/dev/core_impl.cpp:556:
Device with "device" name is not registered in the OpenVINO Runtime
Traceback (most recent call last):
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 166, in main
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 166, in main
supported_properties = benchmark.core.get_property(device, properties.supported_properties())
RuntimeError: Exception from src/inference/src/core.cpp:228:
Exception from src/inference/src/dev/core_impl.cpp:560:
RuntimeError: Exception from src/inference/src/cpp/core.cpp:216:
Exception from src/inference/src/dev/core_impl.cpp:556:
Device with "device" name is not registered in the OpenVINO Runtime

171
docs/notebooks/106-auto-device-with-output.rst
View File

@ -81,13 +81,13 @@ Import modules and create Core
import time
import sys
import openvino as ov
from IPython.display import Markdown, display
core = ov.Core()
if "GPU" not in core.available_devices:
display(Markdown('<div class="alert alert-block alert-danger"><b>Warning: </b> A GPU device is not available. This notebook requires GPU device to have meaningful results. </div>'))
@ -127,11 +127,11 @@ For more information about model conversion API, see this
import torchvision
from pathlib import Path
base_model_dir = Path("./model")
base_model_dir.mkdir(exist_ok=True)
model_path = base_model_dir / "resnet50.xml"
if not model_path.exists():
pt_model = torchvision.models.resnet50(weights="DEFAULT")
ov_model = ov.convert_model(pt_model, input=[[1,3,224,224]])
@ -164,24 +164,25 @@ By default, ``compile_model`` API will select **AUTO** as
# Set LOG_LEVEL to LOG_INFO.
core.set_property("AUTO", {"LOG_LEVEL":"LOG_INFO"})
# Load the model onto the target device.
compiled_model = core.compile_model(ov_model)
if isinstance(compiled_model, ov.CompiledModel):
print("Successfully compiled model without a device_name.")
print("Successfully compiled model without a device_name.")
.. parsed-literal::
[22:41:31.9445]I[plugin.cpp:536][AUTO] device:CPU, config:PERFORMANCE_HINT=LATENCY
[22:41:31.9445]I[plugin.cpp:536][AUTO] device:CPU, config:PERFORMANCE_HINT_NUM_REQUESTS=0
[22:41:31.9445]I[plugin.cpp:536][AUTO] device:CPU, config:PERF_COUNT=NO
[22:41:31.9445]I[plugin.cpp:541][AUTO] device:CPU, priority:0
[22:41:31.9446]I[schedule.cpp:17][AUTO] scheduler starting
[22:41:31.9446]I[auto_schedule.cpp:131][AUTO] select device:CPU
[22:41:32.0858]I[auto_schedule.cpp:109][AUTO] device:CPU compiling model finished
[22:41:32.0860]I[plugin.cpp:569][AUTO] underlying hardware does not support hardware context
[22:24:04.4814]I[plugin.cpp:418][AUTO] device:CPU, config:LOG_LEVEL=LOG_INFO
[22:24:04.4815]I[plugin.cpp:418][AUTO] device:CPU, config:PERFORMANCE_HINT=LATENCY
[22:24:04.4815]I[plugin.cpp:418][AUTO] device:CPU, config:PERFORMANCE_HINT_NUM_REQUESTS=0
[22:24:04.4815]I[plugin.cpp:418][AUTO] device:CPU, config:PERF_COUNT=NO
[22:24:04.4815]I[plugin.cpp:423][AUTO] device:CPU, priority:0
[22:24:04.4815]I[schedule.cpp:17][AUTO] scheduler starting
[22:24:04.4815]I[auto_schedule.cpp:131][AUTO] select device:CPU
[22:24:04.6260]I[auto_schedule.cpp:109][AUTO] device:CPU compiling model finished
[22:24:04.6262]I[plugin.cpp:451][AUTO] underlying hardware does not support hardware context
Successfully compiled model without a device_name.
@ -194,8 +195,8 @@ By default, ``compile_model`` API will select **AUTO** as
.. parsed-literal::
[22:24:04.6373]I[schedule.cpp:303][AUTO] scheduler ending
Deleted compiled_model
[22:41:32.0982]I[schedule.cpp:303][AUTO] scheduler ending
Explicitly pass AUTO as device_name to Core::compile_model API
@ -210,9 +211,9 @@ improve readability of your code.
# Set LOG_LEVEL to LOG_NONE.
core.set_property("AUTO", {"LOG_LEVEL":"LOG_NONE"})
compiled_model = core.compile_model(model=ov_model, device_name="AUTO")
if isinstance(compiled_model, ov.CompiledModel):
print("Successfully compiled model using AUTO.")
@ -271,16 +272,16 @@ function, we will reuse it for preparing input data.
.. code:: ipython3
from PIL import Image
# Download the image from the openvino_notebooks storage
image_filename = download_file(
"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg",
directory="data"
)
image = Image.open(str(image_filename))
input_transform = torchvision.models.ResNet50_Weights.DEFAULT.transforms()
input_tensor = input_transform(image)
input_tensor = input_tensor.unsqueeze(0).numpy()
image
@ -307,14 +308,14 @@ Load the model to GPU device and perform inference
if "GPU" not in core.available_devices:
print(f"A GPU device is not available. Available devices are: {core.available_devices}")
else :
else :
# Start time.
gpu_load_start_time = time.perf_counter()
compiled_model = core.compile_model(model=ov_model, device_name="GPU") # load to GPU
# Execute the first inference.
results = compiled_model(input_tensor)[0]
# Measure time to the first inference.
gpu_fil_end_time = time.perf_counter()
gpu_fil_span = gpu_fil_end_time - gpu_load_start_time
@ -340,11 +341,11 @@ executed on CPU until GPU is ready.
# Start time.
auto_load_start_time = time.perf_counter()
compiled_model = core.compile_model(model=ov_model) # The device_name is AUTO by default.
# Execute the first inference.
results = compiled_model(input_tensor)[0]
# Measure time to the first inference.
auto_fil_end_time = time.perf_counter()
auto_fil_span = auto_fil_end_time - auto_load_start_time
@ -406,11 +407,11 @@ Class and callback definition
"""
self.fps = 0
self.latency = 0
self.start_time = time.perf_counter()
self.latency_list = []
self.interval = interval
def update(self, infer_request: ov.InferRequest) -> bool:
"""
Update the metrics if current ongoing @interval seconds duration is expired. Record the latency only if it is not expired.
@ -432,12 +433,12 @@ Class and callback definition
return True
else :
return False
class InferContext:
"""
Inference context. Record and update peforamnce metrics via @metrics, set @feed_inference to False once @remaining_update_num <=0
:member: metrics: instance of class PerformanceMetrics
:member: metrics: instance of class PerformanceMetrics
:member: remaining_update_num: the remaining times for peforamnce metrics updating.
:member: feed_inference: if feed inference request is required or not.
"""
@ -452,7 +453,7 @@ Class and callback definition
self.metrics = PerformanceMetrics(update_interval)
self.remaining_update_num = num
self.feed_inference = True
def update(self, infer_request: ov.InferRequest):
"""
Update the context. Set @feed_inference to False if the number of remaining performance metric updates (@remaining_update_num) reaches 0
@ -461,13 +462,13 @@ Class and callback definition
"""
if self.remaining_update_num <= 0 :
self.feed_inference = False
if self.metrics.update(infer_request) :
self.remaining_update_num = self.remaining_update_num - 1
if self.remaining_update_num <= 0 :
self.feed_inference = False
def completion_callback(infer_request: ov.InferRequest, context) -> None:
"""
callback for the inference request, pass the @infer_request to @context for updating
@ -476,8 +477,8 @@ Class and callback definition
:returns: None
"""
context.update(infer_request)
# Performance metrics update interval (seconds) and number of times.
metrics_update_interval = 10
metrics_update_num = 6
@ -493,29 +494,29 @@ Loop for inference and update the FPS/Latency every
.. code:: ipython3
THROUGHPUT_hint_context = InferContext(metrics_update_interval, metrics_update_num)
print("Compiling Model for AUTO device with THROUGHPUT hint")
sys.stdout.flush()
compiled_model = core.compile_model(model=ov_model, config={"PERFORMANCE_HINT":"THROUGHPUT"})
infer_queue = ov.AsyncInferQueue(compiled_model, 0) # Setting to 0 will query optimal number by default.
infer_queue.set_callback(completion_callback)
print(f"Start inference, {metrics_update_num: .0f} groups of FPS/latency will be measured over {metrics_update_interval: .0f}s intervals")
sys.stdout.flush()
while THROUGHPUT_hint_context.feed_inference:
infer_queue.start_async(input_tensor, THROUGHPUT_hint_context)
infer_queue.wait_all()
# Take the FPS and latency of the latest period.
THROUGHPUT_hint_fps = THROUGHPUT_hint_context.metrics.fps
THROUGHPUT_hint_latency = THROUGHPUT_hint_context.metrics.latency
print("Done")
del compiled_model
@ -531,32 +532,32 @@ Loop for inference and update the FPS/Latency every
.. parsed-literal::
throughput: 179.69fps, latency: 31.58ms, time interval: 10.00s
throughput: 181.84fps, latency: 31.28ms, time interval: 10.00s
.. parsed-literal::
throughput: 182.30fps, latency: 32.10ms, time interval: 10.00s
throughput: 183.53fps, latency: 31.88ms, time interval: 10.01s
.. parsed-literal::
throughput: 180.62fps, latency: 32.36ms, time interval: 10.02s
throughput: 182.49fps, latency: 32.10ms, time interval: 10.00s
.. parsed-literal::
throughput: 179.76fps, latency: 32.61ms, time interval: 10.00s
throughput: 183.45fps, latency: 31.92ms, time interval: 10.00s
.. parsed-literal::
throughput: 180.36fps, latency: 32.36ms, time interval: 10.02s
throughput: 182.69fps, latency: 32.02ms, time interval: 10.00s
.. parsed-literal::
throughput: 179.77fps, latency: 32.58ms, time interval: 10.00s
throughput: 181.11fps, latency: 32.34ms, time interval: 10.02s
.. parsed-literal::
@ -575,30 +576,30 @@ Loop for inference and update the FPS/Latency for each
.. code:: ipython3
LATENCY_hint_context = InferContext(metrics_update_interval, metrics_update_num)
print("Compiling Model for AUTO Device with LATENCY hint")
sys.stdout.flush()
compiled_model = core.compile_model(model=ov_model, config={"PERFORMANCE_HINT":"LATENCY"})
# Setting to 0 will query optimal number by default.
infer_queue = ov.AsyncInferQueue(compiled_model, 0)
infer_queue.set_callback(completion_callback)
print(f"Start inference, {metrics_update_num: .0f} groups fps/latency will be out with {metrics_update_interval: .0f}s interval")
sys.stdout.flush()
while LATENCY_hint_context.feed_inference:
infer_queue.start_async(input_tensor, LATENCY_hint_context)
infer_queue.wait_all()
# Take the FPS and latency of the latest period.
LATENCY_hint_fps = LATENCY_hint_context.metrics.fps
LATENCY_hint_latency = LATENCY_hint_context.metrics.latency
print("Done")
del compiled_model
@ -614,32 +615,32 @@ Loop for inference and update the FPS/Latency for each
.. parsed-literal::
throughput: 139.27fps, latency: 6.65ms, time interval: 10.00s
throughput: 138.83fps, latency: 6.64ms, time interval: 10.01s
.. parsed-literal::
throughput: 141.22fps, latency: 6.62ms, time interval: 10.01s
throughput: 141.03fps, latency: 6.64ms, time interval: 10.00s
.. parsed-literal::
throughput: 140.71fps, latency: 6.64ms, time interval: 10.01s
throughput: 140.77fps, latency: 6.64ms, time interval: 10.00s
.. parsed-literal::
throughput: 141.11fps, latency: 6.63ms, time interval: 10.01s
throughput: 141.80fps, latency: 6.65ms, time interval: 10.01s
.. parsed-literal::
throughput: 141.26fps, latency: 6.62ms, time interval: 10.00s
throughput: 142.26fps, latency: 6.66ms, time interval: 10.00s
.. parsed-literal::
throughput: 141.18fps, latency: 6.63ms, time interval: 10.00s
throughput: 141.36fps, latency: 6.63ms, time interval: 10.00s
.. parsed-literal::
@ -655,21 +656,21 @@ Difference in FPS and latency
.. code:: ipython3
import matplotlib.pyplot as plt
TPUT = 0
LAT = 1
labels = ["THROUGHPUT hint", "LATENCY hint"]
fig1, ax1 = plt.subplots(1, 1)
fig1, ax1 = plt.subplots(1, 1)
fig1.patch.set_visible(False)
ax1.axis('tight')
ax1.axis('off')
ax1.axis('tight')
ax1.axis('off')
cell_text = []
cell_text.append(['%.2f%s' % (THROUGHPUT_hint_fps," FPS"), '%.2f%s' % (THROUGHPUT_hint_latency, " ms")])
cell_text.append(['%.2f%s' % (LATENCY_hint_fps," FPS"), '%.2f%s' % (LATENCY_hint_latency, " ms")])
table = ax1.table(cellText=cell_text, colLabels=["FPS (Higher is better)", "Latency (Lower is better)"], rowLabels=labels,
table = ax1.table(cellText=cell_text, colLabels=["FPS (Higher is better)", "Latency (Lower is better)"], rowLabels=labels,
rowColours=["deepskyblue"] * 2, colColours=["deepskyblue"] * 2,
cellLoc='center', loc='upper left')
table.auto_set_font_size(False)
@ -677,7 +678,7 @@ Difference in FPS and latency
table.auto_set_column_width(0)
table.auto_set_column_width(1)
table.scale(1, 3)
fig1.tight_layout()
plt.show()
@ -691,28 +692,28 @@ Difference in FPS and latency
# Output the difference.
width = 0.4
fontsize = 14
plt.rc('font', size=fontsize)
fig, ax = plt.subplots(1,2, figsize=(10, 8))
rects1 = ax[0].bar([0], THROUGHPUT_hint_fps, width, label=labels[TPUT], color='#557f2d')
rects2 = ax[0].bar([width], LATENCY_hint_fps, width, label=labels[LAT])
ax[0].set_ylabel("frames per second")
ax[0].set_xticks([width / 2])
ax[0].set_xticks([width / 2])
ax[0].set_xticklabels(["FPS"])
ax[0].set_xlabel("Higher is better")
rects1 = ax[1].bar([0], THROUGHPUT_hint_latency, width, label=labels[TPUT], color='#557f2d')
rects2 = ax[1].bar([width], LATENCY_hint_latency, width, label=labels[LAT])
ax[1].set_ylabel("milliseconds")
ax[1].set_xticks([width / 2])
ax[1].set_xticklabels(["Latency (ms)"])
ax[1].set_xlabel("Lower is better")
fig.suptitle('Performance Hints')
fig.legend(labels, fontsize=fontsize)
fig.tight_layout()
plt.show()

4
docs/notebooks/106-auto-device-with-output_files/106-auto-device-with-output_27_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:acc35ef9a1d6c2acf74782263f65bd81b65d7fd6663a146ffb88c9a9b64e343d
size 26542
oid sha256:8ce150e30514fd5c85f9d29b953b41d9cbd0c4f58c2289513286a4aef99c3984
size 26193

4
docs/notebooks/106-auto-device-with-output_files/106-auto-device-with-output_28_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ab4df6ece8c31c9c5d5ab7d51b7d628ef7fb5df7193d859b8cfd9547273b76e3
size 39977
oid sha256:7b1eaf56574fadd9331d380d2f057dd74775ebacdaa34fff9a1af2fe382128ab
size 40039

264
docs/notebooks/107-speech-recognition-quantization-data2vec-with-output.rst
View File

File diff suppressed because one or more lines are too long

394
docs/notebooks/107-speech-recognition-quantization-wav2vec2-with-output.rst
View File

File diff suppressed because one or more lines are too long

166
docs/notebooks/108-gpu-device-with-output.rst
View File

@ -108,9 +108,9 @@ Install required packages
.. code:: ipython3
%pip install -q "openvino-dev>=2023.1.0"
%pip install -q "openvino-dev>=2024.0.0"
%pip install -q tensorflow
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
@ -148,7 +148,7 @@ appear.
.. code:: ipython3
import openvino as ov
core = ov.Core()
core.available_devices
@ -192,7 +192,7 @@ To get the value of a property, such as the device name, we can use the
.. code:: ipython3
device = "GPU"
core.get_property(device, "FULL_DEVICE_NAME")
@ -216,7 +216,7 @@ for that property.
print(f"{device} SUPPORTED_PROPERTIES:\n")
supported_properties = core.get_property(device, "SUPPORTED_PROPERTIES")
indent = len(max(supported_properties, key=len))
for property_key in supported_properties:
if property_key not in ('SUPPORTED_METRICS', 'SUPPORTED_CONFIG_KEYS', 'SUPPORTED_PROPERTIES'):
try:
@ -229,7 +229,7 @@ for that property.
.. parsed-literal::
GPU SUPPORTED_PROPERTIES:
AVAILABLE_DEVICES : ['0']
RANGE_FOR_ASYNC_INFER_REQUESTS: (1, 2, 1)
RANGE_FOR_STREAMS : (1, 2)
@ -252,7 +252,7 @@ for that property.
GPU_QUEUE_PRIORITY : Priority.MEDIUM
GPU_QUEUE_THROTTLE : Priority.MEDIUM
GPU_ENABLE_LOOP_UNROLLING : True
CACHE_DIR :
CACHE_DIR :
PERFORMANCE_HINT : PerformanceMode.UNDEFINED
COMPILATION_NUM_THREADS : 20
NUM_STREAMS : 1
@ -328,23 +328,23 @@ package is already downloaded.
import sys
import tarfile
from pathlib import Path
sys.path.append("../utils")
import notebook_utils as utils
# A directory where the model will be downloaded.
base_model_dir = Path("./model").expanduser()
model_name = "ssdlite_mobilenet_v2"
archive_name = Path(f"{model_name}_coco_2018_05_09.tar.gz")
# Download the archive
downloaded_model_path = base_model_dir / archive_name
if not downloaded_model_path.exists():
model_url = f"http://download.tensorflow.org/models/object_detection/{archive_name}"
utils.download_file(model_url, downloaded_model_path.name, downloaded_model_path.parent)
# Unpack the model
tf_model_path = base_model_dir / archive_name.with_suffix("").stem / "frozen_inference_graph.pb"
if not tf_model_path.exists():
@ -365,11 +365,11 @@ package is already downloaded.
to the client in order to avoid crashing it.
To change this limit, set the config variable
`--NotebookApp.iopub_msg_rate_limit`.
Current values:
NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)
NotebookApp.rate_limit_window=3.0 (secs)
Convert the Model to OpenVINO IR format
@ -385,15 +385,15 @@ directory. For more details about model conversion, see this
.. code:: ipython3
from openvino.tools.mo.front import tf as ov_tf_front
precision = 'FP16'
# The output path for the conversion.
model_path = base_model_dir / 'ir_model' / f'{model_name}_{precision.lower()}.xml'
trans_config_path = Path(ov_tf_front.__file__).parent / "ssd_v2_support.json"
pipeline_config = base_model_dir / archive_name.with_suffix("").stem / "pipeline.config"
model = None
if not model_path.exists():
model = ov.tools.mo.convert_model(input_model=tf_model_path,
@ -461,17 +461,17 @@ following:
import time
from pathlib import Path
# Create cache folder
cache_folder = Path("cache")
cache_folder.mkdir(exist_ok=True)
start = time.time()
core = ov.Core()
# Set cache folder
core.set_property({'CACHE_DIR': cache_folder})
# Compile the model as before
model = core.read_model(model=model_path)
compiled_model = core.compile_model(model, device)
@ -494,7 +494,7 @@ compile times with caching enabled and disabled as follows:
model = core.read_model(model=model_path)
compiled_model = core.compile_model(model, device)
print(f"Cache enabled - compile time: {time.time() - start}s")
start = time.time()
core = ov.Core()
model = core.read_model(model=model_path)
@ -559,7 +559,7 @@ or by manual specification of the device name as above. When we have
multiple devices, such as an integrated and discrete GPU, we may use
both at the same time to improve the utilization of the resources. In
order to do this, OpenVINO provides a virtual device called
`MULTI <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_Running_on_multiple_devices.html>`__,
`MULTI <https://docs.openvino.ai/2024/openvino-workflow/running-inference/inference-devices-and-modes/multi-device.html>`__,
which is just a combination of the existent devices that knows how to
split inference work between them, leveraging the capabilities of each
device.
@ -628,12 +628,12 @@ with a latency focus:
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ] Device info:
[ INFO ] GPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[Step 4/11] Reading model files
[ INFO ] Loading model files
@ -662,7 +662,7 @@ with a latency focus:
[ INFO ] GPU_QUEUE_PRIORITY: Priority.MEDIUM
[ INFO ] GPU_QUEUE_THROTTLE: Priority.MEDIUM
[ INFO ] GPU_ENABLE_LOOP_UNROLLING: True
[ INFO ] CACHE_DIR:
[ INFO ] CACHE_DIR:
[ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY
[ INFO ] COMPILATION_NUM_THREADS: 20
[ INFO ] NUM_STREAMS: 1
@ -671,7 +671,7 @@ with a latency focus:
[ INFO ] DEVICE_ID: 0
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'image_tensor'!. This input will be filled with random values!
[ INFO ] Fill input 'image_tensor' with random values
[ INFO ] Fill input 'image_tensor' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 1 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 6.17 ms
@ -709,12 +709,12 @@ CPU vs GPU with Latency Hint
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[Step 4/11] Reading model files
[ INFO ] Loading model files
@ -746,7 +746,7 @@ CPU vs GPU with Latency Hint
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'image_tensor'!. This input will be filled with random values!
[ INFO ] Fill input 'image_tensor' with random values
[ INFO ] Fill input 'image_tensor' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 1 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 4.42 ms
@ -773,12 +773,12 @@ CPU vs GPU with Latency Hint
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ] Device info:
[ INFO ] GPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[Step 4/11] Reading model files
[ INFO ] Loading model files
@ -807,7 +807,7 @@ CPU vs GPU with Latency Hint
[ INFO ] GPU_QUEUE_PRIORITY: Priority.MEDIUM
[ INFO ] GPU_QUEUE_THROTTLE: Priority.MEDIUM
[ INFO ] GPU_ENABLE_LOOP_UNROLLING: True
[ INFO ] CACHE_DIR:
[ INFO ] CACHE_DIR:
[ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY
[ INFO ] COMPILATION_NUM_THREADS: 20
[ INFO ] NUM_STREAMS: 1
@ -816,7 +816,7 @@ CPU vs GPU with Latency Hint
[ INFO ] DEVICE_ID: 0
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'image_tensor'!. This input will be filled with random values!
[ INFO ] Fill input 'image_tensor' with random values
[ INFO ] Fill input 'image_tensor' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 1 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 8.79 ms
@ -848,12 +848,12 @@ CPU vs GPU with Throughput Hint
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[Step 4/11] Reading model files
[ INFO ] Loading model files
@ -885,7 +885,7 @@ CPU vs GPU with Throughput Hint
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'image_tensor'!. This input will be filled with random values!
[ INFO ] Fill input 'image_tensor' with random values
[ INFO ] Fill input 'image_tensor' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 5 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 8.15 ms
@ -912,12 +912,12 @@ CPU vs GPU with Throughput Hint
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ] Device info:
[ INFO ] GPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[Step 4/11] Reading model files
[ INFO ] Loading model files
@ -946,7 +946,7 @@ CPU vs GPU with Throughput Hint
[ INFO ] GPU_QUEUE_PRIORITY: Priority.MEDIUM
[ INFO ] GPU_QUEUE_THROTTLE: Priority.MEDIUM
[ INFO ] GPU_ENABLE_LOOP_UNROLLING: True
[ INFO ] CACHE_DIR:
[ INFO ] CACHE_DIR:
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] COMPILATION_NUM_THREADS: 20
[ INFO ] NUM_STREAMS: 2
@ -955,7 +955,7 @@ CPU vs GPU with Throughput Hint
[ INFO ] DEVICE_ID: 0
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'image_tensor'!. This input will be filled with random values!
[ INFO ] Fill input 'image_tensor' with random values
[ INFO ] Fill input 'image_tensor' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 4 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 9.17 ms
@ -987,12 +987,12 @@ Single GPU vs Multiple GPUs
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ] Device info:
[ INFO ] GPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Device GPU.1 does not support performance hint property(-hint).
[ ERROR ] Config for device with 1 ID is not registered in GPU plugin
@ -1016,14 +1016,14 @@ Single GPU vs Multiple GPUs
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ] Device info:
[ INFO ] AUTO
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] GPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Device GPU.1 does not support performance hint property(-hint).
[Step 4/11] Reading model files
@ -1063,14 +1063,14 @@ Single GPU vs Multiple GPUs
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ] Device info:
[ INFO ] GPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] MULTI
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Device GPU.1 does not support performance hint property(-hint).
[Step 4/11] Reading model files
@ -1121,12 +1121,12 @@ Import Necessary Packages
import time
from pathlib import Path
import cv2
import numpy as np
from IPython.display import Video
import openvino as ov
# Instantiate OpenVINO Runtime
core = ov.Core()
core.available_devices
@ -1151,11 +1151,11 @@ Compile the Model
model = core.read_model(model=model_path)
device_name = "GPU"
compiled_model = core.compile_model(model=model, device_name=device_name, config={"PERFORMANCE_HINT": "THROUGHPUT"})
# Get the input and output nodes
input_layer = compiled_model.input(0)
output_layer = compiled_model.output(0)
# Get the input size
num, height, width, channels = input_layer.shape
print('Model input shape:', num, height, width, channels)
@ -1177,7 +1177,7 @@ Load and Preprocess Video Frames
video_file = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/video/Coco%20Walking%20in%20Berkeley.mp4"
video = cv2.VideoCapture(video_file)
framebuf = []
# Go through every frame of video and resize it
print('Loading video...')
while video.isOpened():
@ -1186,18 +1186,18 @@ Load and Preprocess Video Frames
print('Video loaded!')
video.release()
break
# Preprocess frames - convert them to shape expected by model
input_frame = cv2.resize(src=frame, dsize=(width, height), interpolation=cv2.INTER_AREA)
input_frame = np.expand_dims(input_frame, axis=0)
# Append frame to framebuffer
framebuf.append(input_frame)
print('Frame shape: ', framebuf[0].shape)
print('Number of frames: ', len(framebuf))
# Show original video file
# If the video does not display correctly inside the notebook, please open it with your favorite media player
Video(video_file)
@ -1251,10 +1251,10 @@ Callback Definition
global frame_number
stop_time = time.time()
frame_number += 1
predictions = next(iter(infer_request.results.values()))
results[frame_id] = predictions[:10] # Grab first 10 predictions for this frame
total_time = stop_time - start_time
frame_fps[frame_id] = frame_number / total_time
@ -1283,14 +1283,14 @@ Perform Inference
start_time = time.time()
for i, input_frame in enumerate(framebuf):
infer_queue.start_async({0: input_frame}, i)
infer_queue.wait_all() # Wait until all inference requests in the AsyncInferQueue are completed
stop_time = time.time()
# Calculate total inference time and FPS
total_time = stop_time - start_time
fps = len(framebuf) / total_time
time_per_frame = 1 / fps
time_per_frame = 1 / fps
print(f'Total time to infer all frames: {total_time:.3f}s')
print(f'Time per frame: {time_per_frame:.6f}s ({fps:.3f} FPS)')
@ -1310,20 +1310,20 @@ Process Results
# Set minimum detection threshold
min_thresh = .6
# Load video
video = cv2.VideoCapture(video_file)
# Get video parameters
frame_width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
fps = int(video.get(cv2.CAP_PROP_FPS))
fourcc = int(video.get(cv2.CAP_PROP_FOURCC))
# Create folder and VideoWriter to save output video
Path('./output').mkdir(exist_ok=True)
output = cv2.VideoWriter('output/output.mp4', fourcc, fps, (frame_width, frame_height))
# Draw detection results on every frame of video and save as a new video file
while video.isOpened():
current_frame = int(video.get(cv2.CAP_PROP_POS_FRAMES))
@ -1333,12 +1333,12 @@ Process Results
output.release()
video.release()
break
# Draw info at the top left such as current fps, the devices and the performance hint being used
cv2.putText(frame, f"fps {str(round(frame_fps[current_frame], 2))}", (5, 20), cv2.FONT_ITALIC, 0.6, (0, 0, 0), 1, cv2.LINE_AA)
cv2.putText(frame, f"device {device_name}", (5, 40), cv2.FONT_ITALIC, 0.6, (0, 0, 0), 1, cv2.LINE_AA)
cv2.putText(frame, f"device {device_name}", (5, 40), cv2.FONT_ITALIC, 0.6, (0, 0, 0), 1, cv2.LINE_AA)
cv2.putText(frame, f"hint {compiled_model.get_property('PERFORMANCE_HINT')}", (5, 60), cv2.FONT_ITALIC, 0.6, (0, 0, 0), 1, cv2.LINE_AA)
# prediction contains [image_id, label, conf, x_min, y_min, x_max, y_max] according to model
for prediction in np.squeeze(results[current_frame]):
if prediction[2] > min_thresh:
@ -1347,13 +1347,13 @@ Process Results
x_max = int(prediction[5] * frame_width)
y_max = int(prediction[6] * frame_height)
label = classes[int(prediction[1])]
# Draw a bounding box with its label above it
cv2.rectangle(frame, (x_min, y_min), (x_max, y_max), (0, 255, 0), 1, cv2.LINE_AA)
cv2.putText(frame, label, (x_min, y_min - 10), cv2.FONT_ITALIC, 1, (255, 0, 0), 1, cv2.LINE_AA)
output.write(frame)
# Show output video file
# If the video does not display correctly inside the notebook, please open it with your favorite media player
Video("output/output.mp4", width=800, embed=True)
@ -1396,7 +1396,7 @@ corresponding documentation:
- `Model
Caching <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimizing-latency/model-caching-overview.html>`__
- `MULTI Device
Mode <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_Running_on_multiple_devices.html>`__
Mode <https://docs.openvino.ai/2024/openvino-workflow/running-inference/inference-devices-and-modes/multi-device.html>`__
- `Query Device
Properties <https://docs.openvino.ai/2024/openvino-workflow/running-inference/inference-devices-and-modes/query-device-properties.html>`__
- `Configurations for GPUs with

63
docs/notebooks/109-latency-tricks-with-output.rst
View File

@ -141,7 +141,7 @@ requirements of this particular object detection model.
.. parsed-literal::
<DisplayHandle display_id=c7531cff1487c41296f1ac25e2e96b93>
<DisplayHandle display_id=7a4c7ef25e26f28ca0d4358f0a5c8e27>
@ -202,52 +202,21 @@ PyTorch Hub and small enough to see the difference in performance.
.. parsed-literal::
7%|▋ | 272k/3.87M [00:00<00:01, 2.23MB/s]
8%|▊ | 304k/3.87M [00:00<00:01, 3.08MB/s]
.. parsed-literal::
19%|█▉ | 752k/3.87M [00:00<00:00, 3.64MB/s]
64%|██████▎ | 2.47M/3.87M [00:00<00:00, 14.6MB/s]
.. parsed-literal::
29%|██▉ | 1.13M/3.87M [00:00<00:00, 3.87MB/s]
.. parsed-literal::
100%|██████████| 3.87M/3.87M [00:00<00:00, 18.6MB/s]
39%|███▉ | 1.52M/3.87M [00:00<00:00, 3.61MB/s]
.. parsed-literal::
49%|████▉ | 1.89M/3.87M [00:00<00:00, 3.68MB/s]
.. parsed-literal::
61%|██████▏ | 2.38M/3.87M [00:00<00:00, 4.06MB/s]
.. parsed-literal::
72%|███████▏ | 2.77M/3.87M [00:00<00:00, 3.82MB/s]
.. parsed-literal::
83%|████████▎ | 3.23M/3.87M [00:00<00:00, 4.07MB/s]
.. parsed-literal::
94%|█████████▍| 3.63M/3.87M [00:01<00:00, 3.85MB/s]
.. parsed-literal::
100%|██████████| 3.87M/3.87M [00:01<00:00, 3.87MB/s]
@ -465,12 +434,12 @@ optimizations applied. We will treat it as our baseline.
.. parsed-literal::
PyTorch model on CPU. First inference time: 0.0280 seconds
PyTorch model on CPU. First inference time: 0.0266 seconds
.. parsed-literal::
PyTorch model on CPU: 0.0218 seconds per image (45.96 FPS)
PyTorch model on CPU: 0.0216 seconds per image (46.33 FPS)
ONNX model
@ -520,12 +489,12 @@ Representation (IR) to leverage the OpenVINO Runtime.
.. parsed-literal::
ONNX model on CPU. First inference time: 0.0174 seconds
ONNX model on CPU. First inference time: 0.0164 seconds
.. parsed-literal::
ONNX model on CPU: 0.0136 seconds per image (73.63 FPS)
ONNX model on CPU: 0.0125 seconds per image (80.25 FPS)
OpenVINO IR model
@ -562,12 +531,12 @@ accuracy drop. That’s why we skip that step in this notebook.
.. parsed-literal::
OpenVINO model on CPU. First inference time: 0.0153 seconds
OpenVINO model on CPU. First inference time: 0.0163 seconds
.. parsed-literal::
OpenVINO model on CPU: 0.0122 seconds per image (82.17 FPS)
OpenVINO model on CPU: 0.0123 seconds per image (81.26 FPS)
OpenVINO IR model on GPU
@ -628,12 +597,12 @@ If it is the case, don’t use it.
.. parsed-literal::
OpenVINO model + more threads on CPU. First inference time: 0.0150 seconds
OpenVINO model + more threads on CPU. First inference time: 0.0151 seconds
.. parsed-literal::
OpenVINO model + more threads on CPU: 0.0122 seconds per image (81.82 FPS)
OpenVINO model + more threads on CPU: 0.0124 seconds per image (80.94 FPS)
OpenVINO IR model in latency mode
@ -664,12 +633,12 @@ devices as well.
.. parsed-literal::
OpenVINO model on AUTO. First inference time: 0.0153 seconds
OpenVINO model on AUTO. First inference time: 0.0158 seconds
.. parsed-literal::
OpenVINO model on AUTO: 0.0125 seconds per image (80.25 FPS)
OpenVINO model on AUTO: 0.0126 seconds per image (79.53 FPS)
OpenVINO IR model in latency mode + shared memory
@ -704,12 +673,12 @@ performance!
.. parsed-literal::
OpenVINO model + shared memory on AUTO. First inference time: 0.0113 seconds
OpenVINO model + shared memory on AUTO. First inference time: 0.0103 seconds
.. parsed-literal::
OpenVINO model + shared memory on AUTO: 0.0054 seconds per image (186.01 FPS)
OpenVINO model + shared memory on AUTO: 0.0054 seconds per image (185.55 FPS)
Other tricks

4
docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_31_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ae331df2cb3993ba23bef0c441831024024f4f1f946c2da17a2ef6a2e6a68d8f
size 52876
oid sha256:5d89934369311be59343b18178462948179f2338b011e203bd44ba12f2c5f415
size 52981

154
docs/notebooks/109-throughput-tricks-with-output.rst
View File

@ -93,7 +93,7 @@ Prerequisites
import time
from pathlib import Path
from typing import Any, List, Tuple
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
@ -116,24 +116,24 @@ object detection model.
import numpy as np
import cv2
FRAMES_NUMBER = 1024
IMAGE_WIDTH = 640
IMAGE_HEIGHT = 480
# load image
image = utils.load_image("https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_bike.jpg")
image = cv2.resize(image, dsize=(IMAGE_WIDTH, IMAGE_HEIGHT), interpolation=cv2.INTER_AREA)
# preprocess it for YOLOv5
input_image = image / 255.0
input_image = np.transpose(input_image, axes=(2, 0, 1))
input_image = np.expand_dims(input_image, axis=0)
# simulate video with many frames
video_frames = np.tile(input_image, (FRAMES_NUMBER, 1, 1, 1, 1))
# show the image
utils.show_array(image)
@ -146,7 +146,7 @@ object detection model.
.. parsed-literal::
<DisplayHandle display_id=bbb34b7fd1ad545280d19661bf0bd4c3>
<DisplayHandle display_id=6e9bd356e09a9465ea275c04ca5ea451>
@ -164,13 +164,13 @@ PyTorch Hub and small enough to see the difference in performance.
import torch
from IPython.utils import io
# directory for all models
base_model_dir = Path("model")
model_name = "yolov5n"
model_path = base_model_dir / model_name
# load YOLOv5n from PyTorch Hub
pytorch_model = torch.hub.load("ultralytics/yolov5", "custom", path=model_path, device="cpu", skip_validation=True)
# don't print full model architecture
@ -186,12 +186,12 @@ PyTorch Hub and small enough to see the difference in performance.
.. parsed-literal::
YOLOv5 🚀 2023-4-21 Python-3.8.10 torch-2.1.0+cpu CPU
.. parsed-literal::
Fusing layers...
Fusing layers...
.. parsed-literal::
@ -201,7 +201,7 @@ PyTorch Hub and small enough to see the difference in performance.
.. parsed-literal::
Adding AutoShape...
Adding AutoShape...
.. parsed-literal::
@ -223,10 +223,10 @@ benchmarking process.
.. code:: ipython3
import openvino as ov
# initialize OpenVINO
core = ov.Core()
# print available devices
for device in core.available_devices:
device_name = core.get_property(device, "FULL_DEVICE_NAME")
@ -250,8 +250,8 @@ second (FPS).
.. code:: ipython3
from openvino.runtime import AsyncInferQueue
def benchmark_model(model: Any, frames: np.ndarray, async_queue: AsyncInferQueue = None, benchmark_name: str = "OpenVINO model", device_name: str = "CPU") -> float:
"""
Helper function for benchmarking the model. It measures the time and prints results.
@ -264,7 +264,7 @@ second (FPS).
end = time.perf_counter()
first_infer_time = end - start
print(f"{benchmark_name} on {device_name}. First inference time: {first_infer_time :.4f} seconds")
# benchmarking
start = time.perf_counter()
for batch in frames:
@ -273,15 +273,15 @@ second (FPS).
if async_queue:
async_queue.wait_all()
end = time.perf_counter()
# elapsed time
infer_time = end - start
# print second per image and FPS
mean_infer_time = infer_time / FRAMES_NUMBER
mean_fps = FRAMES_NUMBER / infer_time
print(f"{benchmark_name} on {device_name}: {mean_infer_time :.4f} seconds per image ({mean_fps :.2f} FPS)")
return mean_fps
The following functions aim to post-process results and draw boxes on
@ -300,21 +300,21 @@ the image.
"cell phone", "microwave", "oven", "oaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear",
"hair drier", "toothbrush"
]
# Colors for the classes above (Rainbow Color Map).
colors = cv2.applyColorMap(
src=np.arange(0, 255, 255 / len(classes), dtype=np.float32).astype(np.uint8),
colormap=cv2.COLORMAP_RAINBOW,
).squeeze()
def postprocess(detections: np.ndarray) -> List[Tuple]:
"""
Postprocess the raw results from the model.
"""
# candidates - probability > 0.25
detections = detections[detections[..., 4] > 0.25]
boxes = []
labels = []
scores = []
@ -328,22 +328,22 @@ the image.
)
labels.append(int(label))
scores.append(float(score))
# Apply non-maximum suppression to get rid of many overlapping entities.
# See https://paperswithcode.com/method/non-maximum-suppression
# This algorithm returns indices of objects to keep.
indices = cv2.dnn.NMSBoxes(
bboxes=boxes, scores=scores, score_threshold=0.25, nms_threshold=0.5
)
# If there are no boxes.
if len(indices) == 0:
return []
# Filter detected objects.
return [(labels[idx], scores[idx], boxes[idx]) for idx in indices.flatten()]
def draw_boxes(img: np.ndarray, boxes):
"""
Draw detected boxes on the image.
@ -355,7 +355,7 @@ the image.
x2 = box[0] + box[2]
y2 = box[1] + box[3]
cv2.rectangle(img=img, pt1=box[:2], pt2=(x2, y2), color=color, thickness=2)
# Draw a label name inside the box.
cv2.putText(
img=img,
@ -367,17 +367,17 @@ the image.
thickness=1,
lineType=cv2.LINE_AA,
)
def show_result(results: np.ndarray):
"""
Postprocess the raw results, draw boxes and show the image.
"""
output_img = image.copy()
detections = postprocess(results)
draw_boxes(output_img, detections)
utils.show_array(output_img)
Optimizations
@ -400,7 +400,7 @@ optimizations applied. We will treat it as our baseline.
.. code:: ipython3
import torch
with torch.no_grad():
result = pytorch_model(torch.as_tensor(video_frames[0])).detach().numpy()[0]
show_result(result)
@ -413,12 +413,12 @@ optimizations applied. We will treat it as our baseline.
.. parsed-literal::
PyTorch model on CPU. First inference time: 0.0224 seconds
PyTorch model on CPU. First inference time: 0.0262 seconds
.. parsed-literal::
PyTorch model on CPU: 0.0221 seconds per image (45.32 FPS)
PyTorch model on CPU: 0.0205 seconds per image (48.72 FPS)
OpenVINO IR model
@ -438,23 +438,23 @@ step in this notebook.
.. code:: ipython3
onnx_path = base_model_dir / Path(f"{model_name}_{IMAGE_WIDTH}_{IMAGE_HEIGHT}").with_suffix(".onnx")
# export PyTorch model to ONNX if it doesn't already exist
if not onnx_path.exists():
dummy_input = torch.randn(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH)
torch.onnx.export(pytorch_model, dummy_input, onnx_path)
# convert ONNX model to IR, use FP16
ov_model = ov.convert_model(onnx_path)
.. code:: ipython3
ov_cpu_model = core.compile_model(ov_model, device_name="CPU")
result = ov_cpu_model(video_frames[0])[ov_cpu_model.output(0)][0]
show_result(result)
ov_cpu_fps = benchmark_model(model=ov_cpu_model, frames=video_frames, benchmark_name="OpenVINO model")
del ov_cpu_model # release resources
@ -464,12 +464,12 @@ step in this notebook.
.. parsed-literal::
OpenVINO model on CPU. First inference time: 0.0142 seconds
OpenVINO model on CPU. First inference time: 0.0147 seconds
.. parsed-literal::
OpenVINO model on CPU: 0.0071 seconds per image (141.49 FPS)
OpenVINO model on CPU: 0.0070 seconds per image (142.58 FPS)
OpenVINO IR model + bigger batch
@ -488,13 +488,13 @@ hardware and model.
.. code:: ipython3
batch_size = 4
onnx_batch_path = base_model_dir / Path(f"{model_name}_{IMAGE_WIDTH}_{IMAGE_HEIGHT}_batch_{batch_size}").with_suffix(".onnx")
if not onnx_batch_path.exists():
dummy_input = torch.randn(batch_size, 3, IMAGE_HEIGHT, IMAGE_WIDTH)
torch.onnx.export(pytorch_model, dummy_input, onnx_batch_path)
# export the model with the bigger batch size
ov_batch_model = ov.convert_model(onnx_batch_path)
@ -510,13 +510,13 @@ hardware and model.
.. code:: ipython3
ov_cpu_batch_model = core.compile_model(ov_batch_model, device_name="CPU")
batched_video_frames = video_frames.reshape([-1, batch_size, 3, IMAGE_HEIGHT, IMAGE_WIDTH])
result = ov_cpu_batch_model(batched_video_frames[0])[ov_cpu_batch_model.output(0)][0]
show_result(result)
ov_cpu_batch_fps = benchmark_model(model=ov_cpu_batch_model, frames=batched_video_frames, benchmark_name="OpenVINO model + bigger batch")
del ov_cpu_batch_model # release resources
@ -526,12 +526,12 @@ hardware and model.
.. parsed-literal::
OpenVINO model + bigger batch on CPU. First inference time: 0.0486 seconds
OpenVINO model + bigger batch on CPU. First inference time: 0.0514 seconds
.. parsed-literal::
OpenVINO model + bigger batch on CPU: 0.0068 seconds per image (146.44 FPS)
OpenVINO model + bigger batch on CPU: 0.0068 seconds per image (146.52 FPS)
Asynchronous processing
@ -561,17 +561,17 @@ the pipeline.
result = infer_request.get_output_tensor(0).data[0]
show_result(result)
pass
infer_queue = ov.AsyncInferQueue(ov_model)
infer_queue.set_callback(callback) # set callback to post-process (show) results
infer_queue.start_async(video_frames[0])
infer_queue.wait_all()
# don't show output for the remaining frames
infer_queue.set_callback(lambda x, y: {})
fps = benchmark_model(model=infer_queue.start_async, frames=video_frames, async_queue=infer_queue, benchmark_name=benchmark_name, device_name=device_name)
del infer_queue # release resources
return fps
@ -592,9 +592,9 @@ feature, which sets the batch size to the optimal level.
.. code:: ipython3
ov_cpu_through_model = core.compile_model(ov_model, device_name="CPU", config={"PERFORMANCE_HINT": "THROUGHPUT"})
ov_cpu_through_fps = benchmark_async_mode(ov_cpu_through_model, benchmark_name="OpenVINO model", device_name="CPU (THROUGHPUT)")
del ov_cpu_through_model # release resources
@ -604,12 +604,12 @@ feature, which sets the batch size to the optimal level.
.. parsed-literal::
OpenVINO model on CPU (THROUGHPUT). First inference time: 0.0221 seconds
OpenVINO model on CPU (THROUGHPUT). First inference time: 0.0244 seconds
.. parsed-literal::
OpenVINO model on CPU (THROUGHPUT): 0.0040 seconds per image (249.65 FPS)
OpenVINO model on CPU (THROUGHPUT): 0.0040 seconds per image (249.04 FPS)
OpenVINO IR model in throughput mode on GPU
@ -633,9 +633,9 @@ execution.
if "GPU" in core.available_devices:
# compile for GPU
ov_gpu_model = core.compile_model(ov_model, device_name="GPU", config={"PERFORMANCE_HINT": "THROUGHPUT"})
ov_gpu_fps = benchmark_async_mode(ov_gpu_model, benchmark_name="OpenVINO model", device_name="GPU (THROUGHPUT)")
del ov_gpu_model # release resources
OpenVINO IR model in throughput mode on AUTO
@ -651,9 +651,9 @@ performance hint.
.. code:: ipython3
ov_auto_model = core.compile_model(ov_model, device_name="AUTO", config={"PERFORMANCE_HINT": "THROUGHPUT"})
ov_auto_fps = benchmark_async_mode(ov_auto_model, benchmark_name="OpenVINO model", device_name="AUTO (THROUGHPUT)")
del ov_auto_model # release resources
@ -663,12 +663,12 @@ performance hint.
.. parsed-literal::
OpenVINO model on AUTO (THROUGHPUT). First inference time: 0.0235 seconds
OpenVINO model on AUTO (THROUGHPUT). First inference time: 0.0233 seconds
.. parsed-literal::
OpenVINO model on AUTO (THROUGHPUT): 0.0040 seconds per image (249.90 FPS)
OpenVINO model on AUTO (THROUGHPUT): 0.0040 seconds per image (250.58 FPS)
OpenVINO IR model in cumulative throughput mode on AUTO
@ -685,7 +685,7 @@ activate all devices.
.. code:: ipython3
ov_auto_cumulative_model = core.compile_model(ov_model, device_name="AUTO", config={"PERFORMANCE_HINT": "CUMULATIVE_THROUGHPUT"})
ov_auto_cumulative_fps = benchmark_async_mode(ov_auto_cumulative_model, benchmark_name="OpenVINO model", device_name="AUTO (CUMULATIVE THROUGHPUT)")
@ -695,12 +695,12 @@ activate all devices.
.. parsed-literal::
OpenVINO model on AUTO (CUMULATIVE THROUGHPUT). First inference time: 0.0227 seconds
OpenVINO model on AUTO (CUMULATIVE THROUGHPUT). First inference time: 0.0204 seconds
.. parsed-literal::
OpenVINO model on AUTO (CUMULATIVE THROUGHPUT): 0.0040 seconds per image (250.47 FPS)
OpenVINO model on AUTO (CUMULATIVE THROUGHPUT): 0.0040 seconds per image (251.22 FPS)
Other tricks
@ -712,7 +712,7 @@ There are other tricks for performance improvement, such as advanced
options, quantization and pre-post-processing or dedicated to latency
mode. To get even more from your model, please visit `advanced
throughput
options <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimizing-throughput-advanced.html>`__,
options <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimizing-throughput/advanced_throughput_options.html>`__,
`109-latency-tricks <109-latency-tricks-with-output.html-with-output.html>`__,
`111-detection-quantization <111-detection-quantization-with-output.html>`__, and
`118-optimize-preprocessing <118-optimize-preprocessing-with-output.html>`__.
@ -733,20 +733,20 @@ steps, just skip them.
.. code:: ipython3
from matplotlib import pyplot as plt
labels = ["PyTorch model", "OpenVINO IR model", "OpenVINO IR model + bigger batch", "OpenVINO IR model in throughput mode", "OpenVINO IR model in throughput mode on GPU",
"OpenVINO IR model in throughput mode on AUTO", "OpenVINO IR model in cumulative throughput mode on AUTO"]
fps = [pytorch_fps, ov_cpu_fps, ov_cpu_batch_fps, ov_cpu_through_fps, ov_gpu_fps, ov_auto_fps, ov_auto_cumulative_fps]
bar_colors = colors[::10] / 255.0
fig, ax = plt.subplots(figsize=(16, 8))
ax.bar(labels, fps, color=bar_colors)
ax.set_ylabel("Throughput [FPS]")
ax.set_title("Performance difference")
plt.xticks(rotation='vertical')
plt.show()

4
docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_34_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c054b14da27759eeaf1a1791c919f28eade5a92ea995b1f07690c71c435b2092
size 62458
oid sha256:fd3c5e32a2fa420855d0c3afbe63b2766bea7945af42dc014a2c4ef0511b34e9
size 62481

113
docs/notebooks/110-ct-scan-live-inference-with-output.rst
View File

@ -48,7 +48,7 @@ Table of contents:
.. code:: ipython3
%pip install -q "openvino>=2023.1.0" "monai>=0.9.1,<1.0.0" "nncf>=2.5.0"
%pip install -q "openvino>=2023.3.0" "monai>=0.9.1" "nncf>=2.8.0"
.. parsed-literal::
@ -67,27 +67,27 @@ Imports
import sys
import zipfile
from pathlib import Path
import numpy as np
from monai.transforms import LoadImage
import openvino as ov
from custom_segmentation import SegmentationModel
sys.path.append("../utils")
from notebook_utils import download_file
.. parsed-literal::
2024-02-09 22:50:38.323593: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-02-09 22:50:38.357752: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2024-03-12 22:35:25.130181: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-03-12 22:35:25.164310: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
.. parsed-literal::
2024-02-09 22:50:38.922511: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2024-03-12 22:35:25.734617: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
Settings
@ -104,16 +104,16 @@ trained or optimized yourself, adjust the model paths.
# The directory that contains the IR model (xml and bin) files.
models_dir = Path('pretrained_model')
ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/kidney-segmentation-kits19/FP16-INT8/'
ir_model_name_xml = 'quantized_unet_kits19.xml'
ir_model_name_bin = 'quantized_unet_kits19.bin'
download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory=models_dir)
download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory=models_dir)
MODEL_PATH = models_dir / ir_model_name_xml
# Uncomment the next line to use the FP16 model instead of the quantized model.
# MODEL_PATH = "pretrained_model/unet_kits19.xml"
@ -140,7 +140,7 @@ Tool <https://docs.openvino.ai/2024/learn-openvino/openvino-samples/benchmark-to
is a command-line application that can be run in the notebook with
``! benchmark_app`` or ``%sx benchmark_app`` commands.
**NOTE**: The ``benchmark_app`` tool is able to measure the
**Note**: The ``benchmark_app`` tool is able to measure the
performance of the OpenVINO Intermediate Representation (OpenVINO IR)
models only. For more accurate performance, run ``benchmark_app`` in
a terminal/command prompt after closing other applications. Run
@ -154,16 +154,16 @@ is a command-line application that can be run in the notebook with
core = ov.Core()
# By default, benchmark on MULTI:CPU,GPU if a GPU is available, otherwise on CPU.
device_list = ["MULTI:CPU,GPU" if "GPU" in core.available_devices else "AUTO"]
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices + device_list,
value=device_list[0],
description='Device:',
disabled=False,
)
device
@ -187,18 +187,18 @@ is a command-line application that can be run in the notebook with
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] AUTO
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ]
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 13.02 ms
[ INFO ] Read model took 13.37 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input.1 (node: input.1) : f32 / [...] / [1,1,512,512]
@ -216,7 +216,7 @@ is a command-line application that can be run in the notebook with
.. parsed-literal::
[ INFO ] Compile model took 232.64 ms
[ INFO ] Compile model took 259.78 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: pretrained_unet_kits19
@ -228,12 +228,15 @@ is a command-line application that can be run in the notebook with
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] CPU_DENORMALS_OPTIMIZATION: False
[ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
[ INFO ] DYNAMIC_QUANTIZATION_GROUP_SIZE: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] ENABLE_HYPER_THREADING: False
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] INFERENCE_NUM_THREADS: 12
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] KV_CACHE_PRECISION: <Type: 'float16'>
[ INFO ] LOG_LEVEL: Level.NO
[ INFO ] NETWORK_NAME: pretrained_unet_kits19
[ INFO ] NUM_STREAMS: 1
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1
@ -245,28 +248,24 @@ is a command-line application that can be run in the notebook with
[ INFO ] LOADED_FROM_CACHE: False
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'input.1'!. This input will be filled with random values!
[ INFO ] Fill input 'input.1' with random values
[ INFO ] Fill input 'input.1' with random values
[Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
.. parsed-literal::
[ INFO ] First inference took 24.68 ms
[ INFO ] First inference took 26.09 ms
.. parsed-literal::
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 1366 iterations
[ INFO ] Duration: 15004.33 ms
[ INFO ] Count: 1346 iterations
[ INFO ] Duration: 15003.91 ms
[ INFO ] Latency:
[ INFO ] Median: 10.75 ms
[ INFO ] Average: 10.80 ms
[ INFO ] Min: 10.53 ms
[ INFO ] Max: 12.59 ms
[ INFO ] Throughput: 91.04 FPS
[ INFO ] Median: 10.91 ms
[ INFO ] Average: 10.96 ms
[ INFO ] Min: 10.64 ms
[ INFO ] Max: 12.98 ms
[ INFO ] Throughput: 89.71 FPS
Download and Prepare Data
@ -292,9 +291,9 @@ downloaded and extracted in the next cell.
# The CT scan case number. For example: 16 for data from the case_00016 directory.
# Currently only 117 is supported.
CASE = 117
case_path = BASEDIR / f"case_{CASE:05d}"
if not case_path.exists():
filename = download_file(
f"https://storage.openvinotoolkit.org/data/test_data/openvino_notebooks/kits19/case_{CASE:05d}.zip"
@ -371,7 +370,7 @@ to see the implementation.
ie=core, model_path=Path(MODEL_PATH), sigmoid=True, rotate_and_flip=True
)
image_paths = sorted(case_path.glob("imaging_frames/*jpg"))
print(f"{case_path.name}, {len(image_paths)} images")
@ -393,10 +392,10 @@ tutorial.
.. code:: ipython3
framebuf = []
next_frame_id = 0
reader = LoadImage(image_only=True, dtype=np.uint8)
while next_frame_id < len(image_paths) - 1:
image_path = image_paths[next_frame_id]
image = reader(str(image_path))
@ -439,20 +438,20 @@ The ``callback`` function will show the results of inference.
import cv2
import copy
from IPython import display
from typing import Dict, Any
# Define a callback function that runs every time the asynchronous pipeline completes inference on a frame
def completion_callback(infer_request: ov.InferRequest, user_data: Dict[str, Any],) -> None:
preprocess_meta = user_data['preprocess_meta']
raw_outputs = {out.any_name: copy.deepcopy(res.data) for out, res in zip(infer_request.model_outputs, infer_request.output_tensors)}
raw_outputs = {idx: copy.deepcopy(res.data) for idx, (out, res) in enumerate(zip(infer_request.model_outputs, infer_request.output_tensors))}
frame = segmentation_model.postprocess(raw_outputs, preprocess_meta)
_, encoded_img = cv2.imencode(".jpg", frame, params=[cv2.IMWRITE_JPEG_QUALITY, 90])
# Create IPython image
i = display.Image(data=encoded_img)
# Display the image in this notebook
display.clear_output(wait=True)
display.display(i)
@ -465,34 +464,34 @@ Create asynchronous inference queue and perform it
.. code:: ipython3
import time
load_start_time = time.perf_counter()
compiled_model = core.compile_model(segmentation_model.net, device.value)
# Create asynchronous inference queue with optimal number of infer requests
infer_queue = ov.AsyncInferQueue(compiled_model)
infer_queue.set_callback(completion_callback)
load_end_time = time.perf_counter()
results = [None] * len(framebuf)
frame_number = 0
# Perform inference on every frame in the framebuffer
start_time = time.time()
for i, input_frame in enumerate(framebuf):
inputs, preprocessing_meta = segmentation_model.preprocess({segmentation_model.net.input(0): input_frame})
infer_queue.start_async(inputs, {'preprocess_meta': preprocessing_meta})
# Wait until all inference requests in the AsyncInferQueue are completed
infer_queue.wait_all()
stop_time = time.time()
# Calculate total inference time and FPS
total_time = stop_time - start_time
fps = len(framebuf) / total_time
time_per_frame = 1 / fps
time_per_frame = 1 / fps
print(f"Loaded model to {device} in {load_end_time-load_start_time:.2f} seconds.")
print(f'Total time to infer all frames: {total_time:.3f}s')
print(f'Time per frame: {time_per_frame:.6f}s ({fps:.3f} FPS)')
@ -503,7 +502,7 @@ Create asynchronous inference queue and perform it
.. parsed-literal::
Loaded model to Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') in 0.23 seconds.
Total time to infer all frames: 2.588s
Time per frame: 0.038061s (26.274 FPS)
Loaded model to Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') in 0.26 seconds.
Total time to infer all frames: 2.496s
Time per frame: 0.036711s (27.239 FPS)

370
docs/notebooks/110-ct-segmentation-quantize-nncf-with-output.rst
View File

@ -87,7 +87,7 @@ Table of contents:
.. code:: ipython3
%pip install -q "openvino>=2023.1.0" "monai>=0.9.1,<1.0.0" "torchmetrics>=0.11.0" "nncf>=2.6.0" --extra-index-url https://download.pytorch.org/whl/cpu
%pip install -q "openvino>=2023.3.0" "monai>=0.9.1" "torchmetrics>=0.11.0" "nncf>=2.8.0" torch --extra-index-url https://download.pytorch.org/whl/cpu
.. parsed-literal::
@ -100,52 +100,6 @@ Imports
.. code:: ipython3
# On Windows, try to find the directory that contains x64 cl.exe and add it to the PATH to enable PyTorch
# to find the required C++ tools. This code assumes that Visual Studio is installed in the default
# directory. If you have a different C++ compiler, please add the correct path to os.environ["PATH"]
# directly. Note that the C++ Redistributable is not enough to run this notebook.
# Adding the path to os.environ["LIB"] is not always required - it depends on the system's configuration
import sys
if sys.platform == "win32":
import distutils.command.build_ext
import os
from pathlib import Path
if sys.getwindowsversion().build >= 20000: # Windows 11
search_path = "**/Hostx64/x64/cl.exe"
else:
search_path = "**/Hostx86/x64/cl.exe"
VS_INSTALL_DIR_2019 = r"C:/Program Files (x86)/Microsoft Visual Studio"
VS_INSTALL_DIR_2022 = r"C:/Program Files/Microsoft Visual Studio"
cl_paths_2019 = sorted(list(Path(VS_INSTALL_DIR_2019).glob(search_path)))
cl_paths_2022 = sorted(list(Path(VS_INSTALL_DIR_2022).glob(search_path)))
cl_paths = cl_paths_2019 + cl_paths_2022
if len(cl_paths) == 0:
raise ValueError(
"Cannot find Visual Studio. This notebook requires an x64 C++ compiler. If you installed "
"a C++ compiler, please add the directory that contains cl.exe to `os.environ['PATH']`."
)
else:
# If multiple versions of MSVC are installed, get the most recent version
cl_path = cl_paths[-1]
vs_dir = str(cl_path.parent)
os.environ["PATH"] += f"{os.pathsep}{vs_dir}"
# Code for finding the library dirs from
# https://stackoverflow.com/questions/47423246/get-pythons-lib-path
d = distutils.core.Distribution()
b = distutils.command.build_ext.build_ext(d)
b.finalize_options()
os.environ["LIB"] = os.pathsep.join(b.library_dirs)
print(f"Added {vs_dir} to PATH")
.. code:: ipython3
import logging
@ -157,9 +111,9 @@ Imports
import zipfile
from pathlib import Path
from typing import Union
warnings.filterwarnings("ignore", category=UserWarning)
import cv2
import matplotlib.pyplot as plt
import monai
@ -170,26 +124,26 @@ Imports
from monai.transforms import LoadImage
from nncf.common.logging.logger import set_log_level
from torchmetrics import F1Score as F1
set_log_level(logging.ERROR) # Disables all NNCF info and warning messages
from custom_segmentation import SegmentationModel
from async_pipeline import show_live_inference
sys.path.append("../utils")
from notebook_utils import download_file
.. parsed-literal::
2024-02-09 22:51:11.599112: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-02-09 22:51:11.634091: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2024-03-12 22:35:56.579274: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-03-12 22:35:56.613310: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
.. parsed-literal::
2024-02-09 22:51:12.223414: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2024-03-12 22:35:57.179215: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
@ -233,13 +187,13 @@ notebook <pytorch-monai-training.ipynb>`__.
state_dict_url = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/kidney-segmentation-kits19/unet_kits19_state_dict.pth"
state_dict_file = download_file(state_dict_url, directory="pretrained_model")
state_dict = torch.load(state_dict_file, map_location=torch.device("cpu"))
new_state_dict = {}
for k, v in state_dict.items():
new_key = k.replace("_model.", "")
new_state_dict[new_key] = v
new_state_dict.pop("loss_function.pos_weight")
model = monai.networks.nets.BasicUNet(spatial_dims=2, in_channels=1, out_channels=1).eval()
model.load_state_dict(new_state_dict)
@ -317,8 +271,8 @@ method to display the images in the expected orientation:
def rotate_and_flip(image):
"""Rotate `image` by 90 degrees and flip horizontally"""
return cv2.flip(cv2.rotate(image, rotateCode=cv2.ROTATE_90_CLOCKWISE), flipCode=1)
class KitsDataset:
def __init__(self, basedir: str):
"""
@ -327,40 +281,40 @@ method to display the images in the expected orientation:
with each subdirectory containing directories imaging_frames, with jpg images, and
segmentation_frames with segmentation masks as png files.
See https://github.com/openvinotoolkit/openvino_notebooks/blob/main/notebooks/110-ct-segmentation-quantize/data-preparation-ct-scan.ipynb
:param basedir: Directory that contains the prepared CT scans
"""
masks = sorted(BASEDIR.glob("case_*/segmentation_frames/*png"))
self.basedir = basedir
self.dataset = masks
print(
f"Created dataset with {len(self.dataset)} items. "
f"Base directory for data: {basedir}"
)
def __getitem__(self, index):
"""
Get an item from the dataset at the specified index.
:return: (image, segmentation_mask)
"""
mask_path = self.dataset[index]
image_path = str(mask_path.with_suffix(".jpg")).replace(
"segmentation_frames", "imaging_frames"
)
# Load images with MONAI's LoadImage to match data loading in training notebook
mask = LoadImage(image_only=True, dtype=np.uint8)(str(mask_path)).numpy()
img = LoadImage(image_only=True, dtype=np.float32)(str(image_path)).numpy()
if img.shape[:2] != (512, 512):
img = cv2.resize(img.astype(np.uint8), (512, 512)).astype(np.float32)
mask = cv2.resize(mask, (512, 512))
input_image = np.expand_dims(img, axis=0)
return input_image, mask
def __len__(self):
return len(self.dataset)
@ -378,10 +332,10 @@ kidney pixels to verify that the annotations look correct:
image_data, mask = next(item for item in dataset if np.count_nonzero(item[1]) > 5000)
# Remove extra image dimension and rotate and flip the image for visualization
image = rotate_and_flip(image_data.squeeze())
# The data loader returns annotations as (index, mask) and mask in shape (H,W)
mask = rotate_and_flip(mask)
fig, ax = plt.subplots(1, 2, figsize=(12, 6))
ax[0].imshow(image, cmap="gray")
ax[1].imshow(mask, cmap="gray");
@ -393,7 +347,7 @@ kidney pixels to verify that the annotations look correct:
.. image:: 110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_15_1.png
.. image:: 110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_14_1.png
Metric
@ -459,7 +413,7 @@ this notebook.
.. code:: ipython3
fp32_ir_path = MODEL_DIR / Path('unet_kits19_fp32.xml')
fp32_ir_model = ov.convert_model(model, example_input=torch.ones(1, 1, 512, 512, dtype=torch.float32))
ov.save_model(fp32_ir_model, str(fp32_ir_path))
@ -481,7 +435,7 @@ this notebook.
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:179: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if x_e.shape[-i - 1] != x_0.shape[-i - 1]:
@ -489,7 +443,7 @@ this notebook.
advanced algorithms for Neural Networks inference optimization in
OpenVINO with minimal accuracy drop.
**NOTE**: NNCF Post-training Quantization is available in OpenVINO
**Note**: NNCF Post-training Quantization is available in OpenVINO
2023.0 release.
Create a quantized model from the pre-trained ``FP32`` model and the
@ -513,8 +467,8 @@ steps:
"""
images, _ = data_item
return images
data_loader = torch.utils.data.DataLoader(dataset)
calibration_dataset = nncf.Dataset(data_loader, transform_fn)
quantized_model = nncf.quantize(
@ -532,7 +486,9 @@ steps:
.. raw:: html
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"></pre>
@ -551,7 +507,9 @@ steps:
.. raw:: html
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"></pre>
@ -563,29 +521,38 @@ steps:
Export the quantized model to ONNX and then convert it to OpenVINO IR
model and save it.
Convert quantized model to OpenVINO IR model and save it.
.. code:: ipython3
dummy_input = torch.randn(1, 1, 512, 512)
int8_onnx_path = MODEL_DIR / "unet_kits19_int8.onnx"
int8_ir_path = Path(int8_onnx_path).with_suffix(".xml")
torch.onnx.export(quantized_model, dummy_input, int8_onnx_path)
int8_ir_model = ov.convert_model(int8_onnx_path)
int8_ir_model = ov.convert_model(quantized_model, example_input=dummy_input, input=dummy_input.shape)
ov.save_model(int8_ir_model, str(int8_ir_path))
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:334: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:337: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
return self._level_low.item()
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:342: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:345: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
return self._level_high.item()
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:179: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if x_e.shape[-i - 1] != x_0.shape[-i - 1]:
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1093: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:
Tensor-likes are not close!
Mismatched elements: 248084 / 262144 (94.6%)
Greatest absolute difference: 3.0810165405273438 at index (0, 0, 135, 41) (up to 1e-05 allowed)
Greatest relative difference: 29099.340127879757 at index (0, 0, 139, 287) (up to 1e-05 allowed)
_check_trace(
This notebook demonstrates post-training quantization with NNCF.
NNCF also supports quantization-aware training, and other algorithms
@ -607,7 +574,7 @@ Compare File Size
fp32_ir_model_size = fp32_ir_path.with_suffix(".bin").stat().st_size / 1024
quantized_model_size = int8_ir_path.with_suffix(".bin").stat().st_size / 1024
print(f"FP32 IR model size: {fp32_ir_model_size:.2f} KB")
print(f"INT8 model size: {quantized_model_size:.2f} KB")
@ -615,7 +582,38 @@ Compare File Size
.. parsed-literal::
FP32 IR model size: 3864.14 KB
INT8 model size: 1940.41 KB
INT8 model size: 1940.40 KB
Select Inference Device
~~~~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
core = ov.Core()
# By default, benchmark on MULTI:CPU,GPU if a GPU is available, otherwise on CPU.
device_list = ["MULTI:CPU,GPU" if "GPU" in core.available_devices else "AUTO"]
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices + device_list,
value=device_list[0],
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
Compare Metrics for the original model and the quantized model to be sure that there no degradation.
@ -625,11 +623,11 @@ Compare Metrics for the original model and the quantized model to be sure that t
.. code:: ipython3
core = ov.Core()
int8_compiled_model = core.compile_model(int8_ir_model)
int8_compiled_model = core.compile_model(int8_ir_model, device.value)
int8_f1 = compute_f1(int8_compiled_model, dataset)
print(f"FP32 F1: {fp32_f1:.3f}")
print(f"INT8 F1: {int8_f1:.3f}")
@ -664,14 +662,10 @@ be run in the notebook with ``! benchmark_app`` or
# ! benchmark_app --help
.. code:: ipython3
device = "CPU"
.. code:: ipython3
# Benchmark FP32 model
! benchmark_app -m $fp32_ir_path -d $device -t 15 -api sync
! benchmark_app -m $fp32_ir_path -d $device.value -t 15 -api sync
.. parsed-literal::
@ -680,22 +674,18 @@ be run in the notebook with ``! benchmark_app`` or
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ] Device info:
.. parsed-literal::
[ INFO ] CPU
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ]
[ INFO ] AUTO
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.LATENCY.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 26.51 ms
[ INFO ] Read model took 8.66 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] x (node: x) : f32 / [...] / [?,?,?,?]
@ -713,31 +703,42 @@ be run in the notebook with ``! benchmark_app`` or
.. parsed-literal::
[ INFO ] Compile model took 87.61 ms
[ INFO ] Compile model took 136.91 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: Model0
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1
[ INFO ] NUM_STREAMS: 1
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 12
[ INFO ] PERF_COUNT: NO
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: LATENCY
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: False
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] CPU_DENORMALS_OPTIMIZATION: False
[ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
[ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1
[ INFO ] MULTI_DEVICE_PRIORITIES: CPU
[ INFO ] CPU:
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] CPU_DENORMALS_OPTIMIZATION: False
[ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
[ INFO ] DYNAMIC_QUANTIZATION_GROUP_SIZE: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] ENABLE_HYPER_THREADING: False
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] INFERENCE_NUM_THREADS: 12
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] KV_CACHE_PRECISION: <Type: 'float16'>
[ INFO ] LOG_LEVEL: Level.NO
[ INFO ] NETWORK_NAME: Model0
[ INFO ] NUM_STREAMS: 1
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1
[ INFO ] PERFORMANCE_HINT: LATENCY
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] PERF_COUNT: NO
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] MODEL_PRIORITY: Priority.MEDIUM
[ INFO ] LOADED_FROM_CACHE: False
[Step 9/11] Creating infer requests and preparing input tensors
[ ERROR ] Input x is dynamic. Provide data shapes!
Traceback (most recent call last):
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 486, in main
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 486, in main
data_queue = get_input_data(paths_to_input, app_inputs_info)
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/utils/inputs_filling.py", line 123, in get_input_data
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/utils/inputs_filling.py", line 123, in get_input_data
raise Exception(f"Input {info.name} is dynamic. Provide data shapes!")
Exception: Input x is dynamic. Provide data shapes!
@ -745,7 +746,7 @@ be run in the notebook with ``! benchmark_app`` or
.. code:: ipython3
# Benchmark INT8 model
! benchmark_app -m $int8_ir_path -d $device -t 15 -api sync
! benchmark_app -m $int8_ir_path -d $device.value -t 15 -api sync
.. parsed-literal::
@ -754,78 +755,93 @@ be run in the notebook with ``! benchmark_app`` or
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ]
[ INFO ] AUTO
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.LATENCY.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 13.17 ms
.. parsed-literal::
[ INFO ] Read model took 13.46 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] x.1 (node: x.1) : f32 / [...] / [1,1,512,512]
[ INFO ] x (node: x) : f32 / [...] / [1,1,512,512]
[ INFO ] Model outputs:
[ INFO ] 571 (node: 571) : f32 / [...] / [1,1,512,512]
[ INFO ] ***NO_NAME*** (node: __module.final_conv/aten::_convolution/Add) : f32 / [...] / [1,1,512,512]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] x.1 (node: x.1) : f32 / [N,C,H,W] / [1,1,512,512]
[ INFO ] x (node: x) : f32 / [N,C,H,W] / [1,1,512,512]
[ INFO ] Model outputs:
[ INFO ] 571 (node: 571) : f32 / [...] / [1,1,512,512]
[ INFO ] ***NO_NAME*** (node: __module.final_conv/aten::_convolution/Add) : f32 / [...] / [1,1,512,512]
[Step 7/11] Loading the model to the device
.. parsed-literal::
[ INFO ] Compile model took 190.56 ms
[ INFO ] Compile model took 274.86 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: main_graph
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1
[ INFO ] NUM_STREAMS: 1
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 12
[ INFO ] PERF_COUNT: NO
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: LATENCY
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: False
[ INFO ] NETWORK_NAME: Model49
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] CPU_DENORMALS_OPTIMIZATION: False
[ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
[ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1
[ INFO ] MULTI_DEVICE_PRIORITIES: CPU
[ INFO ] CPU:
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] CPU_DENORMALS_OPTIMIZATION: False
[ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
[ INFO ] DYNAMIC_QUANTIZATION_GROUP_SIZE: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] ENABLE_HYPER_THREADING: False
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] INFERENCE_NUM_THREADS: 12
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] KV_CACHE_PRECISION: <Type: 'float16'>
[ INFO ] LOG_LEVEL: Level.NO
[ INFO ] NETWORK_NAME: Model49
[ INFO ] NUM_STREAMS: 1
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1
[ INFO ] PERFORMANCE_HINT: LATENCY
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] PERF_COUNT: NO
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] MODEL_PRIORITY: Priority.MEDIUM
[ INFO ] LOADED_FROM_CACHE: False
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'x.1'!. This input will be filled with random values!
[ INFO ] Fill input 'x.1' with random values
[ WARNING ] No input files were given for input 'x'!. This input will be filled with random values!
[ INFO ] Fill input 'x' with random values
[Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
.. parsed-literal::
[ INFO ] First inference took 30.25 ms
[ INFO ] First inference took 30.24 ms
.. parsed-literal::
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 973 iterations
[ INFO ] Duration: 15006.05 ms
[ INFO ] Count: 972 iterations
[ INFO ] Duration: 15004.50 ms
[ INFO ] Latency:
[ INFO ] Median: 15.16 ms
[ INFO ] Average: 15.22 ms
[ INFO ] Min: 14.87 ms
[ INFO ] Max: 17.88 ms
[ INFO ] Throughput: 64.84 FPS
[ INFO ] Median: 15.20 ms
[ INFO ] Average: 15.24 ms
[ INFO ] Min: 14.91 ms
[ INFO ] Max: 16.93 ms
[ INFO ] Throughput: 64.78 FPS
Visually Compare Inference Results
@ -859,31 +875,31 @@ seed is displayed to enable reproducing specific runs of this cell.
# to binary segmentation masks
def sigmoid(x):
return np.exp(-np.logaddexp(0, -x))
num_images = 4
colormap = "gray"
# Load FP32 and INT8 models
core = ov.Core()
fp_model = core.read_model(fp32_ir_path)
int8_model = core.read_model(int8_ir_path)
compiled_model_fp = core.compile_model(fp_model, device_name="CPU")
compiled_model_int8 = core.compile_model(int8_model, device_name="CPU")
compiled_model_fp = core.compile_model(fp_model, device_name=device.value)
compiled_model_int8 = core.compile_model(int8_model, device_name=device.value)
output_layer_fp = compiled_model_fp.output(0)
output_layer_int8 = compiled_model_int8.output(0)
# Create subset of dataset
background_slices = (item for item in dataset if np.count_nonzero(item[1]) == 0)
kidney_slices = (item for item in dataset if np.count_nonzero(item[1]) > 50)
data_subset = random.sample(list(background_slices), 2) + random.sample(list(kidney_slices), 2)
# Set seed to current time. To reproduce specific results, copy the printed seed
# and manually set `seed` to that value.
seed = int(time.time())
random.seed(seed)
print(f"Visualizing results with seed {seed}")
fig, ax = plt.subplots(nrows=num_images, ncols=4, figsize=(24, num_images * 4))
for i, (image, mask) in enumerate(data_subset):
display_image = rotate_and_flip(image.squeeze())
@ -892,13 +908,13 @@ seed is displayed to enable reproducing specific runs of this cell.
input_image = np.expand_dims(image, 0)
res_fp = compiled_model_fp([input_image])
res_int8 = compiled_model_int8([input_image])
# Process inference outputs and convert to binary segementation masks
result_mask_fp = sigmoid(res_fp[output_layer_fp]).squeeze().round().astype(np.uint8)
result_mask_int8 = sigmoid(res_int8[output_layer_int8]).squeeze().round().astype(np.uint8)
result_mask_fp = rotate_and_flip(result_mask_fp)
result_mask_int8 = rotate_and_flip(result_mask_int8)
# Display images, annotations, FP32 result and INT8 result
ax[i, 0].imshow(display_image, cmap=colormap)
ax[i, 1].imshow(target_mask, cmap=colormap)
@ -910,7 +926,7 @@ seed is displayed to enable reproducing specific runs of this cell.
.. parsed-literal::
Visualizing results with seed 1707515536
Visualizing results with seed 1710279421
@ -952,7 +968,7 @@ overlay of the segmentation mask on the original image/frame.
.. code:: ipython3
CASE = 117
segmentation_model = SegmentationModel(
ie=core, model_path=int8_ir_path, sigmoid=True, rotate_and_flip=True
)
@ -979,11 +995,9 @@ performs inference, and displays the results on the frames loaded in
.. code:: ipython3
# Possible options for device include "CPU", "GPU", "AUTO", "MULTI:CPU,GPU"
device = "CPU"
reader = LoadImage(image_only=True, dtype=np.uint8)
show_live_inference(
ie=core, image_paths=image_paths, model=segmentation_model, device=device, reader=reader
ie=core, image_paths=image_paths, model=segmentation_model, device=device.value, reader=reader
)
@ -993,8 +1007,8 @@ performs inference, and displays the results on the frames loaded in
.. parsed-literal::
Loaded model to CPU in 0.18 seconds.
Total time for 68 frames: 2.68 seconds, fps:25.70
Loaded model to AUTO in 0.22 seconds.
Total time for 68 frames: 2.67 seconds, fps:25.87
References

3
docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_14_1.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f8a243947cb2e6e61c56f5971421b38d61ef231f122442e7e17517f811ffb643
size 158997

3
docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_15_1.png
View File

@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8d5031a8fde68c06c9d1585a57d549809af699f8411c9021d3571718ae037908
size 158997

4
docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_37_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:fa3a987453cb9be2a1e6e3c9d6ef943c29914c0a1f740e2eac55ebad8bc9ac4a
size 387367
oid sha256:343ba28303ca515ea1d2e7a3a11fc71f2da545414422d023c6bb0bf6e5f0d33c
size 387909

264
docs/notebooks/112-pytorch-post-training-quantization-nncf-with-output.rst
View File

@ -58,8 +58,8 @@ Preparations
.. code:: ipython3
# Install openvino package
%pip install -q "openvino>=2023.1.0" torch torchvision --extra-index-url https://download.pytorch.org/whl/cpu
%pip install -q "nncf>=2.6.0"
%pip install -q "openvino>=2024.0.0" torch torchvision --extra-index-url https://download.pytorch.org/whl/cpu
%pip install -q "nncf>=2.9.0"
.. parsed-literal::
@ -67,43 +67,10 @@ Preparations
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
.. code:: ipython3
Note: you may need to restart the kernel to use updated packages.
# On Windows, this script adds the directory that contains cl.exe to the PATH to enable PyTorch to find the
# required C++ tools. This code assumes that Visual Studio 2019 is installed in the default
# directory. If you have a different C++ compiler, add the correct path to os.environ["PATH"]
# directly.
# Adding the path to os.environ["LIB"] is not always required - it depends on the system configuration.
import sys
if sys.platform == "win32":
import distutils.command.build_ext
import os
from pathlib import Path
VS_INSTALL_DIR = r"C:/Program Files (x86)/Microsoft Visual Studio"
cl_paths = sorted(list(Path(VS_INSTALL_DIR).glob("**/Hostx86/x64/cl.exe")))
if len(cl_paths) == 0:
raise ValueError(
"Cannot find Visual Studio. This notebook requires C++. If you installed "
"a C++ compiler, please add the directory that contains cl.exe to "
"`os.environ['PATH']`"
)
else:
# If multiple versions of MSVC are installed, get the most recent one.
cl_path = cl_paths[-1]
vs_dir = str(cl_path.parent)
os.environ["PATH"] += f"{os.pathsep}{vs_dir}"
# The code for finding the library dirs is from
# https://stackoverflow.com/questions/47423246/get-pythons-lib-path
d = distutils.core.Distribution()
b = distutils.command.build_ext.build_ext(d)
b.finalize_options()
os.environ["LIB"] = os.pathsep.join(b.library_dirs)
print(f"Added {vs_dir} to PATH")
Imports
~~~~~~~
@ -113,19 +80,20 @@ Imports
.. code:: ipython3
import os
import sys
import time
import zipfile
from pathlib import Path
from typing import List, Tuple
import nncf
import openvino as ov
import torch
from torchvision.datasets import ImageFolder
from torchvision.models import resnet50
import torchvision.transforms as transforms
sys.path.append("../utils")
from notebook_utils import download_file
@ -144,21 +112,21 @@ Settings
torch_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using {torch_device} device")
MODEL_DIR = Path("model")
OUTPUT_DIR = Path("output")
BASE_MODEL_NAME = "resnet50"
IMAGE_SIZE = [64, 64]
OUTPUT_DIR.mkdir(exist_ok=True)
MODEL_DIR.mkdir(exist_ok=True)
# Paths where PyTorch and OpenVINO IR models will be stored.
fp32_checkpoint_filename = Path(BASE_MODEL_NAME + "_fp32").with_suffix(".pth")
fp32_ir_path = OUTPUT_DIR / Path(BASE_MODEL_NAME + "_fp32").with_suffix(".xml")
int8_ir_path = OUTPUT_DIR / Path(BASE_MODEL_NAME + "_int8").with_suffix(".xml")
fp32_pth_url = "https://storage.openvinotoolkit.org/repositories/nncf/openvino_notebook_ckpts/304_resnet50_fp32.pth"
download_file(fp32_pth_url, directory=MODEL_DIR, filename=fp32_checkpoint_filename)
@ -178,7 +146,7 @@ Settings
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/112-pytorch-post-training-quantization-nncf/model/resnet50_fp32.pth')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/112-pytorch-post-training-quantization-nncf/model/resnet50_fp32.pth')
@ -204,29 +172,29 @@ Download and Prepare Tiny ImageNet dataset
zip_ref.extractall(path=output_dir)
zip_ref.close()
print(f"Successfully downloaded and extracted dataset to: {output_dir}")
def create_validation_dir(dataset_dir: Path):
VALID_DIR = dataset_dir / "val"
val_img_dir = VALID_DIR / "images"
fp = open(VALID_DIR / "val_annotations.txt", "r")
data = fp.readlines()
val_img_dict = {}
for line in data:
words = line.split("\t")
val_img_dict[words[0]] = words[1]
fp.close()
for img, folder in val_img_dict.items():
newpath = val_img_dir / folder
if not newpath.exists():
os.makedirs(newpath)
if (val_img_dir / img).exists():
os.rename(val_img_dir / img, newpath / img)
DATASET_DIR = OUTPUT_DIR / "tiny-imagenet-200"
if not DATASET_DIR.exists():
download_tiny_imagenet_200(OUTPUT_DIR)
@ -256,7 +224,7 @@ process.
class AverageMeter(object):
"""Computes and stores the average and current value"""
def __init__(self, name: str, fmt: str = ":f"):
self.name = name
self.fmt = fmt
@ -264,52 +232,52 @@ process.
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val: float, n: int = 1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def __str__(self):
fmtstr = "{name} {val" + self.fmt + "} ({avg" + self.fmt + "})"
return fmtstr.format(**self.__dict__)
class ProgressMeter(object):
"""Displays the progress of validation process"""
def __init__(self, num_batches: int, meters: List[AverageMeter], prefix: str = ""):
self.batch_fmtstr = self._get_batch_fmtstr(num_batches)
self.meters = meters
self.prefix = prefix
def display(self, batch: int):
entries = [self.prefix + self.batch_fmtstr.format(batch)]
entries += [str(meter) for meter in self.meters]
print("\t".join(entries))
def _get_batch_fmtstr(self, num_batches: int):
num_digits = len(str(num_batches // 1))
fmt = "{:" + str(num_digits) + "d}"
return "[" + fmt + "/" + fmt.format(num_batches) + "]"
def accuracy(output: torch.Tensor, target: torch.Tensor, topk: Tuple[int] = (1,)):
"""Computes the accuracy over the k top predictions for the specified values of k"""
with torch.no_grad():
maxk = max(topk)
batch_size = target.size(0)
_, pred = output.topk(maxk, 1, True, True)
pred = pred.t()
correct = pred.eq(target.view(1, -1).expand_as(pred))
res = []
for k in topk:
correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True)
res.append(correct_k.mul_(100.0 / batch_size))
return res
Validation function
@ -321,8 +289,8 @@ Validation function
from typing import Union
from openvino.runtime.ie_api import CompiledModel
def validate(val_loader: torch.utils.data.DataLoader, model: Union[torch.nn.Module, CompiledModel]):
"""Compute the metrics using data from val_loader for the model"""
batch_time = AverageMeter("Time", ":3.3f")
@ -334,13 +302,13 @@ Validation function
if not isinstance(model, CompiledModel):
model.eval()
model.to(torch_device)
with torch.no_grad():
end = time.time()
for i, (images, target) in enumerate(val_loader):
images = images.to(torch_device)
target = target.to(torch_device)
# Compute the output.
if isinstance(model, CompiledModel):
output_layer = model.output(0)
@ -348,20 +316,20 @@ Validation function
output = torch.from_numpy(output)
else:
output = model(images)
# Measure accuracy and record loss.
acc1, acc5 = accuracy(output, target, topk=(1, 5))
top1.update(acc1[0], images.size(0))
top5.update(acc5[0], images.size(0))
# Measure elapsed time.
batch_time.update(time.time() - end)
end = time.time()
print_frequency = 10
if i % print_frequency == 0:
progress.display(i)
print(
" * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f} Total time: {total_time:.3f}".format(top1=top1, top5=top5, total_time=end - start_time)
)
@ -372,7 +340,7 @@ Create and load original uncompressed model
ResNet-50 from the ```torchivision``
ResNet-50 from the `torchivision
repository <https://github.com/pytorch/vision>`__ is pre-trained on
ImageNet with more prediction classes than Tiny ImageNet, so the model
is adjusted by swapping the last FC layer to one with fewer output
@ -393,8 +361,8 @@ values.
else:
raise RuntimeError("There is no checkpoint to load")
return model
model = create_model(MODEL_DIR / fp32_checkpoint_filename)
Create train and validation DataLoaders
@ -427,7 +395,7 @@ Create train and validation DataLoaders
[transforms.Resize(IMAGE_SIZE), transforms.ToTensor(), normalize]
),
)
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=batch_size,
@ -436,7 +404,7 @@ Create train and validation DataLoaders
pin_memory=True,
sampler=None,
)
val_loader = torch.utils.data.DataLoader(
val_dataset,
batch_size=batch_size,
@ -445,8 +413,8 @@ Create train and validation DataLoaders
pin_memory=True,
)
return train_loader, val_loader
train_loader, val_loader = create_dataloaders()
Model quantization and benchmarking
@ -471,47 +439,47 @@ I. Evaluate the loaded model
.. parsed-literal::
Test: [ 0/79] Time 0.283 (0.283) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
Test: [ 0/79] Time 0.266 (0.266) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
.. parsed-literal::
Test: [10/79] Time 0.242 (0.242) Acc@1 56.25 (66.97) Acc@5 86.72 (87.50)
Test: [10/79] Time 0.237 (0.240) Acc@1 56.25 (66.97) Acc@5 86.72 (87.50)
.. parsed-literal::
Test: [20/79] Time 0.237 (0.241) Acc@1 67.97 (64.29) Acc@5 85.16 (87.35)
Test: [20/79] Time 0.226 (0.238) Acc@1 67.97 (64.29) Acc@5 85.16 (87.35)
.. parsed-literal::
Test: [30/79] Time 0.238 (0.241) Acc@1 53.12 (62.37) Acc@5 77.34 (85.33)
Test: [30/79] Time 0.289 (0.240) Acc@1 53.12 (62.37) Acc@5 77.34 (85.33)
.. parsed-literal::
Test: [40/79] Time 0.244 (0.241) Acc@1 67.19 (60.86) Acc@5 90.62 (84.51)
Test: [40/79] Time 0.239 (0.239) Acc@1 67.19 (60.86) Acc@5 90.62 (84.51)
.. parsed-literal::
Test: [50/79] Time 0.270 (0.245) Acc@1 60.16 (60.80) Acc@5 88.28 (84.42)
Test: [50/79] Time 0.238 (0.239) Acc@1 60.16 (60.80) Acc@5 88.28 (84.42)
.. parsed-literal::
Test: [60/79] Time 0.246 (0.245) Acc@1 66.41 (60.46) Acc@5 86.72 (83.79)
Test: [60/79] Time 0.239 (0.239) Acc@1 66.41 (60.46) Acc@5 86.72 (83.79)
.. parsed-literal::
Test: [70/79] Time 0.263 (0.244) Acc@1 52.34 (60.21) Acc@5 80.47 (83.33)
Test: [70/79] Time 0.237 (0.239) Acc@1 52.34 (60.21) Acc@5 80.47 (83.33)
.. parsed-literal::
* Acc@1 60.740 Acc@5 83.960 Total time: 19.092
* Acc@1 60.740 Acc@5 83.960 Total time: 18.642
Test accuracy of FP32 model: 60.740
@ -540,8 +508,8 @@ Guide <https://docs.openvino.ai/2024/openvino-workflow/model-optimization-guide/
def transform_fn(data_item):
images, _ = data_item
return images
calibration_dataset = nncf.Dataset(train_loader, transform_fn)
2. Create a quantized model from the pre-trained ``FP32`` model and the
@ -554,14 +522,14 @@ Guide <https://docs.openvino.ai/2024/openvino-workflow/model-optimization-guide/
.. parsed-literal::
2024-02-09 22:53:51.860179: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-02-09 22:53:51.891244: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2024-03-12 22:38:49.880763: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-03-12 22:38:49.911905: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
.. parsed-literal::
2024-02-09 22:53:52.407039: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2024-03-12 22:38:50.443391: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
@ -581,7 +549,9 @@ Guide <https://docs.openvino.ai/2024/openvino-workflow/model-optimization-guide/
.. raw:: html
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"></pre>
@ -610,7 +580,9 @@ Guide <https://docs.openvino.ai/2024/openvino-workflow/model-optimization-guide/
.. raw:: html
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"></pre>
@ -635,48 +607,48 @@ Guide <https://docs.openvino.ai/2024/openvino-workflow/model-optimization-guide/
.. parsed-literal::
Test: [ 0/79] Time 0.435 (0.435) Acc@1 82.81 (82.81) Acc@5 92.19 (92.19)
Test: [ 0/79] Time 0.424 (0.424) Acc@1 81.25 (81.25) Acc@5 90.62 (90.62)
.. parsed-literal::
Test: [10/79] Time 0.391 (0.395) Acc@1 54.69 (66.34) Acc@5 85.94 (87.50)
Test: [10/79] Time 0.401 (0.405) Acc@1 53.12 (66.05) Acc@5 86.72 (87.50)
.. parsed-literal::
Test: [20/79] Time 0.389 (0.395) Acc@1 69.53 (63.91) Acc@5 84.38 (87.09)
Test: [20/79] Time 0.401 (0.404) Acc@1 68.75 (63.62) Acc@5 85.94 (87.20)
.. parsed-literal::
Test: [30/79] Time 0.388 (0.395) Acc@1 52.34 (62.22) Acc@5 75.78 (84.90)
Test: [30/79] Time 0.404 (0.403) Acc@1 52.34 (62.00) Acc@5 77.34 (85.13)
.. parsed-literal::
Test: [40/79] Time 0.392 (0.393) Acc@1 67.97 (60.75) Acc@5 89.84 (84.30)
Test: [40/79] Time 0.402 (0.403) Acc@1 66.41 (60.52) Acc@5 90.62 (84.49)
.. parsed-literal::
Test: [50/79] Time 0.398 (0.393) Acc@1 60.16 (60.72) Acc@5 88.28 (84.30)
Test: [50/79] Time 0.404 (0.402) Acc@1 59.38 (60.45) Acc@5 88.28 (84.44)
.. parsed-literal::
Test: [60/79] Time 0.390 (0.393) Acc@1 66.41 (60.27) Acc@5 86.72 (83.75)
Test: [60/79] Time 0.404 (0.402) Acc@1 67.19 (60.17) Acc@5 85.94 (83.88)
.. parsed-literal::
Test: [70/79] Time 0.388 (0.392) Acc@1 54.69 (60.06) Acc@5 80.47 (83.29)
Test: [70/79] Time 0.401 (0.402) Acc@1 53.12 (59.93) Acc@5 80.47 (83.42)
.. parsed-literal::
* Acc@1 60.570 Acc@5 83.950 Total time: 30.736
Accuracy of initialized INT8 model: 60.570
* Acc@1 60.420 Acc@5 84.050 Total time: 31.523
Accuracy of initialized INT8 model: 60.420
It should be noted that the inference time for the quantized PyTorch
@ -700,9 +672,9 @@ For more information about model conversion, refer to this
.. code:: ipython3
dummy_input = torch.randn(128, 3, *IMAGE_SIZE)
model_ir = ov.convert_model(model, example_input=dummy_input, input=[-1, 3, *IMAGE_SIZE])
ov.save_model(model_ir, fp32_ir_path)
@ -719,26 +691,26 @@ For more information about model conversion, refer to this
.. code:: ipython3
quantized_model_ir = ov.convert_model(quantized_model, example_input=dummy_input, input=[-1, 3, *IMAGE_SIZE])
ov.save_model(quantized_model_ir, int8_ir_path)
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:334: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:337: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
return self._level_low.item()
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:342: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:345: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
return self._level_high.item()
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1093: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1093: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:
Tensor-likes are not close!
Mismatched elements: 25587 / 25600 (99.9%)
Greatest absolute difference: 0.5083470344543457 at index (42, 14) (up to 1e-05 allowed)
Greatest relative difference: 79.27243410505909 at index (126, 158) (up to 1e-05 allowed)
Mismatched elements: 25584 / 25600 (99.9%)
Greatest absolute difference: 0.4242134094238281 at index (97, 14) (up to 1e-05 allowed)
Greatest relative difference: 313.25925327299177 at index (60, 158) (up to 1e-05 allowed)
_check_trace(
@ -747,7 +719,7 @@ Select inference device for OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
@ -755,7 +727,7 @@ Select inference device for OpenVINO
description='Device:',
disabled=False,
)
device
@ -779,47 +751,47 @@ Evaluate the FP32 and INT8 models.
.. parsed-literal::
Test: [ 0/79] Time 0.185 (0.185) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
Test: [ 0/79] Time 0.176 (0.176) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
.. parsed-literal::
Test: [10/79] Time 0.136 (0.143) Acc@1 56.25 (66.97) Acc@5 86.72 (87.50)
Test: [10/79] Time 0.138 (0.142) Acc@1 56.25 (66.97) Acc@5 86.72 (87.50)
.. parsed-literal::
Test: [20/79] Time 0.141 (0.141) Acc@1 67.97 (64.29) Acc@5 85.16 (87.35)
Test: [20/79] Time 0.138 (0.140) Acc@1 67.97 (64.29) Acc@5 85.16 (87.35)
.. parsed-literal::
Test: [30/79] Time 0.140 (0.140) Acc@1 53.12 (62.37) Acc@5 77.34 (85.33)
Test: [30/79] Time 0.139 (0.139) Acc@1 53.12 (62.37) Acc@5 77.34 (85.33)
.. parsed-literal::
Test: [40/79] Time 0.140 (0.140) Acc@1 67.19 (60.86) Acc@5 90.62 (84.51)
Test: [40/79] Time 0.137 (0.139) Acc@1 67.19 (60.86) Acc@5 90.62 (84.51)
.. parsed-literal::
Test: [50/79] Time 0.139 (0.139) Acc@1 60.16 (60.80) Acc@5 88.28 (84.42)
Test: [50/79] Time 0.138 (0.139) Acc@1 60.16 (60.80) Acc@5 88.28 (84.42)
.. parsed-literal::
Test: [60/79] Time 0.138 (0.139) Acc@1 66.41 (60.46) Acc@5 86.72 (83.79)
Test: [60/79] Time 0.135 (0.139) Acc@1 66.41 (60.46) Acc@5 86.72 (83.79)
.. parsed-literal::
Test: [70/79] Time 0.136 (0.139) Acc@1 52.34 (60.21) Acc@5 80.47 (83.33)
Test: [70/79] Time 0.138 (0.139) Acc@1 52.34 (60.21) Acc@5 80.47 (83.33)
.. parsed-literal::
* Acc@1 60.740 Acc@5 83.960 Total time: 10.886
* Acc@1 60.740 Acc@5 83.960 Total time: 10.831
Accuracy of FP32 IR model: 60.740
@ -832,48 +804,48 @@ Evaluate the FP32 and INT8 models.
.. parsed-literal::
Test: [ 0/79] Time 0.145 (0.145) Acc@1 82.03 (82.03) Acc@5 91.41 (91.41)
Test: [ 0/79] Time 0.146 (0.146) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
.. parsed-literal::
Test: [10/79] Time 0.076 (0.083) Acc@1 55.47 (66.76) Acc@5 86.72 (87.36)
Test: [10/79] Time 0.078 (0.084) Acc@1 54.69 (66.34) Acc@5 85.94 (87.50)
.. parsed-literal::
Test: [20/79] Time 0.076 (0.080) Acc@1 70.31 (64.43) Acc@5 85.16 (87.02)
Test: [20/79] Time 0.078 (0.081) Acc@1 69.53 (63.88) Acc@5 85.16 (87.17)
.. parsed-literal::
Test: [30/79] Time 0.076 (0.079) Acc@1 53.12 (62.40) Acc@5 75.78 (84.93)
Test: [30/79] Time 0.076 (0.080) Acc@1 53.12 (62.25) Acc@5 75.78 (85.11)
.. parsed-literal::
Test: [40/79] Time 0.077 (0.078) Acc@1 67.19 (60.84) Acc@5 90.62 (84.20)
Test: [40/79] Time 0.076 (0.079) Acc@1 67.19 (60.86) Acc@5 90.62 (84.36)
.. parsed-literal::
Test: [50/79] Time 0.078 (0.078) Acc@1 59.38 (60.83) Acc@5 88.28 (84.15)
Test: [50/79] Time 0.073 (0.078) Acc@1 60.16 (60.80) Acc@5 88.28 (84.28)
.. parsed-literal::
Test: [60/79] Time 0.076 (0.078) Acc@1 64.84 (60.40) Acc@5 87.50 (83.63)
Test: [60/79] Time 0.076 (0.078) Acc@1 65.62 (60.32) Acc@5 86.72 (83.72)
.. parsed-literal::
Test: [70/79] Time 0.077 (0.078) Acc@1 53.12 (60.16) Acc@5 80.47 (83.14)
Test: [70/79] Time 0.075 (0.077) Acc@1 51.56 (60.11) Acc@5 79.69 (83.21)
.. parsed-literal::
* Acc@1 60.680 Acc@5 83.770 Total time: 6.075
Accuracy of INT8 IR model: 60.680
* Acc@1 60.640 Acc@5 83.860 Total time: 6.050
Accuracy of INT8 IR model: 60.640
IV. Compare performance of INT8 model and FP32 model in OpenVINO
@ -916,20 +888,20 @@ throughput (frames per second) values.
"""Prints the output from benchmark_app in human-readable format"""
parsed_output = [line for line in benchmark_output if 'FPS' in line]
print(*parsed_output, sep='\n')
print('Benchmark FP32 model (OpenVINO IR)')
benchmark_output = ! benchmark_app -m "$fp32_ir_path" -d $device.value -api async -t 15 -shape "[1, 3, 512, 512]"
parse_benchmark_output(benchmark_output)
print('Benchmark INT8 model (OpenVINO IR)')
benchmark_output = ! benchmark_app -m "$int8_ir_path" -d $device.value -api async -t 15 -shape "[1, 3, 512, 512]"
parse_benchmark_output(benchmark_output)
print('Benchmark FP32 model (OpenVINO IR) synchronously')
benchmark_output = ! benchmark_app -m "$fp32_ir_path" -d $device.value -api sync -t 15 -shape "[1, 3, 512, 512]"
parse_benchmark_output(benchmark_output)
print('Benchmark INT8 model (OpenVINO IR) synchronously')
benchmark_output = ! benchmark_app -m "$int8_ir_path" -d $device.value -api sync -t 15 -shape "[1, 3, 512, 512]"
parse_benchmark_output(benchmark_output)
@ -942,25 +914,25 @@ throughput (frames per second) values.
.. parsed-literal::
[ INFO ] Throughput: 38.33 FPS
[ INFO ] Throughput: 38.94 FPS
Benchmark INT8 model (OpenVINO IR)
.. parsed-literal::
[ INFO ] Throughput: 155.58 FPS
[ INFO ] Throughput: 154.81 FPS
Benchmark FP32 model (OpenVINO IR) synchronously
.. parsed-literal::
[ INFO ] Throughput: 39.95 FPS
[ INFO ] Throughput: 40.11 FPS
Benchmark INT8 model (OpenVINO IR) synchronously
.. parsed-literal::
[ INFO ] Throughput: 137.77 FPS
[ INFO ] Throughput: 137.11 FPS
Show device Information for reference:
@ -969,7 +941,7 @@ Show device Information for reference:
core = ov.Core()
devices = core.available_devices
for device_name in devices:
device_full_name = core.get_property(device_name, "FULL_DEVICE_NAME")
print(f"{device_name}: {device_full_name}")

2166
docs/notebooks/113-image-classification-quantization-with-output.rst
View File

File diff suppressed because it is too large Load Diff

2
docs/notebooks/113-image-classification-quantization-with-output_files/113-image-classification-quantization-with-output_30_5.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:16f09304ce7269cd0c468bb30705c6f414668fd03616e594e4b89c5734bbd6d4
oid sha256:9990baa1e4f121781deb249b7534c8f31f26ca9b37afd3f269ca50ec0318e7d4
size 14855

405
docs/notebooks/115-async-api-with-output.rst
View File

@ -2,7 +2,7 @@ Asynchronous Inference with OpenVINO™
=====================================
This notebook demonstrates how to use the `Async
API <https://docs.openvino.ai/nightly/openvino_docs_deployment_optimization_guide_common.html>`__
API <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/general-optimizations.html>`__
for asynchronous execution with OpenVINO.
OpenVINO Runtime supports inference in either synchronous or
@ -47,8 +47,19 @@ Imports
.. code:: ipython3
import platform
%pip install -q "openvino>=2023.1.0"
%pip install -q opencv-python matplotlib
%pip install -q opencv-python
if platform.system() != "windows":
%pip install -q "matplotlib>=3.4"
else:
%pip install -q "matplotlib>=3.4,<3.7"
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
@ -69,14 +80,14 @@ Imports
import openvino as ov
from IPython import display
import matplotlib.pyplot as plt
# Fetch the notebook utils script from the openvino_notebooks repo
import urllib.request
urllib.request.urlretrieve(
url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py',
filename='notebook_utils.py'
)
import notebook_utils as utils
Prepare model and data processing
@ -90,15 +101,15 @@ Download test model
We use a pre-trained model from OpenVINO’s `Open Model
Zoo <https://docs.openvino.ai/nightly/model_zoo.html>`__ to start the
test. In this case, the model will be executed to detect the person in
each frame of the video.
Zoo <https://docs.openvino.ai/2024/documentation/legacy-features/model-zoo.html>`__
to start the test. In this case, the model will be executed to detect
the person in each frame of the video.
.. code:: ipython3
# directory where model will be downloaded
base_model_dir = "model"
# model name as named in Open Model Zoo
model_name = "person-detection-0202"
precision = "FP16"
@ -116,205 +127,185 @@ each frame of the video.
.. parsed-literal::
################|| Downloading person-detection-0202 ||################
========== Downloading model/intel/person-detection-0202/FP16/person-detection-0202.xml
.. parsed-literal::
... 12%, 32 KB, 1341 KB/s, 0 seconds passed
... 25%, 64 KB, 1341 KB/s, 0 seconds passed
... 12%, 32 KB, 847 KB/s, 0 seconds passed
.. parsed-literal::
... 38%, 96 KB, 1143 KB/s, 0 seconds passed
... 51%, 128 KB, 1326 KB/s, 0 seconds passed
... 25%, 64 KB, 918 KB/s, 0 seconds passed
... 38%, 96 KB, 1348 KB/s, 0 seconds passed
.. parsed-literal::
... 64%, 160 KB, 1323 KB/s, 0 seconds passed
... 77%, 192 KB, 1444 KB/s, 0 seconds passed
... 89%, 224 KB, 1678 KB/s, 0 seconds passed
... 100%, 248 KB, 1859 KB/s, 0 seconds passed
... 51%, 128 KB, 1218 KB/s, 0 seconds passed
... 64%, 160 KB, 1510 KB/s, 0 seconds passed
... 77%, 192 KB, 1779 KB/s, 0 seconds passed
... 89%, 224 KB, 2058 KB/s, 0 seconds passed
... 100%, 248 KB, 2262 KB/s, 0 seconds passed
========== Downloading model/intel/person-detection-0202/FP16/person-detection-0202.bin
.. parsed-literal::
... 0%, 32 KB, 1301 KB/s, 0 seconds passed
... 0%, 32 KB, 946 KB/s, 0 seconds passed
... 1%, 64 KB, 898 KB/s, 0 seconds passed
... 2%, 96 KB, 1325 KB/s, 0 seconds passed
.. parsed-literal::
... 1%, 64 KB, 1298 KB/s, 0 seconds passed
... 2%, 96 KB, 1907 KB/s, 0 seconds passed
... 3%, 128 KB, 1722 KB/s, 0 seconds passed
... 4%, 160 KB, 2133 KB/s, 0 seconds passed
... 3%, 128 KB, 1272 KB/s, 0 seconds passed
... 4%, 160 KB, 1575 KB/s, 0 seconds passed
... 5%, 192 KB, 1870 KB/s, 0 seconds passed
... 6%, 224 KB, 2155 KB/s, 0 seconds passed
... 7%, 256 KB, 2432 KB/s, 0 seconds passed
.. parsed-literal::
... 5%, 192 KB, 2234 KB/s, 0 seconds passed
... 6%, 224 KB, 2577 KB/s, 0 seconds passed
... 7%, 256 KB, 2312 KB/s, 0 seconds passed
... 8%, 288 KB, 2581 KB/s, 0 seconds passed
... 9%, 320 KB, 2614 KB/s, 0 seconds passed
... 9%, 352 KB, 2855 KB/s, 0 seconds passed
... 8%, 288 KB, 2135 KB/s, 0 seconds passed
... 9%, 320 KB, 2352 KB/s, 0 seconds passed
... 9%, 352 KB, 2561 KB/s, 0 seconds passed
... 10%, 384 KB, 2770 KB/s, 0 seconds passed
... 11%, 416 KB, 2963 KB/s, 0 seconds passed
... 12%, 448 KB, 3064 KB/s, 0 seconds passed
... 13%, 480 KB, 3260 KB/s, 0 seconds passed
... 14%, 512 KB, 3432 KB/s, 0 seconds passed
... 15%, 544 KB, 3629 KB/s, 0 seconds passed
... 16%, 576 KB, 3420 KB/s, 0 seconds passed
... 17%, 608 KB, 3600 KB/s, 0 seconds passed
... 18%, 640 KB, 3782 KB/s, 0 seconds passed
... 18%, 672 KB, 3964 KB/s, 0 seconds passed
... 19%, 704 KB, 4145 KB/s, 0 seconds passed
... 20%, 736 KB, 4326 KB/s, 0 seconds passed
... 21%, 768 KB, 4506 KB/s, 0 seconds passed
... 22%, 800 KB, 4685 KB/s, 0 seconds passed
... 23%, 832 KB, 4865 KB/s, 0 seconds passed
... 24%, 864 KB, 5043 KB/s, 0 seconds passed
... 25%, 896 KB, 5107 KB/s, 0 seconds passed
... 26%, 928 KB, 5275 KB/s, 0 seconds passed
... 27%, 960 KB, 5448 KB/s, 0 seconds passed
.. parsed-literal::
... 10%, 384 KB, 2608 KB/s, 0 seconds passed
... 11%, 416 KB, 2811 KB/s, 0 seconds passed
... 12%, 448 KB, 2817 KB/s, 0 seconds passed
... 13%, 480 KB, 2999 KB/s, 0 seconds passed
... 14%, 512 KB, 2785 KB/s, 0 seconds passed
... 15%, 544 KB, 2949 KB/s, 0 seconds passed
... 27%, 992 KB, 5620 KB/s, 0 seconds passed
... 28%, 1024 KB, 5790 KB/s, 0 seconds passed
... 29%, 1056 KB, 5961 KB/s, 0 seconds passed
... 30%, 1088 KB, 6131 KB/s, 0 seconds passed
... 31%, 1120 KB, 6252 KB/s, 0 seconds passed
... 32%, 1152 KB, 6381 KB/s, 0 seconds passed
... 33%, 1184 KB, 5868 KB/s, 0 seconds passed
... 34%, 1216 KB, 6013 KB/s, 0 seconds passed
... 35%, 1248 KB, 6111 KB/s, 0 seconds passed
... 36%, 1280 KB, 6254 KB/s, 0 seconds passed
... 36%, 1312 KB, 6399 KB/s, 0 seconds passed
... 37%, 1344 KB, 6545 KB/s, 0 seconds passed
... 38%, 1376 KB, 6690 KB/s, 0 seconds passed
... 39%, 1408 KB, 6836 KB/s, 0 seconds passed
... 40%, 1440 KB, 6981 KB/s, 0 seconds passed
... 41%, 1472 KB, 7125 KB/s, 0 seconds passed
... 42%, 1504 KB, 7268 KB/s, 0 seconds passed
... 43%, 1536 KB, 7412 KB/s, 0 seconds passed
... 44%, 1568 KB, 7556 KB/s, 0 seconds passed
... 45%, 1600 KB, 7699 KB/s, 0 seconds passed
... 45%, 1632 KB, 7841 KB/s, 0 seconds passed
... 46%, 1664 KB, 7927 KB/s, 0 seconds passed
... 47%, 1696 KB, 8035 KB/s, 0 seconds passed
... 48%, 1728 KB, 8106 KB/s, 0 seconds passed
... 49%, 1760 KB, 8210 KB/s, 0 seconds passed
... 50%, 1792 KB, 8335 KB/s, 0 seconds passed
... 51%, 1824 KB, 8471 KB/s, 0 seconds passed
... 52%, 1856 KB, 8591 KB/s, 0 seconds passed
... 53%, 1888 KB, 8698 KB/s, 0 seconds passed
... 54%, 1920 KB, 8832 KB/s, 0 seconds passed
... 54%, 1952 KB, 8953 KB/s, 0 seconds passed
... 55%, 1984 KB, 9085 KB/s, 0 seconds passed
... 56%, 2016 KB, 9198 KB/s, 0 seconds passed
... 57%, 2048 KB, 9330 KB/s, 0 seconds passed
... 58%, 2080 KB, 9443 KB/s, 0 seconds passed
... 59%, 2112 KB, 9575 KB/s, 0 seconds passed
... 60%, 2144 KB, 9698 KB/s, 0 seconds passed
... 61%, 2176 KB, 9829 KB/s, 0 seconds passed
... 62%, 2208 KB, 9940 KB/s, 0 seconds passed
... 63%, 2240 KB, 10069 KB/s, 0 seconds passed
... 64%, 2272 KB, 10179 KB/s, 0 seconds passed
... 64%, 2304 KB, 10308 KB/s, 0 seconds passed
... 65%, 2336 KB, 10395 KB/s, 0 seconds passed
.. parsed-literal::
... 16%, 576 KB, 2947 KB/s, 0 seconds passed
... 17%, 608 KB, 3097 KB/s, 0 seconds passed
... 18%, 640 KB, 2903 KB/s, 0 seconds passed
... 18%, 672 KB, 3035 KB/s, 0 seconds passed
... 19%, 704 KB, 3035 KB/s, 0 seconds passed
... 20%, 736 KB, 3158 KB/s, 0 seconds passed
... 66%, 2368 KB, 10012 KB/s, 0 seconds passed
... 67%, 2400 KB, 10057 KB/s, 0 seconds passed
... 68%, 2432 KB, 10171 KB/s, 0 seconds passed
... 69%, 2464 KB, 10276 KB/s, 0 seconds passed
... 70%, 2496 KB, 10393 KB/s, 0 seconds passed
... 71%, 2528 KB, 10511 KB/s, 0 seconds passed
... 72%, 2560 KB, 10594 KB/s, 0 seconds passed
... 73%, 2592 KB, 10707 KB/s, 0 seconds passed
... 73%, 2624 KB, 10823 KB/s, 0 seconds passed
... 74%, 2656 KB, 10940 KB/s, 0 seconds passed
... 75%, 2688 KB, 11057 KB/s, 0 seconds passed
... 76%, 2720 KB, 11175 KB/s, 0 seconds passed
... 77%, 2752 KB, 11293 KB/s, 0 seconds passed
... 78%, 2784 KB, 11410 KB/s, 0 seconds passed
... 79%, 2816 KB, 11527 KB/s, 0 seconds passed
... 80%, 2848 KB, 11643 KB/s, 0 seconds passed
... 81%, 2880 KB, 11760 KB/s, 0 seconds passed
... 82%, 2912 KB, 11875 KB/s, 0 seconds passed
... 82%, 2944 KB, 11990 KB/s, 0 seconds passed
... 83%, 2976 KB, 12106 KB/s, 0 seconds passed
... 84%, 3008 KB, 12221 KB/s, 0 seconds passed
... 85%, 3040 KB, 12335 KB/s, 0 seconds passed
... 86%, 3072 KB, 12450 KB/s, 0 seconds passed
... 87%, 3104 KB, 12564 KB/s, 0 seconds passed
... 88%, 3136 KB, 12677 KB/s, 0 seconds passed
... 89%, 3168 KB, 12791 KB/s, 0 seconds passed
... 90%, 3200 KB, 12904 KB/s, 0 seconds passed
... 91%, 3232 KB, 13017 KB/s, 0 seconds passed
... 91%, 3264 KB, 13131 KB/s, 0 seconds passed
... 92%, 3296 KB, 13243 KB/s, 0 seconds passed
... 93%, 3328 KB, 13356 KB/s, 0 seconds passed
... 94%, 3360 KB, 13468 KB/s, 0 seconds passed
... 95%, 3392 KB, 13577 KB/s, 0 seconds passed
... 96%, 3424 KB, 13688 KB/s, 0 seconds passed
... 97%, 3456 KB, 13799 KB/s, 0 seconds passed
... 98%, 3488 KB, 13911 KB/s, 0 seconds passed
... 99%, 3520 KB, 14023 KB/s, 0 seconds passed
... 100%, 3549 KB, 14120 KB/s, 0 seconds passed
Select inference device
~~~~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='CPU',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
... 21%, 768 KB, 2990 KB/s, 0 seconds passed
... 22%, 800 KB, 3107 KB/s, 0 seconds passed
... 23%, 832 KB, 3100 KB/s, 0 seconds passed
... 24%, 864 KB, 3209 KB/s, 0 seconds passed
.. parsed-literal::
... 25%, 896 KB, 3055 KB/s, 0 seconds passed
... 26%, 928 KB, 3153 KB/s, 0 seconds passed
... 27%, 960 KB, 3148 KB/s, 0 seconds passed
... 27%, 992 KB, 3242 KB/s, 0 seconds passed
... 28%, 1024 KB, 3104 KB/s, 0 seconds passed
... 29%, 1056 KB, 3191 KB/s, 0 seconds passed
.. parsed-literal::
... 30%, 1088 KB, 3186 KB/s, 0 seconds passed
... 31%, 1120 KB, 3271 KB/s, 0 seconds passed
... 32%, 1152 KB, 3144 KB/s, 0 seconds passed
... 33%, 1184 KB, 3225 KB/s, 0 seconds passed
... 34%, 1216 KB, 3217 KB/s, 0 seconds passed
... 35%, 1248 KB, 3293 KB/s, 0 seconds passed
.. parsed-literal::
... 36%, 1280 KB, 3177 KB/s, 0 seconds passed
... 36%, 1312 KB, 3250 KB/s, 0 seconds passed
... 37%, 1344 KB, 3243 KB/s, 0 seconds passed
... 38%, 1376 KB, 3311 KB/s, 0 seconds passed
... 39%, 1408 KB, 3204 KB/s, 0 seconds passed
.. parsed-literal::
... 40%, 1440 KB, 3198 KB/s, 0 seconds passed
... 41%, 1472 KB, 3260 KB/s, 0 seconds passed
... 42%, 1504 KB, 3324 KB/s, 0 seconds passed
... 43%, 1536 KB, 3225 KB/s, 0 seconds passed
... 44%, 1568 KB, 3218 KB/s, 0 seconds passed
... 45%, 1600 KB, 3279 KB/s, 0 seconds passed
... 45%, 1632 KB, 3339 KB/s, 0 seconds passed
.. parsed-literal::
... 46%, 1664 KB, 3246 KB/s, 0 seconds passed
... 47%, 1696 KB, 3234 KB/s, 0 seconds passed
... 48%, 1728 KB, 3291 KB/s, 0 seconds passed
... 49%, 1760 KB, 3349 KB/s, 0 seconds passed
.. parsed-literal::
... 50%, 1792 KB, 3263 KB/s, 0 seconds passed
... 51%, 1824 KB, 3249 KB/s, 0 seconds passed
... 52%, 1856 KB, 3303 KB/s, 0 seconds passed
... 53%, 1888 KB, 3357 KB/s, 0 seconds passed
... 54%, 1920 KB, 3280 KB/s, 0 seconds passed
... 54%, 1952 KB, 3328 KB/s, 0 seconds passed
.. parsed-literal::
... 55%, 1984 KB, 3314 KB/s, 0 seconds passed
... 56%, 2016 KB, 3365 KB/s, 0 seconds passed
... 57%, 2048 KB, 3294 KB/s, 0 seconds passed
... 58%, 2080 KB, 3337 KB/s, 0 seconds passed
... 59%, 2112 KB, 3325 KB/s, 0 seconds passed
... 60%, 2144 KB, 3372 KB/s, 0 seconds passed
.. parsed-literal::
... 61%, 2176 KB, 3306 KB/s, 0 seconds passed
... 62%, 2208 KB, 3345 KB/s, 0 seconds passed
... 63%, 2240 KB, 3335 KB/s, 0 seconds passed
... 64%, 2272 KB, 3379 KB/s, 0 seconds passed
... 64%, 2304 KB, 3316 KB/s, 0 seconds passed
... 65%, 2336 KB, 3351 KB/s, 0 seconds passed
.. parsed-literal::
... 66%, 2368 KB, 3343 KB/s, 0 seconds passed
... 67%, 2400 KB, 3385 KB/s, 0 seconds passed
... 68%, 2432 KB, 3322 KB/s, 0 seconds passed
... 69%, 2464 KB, 3312 KB/s, 0 seconds passed
... 70%, 2496 KB, 3349 KB/s, 0 seconds passed
... 71%, 2528 KB, 3390 KB/s, 0 seconds passed
.. parsed-literal::
... 72%, 2560 KB, 3331 KB/s, 0 seconds passed
... 73%, 2592 KB, 3322 KB/s, 0 seconds passed
... 73%, 2624 KB, 3356 KB/s, 0 seconds passed
... 74%, 2656 KB, 3395 KB/s, 0 seconds passed
.. parsed-literal::
... 75%, 2688 KB, 3338 KB/s, 0 seconds passed
... 76%, 2720 KB, 3328 KB/s, 0 seconds passed
... 77%, 2752 KB, 3363 KB/s, 0 seconds passed
... 78%, 2784 KB, 3399 KB/s, 0 seconds passed
... 79%, 2816 KB, 3344 KB/s, 0 seconds passed
... 80%, 2848 KB, 3335 KB/s, 0 seconds passed
.. parsed-literal::
... 81%, 2880 KB, 3369 KB/s, 0 seconds passed
... 82%, 2912 KB, 3321 KB/s, 0 seconds passed
... 82%, 2944 KB, 3352 KB/s, 0 seconds passed
... 83%, 2976 KB, 3342 KB/s, 0 seconds passed
... 84%, 3008 KB, 3374 KB/s, 0 seconds passed
... 85%, 3040 KB, 3407 KB/s, 0 seconds passed
.. parsed-literal::
... 86%, 3072 KB, 3360 KB/s, 0 seconds passed
... 87%, 3104 KB, 3351 KB/s, 0 seconds passed
... 88%, 3136 KB, 3380 KB/s, 0 seconds passed
... 89%, 3168 KB, 3411 KB/s, 0 seconds passed
... 90%, 3200 KB, 3364 KB/s, 0 seconds passed
.. parsed-literal::
... 91%, 3232 KB, 3357 KB/s, 0 seconds passed
... 91%, 3264 KB, 3384 KB/s, 0 seconds passed
... 92%, 3296 KB, 3342 KB/s, 0 seconds passed
... 93%, 3328 KB, 3368 KB/s, 0 seconds passed
... 94%, 3360 KB, 3358 KB/s, 1 seconds passed
... 95%, 3392 KB, 3388 KB/s, 1 seconds passed
.. parsed-literal::
... 96%, 3424 KB, 3346 KB/s, 1 seconds passed
... 97%, 3456 KB, 3372 KB/s, 1 seconds passed
... 98%, 3488 KB, 3363 KB/s, 1 seconds passed
... 99%, 3520 KB, 3392 KB/s, 1 seconds passed
... 100%, 3549 KB, 3419 KB/s, 1 seconds passed
Dropdown(description='Device:', options=('CPU', 'AUTO'), value='CPU')
@ -327,14 +318,14 @@ Load the model
# initialize OpenVINO runtime
core = ov.Core()
# read the network and corresponding weights from file
model = core.read_model(model=model_path)
# compile the model for the CPU (you can choose manually CPU, GPU etc.)
# or let the engine choose the best available device (AUTO)
compiled_model = core.compile_model(model=model, device_name="CPU")
compiled_model = core.compile_model(model=model, device_name=device.value)
# get input node
input_layer_ir = model.input(0)
N, C, H, W = input_layer_ir.shape
@ -350,7 +341,7 @@ Create functions for data processing
def preprocess(image):
"""
Define the preprocess function for input data
:param: image: the orignal input frame
:returns:
resized_image: the image processed
@ -360,12 +351,12 @@ Create functions for data processing
resized_image = resized_image.transpose((2, 0, 1))
resized_image = np.expand_dims(resized_image, axis=0).astype(np.float32)
return resized_image
def postprocess(result, image, fps):
"""
Define the postprocess function for output data
:param: result: the inference results
image: the orignal input frame
fps: average throughput calculated for each frame
@ -381,7 +372,7 @@ Create functions for data processing
xmax = int(min((xmax * image.shape[1]), image.shape[1] - 10))
ymax = int(min((ymax * image.shape[0]), image.shape[0] - 10))
cv2.rectangle(image, (xmin, ymin), (xmax, ymax), (0, 255, 0), 2)
cv2.putText(image, str(round(fps, 2)) + " fps", (5, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 3)
cv2.putText(image, str(round(fps, 2)) + " fps", (5, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 3)
return image
Get the test video
@ -432,7 +423,7 @@ immediately processed:
def sync_api(source, flip, fps, use_popup, skip_first_frames):
"""
Define the main function for video processing in sync mode
:param: source: the video path or the ID of your webcam
:returns:
sync_fps: the inference throughput in sync mode
@ -456,13 +447,13 @@ immediately processed:
break
resized_frame = preprocess(frame)
infer_request.set_tensor(input_layer_ir, ov.Tensor(resized_frame))
# Start the inference request in synchronous mode
# Start the inference request in synchronous mode
infer_request.infer()
res = infer_request.get_output_tensor(0).data
stop_time = time.time()
total_time = stop_time - start_time
frame_number = frame_number + 1
sync_fps = frame_number / total_time
sync_fps = frame_number / total_time
frame = postprocess(res, frame, sync_fps)
# Display the results
if use_popup:
@ -478,7 +469,7 @@ immediately processed:
i = display.Image(data=encoded_img)
# Display the image in this notebook
display.clear_output(wait=True)
display.display(i)
display.display(i)
# ctrl-c
except KeyboardInterrupt:
print("Interrupted")
@ -505,13 +496,13 @@ Test performance in Sync Mode
.. image:: 115-async-api-with-output_files/115-async-api-with-output_15_0.png
.. image:: 115-async-api-with-output_files/115-async-api-with-output_17_0.png
.. parsed-literal::
Source ended
average throuput in sync mode: 47.04 fps
average throuput in sync mode: 43.30 fps
Async Mode
@ -555,7 +546,7 @@ pipeline (decoding vs inference) and not by the sum of the stages.
def async_api(source, flip, fps, use_popup, skip_first_frames):
"""
Define the main function for video processing in async mode
:param: source: the video path or the ID of your webcam
:returns:
async_fps: the inference throughput in async mode
@ -597,7 +588,7 @@ pipeline (decoding vs inference) and not by the sum of the stages.
stop_time = time.time()
total_time = stop_time - start_time
frame_number = frame_number + 1
async_fps = frame_number / total_time
async_fps = frame_number / total_time
frame = postprocess(res, frame, async_fps)
# Display the results
if use_popup:
@ -617,7 +608,7 @@ pipeline (decoding vs inference) and not by the sum of the stages.
# Swap CURRENT and NEXT frames
frame = next_frame
# Swap CURRENT and NEXT infer requests
curr_request, next_request = next_request, curr_request
curr_request, next_request = next_request, curr_request
# ctrl-c
except KeyboardInterrupt:
print("Interrupted")
@ -644,13 +635,13 @@ Test the performance in Async Mode
.. image:: 115-async-api-with-output_files/115-async-api-with-output_19_0.png
.. image:: 115-async-api-with-output_files/115-async-api-with-output_21_0.png
.. parsed-literal::
Source ended
average throuput in async mode: 74.61 fps
average throuput in async mode: 73.14 fps
Compare the performance
@ -662,25 +653,25 @@ Compare the performance
width = 0.4
fontsize = 14
plt.rc('font', size=fontsize)
fig, ax = plt.subplots(1, 1, figsize=(10, 8))
rects1 = ax.bar([0], sync_fps, width, color='#557f2d')
rects2 = ax.bar([width], async_fps, width)
ax.set_ylabel("frames per second")
ax.set_xticks([0, width])
ax.set_xticks([0, width])
ax.set_xticklabels(["Sync mode", "Async mode"])
ax.set_xlabel("Higher is better")
fig.suptitle('Sync mode VS Async mode')
fig.tight_layout()
plt.show()
.. image:: 115-async-api-with-output_files/115-async-api-with-output_21_0.png
.. image:: 115-async-api-with-output_files/115-async-api-with-output_23_0.png
``AsyncInferQueue``
@ -711,7 +702,7 @@ the possibility of passing runtime values.
def callback(infer_request, info) -> None:
"""
Define the callback function for postprocessing
:param: infer_request: the infer_request object
info: a tuple includes original frame and starts time
:returns:
@ -725,7 +716,7 @@ the possibility of passing runtime values.
total_time = stop_time - start_time
frame_number = frame_number + 1
inferqueue_fps = frame_number / total_time
res = infer_request.get_output_tensor(0).data[0]
frame = postprocess(res, frame, inferqueue_fps)
# Encode numpy array to jpg
@ -741,7 +732,7 @@ the possibility of passing runtime values.
def inferqueue(source, flip, fps, skip_first_frames) -> None:
"""
Define the main function for video processing with async infer queue
:param: source: the video path or the ID of your webcam
:retuns:
None
@ -763,7 +754,7 @@ the possibility of passing runtime values.
print("Source ended")
break
resized_frame = preprocess(frame)
# Start the inference request with async infer queue
# Start the inference request with async infer queue
infer_queue.start_async({input_layer_ir.any_name: resized_frame}, (frame, start_time))
except KeyboardInterrupt:
print("Interrupted")
@ -788,10 +779,10 @@ Test the performance with ``AsyncInferQueue``
.. image:: 115-async-api-with-output_files/115-async-api-with-output_27_0.png
.. image:: 115-async-api-with-output_files/115-async-api-with-output_29_0.png
.. parsed-literal::
average throughput in async mode with async infer queue: 113.01 fps
average throughput in async mode with async infer queue: 112.94 fps

0
docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_15_0.png → docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_17_0.png
View File

4
docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_21_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8c51610ffd478d0bc5358648e0475259dc8eb04d090c6f55cb860d2e9b8a8cb6
size 30356
oid sha256:8ad51650644959fe669787d637d49fe35cfc85c9c8f5637470403470f56f6abb
size 4307

3
docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_23_0.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:93643954352f7013007d0c5a7b47097cbdf533cf5b4aae690069ca9b78625ce2
size 30425

3
docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_27_0.png
View File

@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8ad51650644959fe669787d637d49fe35cfc85c9c8f5637470403470f56f6abb
size 4307

0
docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_19_0.png → docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_29_0.png
View File

111
docs/notebooks/116-sparsity-optimization-with-output.rst
View File

@ -67,7 +67,7 @@ Imports
import shutil
from pathlib import Path
from optimum.intel.openvino import OVModelForSequenceClassification
from transformers import AutoTokenizer, pipeline
from huggingface_hub import hf_hub_download
@ -85,14 +85,14 @@ Imports
.. parsed-literal::
2024-02-09 23:02:05.779349: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-02-09 23:02:05.814537: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2024-03-12 22:46:23.659626: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-03-12 22:46:23.694199: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
.. parsed-literal::
2024-02-09 23:02:06.378496: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2024-03-12 22:46:24.261905: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
Download, quantize and sparsify the model, using Hugging Face Optimum API
@ -112,17 +112,17 @@ model card on Hugging Face.
# The following model has been quantized, sparsified using Optimum-Intel 1.7 which is enabled by OpenVINO and NNCF
# for reproducibility, refer https://huggingface.co/OpenVINO/bert-base-uncased-sst2-int8-unstructured80
model_id = "OpenVINO/bert-base-uncased-sst2-int8-unstructured80"
# The following two steps will set up the model and download them to HF Cache folder
ov_model = OVModelForSequenceClassification.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
# Let's take the model for a spin!
sentiment_classifier = pipeline("text-classification", model=ov_model, tokenizer=tokenizer)
text = "He's a dreadful magician."
outputs = sentiment_classifier(text)
print(outputs)
@ -149,14 +149,14 @@ the IRs into a single folder.
# create a folder
quantized_sparse_dir = Path("bert_80pc_sparse_quantized_ir")
quantized_sparse_dir.mkdir(parents=True, exist_ok=True)
# following return path to specified filename in cache folder (which we've with the
# following return path to specified filename in cache folder (which we've with the
ov_ir_xml_path = hf_hub_download(repo_id=model_id, filename="openvino_model.xml")
ov_ir_bin_path = hf_hub_download(repo_id=model_id, filename="openvino_model.bin")
# copy IRs to the folder
shutil.copy(ov_ir_xml_path, quantized_sparse_dir)
shutil.copy(ov_ir_bin_path, quantized_sparse_dir)
shutil.copy(ov_ir_bin_path, quantized_sparse_dir)
@ -209,17 +209,17 @@ as an example. It is recommended to tune based on your applications.
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ] Device info:
.. parsed-literal::
[ INFO ] CPU
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ]
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
@ -228,7 +228,7 @@ as an example. It is recommended to tune based on your applications.
.. parsed-literal::
[ INFO ] Read model took 62.38 ms
[ INFO ] Read model took 60.53 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input_ids (node: input_ids) : i64 / [...] / [?,?]
@ -239,6 +239,10 @@ as an example. It is recommended to tune based on your applications.
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: 'input_ids': [1,64], 'attention_mask': [1,64], 'token_type_ids': [1,64]
.. parsed-literal::
[ INFO ] Reshape model took 23.14 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
@ -252,7 +256,7 @@ as an example. It is recommended to tune based on your applications.
.. parsed-literal::
[ INFO ] Compile model took 1107.64 ms
[ INFO ] Compile model took 1247.60 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@ -270,35 +274,38 @@ as an example. It is recommended to tune based on your applications.
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] CPU_DENORMALS_OPTIMIZATION: False
[ INFO ] LOG_LEVEL: Level.NO
[ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
[ INFO ] DYNAMIC_QUANTIZATION_GROUP_SIZE: 0
[ INFO ] KV_CACHE_PRECISION: <Type: 'float16'>
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'input_ids'!. This input will be filled with random values!
[ WARNING ] No input files were given for input 'attention_mask'!. This input will be filled with random values!
[ WARNING ] No input files were given for input 'token_type_ids'!. This input will be filled with random values!
[ INFO ] Fill input 'input_ids' with random values
[ INFO ] Fill input 'attention_mask' with random values
[ INFO ] Fill input 'token_type_ids' with random values
[ INFO ] Fill input 'input_ids' with random values
[ INFO ] Fill input 'attention_mask' with random values
[ INFO ] Fill input 'token_type_ids' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 4 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
.. parsed-literal::
[ INFO ] First inference took 30.14 ms
[ INFO ] First inference took 28.98 ms
.. parsed-literal::
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 8852 iterations
[ INFO ] Duration: 60038.32 ms
[ INFO ] Count: 8968 iterations
[ INFO ] Duration: 60028.76 ms
[ INFO ] Latency:
[ INFO ] Median: 26.79 ms
[ INFO ] Average: 26.86 ms
[ INFO ] Min: 24.76 ms
[ INFO ] Max: 42.20 ms
[ INFO ] Throughput: 147.44 FPS
[ INFO ] Median: 26.62 ms
[ INFO ] Average: 26.65 ms
[ INFO ] Min: 25.60 ms
[ INFO ] Max: 42.76 ms
[ INFO ] Throughput: 149.40 FPS
Benchmark quantized sparse inference performance
@ -314,7 +321,7 @@ for which a layer will be enabled.
.. code:: ipython3
# Dump benchmarking config for dense inference
# "CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE" controls minimum sparsity rate for weights to consider
# "CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE" controls minimum sparsity rate for weights to consider
# for sparse optimization at the runtime.
with (quantized_sparse_dir / "perf_config_sparse.json").open("w") as outfile:
outfile.write(
@ -344,17 +351,17 @@ for which a layer will be enabled.
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ] Device info:
.. parsed-literal::
[ INFO ] CPU
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ]
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
@ -363,7 +370,7 @@ for which a layer will be enabled.
.. parsed-literal::
[ INFO ] Read model took 71.12 ms
[ INFO ] Read model took 84.42 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input_ids (node: input_ids) : i64 / [...] / [?,?]
@ -374,10 +381,6 @@ for which a layer will be enabled.
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: 'input_ids': [1,64], 'attention_mask': [1,64], 'token_type_ids': [1,64]
.. parsed-literal::
[ INFO ] Reshape model took 23.54 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
@ -387,20 +390,28 @@ for which a layer will be enabled.
[ INFO ] Model outputs:
[ INFO ] logits (node: logits) : f32 / [...] / [1,2]
[Step 7/11] Loading the model to the device
[ ERROR ] Exception from src/inference/src/core.cpp:99:
[ GENERAL_ERROR ] Exception from src/plugins/intel_cpu/src/config.cpp:158:
Wrong value for property key CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE. Expected only float numbers
.. parsed-literal::
[ ERROR ] Exception from src/inference/src/cpp/core.cpp:106:
Exception from src/inference/src/dev/plugin.cpp:54:
Exception from src/plugins/intel_cpu/src/config.cpp:208:
Wrong value for property key CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE. Expected only float numbers
Traceback (most recent call last):
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 408, in main
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 408, in main
compiled_model = benchmark.core.compile_model(model, benchmark.device, device_config)
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/runtime/ie_api.py", line 547, in compile_model
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/runtime/ie_api.py", line 515, in compile_model
super().compile_model(model, device_name, {} if config is None else config),
RuntimeError: Exception from src/inference/src/core.cpp:99:
[ GENERAL_ERROR ] Exception from src/plugins/intel_cpu/src/config.cpp:158:
RuntimeError: Exception from src/inference/src/cpp/core.cpp:106:
Exception from src/inference/src/dev/plugin.cpp:54:
Exception from src/plugins/intel_cpu/src/config.cpp:208:
Wrong value for property key CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE. Expected only float numbers
When this might be helpful

54
docs/notebooks/117-model-server-with-output.rst
View File

@ -93,10 +93,10 @@ image and a message.
.. parsed-literal::
Hello from Docker!
This message shows that your installation appears to be working correctly.
To generate this message, Docker took the following steps:
1. The Docker client contacted the Docker daemon.
2. The Docker daemon pulled the "hello-world" image from the Docker Hub.
@ -105,16 +105,16 @@ image and a message.
executable that produces the output you are currently reading.
4. The Docker daemon streamed that output to the Docker client, which sent it
to your terminal.
To try something more ambitious, you can run an Ubuntu container with:
$ docker run -it ubuntu bash
Share images, automate workflows, and more with a free Docker ID:
https://hub.docker.com/
For more examples and ideas, visit:
https://docs.docker.com/get-started/
Step 2: Preparing a Model Repository
@ -175,7 +175,7 @@ following rules:
.. code:: ipython3
import os
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
@ -183,18 +183,18 @@ following rules:
filename='notebook_utils.py'
)
from notebook_utils import download_file
dedicated_dir = "models"
model_name = "detection"
model_version = "1"
MODEL_DIR = f"{dedicated_dir}/{model_name}/{model_version}"
XML_PATH = "horizontal-text-detection-0001.xml"
BIN_PATH = "horizontal-text-detection-0001.bin"
os.makedirs(MODEL_DIR, exist_ok=True)
model_xml_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.xml"
model_bin_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.bin"
download_file(model_xml_url, XML_PATH, MODEL_DIR)
download_file(model_bin_url, BIN_PATH, MODEL_DIR)
@ -229,14 +229,14 @@ Searching for an available serving port in local.
.. code:: ipython3
import socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('localhost', 0))
sock.listen(1)
port = sock.getsockname()[1]
sock.close()
print(f"Port {port} is available")
os.environ['port'] = str(port)
@ -737,7 +737,7 @@ Request Model Status
.. code:: ipython3
address = "localhost:" + str(port)
# Bind the grpc address to the client object
client = make_grpc_client(address)
model_status = client.get_model_status(model_name=model_name)
@ -777,16 +777,16 @@ Load input image
"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/intel_rnb.jpg",
directory="data"
)
# Text detection models expect an image in BGR format.
image = cv2.imread(str(image_filename))
fp_image = image.astype("float32")
# Resize the image to meet network expected input sizes.
input_shape = model_metadata['inputs']['image']['shape']
height, width = input_shape[2], input_shape[3]
resized_image = cv2.resize(fp_image, (height, width))
# Reshape to the network input shape.
input_image = np.expand_dims(resized_image.transpose(2, 0, 1), 0)
plt.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
@ -818,10 +818,10 @@ Request Prediction on a Numpy Array
.. code:: ipython3
inputs = {"image": input_image}
# Run inference on model server and receive the result data
boxes = client.predict(inputs=inputs, model_name=model_name)['boxes']
# Remove zero only boxes.
boxes = boxes[~np.all(boxes == 0, axis=1)]
print(boxes)
@ -849,31 +849,31 @@ Visualization
def convert_result_to_image(bgr_image, resized_image, boxes, threshold=0.3, conf_labels=True):
# Define colors for boxes and descriptions.
colors = {"red": (255, 0, 0), "green": (0, 255, 0)}
# Fetch the image shapes to calculate a ratio.
(real_y, real_x), (resized_y, resized_x) = bgr_image.shape[:2], resized_image.shape[:2]
ratio_x, ratio_y = real_x / resized_x, real_y / resized_y
# Convert the base image from BGR to RGB format.
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
# Iterate through non-zero boxes.
for box in boxes:
# Pick a confidence factor from the last place in an array.
conf = box[-1]
if conf > threshold:
# Convert float to int and multiply corner position of each box by x and y ratio.
# If the bounding box is found at the top of the image,
# position the upper box bar little lower to make it visible on the image.
# If the bounding box is found at the top of the image,
# position the upper box bar little lower to make it visible on the image.
(x_min, y_min, x_max, y_max) = [
int(max(corner_position * ratio_y, 10)) if idx % 2
int(max(corner_position * ratio_y, 10)) if idx % 2
else int(corner_position * ratio_x)
for idx, corner_position in enumerate(box[:-1])
]
# Draw a box based on the position, parameters in rectangle function are: image, start_point, end_point, color, thickness.
rgb_image = cv2.rectangle(rgb_image, (x_min, y_min), (x_max, y_max), colors["green"], 3)
# Add text to the image based on position and confidence.
# Parameters in text function are: image, text, bottom-left_corner_textfield, font, font_scale, color, thickness, line_type.
if conf_labels:
@ -887,7 +887,7 @@ Visualization
1,
cv2.LINE_AA,
)
return rgb_image
.. code:: ipython3

126
docs/notebooks/118-optimize-preprocessing-with-output.rst
View File

@ -9,9 +9,9 @@ instrument, that enables integration of preprocessing steps into an
execution graph and performing it on a selected device, which can
improve device utilization. For more information about Preprocessing
API, see this
`overview <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimizie-preprocessing.html>`__
`overview <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimize-preprocessing.html#>`__
and
`details <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimizie-preprocessing/preprocessing-api-details.html>`__
`details <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimize-preprocessing/preprocessing-api-details.html>`__
This tutorial include following steps:
@ -40,7 +40,7 @@ Table of contents:
- `Convert model to OpenVINO IR with model conversion
API <#convert-model-to-openvino-ir-with-model-conversion-api>`__
- `Create ``PrePostProcessor``
- `Create PrePostProcessor
Object <#create-prepostprocessor-object>`__
- `Declare User’s Data Format <#declare-users-data-format>`__
- `Declaring Model Layout <#declaring-model-layout>`__
@ -88,13 +88,13 @@ Imports
import time
from pathlib import Path
import cv2
import matplotlib.pyplot as plt
import numpy as np
import openvino as ov
import tensorflow as tf
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
@ -106,14 +106,14 @@ Imports
.. parsed-literal::
2024-02-09 23:03:22.538807: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-02-09 23:03:22.573455: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2024-03-12 22:47:40.396566: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-03-12 22:47:40.431134: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
.. parsed-literal::
2024-02-09 23:03:23.087260: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2024-03-12 22:47:40.948008: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
Setup image and device
@ -140,7 +140,7 @@ Setup image and device
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
@ -148,7 +148,7 @@ Setup image and device
description='Device:',
disabled=False,
)
device
@ -182,24 +182,24 @@ and save it to the disk.
.. code:: ipython3
model_name = "InceptionResNetV2"
model_dir = Path("model")
model_dir.mkdir(exist_ok=True)
model_path = model_dir / model_name
model = tf.keras.applications.InceptionV3()
model.save(model_path)
.. parsed-literal::
2024-02-09 23:03:26.685646: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW
2024-02-09 23:03:26.685683: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:168] retrieving CUDA diagnostic information for host: iotg-dev-workstation-07
2024-02-09 23:03:26.685688: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:175] hostname: iotg-dev-workstation-07
2024-02-09 23:03:26.685821: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:199] libcuda reported version is: 470.223.2
2024-02-09 23:03:26.685836: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:203] kernel reported version is: 470.182.3
2024-02-09 23:03:26.685841: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration
2024-03-12 22:47:44.291989: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW
2024-03-12 22:47:44.292027: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:168] retrieving CUDA diagnostic information for host: iotg-dev-workstation-07
2024-03-12 22:47:44.292031: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:175] hostname: iotg-dev-workstation-07
2024-03-12 22:47:44.292168: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:199] libcuda reported version is: 470.223.2
2024-03-12 22:47:44.292184: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:203] kernel reported version is: 470.182.3
2024-03-12 22:47:44.292187: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration
.. parsed-literal::
@ -273,7 +273,7 @@ Graph modifications of a model shall be performed after the model is
read from a drive and before it is loaded on the actual device.
Pre-processing support following operations (please, see more details
`here <https://docs.openvino.ai/2024/api/c_cpp_api/group__ov__dev__exec__model.html#_CPPv3N2ov10preprocess15PreProcessStepsE>`__)
`here <https://docs.openvino.ai/2024/api/c_cpp_api/group__ov__dev__exec__model.html#_CPPv3N2ov10preprocess15PreProcessStepsD0Ev>`__)
- Mean/Scale Normalization
- Converting Precision
@ -292,13 +292,13 @@ The options for preprocessing are not required.
.. code:: ipython3
ir_path = model_dir / "ir_model" / f"{model_name}.xml"
ppp_model = None
if ir_path.exists():
ppp_model = core.read_model(model=ir_path)
print(f"Model in OpenVINO format already exists: {ir_path}")
else:
else:
ppp_model = ov.convert_model(model_path,
input=[1,299,299,3])
ov.save_model(ppp_model, str(ir_path))
@ -316,7 +316,7 @@ a model.
.. code:: ipython3
from openvino.preprocess import PrePostProcessor
ppp = PrePostProcessor(ppp_model)
Declare User’s Data Format
@ -334,7 +334,7 @@ about user’s input tensor will be initialized to same data
(type/shape/etc) as model’s input parameter. User application can
override particular parameters according to application’s data. Refer to
the following
`page <https://docs.openvino.ai/2024/api/c_cpp_api/group__ov__dev__exec__model.html#_CPPv4N2ov10preprocess15InputTensorInfoE>`__
`page <https://docs.openvino.ai/2024/api/c_cpp_api/group__ov__dev__exec__model.html#_CPPv3N2ov10preprocess9InputInfo6tensorEv>`__
for more information about parameters for overriding.
Below is all the specified input information:
@ -360,7 +360,7 @@ for mean/scale normalization.
.. parsed-literal::
<openvino._pyopenvino.preprocess.InputTensorInfo at 0x7ff8b271c1b0>
<openvino._pyopenvino.preprocess.InputTensorInfo at 0x7f12e449edf0>
@ -372,13 +372,13 @@ Declaring Model Layout
Model input already has information about precision and shape.
Preprocessing API is not intended to modify this. The only thing that
may be specified is input data
`layout <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimizie-preprocessing/layout-api-overview.html>`__.
`layout <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimize-preprocessing/layout-api-overview.html>`__.
.. code:: ipython3
input_layer_ir = next(iter(ppp_model.inputs))
print(f"The input shape of the model is {input_layer_ir.shape}")
ppp.input().model().set_layout(ov.Layout('NHWC'))
@ -391,7 +391,7 @@ may be specified is input data
.. parsed-literal::
<openvino._pyopenvino.preprocess.InputModelInfo at 0x7ff7b1f9dd70>
<openvino._pyopenvino.preprocess.InputModelInfo at 0x7f12e448f9f0>
@ -421,7 +421,7 @@ then such conversion will be added explicitly.
.. code:: ipython3
from openvino.preprocess import ResizeAlgorithm
ppp.input().preprocess().convert_element_type(ov.Type.f32) \
.resize(ResizeAlgorithm.RESIZE_LINEAR)\
.mean([127.5,127.5,127.5])\
@ -432,7 +432,7 @@ then such conversion will be added explicitly.
.. parsed-literal::
<openvino._pyopenvino.preprocess.PreProcessSteps at 0x7ff7b1f9d230>
<openvino._pyopenvino.preprocess.PreProcessSteps at 0x7f12e448fdf0>
@ -461,7 +461,7 @@ configuration for debugging purposes.
resize to model width/height: ([1,?,?,3], [N,H,W,C], f32) -> ([1,299,299,3], [N,H,W,C], f32)
mean (127.5,127.5,127.5): ([1,299,299,3], [N,H,W,C], f32) -> ([1,299,299,3], [N,H,W,C], f32)
scale (127.5,127.5,127.5): ([1,299,299,3], [N,H,W,C], f32) -> ([1,299,299,3], [N,H,W,C], f32)
Load model and perform inference
@ -475,12 +475,12 @@ Load model and perform inference
image = cv2.imread(image_path)
input_tensor = np.expand_dims(image, 0)
return input_tensor
compiled_model_with_preprocess_api = core.compile_model(model=ppp_model, device_name=device.value)
ppp_output_layer = compiled_model_with_preprocess_api.output(0)
ppp_input_tensor = prepare_image_api_preprocess(image_path)
results = compiled_model_with_preprocess_api(ppp_input_tensor)[ppp_output_layer][0]
@ -508,22 +508,22 @@ Load image and fit it to model input
def manual_image_preprocessing(path_to_image, compiled_model):
input_layer_ir = next(iter(compiled_model.inputs))
# N, H, W, C = batch size, height, width, number of channels
N, H, W, C = input_layer_ir.shape
# load image, image will be resized to model input size and converted to RGB
img = tf.keras.preprocessing.image.load_img(image_path, target_size=(H, W), color_mode='rgb')
x = tf.keras.preprocessing.image.img_to_array(img)
x = np.expand_dims(x, axis=0)
# will scale input pixels between -1 and 1
input_tensor = tf.keras.applications.inception_resnet_v2.preprocess_input(x)
return input_tensor
input_tensor = manual_image_preprocessing(image_path, compiled_model)
print(f"The shape of the image is {input_tensor.shape}")
print(f"The data type of the image is {input_tensor.dtype}")
@ -543,7 +543,7 @@ Perform inference
.. code:: ipython3
output_layer = compiled_model.output(0)
result = compiled_model(input_tensor)[output_layer]
Compare results
@ -560,18 +560,18 @@ Compare results on one image
def check_results(input_tensor, compiled_model, imagenet_classes):
output_layer = compiled_model.output(0)
results = compiled_model(input_tensor)[output_layer][0]
top_indices = np.argsort(results)[-5:][::-1]
top_softmax = results[top_indices]
for index, softmax_probability in zip(top_indices, top_softmax):
print(f"{imagenet_classes[index]}, {softmax_probability:.5f}")
return top_indices, top_softmax
# Convert the inference result to a class name.
imagenet_filename = download_file(
"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_2012.txt",
@ -579,13 +579,13 @@ Compare results on one image
)
imagenet_classes = imagenet_filename.read_text().splitlines()
imagenet_classes = ['background'] + imagenet_classes
# get result for inference with preprocessing api
print("Result of inference with Preprocessing API:")
res = check_results(ppp_input_tensor, compiled_model_with_preprocess_api, imagenet_classes)
print("\n")
# get result for inference with the manual preparing of the image
print("Result of inference with manual image setup:")
res = check_results(input_tensor, compiled_model, imagenet_classes)
@ -605,8 +605,8 @@ Compare results on one image
n02108915 French bulldog, 0.01915
n02111129 Leonberg, 0.00825
n02097047 miniature schnauzer, 0.00294
Result of inference with manual image setup:
n02098413 Lhasa, Lhasa apso, 0.76843
n02099601 golden retriever, 0.19322
@ -624,24 +624,24 @@ Compare performance
def check_performance(compiled_model, preprocessing_function=None):
num_images = 1000
start = time.perf_counter()
for _ in range(num_images):
input_tensor = preprocessing_function(image_path, compiled_model)
compiled_model(input_tensor)
end = time.perf_counter()
time_ir = end - start
return time_ir, num_images
time_ir, num_images = check_performance(compiled_model, manual_image_preprocessing)
print(
f"IR model in OpenVINO Runtime/CPU with manual image preprocessing: {time_ir/num_images:.4f} "
f"seconds per image, FPS: {num_images/time_ir:.2f}"
)
time_ir, num_images = check_performance(compiled_model_with_preprocess_api, prepare_image_api_preprocess)
print(
f"IR model in OpenVINO Runtime/CPU with preprocessing API: {time_ir/num_images:.4f} "
@ -651,10 +651,10 @@ Compare performance
.. parsed-literal::
IR model in OpenVINO Runtime/CPU with manual image preprocessing: 0.0153 seconds per image, FPS: 65.53
IR model in OpenVINO Runtime/CPU with manual image preprocessing: 0.0153 seconds per image, FPS: 65.44
.. parsed-literal::
IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0183 seconds per image, FPS: 54.77
IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0183 seconds per image, FPS: 54.74

2
docs/notebooks/118-optimize-preprocessing-with-output_files/118-optimize-preprocessing-with-output_14_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2dd4338c6c163e7693885ce544e8c9cd2aecedf3b136fa295e22877f37b5634c
oid sha256:5712bd24e962ae0e0267607554ebe1f2869c223b108876ce10e5d20fe6285126
size 387941

124
docs/notebooks/119-tflite-to-openvino-with-output.rst
View File

@ -11,7 +11,7 @@ image classification model to OpenVINO `Intermediate
Representation <https://docs.openvino.ai/2024/documentation/openvino-ir-format/operation-sets.html>`__
(OpenVINO IR) format, using Model Converter. After creating the OpenVINO
IR, load the model in `OpenVINO
Runtime <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_OV_Runtime_User_Guide.html>`__
Runtime <https://docs.openvino.ai/2024/openvino-workflow/running-inference.html>`__
and do inference with a sample image.
Table of contents:
@ -47,7 +47,7 @@ Install requirements
%pip install -q "openvino>=2023.1.0"
%pip install -q opencv-python requests tqdm
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
@ -77,7 +77,7 @@ Imports
import numpy as np
from PIL import Image
import openvino as ov
from notebook_utils import download_file, load_image
Download TFLite model
@ -89,10 +89,10 @@ Download TFLite model
model_dir = Path("model")
tflite_model_path = model_dir / "efficientnet_lite0_fp32_2.tflite"
ov_model_path = tflite_model_path.with_suffix(".xml")
model_url = "https://www.kaggle.com/models/tensorflow/efficientnet/frameworks/tfLite/variations/lite0-fp32/versions/2?lite-format=tflite"
download_file(model_url, tflite_model_path.name, model_dir)
@ -106,7 +106,7 @@ Download TFLite model
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/119-tflite-to-openvino/model/efficientnet_lite0_fp32_2.tflite')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/119-tflite-to-openvino/model/efficientnet_lite0_fp32_2.tflite')
@ -153,7 +153,7 @@ this `tutorial <002-openvino-api-with-output.html>`__.
.. code:: ipython3
core = ov.Core()
ov_model = core.read_model(tflite_model_path)
Run OpenVINO model inference
@ -183,14 +183,14 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
@ -204,18 +204,18 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
compiled_model = core.compile_model(ov_model)
compiled_model = core.compile_model(ov_model, device.value)
predicted_scores = compiled_model(input_tensor)[0]
.. code:: ipython3
imagenet_classes_file_path = download_file("https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_2012.txt")
imagenet_classes = open(imagenet_classes_file_path).read().splitlines()
top1_predicted_cls_id = np.argmax(predicted_scores)
top1_predicted_score = predicted_scores[0][top1_predicted_cls_id]
predicted_label = imagenet_classes[top1_predicted_cls_id]
display(image.resize((640, 512)))
print(f"Predicted label: {predicted_label} with probability {top1_predicted_score :2f}")
@ -252,16 +252,13 @@ GPU.
.. code:: ipython3
print("Benchmark model inference on CPU")
!benchmark_app -m $ov_model_path -d CPU -t 15
if "GPU" in core.available_devices:
print("\n\nBenchmark model inference on GPU")
!benchmark_app -m $ov_model_path -d GPU -t 15
print(f"Benchmark model inference on {device.value}")
!benchmark_app -m $ov_model_path -d $device.value -t 15
.. parsed-literal::
Benchmark model inference on CPU
Benchmark model inference on AUTO
.. parsed-literal::
@ -270,80 +267,91 @@ GPU.
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ]
[ INFO ] AUTO
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
.. parsed-literal::
[ INFO ] Read model took 21.35 ms
[ INFO ] Read model took 9.52 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] images (node: images) : f32 / [...] / [1,224,224,3]
[ INFO ] Model outputs:
[ INFO ] Softmax (node: 63) : f32 / [...] / [1,1000]
[ INFO ] Softmax (node: 61) : f32 / [...] / [1,1000]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] images (node: images) : u8 / [N,H,W,C] / [1,224,224,3]
[ INFO ] Model outputs:
[ INFO ] Softmax (node: 63) : f32 / [...] / [1,1000]
[ INFO ] Softmax (node: 61) : f32 / [...] / [1,1000]
[Step 7/11] Loading the model to the device
.. parsed-literal::
[ INFO ] Compile model took 147.38 ms
[ INFO ] Compile model took 181.00 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: TensorFlow_Lite_Frontend_IR
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 6
[ INFO ] MULTI_DEVICE_PRIORITIES: CPU
[ INFO ] CPU:
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] CPU_DENORMALS_OPTIMIZATION: False
[ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
[ INFO ] DYNAMIC_QUANTIZATION_GROUP_SIZE: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] INFERENCE_NUM_THREADS: 24
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] KV_CACHE_PRECISION: <Type: 'float16'>
[ INFO ] LOG_LEVEL: Level.NO
[ INFO ] NETWORK_NAME: TensorFlow_Lite_Frontend_IR
[ INFO ] NUM_STREAMS: 6
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 6
[ INFO ] PERFORMANCE_HINT: THROUGHPUT
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] PERF_COUNT: NO
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] MODEL_PRIORITY: Priority.MEDIUM
[ INFO ] LOADED_FROM_CACHE: False
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'images'!. This input will be filled with random values!
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
.. parsed-literal::
[ INFO ] NETWORK_NAME: TensorFlow_Lite_Frontend_IR
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 6
[ INFO ] NUM_STREAMS: 6
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 24
[ INFO ] PERF_COUNT: NO
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] CPU_DENORMALS_OPTIMIZATION: False
[ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'images'!. This input will be filled with random values!
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 7.28 ms
[ INFO ] First inference took 7.56 ms
.. parsed-literal::
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 17502 iterations
[ INFO ] Duration: 15007.36 ms
[ INFO ] Count: 17412 iterations
[ INFO ] Duration: 15009.54 ms
[ INFO ] Latency:
[ INFO ] Median: 5.02 ms
[ INFO ] Average: 5.02 ms
[ INFO ] Min: 3.02 ms
[ INFO ] Max: 13.94 ms
[ INFO ] Throughput: 1166.23 FPS
[ INFO ] Median: 5.04 ms
[ INFO ] Average: 5.04 ms
[ INFO ] Min: 3.66 ms
[ INFO ] Max: 13.39 ms
[ INFO ] Throughput: 1160.06 FPS

108
docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output.rst
View File

@ -21,7 +21,7 @@ Representation <https://docs.openvino.ai/2024/documentation/openvino-ir-format/o
(OpenVINO IR) format, using `Model Conversion
API <https://docs.openvino.ai/2024/openvino-workflow/model-preparation.html>`__.
After creating the OpenVINO IR, load the model in `OpenVINO
Runtime <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_OV_Runtime_User_Guide.html>`__
Runtime <https://docs.openvino.ai/2024/openvino-workflow/running-inference.html>`__
and do inference with a sample image.
Table of contents:
@ -60,7 +60,19 @@ Install required packages:
.. code:: ipython3
%pip install -q "openvino>=2023.1.0" "numpy>=1.21.0" "opencv-python" "matplotlib>=3.4"
import platform
%pip install -q "openvino>=2023.1.0" "numpy>=1.21.0" "opencv-python"
if platform.system() != "Windows":
%pip install -q "matplotlib>=3.4"
else:
%pip install -q "matplotlib>=3.4,<3.7"
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
@ -75,7 +87,7 @@ The notebook uses utility functions. The cell below will download the
# Fetch the notebook utils script from the openvino_notebooks repo
import urllib.request
urllib.request.urlretrieve(
url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py",
filename="notebook_utils.py",
@ -90,15 +102,15 @@ Imports
# Standard python modules
from pathlib import Path
# External modules and dependencies
import cv2
import matplotlib.pyplot as plt
import numpy as np
# Notebook utils module
from notebook_utils import download_file
# OpenVINO modules
import openvino as ov
@ -114,21 +126,21 @@ Define model related variables and create corresponding directories:
# Create directories for models files
model_dir = Path("model")
model_dir.mkdir(exist_ok=True)
# Create directory for TensorFlow model
tf_model_dir = model_dir / "tf"
tf_model_dir.mkdir(exist_ok=True)
# Create directory for OpenVINO IR model
ir_model_dir = model_dir / "ir"
ir_model_dir.mkdir(exist_ok=True)
model_name = "mask_rcnn_inception_resnet_v2_1024x1024"
openvino_ir_path = ir_model_dir / f"{model_name}.xml"
tf_model_url = "https://www.kaggle.com/models/tensorflow/mask-rcnn-inception-resnet-v2/frameworks/tensorFlow2/variations/1024x1024/versions/1?tf-hub-format=compressed"
tf_model_archive_filename = f"{model_name}.tar.gz"
Download Model from TensorFlow Hub
@ -161,7 +173,7 @@ archive:
.. code:: ipython3
import tarfile
with tarfile.open(tf_model_dir / tf_model_archive_filename) as file:
file.extractall(path=tf_model_dir)
@ -189,7 +201,7 @@ when the model is run in the future.
.. code:: ipython3
ov_model = ov.convert_model(tf_model_dir)
# Save converted OpenVINO IR model to the corresponding directory
ov.save_model(ov_model, openvino_ir_path)
@ -208,7 +220,7 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
@ -216,7 +228,7 @@ select device from dropdown list for running inference using OpenVINO
description='Device:',
disabled=False,
)
device
@ -272,12 +284,12 @@ the first (and highest) detection score.
model_inputs = compiled_model.inputs
model_outputs = compiled_model.outputs
print("Model inputs count:", len(model_inputs))
print("Model inputs:")
for _input in model_inputs:
print(" ", _input)
print("Model outputs count:", len(model_outputs))
print("Model outputs:")
for output in model_outputs:
@ -326,7 +338,7 @@ Load and save an image:
.. code:: ipython3
image_path = Path("./data/coco_bike.jpg")
download_file(
url="https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_bike.jpg",
filename=image_path.name,
@ -346,16 +358,16 @@ Read the image, resize and convert it to the input shape of the network:
# Read the image
image = cv2.imread(filename=str(image_path))
# The network expects images in RGB format
image = cv2.cvtColor(image, code=cv2.COLOR_BGR2RGB)
# Resize the image to the network input shape
resized_image = cv2.resize(src=image, dsize=(255, 255))
# Add batch dimension to image
network_input_image = np.expand_dims(resized_image, 0)
# Show the image
plt.imshow(image)
@ -364,7 +376,7 @@ Read the image, resize and convert it to the input shape of the network:
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f39e4396eb0>
<matplotlib.image.AxesImage at 0x7fd56a6e5b80>
@ -391,23 +403,23 @@ be extracted from the result. For further model result visualization
detection_boxes = compiled_model.output("detection_boxes")
image_detection_boxes = inference_result[detection_boxes]
print("image_detection_boxes:", image_detection_boxes.shape)
detection_masks = compiled_model.output("detection_masks")
image_detection_masks = inference_result[detection_masks]
print("image_detection_masks:", image_detection_masks.shape)
detection_classes = compiled_model.output("detection_classes")
image_detection_classes = inference_result[detection_classes]
print("image_detection_classes:", image_detection_classes.shape)
detection_scores = compiled_model.output("detection_scores")
image_detection_scores = inference_result[detection_scores]
print("image_detection_scores:", image_detection_scores.shape)
num_detections = compiled_model.output("num_detections")
image_num_detections = inference_result[num_detections]
print("image_detections_num:", image_num_detections)
# Alternatively, inference result data can be extracted by model output name with `.get()` method
assert (inference_result[detection_boxes] == inference_result.get("detection_boxes")).all(), "extracted inference result data should be equal"
@ -432,14 +444,14 @@ Define utility functions to visualize the inference results
import random
from typing import Optional
def add_detection_box(
box: np.ndarray, image: np.ndarray, mask: np.ndarray, label: Optional[str] = None
) -> np.ndarray:
"""
Helper function for adding single bounding box to the image
Parameters
----------
box : np.ndarray
@ -450,18 +462,18 @@ Define utility functions to visualize the inference results
Segmentation mask in format (H, W)
label : str, optional
Detection box label string, if not provided will not be added to result image (default is None)
Returns
-------
np.ndarray
NumPy array including image, detection box, and segmentation mask
"""
ymin, xmin, ymax, xmax = box
point1, point2 = (int(xmin), int(ymin)), (int(xmax), int(ymax))
box_color = [random.randint(0, 255) for _ in range(3)]
line_thickness = round(0.002 * (image.shape[0] + image.shape[1]) / 2) + 1
result = cv2.rectangle(
img=image,
pt1=point1,
@ -470,7 +482,7 @@ Define utility functions to visualize the inference results
thickness=line_thickness,
lineType=cv2.LINE_AA,
)
if label:
font_thickness = max(line_thickness - 1, 1)
font_face = 0
@ -513,12 +525,12 @@ Define utility functions to visualize the inference results
def get_mask_frame(box, frame, mask):
"""
Transform a binary mask to fit within a specified bounding box in a frame using perspective transformation.
Args:
box (tuple): A bounding box represented as a tuple (y_min, x_min, y_max, x_max).
frame (numpy.ndarray): The larger frame or image where the mask will be placed.
mask (numpy.ndarray): A binary mask image to be transformed.
Returns:
numpy.ndarray: A transformed mask image that fits within the specified bounding box in the frame.
"""
@ -543,10 +555,10 @@ Define utility functions to visualize the inference results
.. code:: ipython3
from typing import Dict
from openvino.runtime.utils.data_helpers import OVDict
def visualize_inference_result(
inference_result: OVDict,
image: np.ndarray,
@ -555,7 +567,7 @@ Define utility functions to visualize the inference results
):
"""
Helper function for visualizing inference result on the image
Parameters
----------
inference_result : OVDict
@ -572,13 +584,13 @@ Define utility functions to visualize the inference results
detection_scores = inference_result.get("detection_scores")
num_detections = inference_result.get("num_detections")
detection_masks = inference_result.get("detection_masks")
detections_limit = int(
min(detections_limit, num_detections[0])
if detections_limit is not None
else num_detections[0]
)
# Normalize detection boxes coordinates to original image size
original_image_height, original_image_width, _ = image.shape
normalized_detection_boxes = detection_boxes[0, :detections_limit] * [
@ -598,7 +610,7 @@ Define utility functions to visualize the inference results
result = add_detection_box(
box=normalized_detection_boxes[i], image=result, mask=mask_reframed, label=label
)
plt.imshow(result)
TensorFlow Instance Segmentation model
@ -614,7 +626,7 @@ Zoo <https://github.com/openvinotoolkit/open_model_zoo/>`__:
.. code:: ipython3
coco_labels_file_path = Path("./data/coco_91cl.txt")
download_file(
url="https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/data/dataset_classes/coco_91cl.txt",
filename=coco_labels_file_path.name,
@ -637,7 +649,7 @@ file:
with open(coco_labels_file_path, "r") as file:
coco_labels = file.read().strip().split("\n")
coco_labels_map = dict(enumerate(coco_labels, 1))
print(coco_labels_map)
@ -698,4 +710,4 @@ utilization.
For more information, refer to the `Optimize Preprocessing
tutorial <118-optimize-preprocessing-with-output.html>`__
and to the overview of `Preprocessing
API <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimizie-preprocessing/preprocessing-api-details.html>`__.
API <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimize-preprocessing/preprocessing-api-details.html>`__.

2
docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output_files/120-tensorflow-instance-segmentation-to-openvino-with-output_25_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:62971c6546b399ce0eecbdc81a5cb327c4bebdfa1a6db658e364870db606d761
oid sha256:90f5ec228ab4ebfbb7bbf584f3eb86452f97261c08225f0fd2cb0b4bceadacb1
size 395346

4
docs/notebooks/120-tensorflow-instance-segmentation-to-openvino-with-output_files/120-tensorflow-instance-segmentation-to-openvino-with-output_39_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8e7e8a03ae8a07e661d3a25306fe3dca53093416ecf98222fe478edaf13d07df
size 393160
oid sha256:50bd4a35af276f9e1ead8973e69ac3ae780beb2087b477c4d20929bf2f61b93f
size 389505

216
docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output.rst
View File

@ -59,7 +59,19 @@ Install required packages:
.. code:: ipython3
%pip install -q "openvino>=2023.1.0" "numpy>=1.21.0" "opencv-python" "matplotlib>=3.4"
import platform
%pip install -q "openvino>=2023.1.0" "numpy>=1.21.0" "opencv-python"
if platform.system() != "Windows":
%pip install -q "matplotlib>=3.4"
else:
%pip install -q "matplotlib>=3.4,<3.7"
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
@ -74,7 +86,7 @@ The notebook uses utility functions. The cell below will download the
# Fetch the notebook utils script from the openvino_notebooks repo
import urllib.request
urllib.request.urlretrieve(
url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py",
filename="notebook_utils.py",
@ -89,14 +101,14 @@ Imports
# Standard python modules
from pathlib import Path
# External modules and dependencies
import cv2
import matplotlib.pyplot as plt
import numpy as np
# OpenVINO import
import openvino as ov
# Notebook utils module
from notebook_utils import download_file
@ -112,21 +124,21 @@ Define model related variables and create corresponding directories:
# Create directories for models files
model_dir = Path("model")
model_dir.mkdir(exist_ok=True)
# Create directory for TensorFlow model
tf_model_dir = model_dir / "tf"
tf_model_dir.mkdir(exist_ok=True)
# Create directory for OpenVINO IR model
ir_model_dir = model_dir / "ir"
ir_model_dir.mkdir(exist_ok=True)
model_name = "faster_rcnn_resnet50_v1_640x640"
openvino_ir_path = ir_model_dir / f"{model_name}.xml"
tf_model_url = "https://www.kaggle.com/models/tensorflow/faster-rcnn-resnet-v1/frameworks/tensorFlow2/variations/faster-rcnn-resnet50-v1-640x640/versions/1?tf-hub-format=compressed"
tf_model_archive_filename = f"{model_name}.tar.gz"
Download Model from TensorFlow Hub
@ -157,7 +169,7 @@ from TensorFlow Hub:
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/model/tf/faster_rcnn_resnet50_v1_640x640.tar.gz')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/model/tf/faster_rcnn_resnet50_v1_640x640.tar.gz')
@ -166,7 +178,7 @@ Extract TensorFlow Object Detection model from the downloaded archive:
.. code:: ipython3
import tarfile
with tarfile.open(tf_model_dir / tf_model_archive_filename) as file:
file.extractall(path=tf_model_dir)
@ -196,7 +208,7 @@ support <https://docs.openvino.ai/2024/openvino-workflow/model-preparation/conve
.. code:: ipython3
ov_model = ov.convert_model(tf_model_dir)
# Save converted OpenVINO IR model to the corresponding directory
ov.save_model(ov_model, openvino_ir_path)
@ -215,7 +227,7 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
@ -223,7 +235,7 @@ select device from dropdown list for running inference using OpenVINO
description='Device:',
disabled=False,
)
device
@ -290,10 +302,10 @@ for more information about model inputs, outputs and their formats.
model_inputs = compiled_model.inputs
model_input = compiled_model.input(0)
model_outputs = compiled_model.outputs
print("Model inputs count:", len(model_inputs))
print("Model input:", model_input)
print("Model outputs count:", len(model_outputs))
print("Model outputs:")
for output in model_outputs:
@ -326,7 +338,7 @@ Load and save an image:
.. code:: ipython3
image_path = Path("./data/coco_bike.jpg")
download_file(
url="https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_bike.jpg",
filename=image_path.name,
@ -343,7 +355,7 @@ Load and save an image:
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/data/coco_bike.jpg')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/data/coco_bike.jpg')
@ -353,16 +365,16 @@ Read the image, resize and convert it to the input shape of the network:
# Read the image
image = cv2.imread(filename=str(image_path))
# The network expects images in RGB format
image = cv2.cvtColor(image, code=cv2.COLOR_BGR2RGB)
# Resize the image to the network input shape
resized_image = cv2.resize(src=image, dsize=(255, 255))
# Transpose the image to the network input shape
network_input_image = np.expand_dims(resized_image, 0)
# Show the image
plt.imshow(image)
@ -371,7 +383,7 @@ Read the image, resize and convert it to the input shape of the network:
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f62f45088e0>
<matplotlib.image.AxesImage at 0x7f7847e53880>
@ -396,28 +408,28 @@ outputs will be used.
.. code:: ipython3
_, detection_boxes, detection_classes, _, detection_scores, num_detections, _, _ = model_outputs
image_detection_boxes = inference_result[detection_boxes]
print("image_detection_boxes:", image_detection_boxes)
image_detection_classes = inference_result[detection_classes]
print("image_detection_classes:", image_detection_classes)
image_detection_scores = inference_result[detection_scores]
print("image_detection_scores:", image_detection_scores)
image_num_detections = inference_result[num_detections]
print("image_detections_num:", image_num_detections)
# Alternatively, inference result data can be extracted by model output name with `.get()` method
assert (inference_result[detection_boxes] == inference_result.get("detection_boxes")).all(), "extracted inference result data should be equal"
.. parsed-literal::
image_detection_boxes: [[[0.1645457 0.54601336 0.8953864 0.85500604]
[0.67189544 0.01240015 0.9843237 0.53085935]
[0.49188587 0.0117609 0.98050654 0.8866383 ]
image_detection_boxes: [[[0.16454576 0.54601336 0.8953865 0.85500604]
[0.67189544 0.01240013 0.9843237 0.5308593 ]
[0.4918859 0.0117609 0.98050654 0.8866383 ]
...
[0.43604603 0.59332204 0.4692565 0.6341099 ]
[0.46022677 0.59246916 0.48732638 0.61871874]
@ -439,56 +451,56 @@ outputs will be used.
84. 38. 1. 15. 3. 20. 62. 58. 41. 20. 2. 4. 88. 62. 15. 31. 1. 31.
14. 19. 4. 1. 2. 8. 18. 15. 4. 2. 2. 2. 31. 84. 15. 3. 28. 2.
27. 18. 15. 1. 31. 28. 1. 41. 8. 1. 3. 20.]]
image_detection_scores: [[0.9810079 0.9406672 0.9318088 0.877368 0.8406416 0.590001
0.55449295 0.53957206 0.49390146 0.48142543 0.46272704 0.44070077
0.40116653 0.34708446 0.31795666 0.27489546 0.24746332 0.23632598
0.23248206 0.22401379 0.21871354 0.20231584 0.19377239 0.14768413
0.1455532 0.14337878 0.12709719 0.12582931 0.11867398 0.11002147
0.10564942 0.09225623 0.08963215 0.08887199 0.08704525 0.08072542
0.08002211 0.07911447 0.0666113 0.06338121 0.06100726 0.06005874
0.05798694 0.05364129 0.0520498 0.05011013 0.04850959 0.04709018
0.04469205 0.04128502 0.04075819 0.03989548 0.03523409 0.03272378
0.03108071 0.02970156 0.028723 0.02845931 0.02585638 0.02348842
0.0233041 0.02148155 0.02133748 0.02086138 0.02035652 0.01959795
0.01931953 0.01926655 0.01872199 0.0185623 0.01853302 0.01838779
0.01818969 0.01780701 0.01727104 0.0166365 0.01586579 0.01579063
0.01573381 0.01528252 0.01502847 0.01451413 0.01439992 0.01428944
0.01419329 0.01380476 0.01360496 0.0129911 0.01249144 0.01198867
0.01148862 0.01145841 0.01144459 0.01139607 0.01113943 0.01108592
0.01089338 0.01082358 0.01051232 0.01027328 0.01006837 0.00979451
0.0097324 0.00960593 0.00957182 0.00953105 0.00949826 0.00942655
0.00942555 0.00931226 0.00907306 0.00887798 0.00884452 0.00881256
0.00864548 0.00854316 0.00849879 0.00849662 0.00846909 0.00820138
0.00816586 0.00791354 0.00790157 0.0076993 0.00768906 0.00766408
0.00766065 0.00764457 0.0074557 0.00721993 0.00706666 0.00700596
0.0067884 0.00648049 0.00646963 0.0063817 0.00635814 0.00625102
0.0062297 0.00599666 0.00591931 0.00585055 0.00578007 0.00576511
0.00572359 0.00560452 0.00558355 0.00556507 0.00553867 0.00548295
0.00547356 0.00543471 0.00543378 0.00540831 0.0053792 0.00535764
0.00523385 0.00518935 0.00505314 0.00505005 0.00492085 0.0048256
0.00471783 0.00470318 0.00464703 0.00461124 0.004583 0.00457273
0.00455803 0.00454314 0.00454088 0.00441311 0.00437612 0.00426319
0.00420744 0.00415996 0.00409997 0.00409557 0.00407971 0.00405195
0.00404085 0.00399853 0.00399512 0.00393439 0.00390283 0.00387302
0.0038489 0.00382758 0.00380028 0.00379529 0.00376791 0.00374193
0.00371191 0.0036963 0.00366445 0.00358808 0.00351783 0.00350439
0.00344527 0.00343266 0.00342918 0.0033823 0.00332239 0.00330844
0.00329753 0.00327267 0.00315135 0.0031098 0.00308979 0.00308362
0.00305496 0.00304868 0.00304044 0.00303659 0.00302582 0.00301237
0.00298851 0.00291267 0.00290264 0.00289242 0.00287722 0.00286563
0.0028257 0.00282502 0.00275258 0.00274531 0.0027204 0.00268617
0.00261917 0.00260795 0.00256594 0.00254094 0.00252856 0.00250768
0.00249793 0.00249551 0.00248255 0.00247911 0.00246619 0.00241695
0.00240165 0.00236032 0.00235902 0.00234437 0.00234337 0.0023379
0.00233535 0.00230773 0.00230558 0.00229113 0.00228888 0.0022631
0.00225214 0.00224186 0.00222553 0.00219966 0.00219677 0.00217865
0.00217775 0.00215921 0.0021541 0.00214997 0.00212954 0.00211928
0.0021005 0.00205066 0.0020487 0.00203887 0.00203537 0.00203026
0.00201357 0.00199936 0.00199386 0.00197951 0.00197287 0.00195502
0.00194848 0.00192128 0.00189951 0.00187285 0.0018519 0.0018299
0.00179158 0.00177908 0.00176328 0.00176319 0.00175034 0.00173788
0.00172983 0.00172819 0.00168272 0.0016768 0.00167543 0.00167397
0.0016395 0.00163637 0.00163319 0.00162886 0.00162824 0.00162028]]
image_detection_scores: [[0.981008 0.9406672 0.9318087 0.8773675 0.8406423 0.59000057
0.5544938 0.5395715 0.4939019 0.48142588 0.4627259 0.4407012
0.4011658 0.34708387 0.31795812 0.27489564 0.24746375 0.23632699
0.23248124 0.2240141 0.21871349 0.20231551 0.19377194 0.14768386
0.14555368 0.14337902 0.12709695 0.12582937 0.11867426 0.11002194
0.10564959 0.0922567 0.08963199 0.0888719 0.08704563 0.08072611
0.08002175 0.07911427 0.06661151 0.06338179 0.06100735 0.06005858
0.05798701 0.05364129 0.05204971 0.05011016 0.04850911 0.04709023
0.04469217 0.04128499 0.04075789 0.03989535 0.03523415 0.03272349
0.03108067 0.02970151 0.02872295 0.02845928 0.02585636 0.02348836
0.02330403 0.02148154 0.0213374 0.02086144 0.02035653 0.01959788
0.01931941 0.01926653 0.01872193 0.01856227 0.01853303 0.01838784
0.0181897 0.01780703 0.017271 0.01663653 0.01586576 0.01579067
0.01573383 0.01528259 0.01502851 0.01451424 0.01439989 0.0142894
0.01419323 0.01380469 0.01360497 0.01299106 0.01249145 0.01198861
0.01148866 0.01145843 0.0114446 0.01139614 0.0111394 0.01108592
0.01089339 0.01082359 0.01051234 0.01027329 0.01006839 0.0097945
0.0097324 0.00960594 0.00957183 0.00953107 0.00949827 0.00942658
0.00942553 0.0093122 0.00907309 0.00887799 0.0088445 0.00881257
0.00864545 0.00854312 0.00849879 0.00849659 0.00846911 0.00820138
0.00816589 0.00791355 0.00790155 0.00769932 0.00768909 0.00766407
0.00766063 0.00764461 0.00745569 0.00721991 0.00706666 0.00700593
0.00678841 0.00648049 0.00646962 0.00638172 0.00635816 0.00625101
0.0062297 0.00599664 0.00591933 0.00585052 0.0057801 0.00576511
0.00572357 0.00560453 0.00558353 0.00556504 0.00553866 0.00548296
0.00547356 0.00543473 0.00543379 0.00540833 0.00537916 0.00535765
0.00523385 0.00518937 0.00505316 0.00505005 0.00492084 0.00482558
0.00471782 0.00470318 0.00464702 0.00461124 0.00458301 0.00457273
0.00455803 0.00454314 0.00454089 0.00441312 0.00437611 0.0042632
0.00420743 0.00415999 0.00409998 0.00409558 0.00407969 0.00405196
0.00404087 0.00399854 0.0039951 0.00393439 0.00390283 0.00387302
0.0038489 0.00382759 0.0038003 0.00379529 0.00376794 0.00374193
0.00371189 0.0036963 0.00366447 0.00358808 0.00351783 0.0035044
0.00344527 0.00343266 0.00342917 0.00338231 0.00332238 0.00330844
0.00329753 0.00327268 0.00315135 0.00310979 0.0030898 0.00308362
0.00305496 0.00304868 0.00304045 0.0030366 0.00302583 0.00301238
0.00298852 0.00291268 0.00290265 0.00289242 0.00287723 0.00286562
0.00282571 0.00282504 0.00275257 0.00274531 0.00272039 0.00268618
0.00261918 0.00260795 0.00256593 0.00254094 0.00252855 0.00250768
0.00249794 0.00249551 0.00248254 0.0024791 0.00246619 0.00241695
0.00240167 0.00236033 0.00235902 0.00234437 0.00234337 0.00233791
0.00233533 0.00230773 0.00230558 0.00229113 0.00228889 0.0022631
0.00225215 0.00224185 0.00222553 0.00219966 0.00219676 0.00217864
0.00217775 0.00215921 0.00215411 0.00214996 0.00212955 0.00211928
0.0021005 0.00205065 0.0020487 0.00203887 0.00203538 0.00203026
0.00201357 0.00199935 0.00199386 0.00197949 0.00197287 0.00195501
0.00194847 0.00192128 0.0018995 0.00187285 0.00185189 0.0018299
0.00179158 0.00177908 0.00176327 0.00176319 0.00175033 0.00173788
0.00172983 0.00172819 0.00168272 0.0016768 0.00167542 0.00167398
0.0016395 0.00163637 0.00163319 0.00162886 0.00162823 0.00162028]]
image_detections_num: [300.]
@ -503,12 +515,12 @@ Define utility functions to visualize the inference results
import random
from typing import Optional
def add_detection_box(box: np.ndarray, image: np.ndarray, label: Optional[str] = None) -> np.ndarray:
"""
Helper function for adding single bounding box to the image
Parameters
----------
box : np.ndarray
@ -517,20 +529,20 @@ Define utility functions to visualize the inference results
The image to which detection box is added
label : str, optional
Detection box label string, if not provided will not be added to result image (default is None)
Returns
-------
np.ndarray
NumPy array including both image and detection box
"""
ymin, xmin, ymax, xmax = box
point1, point2 = (int(xmin), int(ymin)), (int(xmax), int(ymax))
box_color = [random.randint(0, 255) for _ in range(3)]
line_thickness = round(0.002 * (image.shape[0] + image.shape[1]) / 2) + 1
cv2.rectangle(img=image, pt1=point1, pt2=point2, color=box_color, thickness=line_thickness, lineType=cv2.LINE_AA)
if label:
font_thickness = max(line_thickness - 1, 1)
font_face = 0
@ -551,14 +563,14 @@ Define utility functions to visualize the inference results
.. code:: ipython3
from typing import Dict
from openvino.runtime.utils.data_helpers import OVDict
def visualize_inference_result(inference_result: OVDict, image: np.ndarray, labels_map: Dict, detections_limit: Optional[int] = None):
"""
Helper function for visualizing inference result on the image
Parameters
----------
inference_result : OVDict
@ -574,13 +586,13 @@ Define utility functions to visualize the inference results
detection_classes: np.ndarray = inference_result.get("detection_classes")
detection_scores: np.ndarray = inference_result.get("detection_scores")
num_detections: np.ndarray = inference_result.get("num_detections")
detections_limit = int(
min(detections_limit, num_detections[0])
if detections_limit is not None
else num_detections[0]
)
# Normalize detection boxes coordinates to original image size
original_image_height, original_image_width, _ = image.shape
normalized_detection_boxex = detection_boxes[::] * [
@ -589,9 +601,9 @@ Define utility functions to visualize the inference results
original_image_height,
original_image_width,
]
image_with_detection_boxex = np.copy(image)
for i in range(detections_limit):
detected_class_name = labels_map[int(detection_classes[0, i])]
score = detection_scores[0, i]
@ -601,7 +613,7 @@ Define utility functions to visualize the inference results
image=image_with_detection_boxex,
label=label,
)
plt.imshow(image_with_detection_boxex)
TensorFlow Object Detection model
@ -617,7 +629,7 @@ Zoo <https://github.com/openvinotoolkit/open_model_zoo/>`__:
.. code:: ipython3
coco_labels_file_path = Path("./data/coco_91cl.txt")
download_file(
url="https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/data/dataset_classes/coco_91cl.txt",
filename=coco_labels_file_path.name,
@ -635,7 +647,7 @@ Zoo <https://github.com/openvinotoolkit/open_model_zoo/>`__:
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/data/coco_91cl.txt')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/data/coco_91cl.txt')
@ -648,7 +660,7 @@ file:
with open(coco_labels_file_path, "r") as file:
coco_labels = file.read().strip().split("\n")
coco_labels_map = dict(enumerate(coco_labels, 1))
print(coco_labels_map)
@ -709,4 +721,4 @@ utilization.
For more information, refer to the `Optimize Preprocessing
tutorial <118-optimize-preprocessing-with-output.html>`__
and to the overview of `Preprocessing
API <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimizie-preprocessing/preprocessing-api-details.html>`__.
API <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimize-preprocessing/preprocessing-api-details.html>`__.

2
docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_25_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:62971c6546b399ce0eecbdc81a5cb327c4bebdfa1a6db658e364870db606d761
oid sha256:90f5ec228ab4ebfbb7bbf584f3eb86452f97261c08225f0fd2cb0b4bceadacb1
size 395346

4
docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_38_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:69ba7f4bdc1e3d85a98b34863980a347ab3bdf2aedb7d9d78a2f8d43484127b6
size 391578
oid sha256:b7c3ef737ce7f0e16dd9d3a392c533b542f408fe58f47c9bcc2b28f8d54e67d8
size 391878

125
docs/notebooks/121-convert-to-openvino-with-output.rst
View File

@ -30,12 +30,12 @@ Table of contents:
# Required imports. Please execute this cell first.
%pip install --upgrade pip
%pip install -q --extra-index-url https://download.pytorch.org/whl/cpu \
"openvino-dev>=2023.2.0" "requests" "tqdm" "transformers[onnx]>=4.21.1" "torch" "torchvision" "tensorflow_hub" "tensorflow"
"openvino-dev>=2024.0.0" "requests" "tqdm" "transformers[onnx]>=4.21.1" "torch" "torchvision" "tensorflow_hub" "tensorflow"
.. parsed-literal::
Requirement already satisfied: pip in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (24.0)
Requirement already satisfied: pip in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (24.0)
.. parsed-literal::
@ -54,14 +54,15 @@ OpenVINO IR format
OpenVINO `Intermediate Representation
(IR) <https://docs.openvino.ai/2024/documentation/openvino-ir-format.html>`__ is the
proprietary model format of OpenVINO. It is produced after converting a
model with model conversion API. Model conversion API translates the
frequently used deep learning operations to their respective similar
representation in OpenVINO and tunes them with the associated weights
and biases from the trained model. The resulting IR contains two files:
an ``.xml`` file, containing information about network topology, and a
``.bin`` file, containing the weights and biases binary data.
(IR) <https://docs.openvino.ai/2024/documentation/openvino-ir-format.html>`__
is the proprietary model format of OpenVINO. It is produced after
converting a model with model conversion API. Model conversion API
translates the frequently used deep learning operations to their
respective similar representation in OpenVINO and tunes them with the
associated weights and biases from the trained model. The resulting IR
contains two files: an ``.xml`` file, containing information about
network topology, and a ``.bin`` file, containing the weights and biases
binary data.
There are two ways to convert a model from the original framework format
to OpenVINO IR: Python conversion API and OVC command-line tool. You can
@ -79,7 +80,7 @@ documentation.
.. code:: ipython3
# OVC CLI tool parameters description
! ovc --help
@ -88,12 +89,12 @@ documentation.
usage: ovc INPUT_MODEL... [-h] [--output_model OUTPUT_MODEL]
[--compress_to_fp16 [True | False]] [--version] [--input INPUT]
[--output OUTPUT] [--extension EXTENSION] [--verbose]
positional arguments:
INPUT_MODEL Input model file(s) from TensorFlow, ONNX,
PaddlePaddle. Use openvino.convert_model in Python to
convert models from PyTorch.
optional arguments:
-h, --help show this help message and exit
--output_model OUTPUT_MODEL
@ -131,9 +132,7 @@ documentation.
output=out_1,out_2
--extension EXTENSION
Paths or a comma-separated list of paths to libraries
(.so or .dll) with extensions. To disable all
extensions including those that are placed at the
default location, pass an empty string.
(.so or .dll) with extensions.
--verbose Print detailed information about conversion.
@ -152,7 +151,7 @@ This notebook uses two models for conversion examples:
.. code:: ipython3
from pathlib import Path
# create a directory for models files
MODEL_DIRECTORY_PATH = Path("model")
MODEL_DIRECTORY_PATH.mkdir(exist_ok=True)
@ -165,9 +164,9 @@ NLP model from Hugging Face and export it in ONNX format:
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from transformers.onnx import export, FeaturesManager
ONNX_NLP_MODEL_PATH = MODEL_DIRECTORY_PATH / "distilbert.onnx"
# download model
hf_model = AutoModelForSequenceClassification.from_pretrained(
"distilbert-base-uncased-finetuned-sst-2-english"
@ -176,14 +175,14 @@ NLP model from Hugging Face and export it in ONNX format:
tokenizer = AutoTokenizer.from_pretrained(
"distilbert-base-uncased-finetuned-sst-2-english"
)
# get model onnx config function for output feature format sequence-classification
model_kind, model_onnx_config = FeaturesManager.check_supported_model_or_raise(
hf_model, feature="sequence-classification"
)
# fill onnx config based on pytorch model config
onnx_config = model_onnx_config(hf_model.config)
# export to onnx format
export(
preprocessor=tokenizer,
@ -196,19 +195,19 @@ NLP model from Hugging Face and export it in ONNX format:
.. parsed-literal::
2024-02-09 23:07:37.212192: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-02-09 23:07:37.247733: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2024-03-12 22:50:34.001865: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-03-12 22:50:34.035988: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
.. parsed-literal::
2024-02-09 23:07:37.883428: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2024-03-12 22:50:34.633732: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:246: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:246: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.
mask, torch.tensor(torch.finfo(scores.dtype).min)
@ -227,7 +226,7 @@ CV classification model from torchvision:
.. code:: ipython3
from torchvision.models import resnet50, ResNet50_Weights
# create model object
pytorch_model = resnet50(weights=ResNet50_Weights.DEFAULT)
# switch model from training to inference mode
@ -423,9 +422,9 @@ Convert PyTorch model to ONNX format:
import torch
import warnings
ONNX_CV_MODEL_PATH = MODEL_DIRECTORY_PATH / "resnet.onnx"
if ONNX_CV_MODEL_PATH.exists():
print(f"ONNX model {ONNX_CV_MODEL_PATH} already exists.")
else:
@ -452,11 +451,11 @@ To convert a model to OpenVINO IR, use the following API:
.. code:: ipython3
import openvino as ov
# ov.convert_model returns an openvino.runtime.Model object
print(ONNX_NLP_MODEL_PATH)
ov_model = ov.convert_model(ONNX_NLP_MODEL_PATH)
# then model can be serialized to *.xml & *.bin files
ov.save_model(ov_model, MODEL_DIRECTORY_PATH / "distilbert.xml")
@ -510,7 +509,7 @@ documentation.
.. code:: ipython3
import openvino as ov
ov_model = ov.convert_model(
ONNX_NLP_MODEL_PATH, input=[("input_ids", [1, 128]), ("attention_mask", [1, 128])]
)
@ -549,7 +548,7 @@ conversion API parameter as ``-1`` or ``?`` when using ``ovc``:
.. code:: ipython3
import openvino as ov
ov_model = ov.convert_model(
ONNX_NLP_MODEL_PATH, input=[("input_ids", [1, -1]), ("attention_mask", [1, -1])]
)
@ -590,10 +589,10 @@ sequence length dimension:
.. code:: ipython3
import openvino as ov
sequence_length_dim = ov.Dimension(10, 128)
ov_model = ov.convert_model(
ONNX_NLP_MODEL_PATH, input=[("input_ids", [1, sequence_length_dim]), ("attention_mask", [1, sequence_length_dim])]
)
@ -636,7 +635,7 @@ disabled by setting ``compress_to_fp16`` flag to ``False``:
.. code:: ipython3
import openvino as ov
ov_model = ov.convert_model(ONNX_NLP_MODEL_PATH)
ov.save_model(ov_model, MODEL_DIRECTORY_PATH / 'distilbert.xml', compress_to_fp16=False)
@ -675,9 +674,9 @@ frameworks conversion guides.
import openvino as ov
import torch
example_input = torch.rand(1, 3, 224, 224)
ov_model = ov.convert_model(pytorch_model, example_input=example_input, input=example_input.shape)
@ -690,21 +689,21 @@ frameworks conversion guides.
import openvino as ov
import tensorflow_hub as hub
model = hub.load("https://www.kaggle.com/models/google/movenet/frameworks/TensorFlow2/variations/singlepose-lightning/versions/4")
movenet = model.signatures['serving_default']
ov_model = ov.convert_model(movenet)
.. parsed-literal::
2024-02-09 23:07:56.008045: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW
2024-02-09 23:07:56.008076: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:168] retrieving CUDA diagnostic information for host: iotg-dev-workstation-07
2024-02-09 23:07:56.008081: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:175] hostname: iotg-dev-workstation-07
2024-02-09 23:07:56.008267: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:199] libcuda reported version is: 470.223.2
2024-02-09 23:07:56.008284: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:203] kernel reported version is: 470.182.3
2024-02-09 23:07:56.008287: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration
2024-03-12 22:50:53.831014: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW
2024-03-12 22:50:53.831046: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:168] retrieving CUDA diagnostic information for host: iotg-dev-workstation-07
2024-03-12 22:50:53.831051: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:175] hostname: iotg-dev-workstation-07
2024-03-12 22:50:53.831226: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:199] libcuda reported version is: 470.223.2
2024-03-12 22:50:53.831242: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:203] kernel reported version is: 470.182.3
2024-03-12 22:50:53.831247: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration
Migration from Legacy conversion API
@ -722,7 +721,7 @@ OVC or can be replaced with functionality from ``ov.PrePostProcessor``
class. Refer to `Optimize Preprocessing
notebook <https://github.com/openvinotoolkit/openvino_notebooks/blob/main/notebooks/118-optimize-preprocessing/118-optimize-preprocessing.ipynb>`__
for more information about `Preprocessing
API <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimizie-preprocessing.html>`__.
API <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimize-preprocessing.html>`__.
Here is the migration guide from legacy model preprocessing to
Preprocessing API.
@ -736,7 +735,7 @@ for both inputs and outputs. Some preprocessing requires to set input
layouts, for example, setting a batch, applying mean or scales, and
reversing input channels (BGR<->RGB). For the layout syntax, check the
`Layout API
overview <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimizie-preprocessing/layout-api-overview.html>`__.
overview <https://docs.openvino.ai/2024/openvino-workflow/running-inference/optimize-inference/optimize-preprocessing/layout-api-overview.html>`__.
To specify the layout, you can use the layout option followed by the
layout value.
@ -747,9 +746,9 @@ Resnet50 model that was exported to the ONNX format:
# Converter API
import openvino as ov
ov_model = ov.convert_model(ONNX_CV_MODEL_PATH)
prep = ov.preprocess.PrePostProcessor(ov_model)
prep.input('input.1').model().set_layout(ov.Layout("nchw"))
ov_model = prep.build()
@ -758,7 +757,7 @@ Resnet50 model that was exported to the ONNX format:
# Legacy Model Optimizer API
from openvino.tools import mo
ov_model = mo.convert_model(ONNX_CV_MODEL_PATH, layout="nchw")
@ -787,9 +786,9 @@ and the layout of an original model:
# Converter API
import openvino as ov
ov_model = ov.convert_model(ONNX_CV_MODEL_PATH)
prep = ov.preprocess.PrePostProcessor(ov_model)
prep.input('input.1').tensor().set_layout(ov.Layout("nhwc"))
prep.input('input.1').model().set_layout(ov.Layout("nchw"))
@ -799,9 +798,9 @@ and the layout of an original model:
# Legacy Model Optimizer API
from openvino.tools import mo
ov_model = mo.convert_model(ONNX_CV_MODEL_PATH, layout="nchw->nhwc")
# alternatively use source_layout and target_layout parameters
ov_model = mo.convert_model(
ONNX_CV_MODEL_PATH, source_layout="nchw", target_layout="nhwc"
@ -823,22 +822,22 @@ for more examples.
# Converter API
import openvino as ov
ov_model = ov.convert_model(ONNX_CV_MODEL_PATH)
prep = ov.preprocess.PrePostProcessor(ov_model)
prep.input("input.1").tensor().set_layout(ov.Layout("nchw"))
prep.input("input.1").preprocess().mean([255 * x for x in [0.485, 0.456, 0.406]])
prep.input("input.1").preprocess().scale([255 * x for x in [0.229, 0.224, 0.225]])
ov_model = prep.build()
.. code:: ipython3
# Legacy Model Optimizer API
from openvino.tools import mo
ov_model = mo.convert_model(
ONNX_CV_MODEL_PATH,
mean_values=[255 * x for x in [0.485, 0.456, 0.406]],
@ -860,9 +859,9 @@ the color channels before inference.
# Converter API
import openvino as ov
ov_model = ov.convert_model(ONNX_CV_MODEL_PATH)
prep = ov.preprocess.PrePostProcessor(ov_model)
prep.input('input.1').tensor().set_layout(ov.Layout("nchw"))
prep.input('input.1').preprocess().reverse_channels()
@ -872,7 +871,7 @@ the color channels before inference.
# Legacy Model Optimizer API
from openvino.tools import mo
ov_model = mo.convert_model(ONNX_CV_MODEL_PATH, reverse_input_channels=True)
Cutting Off Parts of a Model

433
docs/notebooks/121-legacy-mo-convert-to-openvino-with-output.rst
View File

File diff suppressed because it is too large Load Diff

85
docs/notebooks/122-speech-recognition-quantization-wav2vec2-with-output.rst
View File

File diff suppressed because one or more lines are too long

436
docs/notebooks/122-yolov8-quantization-with-accuracy-control-with-output.rst
View File

@ -69,6 +69,14 @@ Install necessary packages.
%pip install -q "nncf>=2.6.0"
%pip install -q "ultralytics==8.0.43" --extra-index-url https://download.pytorch.org/whl/cpu
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
Note: you may need to restart the kernel to use updated packages.
Note: you may need to restart the kernel to use updated packages.
Get Pytorch model and OpenVINO IR model
---------------------------------------
@ -95,35 +103,76 @@ we do not need to do these steps manually.
import os
from pathlib import Path
from ultralytics import YOLO
from ultralytics.yolo.cfg import get_cfg
from ultralytics.yolo.data.utils import check_det_dataset
from ultralytics.yolo.engine.validator import BaseValidator as Validator
from ultralytics.yolo.utils import DATASETS_DIR
from ultralytics.yolo.utils import DEFAULT_CFG
from ultralytics.yolo.utils import ops
from ultralytics.yolo.utils.metrics import ConfusionMatrix
ROOT = os.path.abspath('')
MODEL_NAME = "yolov8n-seg"
model = YOLO(f"{ROOT}/{MODEL_NAME}.pt")
args = get_cfg(cfg=DEFAULT_CFG)
args.data = "coco128-seg.yaml"
.. code:: ipython3
# Fetch the notebook utils script from the openvino_notebooks repo
import urllib.request
urllib.request.urlretrieve(
url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py',
filename='notebook_utils.py'
)
from notebook_utils import download_file
.. code:: ipython3
from zipfile import ZipFile
DATA_URL = "https://www.ultralytics.com/assets/coco128-seg.zip"
CFG_URL = "https://raw.githubusercontent.com/ultralytics/ultralytics/8ebe94d1e928687feaa1fee6d5668987df5e43be/ultralytics/datasets/coco128-seg.yaml" # last compatible format with ultralytics 8.0.43
OUT_DIR = Path('./datasets')
DATA_PATH = OUT_DIR / "coco128-seg.zip"
CFG_PATH = OUT_DIR / "coco128-seg.yaml"
download_file(DATA_URL, DATA_PATH.name, DATA_PATH.parent)
download_file(CFG_URL, CFG_PATH.name, CFG_PATH.parent)
if not (OUT_DIR / "coco128/labels").exists():
with ZipFile(DATA_PATH , "r") as zip_ref:
zip_ref.extractall(OUT_DIR)
.. parsed-literal::
'datasets/coco128-seg.zip' already exists.
.. parsed-literal::
datasets/coco128-seg.yaml: 0%| | 0.00/0.98k [00:00<?, ?B/s]
Load model.
.. code:: ipython3
import openvino as ov
model_path = Path(f"{ROOT}/{MODEL_NAME}_openvino_model/{MODEL_NAME}.xml")
if not model_path.exists():
model.export(format="openvino", dynamic=True, half=False)
ov_model = ov.Core().read_model(model_path)
Define validator and data loader
@ -145,8 +194,8 @@ validator class instance.
validator = model.ValidatorClass(args)
validator.data = check_det_dataset(args.data)
data_loader = validator.get_dataloader(f"{DATASETS_DIR}/coco128-seg", 1)
data_loader = validator.get_dataloader("datasets/coco128-seg", 1)
validator.is_coco = True
validator.class_map = ops.coco80_to_coco91_class()
validator.names = model.model.names
@ -159,7 +208,8 @@ validator class instance.
.. parsed-literal::
val: Scanning /home/ea/work/openvino_notebooks/notebooks/230-yolov8-optimization/datasets/coco128-seg/labels/train2017.cache... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 [00:00<?, ?it/s
val: Scanning datasets/coco128-seg/labels/train2017... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 [00:00<00:00, 964.99it/s]
val: New cache created: datasets/coco128-seg/labels/train2017.cache
Prepare calibration and validation datasets
@ -173,15 +223,15 @@ We can use one dataset as calibration and validation datasets. Name it
.. code:: ipython3
from typing import Dict
import nncf
def transform_fn(data_item: Dict):
input_tensor = validator.preprocess(data_item)["img"].numpy()
return input_tensor
quantization_dataset = nncf.Dataset(data_loader, transform_fn)
@ -198,11 +248,11 @@ Prepare validation function
.. code:: ipython3
from functools import partial
import torch
from nncf.quantization.advanced_parameters import AdvancedAccuracyRestorerParameters
def validation_ac(
compiled_model: ov.CompiledModel,
validation_loader: torch.utils.data.DataLoader,
@ -216,7 +266,7 @@ Prepare validation function
validator.batch_i = 1
validator.confusion_matrix = ConfusionMatrix(nc=validator.nc)
num_outputs = len(compiled_model.outputs)
counter = 0
for batch_i, batch in enumerate(validation_loader):
if num_samples is not None and batch_i == num_samples:
@ -240,10 +290,10 @@ Prepare validation function
stats_metrics = stats["metrics/mAP50-95(M)"]
if log:
print(f"Validate: dataset length = {counter}, metric value = {stats_metrics:.3f}")
return stats_metrics
validation_fn = partial(validation_ac, validator=validator, log=False)
Run quantization with accuracy control
@ -285,52 +335,115 @@ value 25 to speed up the execution.
.. parsed-literal::
2023-10-10 09:55:44.477778: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-10-10 09:55:44.516624: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2024-02-28 13:33:46.187903: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-02-28 13:33:46.189894: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.
2024-02-28 13:33:46.226943: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-10-10 09:55:45.324364: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
Statistics collection: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 128/128 [00:16<00:00, 7.79it/s]
Applying Fast Bias correction: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 75/75 [00:03<00:00, 18.84it/s]
2024-02-28 13:33:46.942396: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
Output()
.. raw:: html
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"></pre>
.. raw:: html
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace">
</pre>
.. parsed-literal::
Output()
.. raw:: html
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"></pre>
.. raw:: html
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace">
</pre>
.. parsed-literal::
INFO:nncf:Validation of initial model was started
INFO:nncf:Elapsed Time: 00:00:00
INFO:nncf:Elapsed Time: 00:00:05
INFO:nncf:Metric of initial model: 0.36611468358574506
INFO:nncf:Collecting values for each data item using the initial model
INFO:nncf:Elapsed Time: 00:00:06
INFO:nncf:Validation of quantized model was started
INFO:nncf:Elapsed Time: 00:00:00
INFO:nncf:Elapsed Time: 00:00:05
INFO:nncf:Metric of quantized model: 0.3406029678292
INFO:nncf:Collecting values for each data item using the quantized model
INFO:nncf:Elapsed Time: 00:00:06
INFO:nncf:Accuracy drop: 0.02551171575654504 (absolute)
INFO:nncf:Accuracy drop: 0.02551171575654504 (absolute)
INFO:nncf:Total number of quantized operations in the model: 91
INFO:nncf:Number of parallel workers to rank quantized operations: 1
INFO:nncf:ORIGINAL metric is used to rank quantizers
.. parsed-literal::
INFO:nncf:Elapsed Time: 00:00:00
INFO:nncf:Elapsed Time: 00:00:05
INFO:nncf:Metric of initial model: 0.366118260036709
INFO:nncf:Collecting values for each data item using the initial model
INFO:nncf:Elapsed Time: 00:00:06
INFO:nncf:Validation of quantized model was started
INFO:nncf:Elapsed Time: 00:00:01
INFO:nncf:Elapsed Time: 00:00:04
INFO:nncf:Metric of quantized model: 0.3418411101103462
INFO:nncf:Collecting values for each data item using the quantized model
INFO:nncf:Elapsed Time: 00:00:05
INFO:nncf:Accuracy drop: 0.024277149926362818 (DropType.ABSOLUTE)
INFO:nncf:Accuracy drop: 0.024277149926362818 (DropType.ABSOLUTE)
INFO:nncf:Total number of quantized operations in the model: 91
INFO:nncf:Number of parallel processes to rank quantized operations: 6
INFO:nncf:ORIGINAL metric is used to rank quantizers
INFO:nncf:Calculating ranking score for groups of quantizers
INFO:nncf:Elapsed Time: 00:02:16
Output()
.. raw:: html
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"></pre>
.. raw:: html
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace">
</pre>
.. parsed-literal::
INFO:nncf:Elapsed Time: 00:02:25
INFO:nncf:Changing the scope of quantizer nodes was started
INFO:nncf:Reverted 1 operations to the floating-point precision:
INFO:nncf:Reverted 2 operations to the floating-point precision:
/model.22/Add_11
/model.22/Sub_1
INFO:nncf:Accuracy drop with the new quantization scope is 0.013524778006655136 (absolute)
INFO:nncf:Reverted 1 operations to the floating-point precision:
/model.22/Mul_5
INFO:nncf:Accuracy drop with the new quantization scope is 0.013359187935064742 (DropType.ABSOLUTE)
INFO:nncf:Reverted 1 operations to the floating-point precision:
/model.1/conv/Conv/WithoutBiases
INFO:nncf:Accuracy drop with the new quantization scope is 0.01287864227202773 (DropType.ABSOLUTE)
INFO:nncf:Reverted 1 operations to the floating-point precision:
INFO:nncf:Accuracy drop with the new quantization scope is 0.011937545450662279 (absolute)
INFO:nncf:Reverted 1 operations to the floating-point precision:
/model.2/cv1/conv/Conv/WithoutBiases
INFO:nncf:Algorithm completed: achieved required accuracy drop 0.007027355074555763 (DropType.ABSOLUTE)
INFO:nncf:3 out of 91 were reverted back to the floating-point precision:
INFO:nncf:Algorithm completed: achieved required accuracy drop 0.00905169821338292 (absolute)
INFO:nncf:4 out of 91 were reverted back to the floating-point precision:
/model.22/Add_11
/model.22/Sub_1
/model.22/Mul_5
/model.1/conv/Conv/WithoutBiases
/model.2/cv1/conv/Conv/WithoutBiases
@ -345,24 +458,51 @@ is not exceeded.
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
quantized_compiled_model = core.compile_model(model=quantized_model, device_name='CPU')
compiled_ov_model = core.compile_model(model=ov_model, device_name='CPU')
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
.. code:: ipython3
core = ov.Core()
ov_config = {}
if "GPU" in device.value or ("AUTO" in device.value and "GPU" in core.available_devices):
ov_config = {"GPU_DISABLE_WINOGRAD_CONVOLUTION": "YES"}
quantized_compiled_model = core.compile_model(model=quantized_model, device_name=device.value, ov_config)
compiled_ov_model = core.compile_model(model=ov_model, device_name=device.value, ov_config)
pt_result = validation_ac(compiled_ov_model, data_loader, validator)
quantized_result = validation_ac(quantized_compiled_model, data_loader, validator)
print(f'[Original OpenVino]: {pt_result:.4f}')
print(f'[Quantized OpenVino]: {quantized_result:.4f}')
print(f'[Original OpenVINO]: {pt_result:.4f}')
print(f'[Quantized OpenVINO]: {quantized_result:.4f}')
.. parsed-literal::
Validate: dataset length = 128, metric value = 0.368
Validate: dataset length = 128, metric value = 0.360
[Original OpenVino]: 0.3677
[Quantized OpenVino]: 0.3602
Validate: dataset length = 128, metric value = 0.357
[Original OpenVINO]: 0.3677
[Quantized OpenVINO]: 0.3570
And compare performance.
@ -373,10 +513,10 @@ And compare performance.
# Set model directory
MODEL_DIR = Path("model")
MODEL_DIR.mkdir(exist_ok=True)
ir_model_path = MODEL_DIR / 'ir_model.xml'
quantized_model_path = MODEL_DIR / 'quantized_model.xml'
# Save models to use them in the commandline banchmark app
ov.save_model(ov_model, ir_model_path, compress_to_fp16=False)
ov.save_model(quantized_model, quantized_model_path, compress_to_fp16=False)
@ -384,7 +524,7 @@ And compare performance.
.. code:: ipython3
# Inference Original model (OpenVINO IR)
! benchmark_app -m $ir_model_path -shape "[1,3,640,640]" -d CPU -api async
! benchmark_app -m $ir_model_path -shape "[1,3,640,640]" -d $device.value -api async
.. parsed-literal::
@ -392,19 +532,20 @@ And compare performance.
[Step 1/11] Parsing and validating input arguments
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ WARNING ] Default duration 120 seconds is used for unknown device AUTO
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2023.2.0-12713-47c2a91b6b6
[ INFO ]
[ INFO ] Build ................................. 2024.1.0-14589-0ef2fab3490
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2023.2.0-12713-47c2a91b6b6
[ INFO ]
[ INFO ]
[ INFO ] AUTO
[ INFO ] Build ................................. 2024.1.0-14589-0ef2fab3490
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 27.11 ms
[ INFO ] Read model took 17.83 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] images (node: images) : f32 / [...] / [?,3,?,?]
@ -414,7 +555,7 @@ And compare performance.
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: 'images': [1,3,640,640]
[ INFO ] Reshape model took 13.41 ms
[ INFO ] Reshape model took 13.50 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] images (node: images) : u8 / [N,C,H,W] / [1,3,640,640]
@ -422,47 +563,58 @@ And compare performance.
[ INFO ] output0 (node: output0) : f32 / [...] / [1,116,8400]
[ INFO ] output1 (node: output1) : f32 / [...] / [1,32,160,160]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 274.70 ms
[ INFO ] Compile model took 320.32 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 12
[ INFO ] NUM_STREAMS: 12
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 36
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] CPU_DENORMALS_OPTIMIZATION: False
[ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 12
[ INFO ] MULTI_DEVICE_PRIORITIES: CPU
[ INFO ] CPU:
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] CPU_DENORMALS_OPTIMIZATION: False
[ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
[ INFO ] DYNAMIC_QUANTIZATION_GROUP_SIZE: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] INFERENCE_NUM_THREADS: 36
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] KV_CACHE_PRECISION: <Type: 'float16'>
[ INFO ] LOG_LEVEL: Level.NO
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] NUM_STREAMS: 12
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 12
[ INFO ] PERFORMANCE_HINT: THROUGHPUT
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] PERF_COUNT: NO
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] MODEL_PRIORITY: Priority.MEDIUM
[ INFO ] LOADED_FROM_CACHE: False
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'images'!. This input will be filled with random values!
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 60000 ms duration)
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 120000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 42.88 ms
[ INFO ] First inference took 44.16 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 7716 iterations
[ INFO ] Duration: 60104.07 ms
[ INFO ] Count: 14820 iterations
[ INFO ] Duration: 120139.10 ms
[ INFO ] Latency:
[ INFO ] Median: 88.61 ms
[ INFO ] Average: 93.27 ms
[ INFO ] Min: 52.72 ms
[ INFO ] Max: 181.74 ms
[ INFO ] Throughput: 128.38 FPS
[ INFO ] Median: 95.27 ms
[ INFO ] Average: 97.10 ms
[ INFO ] Min: 72.93 ms
[ INFO ] Max: 164.81 ms
[ INFO ] Throughput: 123.36 FPS
.. code:: ipython3
# Inference Quantized model (OpenVINO IR)
! benchmark_app -m $quantized_model_path -shape "[1,3,640,640]" -d CPU -api async
! benchmark_app -m $quantized_model_path -shape "[1,3,640,640]" -d $device.value -api async
.. parsed-literal::
@ -470,19 +622,20 @@ And compare performance.
[Step 1/11] Parsing and validating input arguments
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ WARNING ] Default duration 120 seconds is used for unknown device AUTO
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2023.2.0-12713-47c2a91b6b6
[ INFO ]
[ INFO ] Build ................................. 2024.1.0-14589-0ef2fab3490
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2023.2.0-12713-47c2a91b6b6
[ INFO ]
[ INFO ]
[ INFO ] AUTO
[ INFO ] Build ................................. 2024.1.0-14589-0ef2fab3490
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 32.74 ms
[ INFO ] Read model took 28.33 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] images (node: images) : f32 / [...] / [?,3,?,?]
@ -492,7 +645,7 @@ And compare performance.
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: 'images': [1,3,640,640]
[ INFO ] Reshape model took 18.09 ms
[ INFO ] Reshape model took 17.60 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] images (node: images) : u8 / [N,C,H,W] / [1,3,640,640]
@ -500,39 +653,50 @@ And compare performance.
[ INFO ] output0 (node: output0) : f32 / [...] / [1,116,8400]
[ INFO ] output1 (node: output1) : f32 / [...] / [1,32,160,160]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 574.58 ms
[ INFO ] Compile model took 605.73 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 12
[ INFO ] NUM_STREAMS: 12
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 36
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] CPU_DENORMALS_OPTIMIZATION: False
[ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 12
[ INFO ] MULTI_DEVICE_PRIORITIES: CPU
[ INFO ] CPU:
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] CPU_DENORMALS_OPTIMIZATION: False
[ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
[ INFO ] DYNAMIC_QUANTIZATION_GROUP_SIZE: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] INFERENCE_NUM_THREADS: 36
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] KV_CACHE_PRECISION: <Type: 'float16'>
[ INFO ] LOG_LEVEL: Level.NO
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] NUM_STREAMS: 12
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 12
[ INFO ] PERFORMANCE_HINT: THROUGHPUT
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] PERF_COUNT: NO
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] MODEL_PRIORITY: Priority.MEDIUM
[ INFO ] LOADED_FROM_CACHE: False
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'images'!. This input will be filled with random values!
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 60000 ms duration)
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 120000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 31.29 ms
[ INFO ] First inference took 22.24 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 15900 iterations
[ INFO ] Duration: 60077.25 ms
[ INFO ] Count: 33624 iterations
[ INFO ] Duration: 120057.46 ms
[ INFO ] Latency:
[ INFO ] Median: 42.02 ms
[ INFO ] Average: 45.18 ms
[ INFO ] Min: 25.42 ms
[ INFO ] Max: 117.81 ms
[ INFO ] Throughput: 264.66 FPS
[ INFO ] Median: 41.97 ms
[ INFO ] Average: 42.70 ms
[ INFO ] Min: 31.11 ms
[ INFO ] Max: 86.51 ms
[ INFO ] Throughput: 280.07 FPS

4
docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_22_0.jpg
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:97f44896da4cb3dee863ed6869c16015702bf26050467e2ff84110920318665b
size 58781
oid sha256:5ee75a57191b0022ebefe5c73eb05071f8c949587cabd740436929ed0e1cc5da
size 59201

4
docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_22_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4398d08f29dcb59dd57f4cd453308897ac75f6f8b860558fb21cfdbcd479b6e3
size 509675
oid sha256:58984d873a9bf772f576c037fdf50e015947a691e82eff7b2459c145a83501ef
size 509755

4
docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_32_0.jpg
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ae3a4df87c84baa406e0d2ed491e25f28bd9d8e6e9fe649bd74c0b7eae35ac07
size 55630
oid sha256:5d80a14442c132a885cab814c149f7a93b3a53a743cf9370e64a4024a79e1999
size 55628

4
docs/notebooks/123-detectron2-to-openvino-with-output_files/123-detectron2-to-openvino-with-output_32_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c832d80fe116434db304523dbc29d6e292e7a5733b70ae709e8eb2a0674506e6
size 457756
oid sha256:73833360f08fdb6c5ff39faa0735fd718cb7a36175bfc1b87ef3031ac7139029
size 458385

129
docs/notebooks/124-hugging-face-hub-with-output.rst
View File

@ -1,5 +1,5 @@
Hugging Face Model Hub with OpenVINO™
=========================================
=======================================
The Hugging Face (HF) `Model Hub <https://huggingface.co/models>`__ is a
central repository for pre-trained deep learning models. It allows
@ -93,10 +93,10 @@ Imports
.. code:: ipython3
from pathlib import Path
import numpy as np
import torch
from transformers import AutoModelForSequenceClassification
from transformers import AutoTokenizer
@ -119,9 +119,9 @@ tutorials <https://huggingface.co/learn/nlp-course/chapter2/2?fw=pt#behind-the-p
.. code:: ipython3
MODEL = "cardiffnlp/twitter-roberta-base-sentiment-latest"
tokenizer = AutoTokenizer.from_pretrained(MODEL, return_dict=True)
# The torchscript=True flag is used to ensure the model outputs are tuples
# instead of ModelOutput (which causes JIT errors).
model = AutoModelForSequenceClassification.from_pretrained(MODEL, torchscript=True)
@ -129,7 +129,7 @@ tutorials <https://huggingface.co/learn/nlp-course/chapter2/2?fw=pt#behind-the-p
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly. To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly. To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
return self.fget.__get__(instance, owner)()
@ -150,18 +150,18 @@ Let’s do a classification of a simple prompt below.
.. code:: ipython3
text = "HF models run perfectly with OpenVINO!"
encoded_input = tokenizer(text, return_tensors='pt')
output = model(**encoded_input)
scores = output[0][0]
scores = torch.softmax(scores, dim=0).numpy(force=True)
def print_prediction(scores):
for i, descending_index in enumerate(scores.argsort()[::-1]):
label = model.config.id2label[descending_index]
score = np.round(float(scores[descending_index]), 4)
print(f"{i+1}) {label} {score}")
print_prediction(scores)
@ -177,7 +177,7 @@ Converting the Model to OpenVINO IR format
We use the OpenVINO `Model
conversion
API <https://docs.openvino.ai/2024/openvino-workflow/model-preparation.html#convert-a-model-in-python-convert-model>`__
API <https://docs.openvino.ai/2024/openvino-workflow/model-preparation.html#convert-a-model-with-python-convert-model>`__
to convert the model (this one is implemented in PyTorch) to OpenVINO
Intermediate Representation (IR).
@ -187,13 +187,20 @@ Note how we reuse our real ``encoded_input``, passing it to the
.. code:: ipython3
import openvino as ov
save_model_path = Path('./models/model.xml')
if not save_model_path.exists():
ov_model = ov.convert_model(model, example_input=dict(encoded_input))
ov.save_model(ov_model, save_model_path)
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4193: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead
warnings.warn(
Converted Model Inference
~~~~~~~~~~~~~~~~~~~~~~~~~
@ -204,16 +211,16 @@ First, we pick a device to do the model inference
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
@ -231,12 +238,12 @@ model inference.
.. code:: ipython3
compiled_model = core.compile_model(save_model_path, device.value)
# Compiled model call is performed using the same parameters as for the original model
scores_ov = compiled_model(encoded_input.data)[0]
scores_ov = torch.softmax(torch.tensor(scores_ov[0]), dim=0).detach().numpy()
print_prediction(scores_ov)
@ -343,14 +350,14 @@ documentation <https://huggingface.co/docs/optimum/intel/inference>`__.
.. parsed-literal::
2024-02-09 23:10:50.826096: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-02-09 23:10:50.861099: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2024-03-12 22:53:48.040478: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-03-12 22:53:48.076125: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
.. parsed-literal::
2024-02-09 23:10:51.428729: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2024-03-12 22:53:48.672121: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
Initialize and Convert the Model Automatically using OVModel class
@ -379,14 +386,14 @@ inference run.
.. code:: ipython3
model = OVModelForSequenceClassification.from_pretrained(MODEL, export=True, device=device.value)
# The save_pretrained() method saves the model weights to avoid conversion on the next load.
model.save_pretrained('./models/optimum_model')
.. parsed-literal::
Framework not specified. Using pt to export to ONNX.
Framework not specified. Using pt to export the model.
.. parsed-literal::
@ -422,6 +429,12 @@ inference run.
WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11.
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4193: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead
warnings.warn(
.. parsed-literal::
Compiling the model to AUTO ...
@ -478,7 +491,7 @@ Full list of supported arguments available via ``--help``
.. parsed-literal::
2024-02-09 23:11:03.409282: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2024-03-12 22:54:00.822899: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
@ -488,39 +501,41 @@ Full list of supported arguments available via ``--help``
[--framework {pt,tf}] [--trust-remote-code]
[--pad-token-id PAD_TOKEN_ID] [--fp16]
[--int8]
[--weight-format {fp32,fp16,int8,int4_sym_g128,int4_asym_g128,int4_sym_g64,int4_asym_g64}]
[--ratio RATIO] [--disable-stateful]
[--convert-tokenizer]
[--weight-format {fp32,fp16,int8,int4,int4_sym_g128,int4_asym_g128,int4_sym_g64,int4_asym_g64}]
[--ratio RATIO] [--sym]
[--group-size GROUP_SIZE]
[--disable-stateful] [--convert-tokenizer]
output
optional arguments:
-h, --help show this help message and exit
Required arguments:
-m MODEL, --model MODEL
Model ID on huggingface.co or path on disk to load
model from.
output Path indicating the directory where to store the
generated OV model.
Optional arguments:
--task TASK The task to export the model for. If not specified,
the task will be auto-inferred based on the model.
Available tasks depend on the model, but are among:
['sentence-similarity', 'object-detection', 'question-
answering', 'text-to-audio', 'audio-xvector', 'stable-
diffusion-xl', 'feature-extraction', 'image-to-image',
'text-generation', 'mask-generation', 'text-
classification', 'image-segmentation', 'automatic-
speech-recognition', 'text2text-generation', 'stable-
diffusion', 'audio-classification', 'semantic-
segmentation', 'fill-mask', 'depth-estimation', 'zero-
shot-image-classification', 'image-to-text', 'zero-
shot-object-detection', 'multiple-choice',
'conversational', 'image-classification', 'masked-im',
'audio-frame-classification', 'token-classification'].
For decoder models, use `xxx-with-past` to export the
model using past key values in the decoder.
['audio-frame-classification', 'depth-estimation',
'multiple-choice', 'automatic-speech-recognition',
'text-to-audio', 'image-segmentation', 'feature-
extraction', 'stable-diffusion-xl', 'text2text-
generation', 'token-classification', 'stable-
diffusion', 'audio-xvector', 'image-to-text',
'semantic-segmentation', 'text-classification',
'image-classification', 'sentence-similarity',
'question-answering', 'mask-generation', 'object-
detection', 'zero-shot-image-classification', 'image-
to-image', 'fill-mask', 'zero-shot-object-detection',
'conversational', 'masked-im', 'text-generation',
'audio-classification']. For decoder models, use `xxx-
with-past` to export the model using past key values
in the decoder.
--cache_dir CACHE_DIR
Path indicating where to store cache.
--framework {pt,tf} The framework to use for the export. If not provided,
@ -537,7 +552,7 @@ Full list of supported arguments available via ``--help``
it.
--fp16 Compress weights to fp16
--int8 Compress weights to int8
--weight-format {fp32,fp16,int8,int4_sym_g128,int4_asym_g128,int4_sym_g64,int4_asym_g64}
--weight-format {fp32,fp16,int8,int4,int4_sym_g128,int4_asym_g128,int4_sym_g64,int4_asym_g64}
The weight format of the exporting model, e.g. f32
stands for float32 weights, f16 - for float16 weights,
i8 - INT8 weights, int4_* - for INT4 compressed
@ -547,6 +562,10 @@ Full list of supported arguments available via ``--help``
sensitivity and keeps the most impactful layers in
INT8precision (by default 20% in INT8). This helps to
achieve better accuracy after weight compression.
--sym Whether to apply symmetric quantization
--group-size GROUP_SIZE
The group size to use for quantization. Recommended
value is 128 and -1 uses per-column quantization.
--disable-stateful Disable stateful converted models, stateless models
will be generated instead. Stateful models are
produced by default when this key is not used. In
@ -580,7 +599,17 @@ compression:
.. parsed-literal::
2024-02-09 23:11:07.691775: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2024-03-12 22:54:05.156908: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino
.. parsed-literal::
No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda'
.. parsed-literal::
@ -590,12 +619,12 @@ compression:
.. parsed-literal::
Framework not specified. Using pt to export to ONNX.
Framework not specified. Using pt to export the model.
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly. To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly. To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
return self.fget.__get__(instance, owner)()
@ -617,6 +646,8 @@ compression:
Using framework PyTorch: 2.1.0+cpu
Overriding 1 configuration item(s)
- use_cache -> False
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4193: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead
warnings.warn(
After export, model will be available in the specified directory and can
@ -649,7 +680,7 @@ Model inference is exactly the same as for the original model!
output = model(**encoded_input)
scores = output[0][0]
scores = torch.softmax(scores, dim=0).numpy(force=True)
print_prediction(scores)

48
docs/notebooks/125-convnext-classification-with-output.rst
View File

@ -67,11 +67,11 @@ image from an open dataset.
.. code:: ipython3
import urllib.request
from torchvision.io import read_image
import torchvision.transforms as transforms
img_path = 'cats_image.jpeg'
urllib.request.urlretrieve(
url='https://huggingface.co/datasets/huggingface/cats-image/resolve/main/cats_image.jpeg',
@ -95,12 +95,12 @@ models <https://pytorch.org/vision/stable/models.html#listing-and-retrieving-ava
.. code:: ipython3
import torchvision.models as models
# List available models
all_models = models.list_models()
# List of models by type. Classification models are in the parent module.
classification_models = models.list_models(module=models)
print(classification_models)
@ -136,17 +136,17 @@ wight <https://pytorch.org/vision/stable/models.html#using-the-pre-trained-model
.. code:: ipython3
import torch
preprocess = models.ConvNeXt_Tiny_Weights.DEFAULT.transforms()
input_data = preprocess(image)
input_data = torch.stack([input_data], dim=0)
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchvision/transforms/functional.py:1603: UserWarning: The default value of the antialias parameter of all the resizing transforms (Resize(), RandomResizedCrop(), etc.) will change from None to True in v0.17, in order to be consistent across the PIL and Tensor backends. To suppress this warning, directly pass antialias=True (recommended, future default), antialias=None (current default, which means False for Tensors and True for PIL), or antialias=False (only works on Tensors - PIL will still use antialiasing). This also applies if you are using the inference transforms from the models weights: update the call to weights.transforms(antialias=True).
warnings.warn(
@ -164,8 +164,8 @@ And print results
.. code:: ipython3
import urllib.request
# download class number to class label mapping
imagenet_classes_file_path = "imagenet_2012.txt"
urllib.request.urlretrieve(
@ -173,12 +173,12 @@ And print results
filename=imagenet_classes_file_path
)
imagenet_classes = open(imagenet_classes_file_path).read().splitlines()
def print_results(outputs: torch.Tensor):
_, predicted_class = outputs.max(1)
predicted_probability = torch.softmax(outputs, dim=1)[0, predicted_class].item()
print(f"Predicted Class: {predicted_class.item()}")
print(f"Predicted Label: {imagenet_classes[predicted_class.item()]}")
print(f"Predicted Probability: {predicted_probability}")
@ -192,7 +192,7 @@ And print results
Predicted Class: 281
Predicted Label: n02123045 tabby, tabby cat
Predicted Probability: 0.5800774693489075
Predicted Probability: 0.4724695086479187
Convert the model to OpenVINO Intermediate representation format
@ -212,12 +212,12 @@ interface. However, it can also be saved on disk using
.. code:: ipython3
from pathlib import Path
import openvino as ov
import openvino as ov
ov_model_xml_path = Path('models/ov_convnext_model.xml')
if not ov_model_xml_path.exists():
ov_model_xml_path.parent.mkdir(parents=True, exist_ok=True)
converted_model = ov.convert_model(model, example_input=torch.randn(1, 3, 224, 224))
@ -238,7 +238,7 @@ Select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
@ -246,7 +246,7 @@ Select device from dropdown list for running inference using OpenVINO
description='Device:',
disabled=False,
)
device
@ -261,7 +261,7 @@ Select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
core = ov.Core()
compiled_model = core.compile_model(ov_model_xml_path, device_name=device.value)
Use the OpenVINO IR model to run an inference
@ -279,5 +279,5 @@ Use the OpenVINO IR model to run an inference
Predicted Class: 281
Predicted Label: n02123045 tabby, tabby cat
Predicted Probability: 0.6132654547691345
Predicted Probability: 0.6132656931877136

53
docs/notebooks/125-lraspp-segmentation-with-output.rst
View File

@ -47,8 +47,15 @@ Prerequisites
.. code:: ipython3
import platform
%pip install -q --extra-index-url https://download.pytorch.org/whl/cpu torch torchvision
%pip install -q matplotlib
if platform.system() != "Windows":
%pip install -q "matplotlib>=3.4"
else:
%pip install -q "matplotlib>=3.4,<3.7"
%pip install -q "openvino>=2023.2.0"
@ -70,7 +77,7 @@ Prerequisites
.. code:: ipython3
from pathlib import Path
import openvino as ov
import torch
@ -84,11 +91,11 @@ First of all lets get a test image from an open dataset.
.. code:: ipython3
import urllib.request
from torchvision.io import read_image
import torchvision.transforms as transforms
img_path = 'cats_image.jpeg'
urllib.request.urlretrieve(
url='https://huggingface.co/datasets/huggingface/cats-image/resolve/main/cats_image.jpeg',
@ -122,12 +129,12 @@ models <https://pytorch.org/vision/stable/models.html#listing-and-retrieving-ava
.. code:: ipython3
import torchvision.models as models
# List available models
all_models = models.list_models()
# List of models by type
segmentation_models = models.list_models(module=models.segmentation)
print(segmentation_models)
@ -167,11 +174,11 @@ wight <https://pytorch.org/vision/stable/models.html#using-the-pre-trained-model
.. code:: ipython3
import numpy as np
preprocess = models.segmentation.LRASPP_MobileNet_V3_Large_Weights.COCO_WITH_VOC_LABELS_V1.transforms()
preprocess.resize_size = (IMAGE_HEIGHT, IMAGE_WIDTH) # change to an image size
input_data = preprocess(image)
input_data = np.expand_dims(input_data, axis=0)
@ -199,8 +206,8 @@ directory. For more information on how to convert models, see this
.. code:: ipython3
ov_model_xml_path = Path('models/ov_lraspp_model.xml')
if not ov_model_xml_path.exists():
ov_model_xml_path.parent.mkdir(parents=True, exist_ok=True)
dummy_input = torch.randn(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH)
@ -219,7 +226,7 @@ Select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
@ -227,7 +234,7 @@ Select device from dropdown list for running inference using OpenVINO
description='Device:',
disabled=False,
)
device
@ -266,13 +273,13 @@ visualize the image with a ``cat`` mask for the PyTorch model.
import torch
import matplotlib.pyplot as plt
import torchvision.transforms.functional as F
plt.rcParams["savefig.bbox"] = 'tight'
def show(imgs):
if not isinstance(imgs, list):
imgs = [imgs]
@ -293,11 +300,11 @@ Prepare and display a cat mask.
'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor'
]
sem_class_to_idx = {cls: idx for (idx, cls) in enumerate(sem_classes)}
normalized_mask = torch.nn.functional.softmax(result_torch, dim=1)
cat_mask = normalized_mask[0, sem_class_to_idx['cat']]
show(cat_mask)
@ -322,7 +329,7 @@ And now we can plot a boolean mask on top of the original image.
.. code:: ipython3
from torchvision.utils import draw_segmentation_masks
show(draw_segmentation_masks(image, masks=boolean_cat_mask, alpha=0.7, colors='yellow'))

2
docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_28_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9b5a1c7f29fc97014c6bdd4eb2604ace96556a4864b7b2175b960a5ed1c62896
oid sha256:a12219cd1386fa59a98712d2e6145f3ebf5e3e52282b0ab6b65fb2d49b8dc36a
size 87576

2
docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_32_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3ae70e272473dddb3c96182bbe79ca0fe1a0d9867ea6b54e91d3322273753233
oid sha256:57f2b4a6d2402555f9d88c6e19274f79d1408419d193457c2983cbb20ebacfe9
size 385522

2
docs/notebooks/125-lraspp-segmentation-with-output_files/125-lraspp-segmentation-with-output_34_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:04f1d79111e4b949ceb3bd19a51c7453f1bdae0ce5b8b21b748717a93e89891e
oid sha256:e37cfa9b4d60c9d0ecfb0d50e4ce014a53aa5277f65ad1f38c1c86f89bd57583
size 385520

51
docs/notebooks/126-tensorflow-hub-with-output.rst
View File

@ -1,6 +1,8 @@
Convert of TensorFlow Hub models to OpenVINO Intermediate Representation (IR)
=============================================================================
|Colab| |Binder|
This tutorial demonstrates step-by-step instructions on how to convert
models loaded from TensorFlow Hub using OpenVINO Runtime.
@ -44,6 +46,11 @@ Table of contents:
- `Select inference device <#select-inference-device>`__
- `Inference <#inference>`__
.. |Colab| image:: https://colab.research.google.com/assets/colab-badge.svg
:target: https://colab.research.google.com/github/openvinotoolkit/openvino_notebooks/blob/main/notebooks/126-tensorflow-hub/126-tensorflow-hub.ipynb
.. |Binder| image:: https://mybinder.org/badge_logo.svg
:target: https://mybinder.org/v2/gh/eaidova/openvino_notebooks_binder.git/main?urlpath=git-pull%3Frepo%3Dhttps%253A%252F%252Fgithub.com%252Fopenvinotoolkit%252Fopenvino_notebooks%26urlpath%3Dtree%252Fopenvino_notebooks%252Fnotebooks%2F126-tensorflow-hub%2F126-tensorflow-hub.ipynb
Image classification
--------------------
@ -74,8 +81,20 @@ Install required packages
.. code:: ipython3
%pip install -q tensorflow_hub tensorflow pillow numpy matplotlib
import platform
%pip install -q tensorflow_hub tensorflow pillow numpy
%pip install -q "openvino>=2023.2.0"
if platform.system() != "Windows":
%pip install -q "matplotlib>=3.4"
else:
%pip install -q "matplotlib>=3.4,<3.7"
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
@ -99,15 +118,15 @@ Import libraries
import os
from urllib.request import urlretrieve
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
import tensorflow_hub as hub
import tensorflow as tf
import PIL
import numpy as np
import matplotlib.pyplot as plt
import openvino as ov
tf.get_logger().setLevel("ERROR")
.. code:: ipython3
@ -131,8 +150,8 @@ and wrap it as a Keras layer with ``hub.KerasLayer``.
.. parsed-literal::
2024-02-09 23:12:03.569013: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW
2024-02-09 23:12:03.569190: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration
2024-03-12 22:55:04.217869: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW
2024-03-12 22:55:04.218046: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration
Download a single image to try the model on
@ -201,16 +220,16 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
@ -342,9 +361,9 @@ Install required packages
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
from urllib.request import urlretrieve
from pathlib import Path
import openvino as ov
import tensorflow_hub as hub
import tensorflow as tf
import cv2
@ -355,10 +374,10 @@ Install required packages
CONTENT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/2/26/YellowLabradorLooking_new.jpg"
CONTENT_IMAGE_PATH = "./data/YellowLabradorLooking_new.jpg"
STYLE_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/b/b4/Vassily_Kandinsky%2C_1913_-_Composition_7.jpg"
STYLE_IMAGE_PATH = "./data/Vassily_Kandinsky%2C_1913_-_Composition_7.jpg"
MODEL_URL = "https://www.kaggle.com/models/google/arbitrary-image-stylization-v1/frameworks/tensorFlow1/variations/256/versions/2"
MODEL_PATH = "./models/arbitrary-image-stylization-v1-256.xml"
@ -409,16 +428,16 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device

2
docs/notebooks/126-tensorflow-hub-with-output_files/126-tensorflow-hub-with-output_26_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3050fd069390a5410693d7649a6d666fe41504c7b1543897a83170e6cd8a6a5f
oid sha256:b0fae96c93f954f1b2eed2f3d236f96f1b4300f8b8f50599a01ba45d46b9de32
size 203738

2
docs/notebooks/126-tensorflow-hub-with-output_files/126-tensorflow-hub-with-output_45_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f34ce33eec0d7a7db4163c8ef99cadbba0249073fa82e923dfc21c3c30c665d0
oid sha256:fc8e4a16cad1ae9fcabdb459a6d05280ee9c3dc7974c0a10517d9ec9723c95b4
size 538743

5525
docs/notebooks/127-tensorflow-bit-image-classification-nncf-quantization-with-output.rst
View File

File diff suppressed because it is too large Load Diff

3
docs/notebooks/127-tensorflow-bit-image-classification-nncf-quantization-with-output_files/127-tensorflow-bit-image-classification-nncf-quantization-with-output_10_1.png
View File

@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:37f1a76ccf411397019d83e8344b79cdebf644f0f6ba6598141e72c6ee50c62b
size 224517

3
docs/notebooks/127-tensorflow-bit-image-classification-nncf-quantization-with-output_files/127-tensorflow-bit-image-classification-nncf-quantization-with-output_10_2.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b063ced731371a1753f01e4f511652de360ac28d5829d5957571165c1a30ef00
size 224517

3
docs/notebooks/127-tensorflow-bit-image-classification-nncf-quantization-with-output_files/127-tensorflow-bit-image-classification-nncf-quantization-with-output_26_3.png
View File

@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:33f7798bd5dfbfa423dcca1d37eb7e29d0474e331105347cf0efdcfd695b850a
size 335047

3
docs/notebooks/127-tensorflow-bit-image-classification-nncf-quantization-with-output_files/127-tensorflow-bit-image-classification-nncf-quantization-with-output_27_3.png Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b1b7b928b0bd02bf88ae7295ba70ab47d6dc7727ab61adfcb409a5a4ef6077d9
size 335047

2
docs/notebooks/127-tensorflow-bit-image-classification-nncf-quantization-with-output_files/127-tensorflow-bit-image-classification-nncf-quantization-with-output_9_2.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:6b354964f6b453eb9454e0e21ccb3e1275071464698af4693fc94235a1ca563e
oid sha256:d203e09f1ce8e5d011e912bd9987ba1c343c353a543285130eb099b8f958b280
size 296205

611
docs/notebooks/128-opnevino-tokenizers-with-output.rst Normal file
View File

@ -0,0 +1,611 @@
OpenVINO Tokenizers: Incorporate Text Processing Into OpenVINO Pipelines
========================================================================
.. raw:: html
<center>
.. raw:: html
</center>
OpenVINO Tokenizers is an OpenVINO extension and a Python library
designed to streamline tokenizer conversion for seamless integration
into your projects. It supports Python and C++ environments and is
compatible with all major platforms: Linux, Windows, and MacOS.
Table of contents:
^^^^^^^^^^^^^^^^^^
- `Tokenization Basics <#tokenization-basics>`__
- `Acquiring OpenVINO Tokenizers <#acquiring-openvino-tokenizers>`__
- `Convert Tokenizer from HuggingFace Hub with CLI
Tool <#convert-tokenizer-from_huggingface-hub-with-cli-tool>`__
- `Convert Tokenizer from HuggingFace Hub with Python
API <#convert-tokenizer-from-huggingface-hub-with-python-api>`__
- `Text Generation Pipeline with OpenVINO
Tokenizers <#text-generation-pipeline-with-openvino-tokenizers>`__
- `Merge Tokenizer into a Model <#merge-tokenizer-into-a-model>`__
- `Conclusion <#conclusion>`__
- `Links <#links>`__
Tokenization Basics
-------------------
One does not simply put text into a neural network, only numbers. The
process of transforming text into a sequence of numbers is called
**tokenization**. It usually contains several steps that transform the
original string, splitting it into parts - tokens - with an associated
number in a dictionary. You can check the `interactive GPT-4
tokenizer <https://platform.openai.com/tokenizer>`__ to gain an
intuitive understanding of the principles of tokenizer work.
.. raw:: html
<center>
.. raw:: html
</center>
There are two important points in the tokenizer-model relation: 1. Every
neural network with text input is paired with a tokenizer and *cannot be
used without it*. 2. To reproduce the model’s accuracy on a specific
task, it is essential to *utilize the same tokenizer employed during the
model training*.
That is why almost all model repositories on `HuggingFace
Hub <https://HuggingFace.co/models>`__ also contain tokenizer files
(``tokenizer.json``, ``vocab.txt``, ``merges.txt``, etc.).
The process of transforming a sequence of numbers into a string is
called **detokenization**. Detokenizer can share the token dictionary
with a tokenizer, like any LLM chat model, or operate with an entirely
distinct dictionary. For instance, translation models dealing with
different source and target languages often necessitate separate
dictionaries.
.. raw:: html
<center>
.. raw:: html
</center>
Some tasks only need a tokenizer, like text classification, named entity
recognition, question answering, and feature extraction. On the other
hand, for tasks such as text generation, chat, translation, and
abstractive summarization, both a tokenizer and a detokenizer are
required.
Acquiring OpenVINO Tokenizers
-----------------------------
OpenVINO Tokenizers Python library allows you to convert HuggingFace
tokenizers into OpenVINO models. To install all required dependencies
use ``pip install openvino-tokenizers[transformers]``.
.. code:: ipython3
%pip install -Uq pip
%pip uninstall -y openvino openvino-nightly openvino-dev
%pip install -Uq openvino-tokenizers[transformers]
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
Found existing installation: openvino 2024.0.0
Uninstalling openvino-2024.0.0:
.. parsed-literal::
Successfully uninstalled openvino-2024.0.0
WARNING: Skipping openvino-nightly as it is not installed.
Found existing installation: openvino-dev 2024.0.0
.. parsed-literal::
Uninstalling openvino-dev-2024.0.0:
Successfully uninstalled openvino-dev-2024.0.0
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
.. code:: ipython3
from pathlib import Path
tokenizer_dir = Path("tokenizer/")
model_id = "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T"
Convert Tokenizer from HuggingFace Hub with CLI Tool
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The first way is to use the CLI utility, bundled with OpenVINO
Tokenizers. Use ``--with-detokenizer`` flag to add the detokenizer model
to the output. By setting ``--clean-up-tokenization-spaces=False`` we
ensure that the detokenizer correctly decodes a code-generation model
output. ``--trust-remote-code`` flag works the same way as passing
``trust_remote_code=True`` to ``AutoTokenizer.from_pretrained``
constructor.
.. code:: ipython3
!convert_tokenizer $model_id --with-detokenizer -o $tokenizer_dir
.. parsed-literal::
Loading Huggingface Tokenizer...
.. parsed-literal::
Converting Huggingface Tokenizer to OpenVINO...
.. parsed-literal::
Saved OpenVINO Tokenizer: tokenizer/openvino_tokenizer.xml, tokenizer/openvino_tokenizer.bin
Saved OpenVINO Detokenizer: tokenizer/openvino_detokenizer.xml, tokenizer/openvino_detokenizer.bin
⚠️ If you have any problems with the command above on MacOS, try to
`install tbb <https://formulae.brew.sh/formula/tbb#default>`__.
The result is two OpenVINO models: ``openvino_tokenizer`` and
``openvino_detokenizer``. Both can be interacted with using
``read_model``, ``compile_model`` and ``save_model``, similar to any
other OpenVINO model.
Convert Tokenizer from HuggingFace Hub with Python API
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The other method is to pass HuggingFace ``hf_tokenizer`` object to
``convert_tokenizer`` function:
.. code:: ipython3
from transformers import AutoTokenizer
from openvino_tokenizers import convert_tokenizer
hf_tokenizer = AutoTokenizer.from_pretrained(model_id)
ov_tokenizer, ov_detokenizer = convert_tokenizer(hf_tokenizer, with_detokenizer=True)
ov_tokenizer, ov_detokenizer
.. parsed-literal::
(<Model: 'tokenizer'
inputs[
<ConstOutput: names[string_input] shape[?] type: string>
]
outputs[
<ConstOutput: names[input_ids] shape[?,?] type: i64>,
<ConstOutput: names[attention_mask] shape[?,?] type: i64>
]>,
<Model: 'detokenizer'
inputs[
<ConstOutput: names[Parameter_19] shape[?,?] type: i64>
]
outputs[
<ConstOutput: names[string_output] shape[?] type: string>
]>)
That way you get OpenVINO model objects. Use ``save_model`` function
from OpenVINO to reuse converted tokenizers later:
.. code:: ipython3
from openvino import save_model
save_model(ov_tokenizer, tokenizer_dir / "openvino_tokenizer.xml")
save_model(ov_detokenizer, tokenizer_dir / "openvino_detokenizer.xml")
To use the tokenizer, compile the converted model and input a list of
strings. It’s essential to be aware that not all original tokenizers
support multiple strings (also called batches) as input. This limitation
arises from the requirement for all resulting number sequences to
maintain the same length. To address this, a padding token must be
specified, which will be appended to shorter tokenized strings. In cases
where no padding token is determined in the original tokenizer, OpenVINO
Tokenizers defaults to using :math:`0` for padding. Presently, *only
right-side padding is supported*, typically used for classification
tasks, but not suitable for text generation.
.. code:: ipython3
from openvino import compile_model
tokenizer, detokenizer = compile_model(ov_tokenizer), compile_model(ov_detokenizer)
test_strings = ["Test", "strings"]
token_ids = tokenizer(test_strings)["input_ids"]
print(f"Token ids: {token_ids}")
detokenized_text = detokenizer(token_ids)["string_output"]
print(f"Detokenized text: {detokenized_text}")
.. parsed-literal::
Token ids: [[ 1 4321]
[ 1 6031]]
Detokenized text: ['<s> Test' '<s> strings']
We can compare the result of converted (de)tokenizer with the original
one:
.. code:: ipython3
hf_token_ids = hf_tokenizer(test_strings).input_ids
print(f"Token ids: {hf_token_ids}")
hf_detokenized_text = hf_tokenizer.batch_decode(hf_token_ids)
print(f"Detokenized text: {hf_detokenized_text}")
.. parsed-literal::
Token ids: [[1, 4321], [1, 6031]]
.. parsed-literal::
2024-03-12 23:13:39.058345: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-03-12 23:13:39.093394: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
.. parsed-literal::
2024-03-12 23:13:39.592323: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
Detokenized text: ['<s> Test', '<s> strings']
Text Generation Pipeline with OpenVINO Tokenizers
-------------------------------------------------
Let’s build a text generation pipeline with OpenVINO Tokenizers and
minimal dependencies. To obtain an OpenVINO model we will use the
Optimum library. The latest version allows you to get a so-called
`stateful
model <https://docs.openvino.ai/2024/openvino-workflow/running-inference/stateful-models.html>`__.
The original ``TinyLlama-1.1B-intermediate-step-1431k-3T`` model is
4.4Gb. To reduce network and disk usage we will load a converted model
which has also been compressed to ``int8``. The original conversion
command is commented.
.. code:: ipython3
model_dir = Path(Path(model_id).name)
if not model_dir.exists():
# converting the original model
# %pip install -U "git+https://github.com/huggingface/optimum-intel.git" "nncf>=2.8.0" onnx
# %optimum-cli export openvino -m $model_id --task text-generation-with-past $model_dir
# load already converted model
from huggingface_hub import hf_hub_download
hf_hub_download(
"chgk13/TinyLlama-1.1B-intermediate-step-1431k-3T",
filename="openvino_model.xml",
local_dir=model_dir,
)
hf_hub_download(
"chgk13/TinyLlama-1.1B-intermediate-step-1431k-3T",
filename="openvino_model.bin",
local_dir=model_dir,
)
.. code:: ipython3
import numpy as np
from tqdm.notebook import trange
from pathlib import Path
import openvino_tokenizers # noqa: F401
from openvino import compile_model
compiled_model = compile_model(model_dir / "openvino_model.xml")
infer_request = compiled_model.create_infer_request()
The ``infer_reques``\ t object provides control over the model’s state -
a Key-Value cache that speeds up inference by reducing computations
Multiple inference requests can be created, and each request maintains a
distinct and separate state..
.. code:: ipython3
text_input = ["Quick brown fox jumped"]
model_input = {name.any_name: output for name, output in tokenizer(text_input).items()}
if "position_ids" in (input.any_name for input in infer_request.model_inputs):
model_input["position_ids"] = np.arange(model_input["input_ids"].shape[1], dtype=np.int64)[np.newaxis, :]
# no beam search, set idx to 0
model_input["beam_idx"] = np.array([0], dtype=np.int32)
# end of sentence token is that model signifies the end of text generation will be available in next version,
# for now, can be obtained from the original tokenizer `original_tokenizer.eos_token_id`
eos_token = 2
tokens_result = np.array([[]], dtype=np.int64)
# reset KV cache inside the model before inference
infer_request.reset_state()
max_infer = 10
for _ in trange(max_infer):
infer_request.start_async(model_input)
infer_request.wait()
# use greedy decoding to get the most probable token as the model prediction
output_token = np.argmax(infer_request.get_output_tensor().data[:, -1, :], axis=-1, keepdims=True)
tokens_result = np.hstack((tokens_result, output_token))
if output_token[0][0] == eos_token:
break
# prepare input for new inference
model_input["input_ids"] = output_token
model_input["attention_mask"] = np.hstack((model_input["attention_mask"].data, [[1]]))
model_input["position_ids"] = np.hstack(
(model_input["position_ids"].data, [[model_input["position_ids"].data.shape[-1]]])
)
text_result = detokenizer(tokens_result)["string_output"]
print(f"Prompt:\n{text_input[0]}")
print(f"Generated:\n{text_result[0]}")
.. parsed-literal::
0%| | 0/10 [00:00<?, ?it/s]
.. parsed-literal::
Prompt:
Quick brown fox jumped
Generated:
over the fence.
Merge Tokenizer into a Model
----------------------------
Packages like ``tensorflow-text`` offer the convenience of integrating
text processing directly into the model, streamlining both distribution
and usage. Similarly, with OpenVINO Tokenizers, you can create models
that combine a converted tokenizer and a model. It’s important to note
that not all scenarios benefit from this merge. In cases where a
tokenizer is used once and a model is inferred multiple times, as seen
in the earlier text generation example, maintaining a separate
(de)tokenizer and model is advisable to prevent unnecessary
tokenization-detokenization cycles during inference. Conversely, if both
a tokenizer and a model are used once in each pipeline inference,
merging simplifies the workflow and aids in avoiding the creation of
intermediate objects:
.. raw:: html
<center>
.. raw:: html
</center>
The OpenVINO Python API allows you to avoid this by using the
``share_inputs`` option during inference, but it requires additional
input from a developer every time the model is inferred. Combining the
models and tokenizers simplifies memory management.
.. code:: ipython3
model_id = "mrm8488/bert-tiny-finetuned-sms-spam-detection"
model_dir = Path(Path(model_id).name)
if not model_dir.exists():
%pip install -qU git+https://github.com/huggingface/optimum-intel.git onnx
!optimum-cli export openvino --model $model_id --task text-classification $model_dir
!convert_tokenizer $model_id -o $model_dir
.. parsed-literal::
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
- Avoid using `tokenizers` before the fork if possible
- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
- Avoid using `tokenizers` before the fork if possible
- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
.. parsed-literal::
2024-03-12 23:13:56.117320: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
OpenVINO Tokenizer version is not compatible with OpenVINO version. Installed OpenVINO version: 2024.0.0,OpenVINO Tokenizers requires . OpenVINO Tokenizers models will not be added during export.
.. parsed-literal::
INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino
.. parsed-literal::
No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda'
.. parsed-literal::
Framework not specified. Using pt to export the model.
.. parsed-literal::
Using the export variant default. Available variants are:
- default: The default ONNX variant.
.. parsed-literal::
Using framework PyTorch: 2.1.0+cpu
Overriding 1 configuration item(s)
- use_cache -> False
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4193: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead
warnings.warn(
.. parsed-literal::
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
- Avoid using `tokenizers` before the fork if possible
- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
.. parsed-literal::
Loading Huggingface Tokenizer...
.. parsed-literal::
Converting Huggingface Tokenizer to OpenVINO...
RegexNormalization pattern is not supported, operation output might differ from the original tokenizer.
.. parsed-literal::
/tmp/tmpctejxcqg/build/third_party/re2/src/extern_re2/re2/re2.cc:205: Error parsing '((?=[^\n\t\r])\p{Cc})|((?=[^\n\t\r])\p{Cf})': invalid perl operator: (?=
.. parsed-literal::
Saved OpenVINO Tokenizer: bert-tiny-finetuned-sms-spam-detection/openvino_tokenizer.xml, bert-tiny-finetuned-sms-spam-detection/openvino_tokenizer.bin
.. code:: ipython3
from openvino import Core, save_model
from openvino_tokenizers import connect_models
core = Core()
text_input = ["Free money!!!"]
ov_tokenizer = core.read_model(model_dir / "openvino_tokenizer.xml")
ov_model = core.read_model(model_dir / "openvino_model.xml")
combined_model = connect_models(ov_tokenizer, ov_model)
save_model(combined_model, model_dir / "combined_openvino_model.xml")
compiled_combined_model = core.compile_model(combined_model)
openvino_output = compiled_combined_model(text_input)
print(f"Logits: {openvino_output['logits']}")
.. parsed-literal::
/tmp/tmpctejxcqg/build/third_party/re2/src/extern_re2/re2/re2.cc:205: Error parsing '((?=[^\n\t\r])\p{Cc})|((?=[^\n\t\r])\p{Cf})': invalid perl operator: (?=
/tmp/tmpctejxcqg/build/third_party/re2/src/extern_re2/re2/re2.cc:205: Error parsing '((?=[^\n\t\r])\p{Cc})|((?=[^\n\t\r])\p{Cf})': invalid perl operator: (?=
.. parsed-literal::
Logits: [[ 1.2007061 -1.4698029]]
Conclusion
----------
The OpenVINO Tokenizers integrate text processing operations into the
OpenVINO ecosystem. Enabling the conversion of HuggingFace tokenizers
into OpenVINO models, the library allows efficient deployment of deep
learning pipelines across varied environments. The feature of combining
tokenizers and models not only simplifies memory management but also
helps to streamline model usage and deployment.
Links
-----
- `Installation instructions for different
environments <https://github.com/openvinotoolkit/openvino_tokenizers?tab=readme-ov-file#installation>`__
- `Supported Tokenizer
Types <https://github.com/openvinotoolkit/openvino_tokenizers?tab=readme-ov-file#supported-tokenizer-types>`__
- `OpenVINO.GenAI repository with the C++ example of OpenVINO
Tokenizers
usage <https://github.com/openvinotoolkit/openvino.genai/tree/master/text_generation/causal_lm/cpp>`__
- `HuggingFace Tokenizers Comparison
Table <https://github.com/openvinotoolkit/openvino_tokenizers?tab=readme-ov-file#output-match-by-model>`__

97
docs/notebooks/201-vision-monodepth-with-output.rst
View File

@ -67,9 +67,16 @@ Install requirements
.. code:: ipython3
import platform
%pip install -q "openvino>=2023.1.0"
%pip install -q matplotlib opencv-python requests tqdm
%pip install -q opencv-python requests tqdm
if platform.system() != "Windows":
%pip install -q "matplotlib>=3.4"
else:
%pip install -q "matplotlib>=3.4,<3.7"
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
@ -88,11 +95,16 @@ Install requirements
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
('notebook_utils.py', <http.client.HTTPMessage at 0x7fe5c4f7e130>)
('notebook_utils.py', <http.client.HTTPMessage at 0x7eff217dc7c0>)
@ -105,7 +117,7 @@ Imports
import time
from pathlib import Path
import cv2
import matplotlib.cm
import matplotlib.pyplot as plt
@ -120,7 +132,7 @@ Imports
display,
)
import openvino as ov
from notebook_utils import download_file, load_image
Download the model
@ -129,19 +141,20 @@ Download the model
The model is in the `OpenVINO Intermediate Representation
(IR) <https://docs.openvino.ai/nightly/openvino_ir.html>`__ format.
(IR) <https://docs.openvino.ai/2024/documentation/openvino-ir-format.html>`__
format.
.. code:: ipython3
model_folder = Path('model')
ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/depth-estimation-midas/FP32/'
ir_model_name_xml = 'MiDaS_small.xml'
ir_model_name_bin = 'MiDaS_small.bin'
download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory=model_folder)
download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory=model_folder)
model_xml_path = model_folder / ir_model_name_xml
@ -167,13 +180,13 @@ Functions
def normalize_minmax(data):
"""Normalizes the values in `data` between 0 and 1"""
return (data - data.min()) / (data.max() - data.min())
def convert_result_to_image(result, colormap="viridis"):
"""
Convert network result of floating point numbers to an RGB image with
integer values from 0-255 by applying a colormap.
`result` is expected to be a single network result in 1,H,W shape
`colormap` is a matplotlib colormap.
See https://matplotlib.org/stable/tutorials/colors/colormaps.html
@ -184,8 +197,8 @@ Functions
result = cmap(result)[:, :, :3] * 255
result = result.astype(np.uint8)
return result
def to_rgb(image_data) -> np.ndarray:
"""
Convert image_data from BGR to RGB
@ -202,7 +215,7 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
@ -210,7 +223,7 @@ select device from dropdown list for running inference using OpenVINO
description='Device:',
disabled=False,
)
device
@ -237,10 +250,10 @@ output keys and the expected input shape for the model.
core.set_property({'CACHE_DIR': '../cache'})
model = core.read_model(model_xml_path)
compiled_model = core.compile_model(model=model, device_name=device.value)
input_key = compiled_model.input(0)
output_key = compiled_model.output(0)
network_input_shape = list(input_key.shape)
network_image_height, network_image_width = network_input_shape[2:]
@ -262,10 +275,10 @@ H=height, W=width).
IMAGE_FILE = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_bike.jpg"
image = load_image(path=IMAGE_FILE)
# Resize to input shape for network.
resized_image = cv2.resize(src=image, dsize=(network_image_height, network_image_width))
# Reshape the image to network input shape NCHW.
input_image = np.expand_dims(np.transpose(resized_image, (2, 0, 1)), 0)
@ -280,11 +293,11 @@ original image shape.
.. code:: ipython3
result = compiled_model([input_image])[output_key]
# Convert the network result of disparity map to an image that shows
# distance as colors.
result_image = convert_result_to_image(result=result)
# Resize back to original image shape. The `cv2.resize` function expects shape
# in (width, height), [::-1] reverses the (height, width) shape to match this.
result_image = cv2.resize(result_image, image.shape[:2][::-1])
@ -292,7 +305,7 @@ original image shape.
.. parsed-literal::
/tmp/ipykernel_2841459/2076527990.py:15: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
/tmp/ipykernel_3058046/2076527990.py:15: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
cmap = matplotlib.cm.get_cmap(colormap)
@ -346,7 +359,7 @@ Video Settings
# Try the `THEO` encoding if you have FFMPEG installed.
# FOURCC = cv2.VideoWriter_fourcc(*"THEO")
FOURCC = cv2.VideoWriter_fourcc(*"vp09")
# Create Path objects for the input video and the result video.
output_directory = Path("output")
output_directory.mkdir(exist_ok=True)
@ -369,11 +382,11 @@ compute values for these properties for the monodepth video.
raise ValueError(f"The video at {VIDEO_FILE} cannot be read.")
input_fps = cap.get(cv2.CAP_PROP_FPS)
input_video_frame_height, input_video_frame_width = image.shape[:2]
target_fps = input_fps / ADVANCE_FRAMES
target_frame_height = int(input_video_frame_height * SCALE_OUTPUT)
target_frame_width = int(input_video_frame_width * SCALE_OUTPUT)
cap.release()
print(
f"The input video has a frame width of {input_video_frame_width}, "
@ -403,10 +416,10 @@ Do Inference on a Video and Create Monodepth Video
input_video_frame_nr = 0
start_time = time.perf_counter()
total_inference_duration = 0
# Open the input video
cap = cv2.VideoCapture(str(VIDEO_FILE))
# Create a result video.
out_video = cv2.VideoWriter(
str(result_video_path),
@ -414,36 +427,36 @@ Do Inference on a Video and Create Monodepth Video
target_fps,
(target_frame_width * 2, target_frame_height),
)
num_frames = int(NUM_SECONDS * input_fps)
total_frames = cap.get(cv2.CAP_PROP_FRAME_COUNT) if num_frames == 0 else num_frames
progress_bar = ProgressBar(total=total_frames)
progress_bar.display()
try:
while cap.isOpened():
ret, image = cap.read()
if not ret:
cap.release()
break
if input_video_frame_nr >= total_frames:
break
# Only process every second frame.
# Prepare a frame for inference.
# Resize to the input shape for network.
resized_image = cv2.resize(src=image, dsize=(network_image_height, network_image_width))
# Reshape the image to network input shape NCHW.
input_image = np.expand_dims(np.transpose(resized_image, (2, 0, 1)), 0)
# Do inference.
inference_start_time = time.perf_counter()
result = compiled_model([input_image])[output_key]
inference_stop_time = time.perf_counter()
inference_duration = inference_stop_time - inference_start_time
total_inference_duration += inference_duration
if input_video_frame_nr % (10 * ADVANCE_FRAMES) == 0:
clear_output(wait=True)
progress_bar.display()
@ -457,7 +470,7 @@ Do Inference on a Video and Create Monodepth Video
f"({1/inference_duration:.2f} FPS)"
)
)
# Transform the network result to a RGB image.
result_frame = to_rgb(convert_result_to_image(result))
# Resize the image and the result to a target frame shape.
@ -467,13 +480,13 @@ Do Inference on a Video and Create Monodepth Video
stacked_frame = np.hstack((image, result_frame))
# Save a frame to the video.
out_video.write(stacked_frame)
input_video_frame_nr = input_video_frame_nr + ADVANCE_FRAMES
cap.set(1, input_video_frame_nr)
progress_bar.progress = input_video_frame_nr
progress_bar.update()
except KeyboardInterrupt:
print("Processing interrupted.")
finally:
@ -483,7 +496,7 @@ Do Inference on a Video and Create Monodepth Video
cap.release()
end_time = time.perf_counter()
duration = end_time - start_time
print(
f"Processed {processed_frames} frames in {duration:.2f} seconds. "
f"Total FPS (including video processing): {processed_frames/duration:.2f}."
@ -494,7 +507,7 @@ Do Inference on a Video and Create Monodepth Video
.. parsed-literal::
Processed 60 frames in 37.50 seconds. Total FPS (including video processing): 1.60.Inference FPS: 43.40
Processed 60 frames in 26.30 seconds. Total FPS (including video processing): 2.28.Inference FPS: 43.50
Monodepth Video saved to 'output/Coco%20Walking%20in%20Berkeley_monodepth.mp4'.
@ -524,8 +537,8 @@ Display Monodepth Video
.. parsed-literal::
Showing monodepth video saved at
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/201-vision-monodepth/output/Coco%20Walking%20in%20Berkeley_monodepth.mp4
If you cannot see the video in your browser, please click on the following link to download the video
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/201-vision-monodepth/output/Coco%20Walking%20in%20Berkeley_monodepth.mp4
If you cannot see the video in your browser, please click on the following link to download the video

2
docs/notebooks/201-vision-monodepth-with-output_files/201-vision-monodepth-with-output_18_0.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:bfb6d38aa3628e614cd15dedb22763f7d7ada417b59965d777dfebcd9cd833e0
oid sha256:b34292be5a7f36c97469fb2faade2c42313b9aca447b82000dc7f200c27d6936
size 959858

144
docs/notebooks/202-vision-superresolution-image-with-output.rst
View File

@ -65,9 +65,21 @@ Install requirements
.. code:: ipython3
import platform
%pip install -q "openvino>=2023.1.0"
%pip install -q opencv-python
%pip install -q pillow matplotlib
%pip install -q pillow
if platform.system() != "Windows":
%pip install -q "matplotlib>=3.4"
else:
%pip install -q "matplotlib>=3.4,<3.7"
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
@ -95,7 +107,7 @@ Imports
import os
import time
from pathlib import Path
import cv2
import matplotlib.pyplot as plt
import numpy as np
@ -129,7 +141,7 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
@ -137,7 +149,7 @@ select device from dropdown list for running inference using OpenVINO
description='Device:',
disabled=False,
)
device
@ -153,20 +165,20 @@ select device from dropdown list for running inference using OpenVINO
# 1032: 4x superresolution, 1033: 3x superresolution
model_name = 'single-image-super-resolution-1032'
base_model_dir = Path("./model").expanduser()
model_xml_name = f'{model_name}.xml'
model_bin_name = f'{model_name}.bin'
model_xml_path = base_model_dir / model_xml_name
model_bin_path = base_model_dir / model_bin_name
if not model_xml_path.exists():
base_url = f'https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/{model_name}/FP16/'
model_xml_url = base_url + model_xml_name
model_bin_url = base_url + model_bin_name
download_file(model_xml_url, model_xml_path)
download_file(model_bin_url, model_bin_path)
else:
@ -183,7 +195,7 @@ Functions
"""
Write the specified text in the top left corner of the image
as white text with a black border.
:param image: image as numpy arry with HWC shape, RGB or BGR
:param text: text to write
:return: image with written text, as numpy array
@ -196,11 +208,11 @@ Functions
font_thickness = 2
text_color_bg = (0, 0, 0)
x, y = org
image = cv2.UMat(image)
(text_w, text_h), _ = cv2.getTextSize(text, font, font_scale, font_thickness)
result_im = cv2.rectangle(image, org, (x + text_w, y + text_h), text_color_bg, -1)
textim = cv2.putText(
result_im,
text,
@ -212,13 +224,13 @@ Functions
line_type,
)
return textim.get()
def convert_result_to_image(result) -> np.ndarray:
"""
Convert network result of floating point numbers to image with integer
values from 0-255. Values outside this range are clipped to 0 and 255.
:param result: a single superresolution network result in N,C,H,W shape
"""
result = result.squeeze(0).transpose(1, 2, 0)
@ -227,8 +239,8 @@ Functions
result[result > 255] = 255
result = result.astype(np.uint8)
return result
def to_rgb(image_data) -> np.ndarray:
"""
Convert image_data from BGR to RGB
@ -254,23 +266,23 @@ information about the network inputs and outputs.
core = ov.Core()
model = core.read_model(model=model_xml_path)
compiled_model = core.compile_model(model=model, device_name=device.value)
# Network inputs and outputs are dictionaries. Get the keys for the
# dictionaries.
original_image_key, bicubic_image_key = compiled_model.inputs
output_key = compiled_model.output(0)
# Get the expected input and target shape. The `.dims[2:]` returns the height
# and width. The `resize` function of OpenCV expects the shape as (width, height),
# so reverse the shape with `[::-1]` and convert it to a tuple.
input_height, input_width = list(original_image_key.shape)[2:]
target_height, target_width = list(bicubic_image_key.shape)[2:]
upsample_factor = int(target_height / input_height)
print(f"The network expects inputs with a width of {input_width}, " f"height of {input_height}")
print(f"The network returns images with a width of {target_width}, " f"height of {target_height}")
print(
f"The image sides are upsampled by a factor of {upsample_factor}. "
f"The new image is {upsample_factor**2} times as large as the "
@ -298,17 +310,17 @@ Load and Show the Input Image
IMAGE_PATH = Path("./data/tower.jpg")
OUTPUT_PATH = Path("output/")
os.makedirs(str(OUTPUT_PATH), exist_ok=True)
download_file('https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/tower.jpg', IMAGE_PATH)
full_image = cv2.imread(str(IMAGE_PATH))
# Uncomment these lines to load a raw image as BGR.
# import rawpy
# with rawpy.imread(IMAGE_PATH) as raw:
# full_image = raw.postprocess()[:,:,(2,1,0)]
plt.imshow(to_rgb(full_image))
print(f"Showing full image with width {full_image.shape[1]} " f"and height {full_image.shape[0]}")
@ -353,17 +365,17 @@ as the crop size.
# Set it to 1 to crop the image with the exact input size.
CROP_FACTOR = 2
adjusted_upsample_factor = upsample_factor // CROP_FACTOR
image_id = "flag" # A tag to recognize the saved images.
starty = 3200
startx = 0
# Perform the crop.
image_crop = full_image[
starty : starty + input_height * CROP_FACTOR,
startx : startx + input_width * CROP_FACTOR,
]
# Show the cropped image.
print(f"Showing image crop with width {image_crop.shape[1]} and " f"height {image_crop.shape[0]}.")
plt.imshow(to_rgb(image_crop));
@ -394,11 +406,11 @@ interpolation. This bicubic image is the second input to the network.
bicubic_image = cv2.resize(
src=image_crop, dsize=(target_width, target_height), interpolation=cv2.INTER_CUBIC
)
# If required, resize the image to the input image shape.
if CROP_FACTOR > 1:
image_crop = cv2.resize(src=image_crop, dsize=(input_width, input_height))
# Reshape the images from (H,W,C) to (N,C,H,W).
input_image_original = np.expand_dims(image_crop.transpose(2, 0, 1), axis=0)
input_image_bicubic = np.expand_dims(bicubic_image.transpose(2, 0, 1), axis=0)
@ -418,7 +430,7 @@ Do inference and convert the inference result to an ``RGB`` image.
bicubic_image_key.any_name: input_image_bicubic,
}
)[output_key]
# Get inference result as numpy array and reshape to image shape and data type
result_image = convert_result_to_image(result)
@ -460,7 +472,7 @@ Save Superresolution and Bicubic Image Crop
# Add a text with "SUPER" or "BICUBIC" to the superresolution or bicubic image.
image_super = write_text_on_image(image=result_image, text="SUPER")
image_bicubic = write_text_on_image(image=bicubic_image, text="BICUBIC")
# Store the image and the results.
crop_image_path = Path(f"{OUTPUT_PATH.stem}/{image_id}_{adjusted_upsample_factor}x_crop.png")
superres_image_path = Path(
@ -493,11 +505,11 @@ Write Animated GIF with Bicubic/Superresolution Comparison
print(image_bicubic.shape)
print(image_super.shape)
result_pil = Image.fromarray(to_rgb(image_super))
bicubic_pil = Image.fromarray(to_rgb(image_bicubic))
gif_image_path = Path(f"{OUTPUT_PATH.stem}/{image_id}_comparison_{adjusted_upsample_factor}x.gif")
result_pil.save(
fp=str(gif_image_path),
format="GIF",
@ -506,7 +518,7 @@ Write Animated GIF with Bicubic/Superresolution Comparison
duration=1000,
loop=0,
)
# The `DisplayImage(str(gif_image_path))` function does not work in Colab.
DisplayImage(data=open(gif_image_path, "rb").read(), width=1920 // 2)
@ -540,7 +552,7 @@ the ``Files`` tool.
.. code:: ipython3
FOURCC = cv2.VideoWriter_fourcc(*"MJPG")
result_video_path = Path(
f"{OUTPUT_PATH.stem}/{image_id}_crop_comparison_{adjusted_upsample_factor}x.avi"
)
@ -548,22 +560,22 @@ the ``Files`` tool.
result_image.shape[0] // 2,
result_image.shape[1] // 2,
)
out_video = cv2.VideoWriter(
filename=str(result_video_path),
fourcc=FOURCC,
fps=90,
frameSize=(video_target_width, video_target_height),
)
resized_result_image = cv2.resize(src=result_image, dsize=(video_target_width, video_target_height))
resized_bicubic_image = cv2.resize(
src=bicubic_image, dsize=(video_target_width, video_target_height)
)
progress_bar = ProgressBar(total=video_target_width)
progress_bar.display()
for i in range(video_target_width):
# Create a frame where the left part (until i pixels width) contains the
# superresolution image, and the right part (from i pixels width) contains
@ -582,7 +594,7 @@ the ``Files`` tool.
progress_bar.update()
out_video.release()
clear_output()
video_link = FileLink(result_video_path)
video_link.html_link_str = "<a href='%s' download>%s</a>"
display(HTML(f"The video has been saved to {video_link._repr_html_()}"))
@ -619,9 +631,9 @@ Compute patches
CROPLINES = 10
# See Superresolution on one crop of the image for description of `CROP_FACTOR`.
CROP_FACTOR = 2
full_image_height, full_image_width = full_image.shape[:2]
# Compute x and y coordinates of left top of image tiles.
x_coords = list(range(0, full_image_width, input_width * CROP_FACTOR - CROPLINES * 2))
while full_image_width - x_coords[-1] < input_width * CROP_FACTOR:
@ -629,12 +641,12 @@ Compute patches
y_coords = list(range(0, full_image_height, input_height * CROP_FACTOR - CROPLINES * 2))
while full_image_height - y_coords[-1] < input_height * CROP_FACTOR:
y_coords.pop(-1)
# Compute the width and height to crop the full image. The full image is
# cropped at the border to tiles of the input size.
crop_width = x_coords[-1] + input_width * CROP_FACTOR
crop_height = y_coords[-1] + input_height * CROP_FACTOR
# Compute the width and height of the target superresolution image.
new_width = (
x_coords[-1] * (upsample_factor // CROP_FACTOR)
@ -677,62 +689,62 @@ as total time to process each patch.
num_patches = len(x_coords) * len(y_coords)
progress_bar = ProgressBar(total=num_patches)
progress_bar.display()
# Crop image to fit tiles of the input size.
full_image_crop = full_image.copy()[:crop_height, :crop_width, :]
# Create an empty array of the target size.
full_superresolution_image = np.empty((new_height, new_width, 3), dtype=np.uint8)
# Create a bicubic upsampled image of the target size for comparison.
full_bicubic_image = cv2.resize(
src=full_image_crop[CROPLINES:-CROPLINES, CROPLINES:-CROPLINES, :],
dsize=(new_width, new_height),
interpolation=cv2.INTER_CUBIC,
)
total_inference_duration = 0
for y in y_coords:
for x in x_coords:
patch_nr += 1
# Crop the input image.
image_crop = full_image_crop[
y : y + input_height * CROP_FACTOR,
x : x + input_width * CROP_FACTOR,
]
# Resize the images to the target shape with bicubic interpolation
bicubic_image = cv2.resize(
src=image_crop,
dsize=(target_width, target_height),
interpolation=cv2.INTER_CUBIC,
)
if CROP_FACTOR > 1:
image_crop = cv2.resize(src=image_crop, dsize=(input_width, input_height))
input_image_original = np.expand_dims(image_crop.transpose(2, 0, 1), axis=0)
input_image_bicubic = np.expand_dims(bicubic_image.transpose(2, 0, 1), axis=0)
# Do inference.
inference_start_time = time.perf_counter()
result = compiled_model(
{
original_image_key.any_name: input_image_original,
bicubic_image_key.any_name: input_image_bicubic,
}
)[output_key]
inference_stop_time = time.perf_counter()
inference_duration = inference_stop_time - inference_start_time
total_inference_duration += inference_duration
# Reshape an inference result to the image shape and the data type.
result_image = convert_result_to_image(result)
# Add the inference result of this patch to the full superresolution
# image.
adjusted_upsample_factor = upsample_factor // CROP_FACTOR
@ -746,10 +758,10 @@ as total time to process each patch.
CROPLINES * adjusted_upsample_factor : -CROPLINES * adjusted_upsample_factor,
:,
]
progress_bar.progress = patch_nr
progress_bar.update()
if patch_nr % 10 == 0:
clear_output(wait=True)
progress_bar.display()
@ -760,7 +772,7 @@ as total time to process each patch.
f"({1/inference_duration:.2f} FPS)"
)
)
end_time = time.perf_counter()
duration = end_time - start_time
clear_output(wait=True)
@ -774,8 +786,8 @@ as total time to process each patch.
.. parsed-literal::
Processed 42 patches in 4.64 seconds. Total patches per second (including processing): 9.05.
Inference patches per second: 17.92
Processed 42 patches in 4.63 seconds. Total patches per second (including processing): 9.08.
Inference patches per second: 17.95
Save superresolution image and the bicubic image
@ -789,7 +801,7 @@ Save superresolution image and the bicubic image
f"{OUTPUT_PATH.stem}/full_superres_{adjusted_upsample_factor}x.jpg"
)
full_bicubic_image_path = Path(f"{OUTPUT_PATH.stem}/full_bicubic_{adjusted_upsample_factor}x.jpg")
cv2.imwrite(str(full_superresolution_image_path), full_superresolution_image)
cv2.imwrite(str(full_bicubic_image_path), full_bicubic_image);

2
docs/notebooks/202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_15_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:494df30726447340860e42ec8216d047eb5de907d9fafde7b1341e5eb7443752
oid sha256:1f3ddf966495258f0b36306e712331bb834c9942b673c30732869fe36a7b2870
size 272963

2
docs/notebooks/202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_17_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e7b9ba119cdc8b86e8c7ff8891ed51572ef6dd8bad5a71988fcc0abeb65c7d8b
oid sha256:302db7065eda5e95c2a5ace7947e0b249607176f03c675cbecb4444863f61bda
size 356735

2
docs/notebooks/202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_23_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c1a121245becc489e58a7c85fa41e2a3fa7c606f63c4f46b509246008bdbeaee
oid sha256:b67df1028d23d7d8fb290ba823cf2aaa90e8f85cb7f12b4369b6e33097c60598
size 2896276

68
docs/notebooks/202-vision-superresolution-video-with-output.rst
View File

@ -81,7 +81,7 @@ Imports
import time
from pathlib import Path
import cv2
import numpy as np
from IPython.display import (
@ -120,7 +120,7 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
@ -128,7 +128,7 @@ select device from dropdown list for running inference using OpenVINO
description='Device:',
disabled=False,
)
device
@ -144,20 +144,20 @@ select device from dropdown list for running inference using OpenVINO
# 1032: 4x superresolution, 1033: 3x superresolution
model_name = 'single-image-super-resolution-1032'
base_model_dir = Path('./model').expanduser()
model_xml_name = f'{model_name}.xml'
model_bin_name = f'{model_name}.bin'
model_xml_path = base_model_dir / model_xml_name
model_bin_path = base_model_dir / model_bin_name
if not model_xml_path.exists():
base_url = f'https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/{model_name}/FP16/'
model_xml_url = base_url + model_xml_name
model_bin_url = base_url + model_bin_name
download_file(model_xml_url, model_xml_path)
download_file(model_bin_url, model_bin_path)
else:
@ -180,7 +180,7 @@ Functions
"""
Convert network result of floating point numbers to image with integer
values from 0-255. Values outside this range are clipped to 0 and 255.
:param result: a single superresolution network result in N,C,H,W shape
"""
result = result.squeeze(0).transpose(1, 2, 0)
@ -215,18 +215,18 @@ resolution version of the image in 1920x1080.
# dictionaries.
original_image_key, bicubic_image_key = compiled_model.inputs
output_key = compiled_model.output(0)
# Get the expected input and target shape. The `.dims[2:]` function returns the height
# and width.The `resize` function of OpenCV expects the shape as (width, height),
# so reverse the shape with `[::-1]` and convert it to a tuple.
input_height, input_width = list(original_image_key.shape)[2:]
target_height, target_width = list(bicubic_image_key.shape)[2:]
upsample_factor = int(target_height / input_height)
print(f"The network expects inputs with a width of {input_width}, " f"height of {input_height}")
print(f"The network returns images with a width of {target_width}, " f"height of {target_height}")
print(
f"The image sides are upsampled by a factor of {upsample_factor}. "
f"The new image is {upsample_factor**2} times as large as the "
@ -264,7 +264,7 @@ Settings
.. code:: ipython3
OUTPUT_DIR = "output"
Path(OUTPUT_DIR).mkdir(exist_ok=True)
# Maximum number of frames to read from the input video. Set to 0 to read all frames.
NUM_FRAMES = 100
@ -290,10 +290,10 @@ Download and Prepare Video
filename = Path(stream.default_filename.encode("ascii", "ignore").decode("ascii")).stem
stream.download(output_path=OUTPUT_DIR, filename=filename)
print(f"Video {filename} downloaded to {OUTPUT_DIR}")
# Create Path objects for the input video and the resulting videos.
video_path = Path(stream.get_file_path(filename, OUTPUT_DIR))
# Path names for the result videos.
superres_video_path = Path(f"{OUTPUT_DIR}/{video_path.stem}_superres.mp4")
bicubic_video_path = Path(f"{OUTPUT_DIR}/{video_path.stem}_bicubic.mp4")
@ -314,14 +314,14 @@ Download and Prepare Video
raise ValueError(f"The video at '{video_path}' cannot be read.")
fps = cap.get(cv2.CAP_PROP_FPS)
frame_count = cap.get(cv2.CAP_PROP_FRAME_COUNT)
if NUM_FRAMES == 0:
total_frames = frame_count
else:
total_frames = min(frame_count, NUM_FRAMES)
original_frame_height, original_frame_width = image.shape[:2]
cap.release()
print(
f"The input video has a frame width of {original_frame_width}, "
@ -389,10 +389,10 @@ video.
start_time = time.perf_counter()
frame_nr = 0
total_inference_duration = 0
progress_bar = ProgressBar(total=total_frames)
progress_bar.display()
cap = cv2.VideoCapture(filename=str(video_path))
try:
while cap.isOpened():
@ -400,22 +400,22 @@ video.
if not ret:
cap.release()
break
if frame_nr >= total_frames:
break
# Resize the input image to the network shape and convert it from (H,W,C) to
# (N,C,H,W).
resized_image = cv2.resize(src=image, dsize=(input_width, input_height))
input_image_original = np.expand_dims(resized_image.transpose(2, 0, 1), axis=0)
# Resize and reshape the image to the target shape with bicubic
# interpolation.
bicubic_image = cv2.resize(
src=image, dsize=(target_width, target_height), interpolation=cv2.INTER_CUBIC
)
input_image_bicubic = np.expand_dims(bicubic_image.transpose(2, 0, 1), axis=0)
# Do inference.
inference_start_time = time.perf_counter()
result = compiled_model(
@ -427,19 +427,19 @@ video.
inference_stop_time = time.perf_counter()
inference_duration = inference_stop_time - inference_start_time
total_inference_duration += inference_duration
# Transform the inference result into an image.
result_frame = convert_result_to_image(result=result)
# Write the result image and the bicubic image to a video file.
superres_video.write(image=result_frame)
bicubic_video.write(image=bicubic_image)
stacked_frame = np.hstack((bicubic_image, result_frame))
comparison_video.write(image=stacked_frame)
frame_nr = frame_nr + 1
# Update the progress bar and the status message.
progress_bar.progress = frame_nr
progress_bar.update()
@ -453,8 +453,8 @@ video.
f"({1/inference_duration:.2f} FPS)"
)
)
except KeyboardInterrupt:
print("Processing interrupted.")
finally:
@ -480,13 +480,13 @@ video.
.. parsed-literal::
Processed frame 100. Inference time: 0.06 seconds (17.00 FPS)
Processed frame 100. Inference time: 0.05 seconds (20.76 FPS)
.. parsed-literal::
Video's saved to output directory.
Processed 100 frames in 243.08 seconds. Total FPS (including video processing): 0.41. Inference FPS: 17.69.
Processed 100 frames in 242.92 seconds. Total FPS (including video processing): 0.41. Inference FPS: 18.04.
Show Side-by-Side Video of Bicubic and Superresolution Version

212
docs/notebooks/203-meter-reader-with-output.rst
View File

@ -43,8 +43,20 @@ Table of contents:
.. code:: ipython3
import platform
# Install openvino package
%pip install -q "openvino>=2023.1.0" matplotlib
%pip install -q "openvino>=2023.1.0"
if platform.system() != "Windows":
%pip install -q "matplotlib>=3.4"
else:
%pip install -q "matplotlib>=3.4,<3.7"
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
@ -68,7 +80,7 @@ Import
import tarfile
import matplotlib.pyplot as plt
import openvino as ov
sys.path.append("../utils")
from notebook_utils import download_file, segmentation_map_to_image
@ -89,9 +101,9 @@ DeepLabV3P pre-trained models from PaddlePaddle community.
IMG_LINK = "https://user-images.githubusercontent.com/91237924/170696219-f68699c6-1e82-46bf-aaed-8e2fc3fa5f7b.jpg"
IMG_FILE_NAME = IMG_LINK.split("/")[-1]
IMG_PATH = Path(f"{DATA_DIR}/{IMG_FILE_NAME}")
os.makedirs(MODEL_DIR, exist_ok=True)
download_file(DET_MODEL_LINK, directory=MODEL_DIR, show_progress=True)
file = tarfile.open(f"model/{DET_FILE_NAME}")
res = file.extractall("model")
@ -99,7 +111,7 @@ DeepLabV3P pre-trained models from PaddlePaddle community.
print(f"Detection Model Extracted to \"./{MODEL_DIR}\".")
else:
print("Error Extracting the Detection model. Please check the network.")
download_file(SEG_MODEL_LINK, directory=MODEL_DIR, show_progress=True)
file = tarfile.open(f"model/{SEG_FILE_NAME}")
res = file.extractall("model")
@ -107,7 +119,7 @@ DeepLabV3P pre-trained models from PaddlePaddle community.
print(f"Segmentation Model Extracted to \"./{MODEL_DIR}\".")
else:
print("Error Extracting the Segmentation model. Please check the network.")
download_file(IMG_LINK, directory=DATA_DIR, show_progress=True)
if IMG_PATH.is_file():
print(f"Test Image Saved to \"./{DATA_DIR}\".")
@ -156,15 +168,15 @@ reading calculation.
.. code:: ipython3
METER_SHAPE = [512, 512]
CIRCLE_CENTER = [256, 256]
METER_SHAPE = [512, 512]
CIRCLE_CENTER = [256, 256]
CIRCLE_RADIUS = 250
PI = math.pi
RECTANGLE_HEIGHT = 120
RECTANGLE_WIDTH = 1570
TYPE_THRESHOLD = 40
COLORMAP = np.array([[28, 28, 28], [238, 44, 44], [250, 250, 250]])
# There are 2 types of meters in test image datasets
METER_CONFIG = [{
'scale_interval_value': 25.0 / 50.0,
@ -175,7 +187,7 @@ reading calculation.
'range': 1.6,
'unit': "(MPa)"
}]
SEG_LABEL = {'background': 0, 'pointer': 1, 'scale': 2}
Load the Models
@ -188,34 +200,34 @@ loading and inference
# Initialize OpenVINO Runtime
core = ov.Core()
class Model:
"""
This class represents a OpenVINO model object.
"""
def __init__(self, model_path, new_shape, device="CPU"):
"""
Initialize the model object
Param:
Param:
model_path (string): path of inference model
new_shape (dict): new shape of model input
"""
self.model = core.read_model(model=model_path)
self.model.reshape(new_shape)
self.compiled_model = core.compile_model(model=self.model, device_name=device)
self.output_layer = self.compiled_model.output(0)
def predict(self, input_image):
"""
Run inference
Param:
Param:
input_image (np.array): input data
Retuns:
result (np.array)): model output data
"""
@ -233,13 +245,13 @@ postprocessing tasks of each model.
def det_preprocess(input_image, target_size):
"""
Preprocessing the input data for detection task
Param:
Param:
input_image (np.array): input data
size (int): the image size required by model input layer
Retuns:
img.astype (np.array): preprocessed image
"""
img = cv2.resize(input_image, (target_size, target_size))
img = np.transpose(img, [2, 0, 1]) / 255
@ -249,41 +261,41 @@ postprocessing tasks of each model.
img -= img_mean
img /= img_std
return img.astype(np.float32)
def filter_bboxes(det_results, score_threshold):
"""
Filter out the detection results with low confidence
Param：
det_results (list[dict]): detection results
score_threshold (float)： confidence threshold
Retuns：
filtered_results (list[dict]): filter detection results
"""
filtered_results = []
for i in range(len(det_results)):
if det_results[i, 1] > score_threshold:
filtered_results.append(det_results[i])
return filtered_results
def roi_crop(image, results, scale_x, scale_y):
"""
Crop the area of detected meter of original image
Param：
img (np.array)：original image。
det_results (list[dict]): detection results
scale_x (float): the scale value in x axis
scale_y (float): the scale value in y axis
Retuns：
roi_imgs (list[np.array]): the list of meter images
loc (list[int]): the list of meter locations
"""
roi_imgs = []
loc = []
@ -294,22 +306,22 @@ postprocessing tasks of each model.
roi_imgs.append(sub_img)
loc.append([xmin, ymin, xmax, ymax])
return roi_imgs, loc
def roi_process(input_images, target_size, interp=cv2.INTER_LINEAR):
"""
Prepare the roi image of detection results data
Preprocessing the input data for segmentation task
Param：
input_images (list[np.array])：the list of meter images
target_size (list|tuple)： height and width of resized image， e.g [heigh,width]
interp (int)：the interp method for image reszing
Retuns：
img_list (list[np.array])：the list of processed images
resize_img (list[np.array]): for visualization
"""
img_list = list()
resize_list = list()
@ -326,48 +338,48 @@ postprocessing tasks of each model.
resize_img /= img_std
img_list.append(resize_img)
return img_list, resize_list
def erode(seg_results, erode_kernel):
"""
Erode the segmentation result to get the more clear instance of pointer and scale
Param：
seg_results (list[dict])：segmentation results
erode_kernel (int): size of erode_kernel
Return：
eroded_results (list[dict])： the lab map of eroded_results
"""
kernel = np.ones((erode_kernel, erode_kernel), np.uint8)
eroded_results = seg_results
for i in range(len(seg_results)):
eroded_results[i] = cv2.erode(seg_results[i].astype(np.uint8), kernel)
return eroded_results
def circle_to_rectangle(seg_results):
"""
Switch the shape of label_map from circle to rectangle
Param：
seg_results (list[dict])：segmentation results
Return：
rectangle_meters (list[np.array])：the rectangle of label map
"""
rectangle_meters = list()
for i, seg_result in enumerate(seg_results):
label_map = seg_result
# The size of rectangle_meter is determined by RECTANGLE_HEIGHT and RECTANGLE_WIDTH
rectangle_meter = np.zeros((RECTANGLE_HEIGHT, RECTANGLE_WIDTH), dtype=np.uint8)
for row in range(RECTANGLE_HEIGHT):
for col in range(RECTANGLE_WIDTH):
theta = PI * 2 * (col + 1) / RECTANGLE_WIDTH
# The radius of meter circle will be mapped to the height of rectangle image
rho = CIRCLE_RADIUS - row - 1
y = int(CIRCLE_CENTER[0] + rho * math.cos(theta) + 0.5)
@ -375,19 +387,19 @@ postprocessing tasks of each model.
rectangle_meter[row, col] = label_map[y, x]
rectangle_meters.append(rectangle_meter)
return rectangle_meters
def rectangle_to_line(rectangle_meters):
"""
Switch the dimension of rectangle label map from 2D to 1D
Param：
rectangle_meters (list[np.array])：2D rectangle OF label_map。
Return：
line_scales (list[np.array])： the list of scales value
line_pointers (list[np.array])：the list of pointers value
"""
line_scales = list()
line_pointers = list()
@ -404,18 +416,18 @@ postprocessing tasks of each model.
line_scales.append(line_scale)
line_pointers.append(line_pointer)
return line_scales, line_pointers
def mean_binarization(data_list):
"""
Binarize the data
Param：
data_list (list[np.array])：input data
Return：
binaried_data_list (list[np.array])：output data。
"""
batch_size = len(data_list)
binaried_data_list = data_list
@ -428,18 +440,18 @@ postprocessing tasks of each model.
else:
binaried_data_list[i][col] = 1
return binaried_data_list
def locate_scale(line_scales):
"""
Find location of center of each scale
Param：
line_scales (list[np.array])：the list of binaried scales value
Return：
scale_locations (list[list])：location of each scale
"""
batch_size = len(line_scales)
scale_locations = list()
@ -465,18 +477,18 @@ postprocessing tasks of each model.
find_start = False
scale_locations.append(locations)
return scale_locations
def locate_pointer(line_pointers):
"""
Find location of center of pointer
Param：
line_scales (list[np.array])：the list of binaried pointer value
Return：
scale_locations (list[list])：location of pointer
"""
batch_size = len(line_pointers)
pointer_locations = list()
@ -500,21 +512,21 @@ postprocessing tasks of each model.
break
pointer_locations.append(location)
return pointer_locations
def get_relative_location(scale_locations, pointer_locations):
"""
Match location of pointer and scales
Param：
scale_locations (list[list])：location of each scale
pointer_locations (list[list])：location of pointer
Return：
pointed_scales (list[dict])： a list of dict with:
'num_scales': total number of scales
'pointed_scale': predicted number of scales
"""
pointed_scales = list()
for scale_location, pointer_location in zip(scale_locations,
@ -528,18 +540,18 @@ postprocessing tasks of each model.
result = {'num_scales': num_scales, 'pointed_scale': pointed_scale}
pointed_scales.append(result)
return pointed_scales
def calculate_reading(pointed_scales):
"""
Calculate the value of meter according to the type of meter
Param：
pointed_scales (list[list])：predicted number of scales
Return：
readings (list[float])： the list of values read from meter
"""
readings = list()
batch_size = len(pointed_scales)
@ -568,14 +580,14 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
@ -603,16 +615,16 @@ bounds of input batch size.
det_model_shape = {'image': [1, 3, 608, 608], 'im_shape': [1, 2], 'scale_factor': [1, 2]}
seg_model_path = f"{MODEL_DIR}/meter_seg_model/model.pdmodel"
seg_model_shape = {'image': [ov.Dimension(1, 2), 3, 512, 512]}
erode_kernel = 4
score_threshold = 0.5
seg_batch_size = 2
input_shape = 608
# Intialize the model objects
detector = Model(det_model_path, det_model_shape, device.value)
segmenter = Model(seg_model_path, seg_model_shape, device.value)
# Visulize a original input photo
image = cv2.imread(img_file)
rgb_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
@ -623,7 +635,7 @@ bounds of input batch size.
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f476c1d6fd0>
<matplotlib.image.AxesImage at 0x7fcb0dada370>
@ -644,24 +656,24 @@ meter and prepare the ROI images for segmentation.
scale_factor = np.array([[1, 2]]).astype('float32')
input_image = det_preprocess(image, input_shape)
inputs_dict = {'image': input_image, "im_shape": im_shape, "scale_factor": scale_factor}
# Run meter detection model
det_results = detector.predict(inputs_dict)
# Filter out the bounding box with low confidence
filtered_results = filter_bboxes(det_results, score_threshold)
# Prepare the input data for meter segmentation model
scale_x = image.shape[1] / input_shape * 2
scale_y = image.shape[0] / input_shape
# Create the individual picture for each detected meter
roi_imgs, loc = roi_crop(image, filtered_results, scale_x, scale_y)
roi_imgs, resize_imgs = roi_process(roi_imgs, METER_SHAPE)
# Create the pictures of detection results
roi_stack = np.hstack(resize_imgs)
if cv2.imwrite(f"{DATA_DIR}/detection_results.jpg", roi_stack):
print("The detection result image has been saved as \"detection_results.jpg\" in data")
plt.imshow(cv2.cvtColor(roi_stack, cv2.COLOR_BGR2RGB))
@ -687,7 +699,7 @@ task on detected ROI.
seg_results = list()
mask_list = list()
num_imgs = len(roi_imgs)
# Run meter segmentation model on all detected meters
for i in range(0, num_imgs, seg_batch_size):
batch = roi_imgs[i : min(num_imgs, i + seg_batch_size)]
@ -695,14 +707,14 @@ task on detected ROI.
seg_results.extend(seg_result)
results = []
for i in range(len(seg_results)):
results.append(np.argmax(seg_results[i], axis=0))
results.append(np.argmax(seg_results[i], axis=0))
seg_results = erode(results, erode_kernel)
# Create the pictures of segmentation results
for i in range(len(seg_results)):
mask_list.append(segmentation_map_to_image(seg_results[i], COLORMAP))
mask_stack = np.hstack(mask_list)
if cv2.imwrite(f"{DATA_DIR}/segmentation_results.jpg", cv2.cvtColor(mask_stack, cv2.COLOR_RGB2BGR)):
print("The segmentation result image has been saved as \"segmentation_results.jpg\" in data")
plt.imshow(mask_stack)
@ -725,7 +737,7 @@ location of the pointer in a scale map.
.. code:: ipython3
# Find the pointer location in scale map and calculate the meters reading
# Find the pointer location in scale map and calculate the meters reading
rectangle_meters = circle_to_rectangle(seg_results)
line_scales, line_pointers = rectangle_to_line(rectangle_meters)
binaried_scales = mean_binarization(line_scales)
@ -734,13 +746,13 @@ location of the pointer in a scale map.
pointer_locations = locate_pointer(binaried_pointers)
pointed_scales = get_relative_location(scale_locations, pointer_locations)
meter_readings = calculate_reading(pointed_scales)
rectangle_list = list()
# Plot the rectangle meters
for i in range(len(rectangle_meters)):
rectangle_list.append(segmentation_map_to_image(rectangle_meters[i], COLORMAP))
rectangle_meters_stack = np.hstack(rectangle_list)
if cv2.imwrite(f"{DATA_DIR}/rectangle_meters.jpg", cv2.cvtColor(rectangle_meters_stack, cv2.COLOR_RGB2BGR)):
print("The rectangle_meters result image has been saved as \"rectangle_meters.jpg\" in data")
plt.imshow(rectangle_meters_stack)
@ -765,7 +777,7 @@ Get the reading result on the meter picture
# Create a final result photo with reading
for i in range(len(meter_readings)):
print("Meter {}: {:.3f}".format(i + 1, meter_readings[i]))
result_image = image.copy()
for i in range(len(loc)):
cv2.rectangle(result_image,(loc[i][0], loc[i][1]), (loc[i][2], loc[i][3]), (0, 150, 0), 3)

2
docs/notebooks/203-meter-reader-with-output_files/203-meter-reader-with-output_16_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:30ad77998245f764b4970ffe4588735cede277f1e5675ba87ce1bd9f9f44863e
oid sha256:08c5ae3bb47e095d707bdaa7f8008bed7eeb1f672c82ae4d63334e665ec3e4d8
size 170121

2
docs/notebooks/203-meter-reader-with-output_files/203-meter-reader-with-output_18_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b160de7f59d0a7e363df9c00873a7cdc273df7d2d1e71a2a8bd5431820b18230
oid sha256:6433ef738eeb00f8d0dc4343ab289073c76321d2e12fe46318fbe374b0f745e2
size 190271

2
docs/notebooks/203-meter-reader-with-output_files/203-meter-reader-with-output_20_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9af2ae1ad9c76985e7bfbbb9ef05c71d9f01221524f2ebe385b671b3810532a6
oid sha256:3d67df91f05c9aeb0442a1c4aaef7527cf27e9be0938642eed807f8b5342aa7b
size 26914

2
docs/notebooks/203-meter-reader-with-output_files/203-meter-reader-with-output_22_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c75189b084fe90902d97c519cc02e065c2af5cd71f9907bcc835f639a1a88c58
oid sha256:50b9f932b844d99b59b51f2c6947dd048f96bf1553fe36de3975d3a3ad1715e4
size 8966

2
docs/notebooks/203-meter-reader-with-output_files/203-meter-reader-with-output_24_1.png
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:12e6831c646fd5431868f0df9c4c3f0ed93c13f8e492d5a440d351e84aeab600
oid sha256:ad7114f80f8925643c865222d0fe0e05d4f65ab54e0b0d354edebe3e5c1ade7c
size 170338

444
docs/notebooks/204-segmenter-semantic-segmentation-with-output.rst
View File

@ -90,13 +90,13 @@ Prerequisites
import sys
from pathlib import Path
# clone Segmenter repo
if not Path("segmenter").exists():
!git clone https://github.com/rstrudel/segmenter
else:
print("Segmenter repo already cloned")
# include path to Segmenter repo to use its functions
sys.path.append("./segmenter")
@ -119,7 +119,10 @@ Receiving objects: 6% (17/268)
Receiving objects: 7% (19/268)
Receiving objects: 8% (22/268)
Receiving objects: 9% (25/268)
Receiving objects: 10% (27/268)
.. parsed-literal::
Receiving objects: 10% (27/268)
Receiving objects: 11% (30/268)
Receiving objects: 12% (33/268)
Receiving objects: 13% (35/268)
@ -136,145 +139,120 @@ Receiving objects: 22% (59/268)
.. parsed-literal::
Receiving objects: 23% (62/268)
Receiving objects: 24% (65/268)
.. parsed-literal::
Receiving objects: 24% (65/268)
Receiving objects: 25% (67/268), 4.67 MiB | 9.20 MiB/s
.. parsed-literal::
Receiving objects: 24% (65/268), 3.68 MiB | 3.63 MiB/s
Receiving objects: 26% (70/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 27% (73/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 28% (76/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 29% (78/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 30% (81/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 31% (84/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 32% (86/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 33% (89/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 34% (92/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 35% (94/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 36% (97/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 37% (100/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 38% (102/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 39% (105/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 40% (108/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 41% (110/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 42% (113/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 43% (116/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 44% (118/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 45% (121/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 46% (124/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 47% (126/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 48% (129/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 49% (132/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 50% (134/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 51% (137/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 52% (140/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 53% (143/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 54% (145/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 55% (148/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 56% (151/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 57% (153/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 58% (156/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 59% (159/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 60% (161/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 61% (164/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 62% (167/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 63% (169/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 64% (172/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 65% (175/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 66% (177/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 67% (180/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 68% (183/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 69% (185/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 70% (188/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 71% (191/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 72% (193/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 73% (196/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 74% (199/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 75% (201/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 76% (204/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 77% (207/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 78% (210/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 79% (212/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 80% (215/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 81% (218/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 82% (220/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 83% (223/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 84% (226/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 85% (228/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 86% (231/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 87% (234/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 88% (236/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 89% (239/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 90% (242/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 91% (244/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 92% (247/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 93% (250/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 94% (252/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 95% (255/268), 4.67 MiB | 9.20 MiB/s
Receiving objects: 96% (258/268), 4.67 MiB | 9.20 MiB/s
.. parsed-literal::
Receiving objects: 24% (66/268), 7.47 MiB | 3.70 MiB/s
.. parsed-literal::
Receiving objects: 25% (67/268), 7.47 MiB | 3.70 MiB/s
.. parsed-literal::
Receiving objects: 25% (68/268), 11.26 MiB | 3.73 MiB/s
.. parsed-literal::
Receiving objects: 26% (70/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 27% (73/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 28% (76/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 29% (78/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 30% (81/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 31% (84/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 32% (86/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 33% (89/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 34% (92/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 35% (94/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 36% (97/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 37% (100/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 38% (102/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 39% (105/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 40% (108/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 41% (110/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 42% (113/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 43% (116/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 44% (118/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 45% (121/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 46% (124/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 47% (126/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 48% (129/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 49% (132/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 50% (134/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 51% (137/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 52% (140/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 53% (143/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 54% (145/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 55% (148/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 56% (151/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 57% (153/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 58% (156/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 59% (159/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 60% (161/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 61% (164/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 62% (167/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 63% (169/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 64% (172/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 65% (175/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 66% (177/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 67% (180/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 68% (183/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 69% (185/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 70% (188/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 71% (191/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 72% (193/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 73% (196/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 74% (199/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 75% (201/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 76% (204/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 77% (207/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 78% (210/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 79% (212/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 80% (215/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 81% (218/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 82% (220/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 83% (223/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 84% (226/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 85% (228/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 86% (231/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 87% (234/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 88% (236/268), 11.26 MiB | 3.73 MiB/s
Receiving objects: 89% (239/268), 11.26 MiB | 3.73 MiB/s
.. parsed-literal::
Receiving objects: 90% (242/268), 11.26 MiB | 3.73 MiB/s
.. parsed-literal::
Receiving objects: 91% (244/268), 11.26 MiB | 3.73 MiB/s
.. parsed-literal::
Receiving objects: 92% (247/268), 13.11 MiB | 3.72 MiB/s
Receiving objects: 93% (250/268), 13.11 MiB | 3.72 MiB/s
Receiving objects: 94% (252/268), 13.11 MiB | 3.72 MiB/s
Receiving objects: 95% (255/268), 13.11 MiB | 3.72 MiB/s
Receiving objects: 96% (258/268), 13.11 MiB | 3.72 MiB/s
.. parsed-literal::
Receiving objects: 97% (260/268), 13.11 MiB | 3.72 MiB/s
Receiving objects: 98% (263/268), 13.11 MiB | 3.72 MiB/s
Receiving objects: 99% (266/268), 13.11 MiB | 3.72 MiB/s
.. parsed-literal::
Receiving objects: 99% (267/268), 15.03 MiB | 3.73 MiB/s
Receiving objects: 96% (259/268), 13.36 MiB | 13.12 MiB/s
Receiving objects: 97% (260/268), 13.36 MiB | 13.12 MiB/s
Receiving objects: 98% (263/268), 13.36 MiB | 13.12 MiB/s
Receiving objects: 99% (266/268), 13.36 MiB | 13.12 MiB/s
.. parsed-literal::
remote: Total 268 (delta 0), reused 0 (delta 0), pack-reused 268
Receiving objects: 100% (268/268), 15.03 MiB | 3.73 MiB/s
Receiving objects: 100% (268/268), 15.34 MiB | 3.73 MiB/s, done.
Receiving objects: 100% (268/268), 13.36 MiB | 13.12 MiB/s
Receiving objects: 100% (268/268), 15.34 MiB | 13.80 MiB/s, done.
Resolving deltas: 0% (0/117)
Resolving deltas: 1% (2/117)
Resolving deltas: 2% (3/117)
Resolving deltas: 3% (4/117)
Resolving deltas: 5% (6/117)
Resolving deltas: 4% (5/117)
Resolving deltas: 6% (8/117)
Resolving deltas: 7% (9/117)
Resolving deltas: 8% (10/117)
Resolving deltas: 9% (11/117)
Resolving deltas: 10% (12/117)
Resolving deltas: 11% (14/117)
Resolving deltas: 11% (13/117)
Resolving deltas: 13% (16/117)
Resolving deltas: 15% (18/117)
Resolving deltas: 19% (23/117)
Resolving deltas: 26% (31/117)
Resolving deltas: 33% (39/117)
Resolving deltas: 54% (64/117)
Resolving deltas: 56% (66/117)
Resolving deltas: 76% (90/117)
Resolving deltas: 40% (47/117)
Resolving deltas: 43% (51/117)
Resolving deltas: 45% (53/117)
Resolving deltas: 66% (78/117)
Resolving deltas: 69% (81/117)
Resolving deltas: 70% (83/117)
Resolving deltas: 71% (84/117)
Resolving deltas: 80% (94/117)
Resolving deltas: 81% (95/117)
Resolving deltas: 82% (96/117)
Resolving deltas: 100% (117/117)
Resolving deltas: 100% (117/117), done.
@ -293,9 +271,9 @@ Resolving deltas: 100% (117/117), done.
.. parsed-literal::
Requirement already satisfied: torch in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from -r segmenter/requirements.txt (line 1)) (2.1.0+cpu)
Requirement already satisfied: click in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from -r segmenter/requirements.txt (line 2)) (8.1.7)
Requirement already satisfied: numpy in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from -r segmenter/requirements.txt (line 3)) (1.23.5)
Requirement already satisfied: torch in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from -r segmenter/requirements.txt (line 1)) (2.1.0+cpu)
Requirement already satisfied: click in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from -r segmenter/requirements.txt (line 2)) (8.1.7)
Requirement already satisfied: numpy in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from -r segmenter/requirements.txt (line 3)) (1.23.5)
.. parsed-literal::
@ -308,15 +286,19 @@ Resolving deltas: 100% (117/117), done.
Collecting python-hostlist (from -r segmenter/requirements.txt (line 5))
Using cached python_hostlist-1.23.0-py3-none-any.whl
Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from -r segmenter/requirements.txt (line 6)) (4.66.1)
Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from -r segmenter/requirements.txt (line 7)) (2.31.0)
Requirement already satisfied: pyyaml in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from -r segmenter/requirements.txt (line 8)) (6.0.1)
.. parsed-literal::
Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from -r segmenter/requirements.txt (line 6)) (4.66.2)
Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from -r segmenter/requirements.txt (line 7)) (2.31.0)
Requirement already satisfied: pyyaml in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from -r segmenter/requirements.txt (line 8)) (6.0.1)
.. parsed-literal::
Collecting timm==0.4.12 (from -r segmenter/requirements.txt (line 9))
Using cached timm-0.4.12-py3-none-any.whl (376 kB)
Using cached timm-0.4.12-py3-none-any.whl.metadata (30 kB)
.. parsed-literal::
@ -328,84 +310,85 @@ Resolving deltas: 100% (117/117), done.
.. parsed-literal::
Collecting mmsegmentation==0.14.1 (from -r segmenter/requirements.txt (line 11))
Using cached mmsegmentation-0.14.1-py3-none-any.whl.metadata (8.3 kB)
Requirement already satisfied: torchvision in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from timm==0.4.12->-r segmenter/requirements.txt (line 9)) (0.16.0+cpu)
.. parsed-literal::
Using cached mmsegmentation-0.14.1-py3-none-any.whl (201 kB)
Requirement already satisfied: torchvision in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from timm==0.4.12->-r segmenter/requirements.txt (line 9)) (0.16.0+cpu)
Requirement already satisfied: addict in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from mmcv==1.3.8->-r segmenter/requirements.txt (line 10)) (2.4.0)
Requirement already satisfied: Pillow in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from mmcv==1.3.8->-r segmenter/requirements.txt (line 10)) (10.2.0)
Collecting addict (from mmcv==1.3.8->-r segmenter/requirements.txt (line 10))
Using cached addict-2.4.0-py3-none-any.whl.metadata (1.0 kB)
Requirement already satisfied: Pillow in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from mmcv==1.3.8->-r segmenter/requirements.txt (line 10)) (10.2.0)
.. parsed-literal::
Collecting yapf (from mmcv==1.3.8->-r segmenter/requirements.txt (line 10))
.. parsed-literal::
Using cached yapf-0.40.2-py3-none-any.whl.metadata (45 kB)
Requirement already satisfied: matplotlib in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (3.7.4)
Requirement already satisfied: prettytable in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (3.9.0)
Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch->-r segmenter/requirements.txt (line 1)) (3.13.1)
Requirement already satisfied: typing-extensions in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch->-r segmenter/requirements.txt (line 1)) (4.9.0)
Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch->-r segmenter/requirements.txt (line 1)) (1.12)
Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch->-r segmenter/requirements.txt (line 1)) (3.1)
Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch->-r segmenter/requirements.txt (line 1)) (3.1.3)
Requirement already satisfied: fsspec in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch->-r segmenter/requirements.txt (line 1)) (2023.10.0)
.. parsed-literal::
Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->-r segmenter/requirements.txt (line 7)) (3.3.2)
Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->-r segmenter/requirements.txt (line 7)) (3.6)
Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->-r segmenter/requirements.txt (line 7)) (2.2.0)
Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->-r segmenter/requirements.txt (line 7)) (2024.2.2)
Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch->-r segmenter/requirements.txt (line 1)) (2.1.5)
Requirement already satisfied: matplotlib in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (3.7.5)
Requirement already satisfied: prettytable in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (3.10.0)
Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch->-r segmenter/requirements.txt (line 1)) (3.13.1)
Requirement already satisfied: typing-extensions in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch->-r segmenter/requirements.txt (line 1)) (4.10.0)
Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch->-r segmenter/requirements.txt (line 1)) (1.12)
Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch->-r segmenter/requirements.txt (line 1)) (3.1)
Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch->-r segmenter/requirements.txt (line 1)) (3.1.3)
Requirement already satisfied: fsspec in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch->-r segmenter/requirements.txt (line 1)) (2024.2.0)
Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->-r segmenter/requirements.txt (line 7)) (3.3.2)
Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->-r segmenter/requirements.txt (line 7)) (3.6)
Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->-r segmenter/requirements.txt (line 7)) (2.2.1)
Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->-r segmenter/requirements.txt (line 7)) (2024.2.2)
.. parsed-literal::
Requirement already satisfied: contourpy>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (1.1.1)
Requirement already satisfied: cycler>=0.10 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (0.12.1)
Requirement already satisfied: fonttools>=4.22.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (4.48.1)
Requirement already satisfied: kiwisolver>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (1.4.5)
Requirement already satisfied: packaging>=20.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (23.2)
Requirement already satisfied: pyparsing>=2.3.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (3.1.1)
Requirement already satisfied: python-dateutil>=2.7 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (2.8.2)
Requirement already satisfied: importlib-resources>=3.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (6.1.1)
Requirement already satisfied: wcwidth in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from prettytable->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (0.2.13)
Requirement already satisfied: mpmath>=0.19 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch->-r segmenter/requirements.txt (line 1)) (1.3.0)
Requirement already satisfied: importlib-metadata>=6.6.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from yapf->mmcv==1.3.8->-r segmenter/requirements.txt (line 10)) (7.0.1)
Requirement already satisfied: platformdirs>=3.5.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from yapf->mmcv==1.3.8->-r segmenter/requirements.txt (line 10)) (4.2.0)
Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch->-r segmenter/requirements.txt (line 1)) (2.1.5)
Requirement already satisfied: contourpy>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (1.1.1)
Requirement already satisfied: cycler>=0.10 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (0.12.1)
Requirement already satisfied: fonttools>=4.22.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (4.49.0)
Requirement already satisfied: kiwisolver>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (1.4.5)
Requirement already satisfied: packaging>=20.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (24.0)
Requirement already satisfied: pyparsing>=2.3.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (3.1.2)
Requirement already satisfied: python-dateutil>=2.7 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (2.9.0.post0)
Requirement already satisfied: importlib-resources>=3.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (6.3.0)
.. parsed-literal::
Requirement already satisfied: wcwidth in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from prettytable->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (0.2.13)
Requirement already satisfied: mpmath>=0.19 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch->-r segmenter/requirements.txt (line 1)) (1.3.0)
Requirement already satisfied: importlib-metadata>=6.6.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from yapf->mmcv==1.3.8->-r segmenter/requirements.txt (line 10)) (7.0.2)
Requirement already satisfied: platformdirs>=3.5.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from yapf->mmcv==1.3.8->-r segmenter/requirements.txt (line 10)) (4.2.0)
.. parsed-literal::
Collecting tomli>=2.0.1 (from yapf->mmcv==1.3.8->-r segmenter/requirements.txt (line 10))
Using cached tomli-2.0.1-py3-none-any.whl (12 kB)
Using cached tomli-2.0.1-py3-none-any.whl.metadata (8.9 kB)
.. parsed-literal::
Requirement already satisfied: zipp>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from importlib-metadata>=6.6.0->yapf->mmcv==1.3.8->-r segmenter/requirements.txt (line 10)) (3.17.0)
.. parsed-literal::
Requirement already satisfied: six>=1.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from python-dateutil>=2.7->matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (1.16.0)
Requirement already satisfied: zipp>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from importlib-metadata>=6.6.0->yapf->mmcv==1.3.8->-r segmenter/requirements.txt (line 10)) (3.17.0)
Requirement already satisfied: six>=1.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from python-dateutil>=2.7->matplotlib->mmsegmentation==0.14.1->-r segmenter/requirements.txt (line 11)) (1.16.0)
.. parsed-literal::
Using cached timm-0.4.12-py3-none-any.whl (376 kB)
Using cached mmsegmentation-0.14.1-py3-none-any.whl (201 kB)
Using cached einops-0.7.0-py3-none-any.whl (44 kB)
Using cached addict-2.4.0-py3-none-any.whl (3.8 kB)
Using cached yapf-0.40.2-py3-none-any.whl (254 kB)
Using cached tomli-2.0.1-py3-none-any.whl (12 kB)
.. parsed-literal::
Installing collected packages: python-hostlist, tomli, einops, yapf, mmsegmentation, mmcv, timm
Installing collected packages: python-hostlist, addict, tomli, einops, yapf, mmsegmentation, mmcv, timm
.. parsed-literal::
@ -420,7 +403,7 @@ Resolving deltas: 100% (117/117), done.
ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
black 21.7b0 requires tomli<2.0.0,>=0.2.6, but you have tomli 2.0.1 which is incompatible.
Successfully installed einops-0.7.0 mmcv-1.3.8 mmsegmentation-0.14.1 python-hostlist-1.23.0 timm-0.4.12 tomli-2.0.1 yapf-0.40.2
Successfully installed addict-2.4.0 einops-0.7.0 mmcv-1.3.8 mmsegmentation-0.14.1 python-hostlist-1.23.0 timm-0.4.12 tomli-2.0.1 yapf-0.40.2
.. parsed-literal::
@ -432,7 +415,7 @@ Resolving deltas: 100% (117/117), done.
import numpy as np
import yaml
# Fetch the notebook utils script from the openvino_notebooks repo
import urllib.request
urllib.request.urlretrieve(
@ -453,13 +436,13 @@ config for our model.
# here we use tiny model, there are also better but larger models available in repository
WEIGHTS_LINK = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/segmenter/checkpoints/ade20k/seg_tiny_mask/checkpoint.pth"
CONFIG_LINK = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/segmenter/checkpoints/ade20k/seg_tiny_mask/variant.yml"
MODEL_DIR = Path("model/")
MODEL_DIR.mkdir(exist_ok=True)
download_file(WEIGHTS_LINK, directory=MODEL_DIR, show_progress=True)
download_file(CONFIG_LINK, directory=MODEL_DIR, show_progress=True)
WEIGHT_PATH = MODEL_DIR / "checkpoint.pth"
CONFIG_PATH = MODEL_DIR / "variant.yaml"
@ -497,7 +480,7 @@ initialize the model.
.. code:: ipython3
from segmenter.segm.model.factory import load_model
pytorch_model, config = load_model(WEIGHT_PATH)
# put model into eval mode, to set it for inference
pytorch_model.eval()
@ -560,12 +543,12 @@ normalized with given mean and standard deviation provided in
from PIL import Image
import torch
import torchvision.transforms.functional as F
def preprocess(im: Image, normalization: dict) -> torch.Tensor:
"""
Preprocess image: scale, normalize and unsqueeze
:param im: input image
:param normalization: dictionary containing normalization data from config file
:return:
@ -577,7 +560,7 @@ normalized with given mean and standard deviation provided in
im = F.normalize(im, normalization["mean"], normalization["std"])
# change dim from [C, H, W] to [1, C, H, W]
im = im.unsqueeze(0)
return im
Visualization
@ -601,29 +584,29 @@ corresponding to the inferred labels.
from segmenter.segm.data.utils import dataset_cat_description, seg_to_rgb
from segmenter.segm.data.ade20k import ADE20K_CATS_PATH
def apply_segmentation_mask(pil_im: Image, results: torch.Tensor) -> Image:
"""
Combine segmentation masks with the image
:param pil_im: original input image
:param results: tensor containing segmentation masks for each pixel
:return:
pil_blend: image with colored segmentation masks overlay
"""
cat_names, cat_colors = dataset_cat_description(ADE20K_CATS_PATH)
# 3D array, where each pixel has values for all classes, take index of max as label
seg_map = results.argmax(0, keepdim=True)
# transform label id to colors
seg_rgb = seg_to_rgb(seg_map, cat_colors)
seg_rgb = (255 * seg_rgb.cpu().numpy()).astype(np.uint8)
pil_seg = Image.fromarray(seg_rgb[0])
# overlay segmentation mask over original image
pil_blend = Image.blend(pil_im, pil_seg, 0.5).convert("RGB")
return pil_blend
Validation of inference of original model
@ -637,15 +620,15 @@ example image ``coco_hollywood.jpg``.
.. code:: ipython3
from segmenter.segm.model.utils import inference
# load image with PIL
image = load_image("https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_hollywood.jpg")
# load_image reads the image in BGR format, [:,:,::-1] reshape transfroms it to RGB
pil_image = Image.fromarray(image[:,:,::-1])
# preprocess image with normalization params loaded in previous steps
image = preprocess(pil_image, normalization)
# inference function needs some meta parameters, where we specify that we don't flip images in inference mode
im_meta = dict(flip=False)
# perform inference with function from repository
@ -665,7 +648,7 @@ previous steps.
# combine segmentation mask with image
blended_image = apply_segmentation_mask(pil_image, original_results)
# show image with segmentation mask overlay
blended_image
@ -712,15 +695,15 @@ they are not a problem.
.. code:: ipython3
import openvino as ov
# get input sizes from config file
batch_size = 2
channels = 3
image_size = config["dataset_kwargs"]["image_size"]
# make dummy input with correct shapes obtained from config file
dummy_input = torch.randn(batch_size, channels, image_size, image_size)
model = ov.convert_model(pytorch_model, example_input=dummy_input, input=([batch_size, channels, image_size, image_size], ))
# serialize model for saving IR
ov.save_model(model, MODEL_DIR / "segmenter.xml")
@ -728,21 +711,21 @@ they are not a problem.
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:69: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:69: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if H % patch_size > 0:
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:71: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:71: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if W % patch_size > 0:
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/vit.py:122: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/vit.py:122: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if x.shape[1] != pos_embed.shape[1]:
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/decoder.py:100: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/decoder.py:100: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
masks = rearrange(masks, "b (h w) n -> b n h w", h=int(GS))
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if extra_h > 0:
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:87: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-632/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:87: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if extra_w > 0:
@ -763,7 +746,7 @@ any additional custom code required to process input.
class SegmenterOV:
"""
Class containing OpenVINO model with all attributes required to work with inference function.
:param model: compiled OpenVINO model
:type model: CompiledModel
:param output_blob: output blob used in inference
@ -774,14 +757,14 @@ any additional custom code required to process input.
:type n_cls: int
:param normalization:
:type normalization: dict
"""
def __init__(self, model_path: Path, device:str = "CPU"):
"""
Constructor method.
Initializes OpenVINO model and sets all required attributes
:param model_path: path to model's .xml file, also containing variant.yml
:param device: device string for selecting inference device
"""
@ -791,23 +774,23 @@ any additional custom code required to process input.
model_xml = core.read_model(model_path)
self.model = core.compile_model(model_xml, device)
self.output_blob = self.model.output(0)
# load model configs
variant_path = Path(model_path).parent / "variant.yml"
with open(variant_path, "r") as f:
self.config = yaml.load(f, Loader=yaml.FullLoader)
# load normalization specs from config
normalization_name = self.config["dataset_kwargs"]["normalization"]
self.normalization = STATS[normalization_name]
# load number of classes from config
self.n_cls = self.config["net_kwargs"]["n_cls"]
def forward(self, data: torch.Tensor) -> torch.Tensor:
"""
Perform inference on data and return the result in Tensor format
:param data: input data to model
:return: data inferred by model
"""
@ -826,7 +809,7 @@ select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
@ -834,7 +817,7 @@ select device from dropdown list for running inference using OpenVINO
description='Device:',
disabled=False,
)
device
@ -866,7 +849,7 @@ select device from dropdown list for running inference using OpenVINO
# combine segmentation mask with image
converted_blend = apply_segmentation_mask(pil_image, results)
# show image with segmentation mask overlay
converted_blend
@ -927,18 +910,22 @@ to measure the inference performance of the model.
[Step 2/11] Loading OpenVINO Runtime
[ WARNING ] Default duration 120 seconds is used for unknown device AUTO
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] AUTO
[ INFO ] Build ................................. 2023.3.0-13775-ceeafaf64f3-releases/2023/3
[ INFO ]
[ INFO ]
[ INFO ] Build ................................. 2024.0.0-14509-34caeefd078-releases/2024/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 23.09 ms
.. parsed-literal::
[ INFO ] Read model took 24.36 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] im (node: im) : f32 / [...] / [2,3,512,512]
@ -956,9 +943,13 @@ to measure the inference performance of the model.
.. parsed-literal::
[ INFO ] Compile model took 385.39 ms
[ INFO ] Compile model took 341.36 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
.. parsed-literal::
[ INFO ] NETWORK_NAME: Model0
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
@ -968,12 +959,15 @@ to measure the inference performance of the model.
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] CPU_DENORMALS_OPTIMIZATION: False
[ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
[ INFO ] DYNAMIC_QUANTIZATION_GROUP_SIZE: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] INFERENCE_NUM_THREADS: 24
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] KV_CACHE_PRECISION: <Type: 'float16'>
[ INFO ] LOG_LEVEL: Level.NO
[ INFO ] NETWORK_NAME: Model0
[ INFO ] NUM_STREAMS: 6
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 6
@ -985,30 +979,26 @@ to measure the inference performance of the model.
[ INFO ] LOADED_FROM_CACHE: False
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'im'!. This input will be filled with random values!
[ INFO ] Fill input 'im' with random values
.. parsed-literal::
[ INFO ] Fill input 'im' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 120000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
.. parsed-literal::
[ INFO ] First inference took 210.45 ms
[ INFO ] First inference took 203.29 ms
.. parsed-literal::
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 1686 iterations
[ INFO ] Duration: 120531.12 ms
[ INFO ] Count: 1728 iterations
[ INFO ] Duration: 120391.49 ms
[ INFO ] Latency:
[ INFO ] Median: 429.25 ms
[ INFO ] Average: 428.34 ms
[ INFO ] Min: 354.96 ms
[ INFO ] Max: 506.55 ms
[ INFO ] Throughput: 27.98 FPS
[ INFO ] Median: 416.77 ms
[ INFO ] Average: 417.75 ms
[ INFO ] Min: 333.14 ms
[ INFO ] Max: 505.65 ms
[ INFO ] Throughput: 28.71 FPS

499
docs/notebooks/205-vision-background-removal-with-output.rst
View File

@ -1,499 +0,0 @@
Image Background Removal with U^2-Net and OpenVINO™
===================================================
This notebook demonstrates background removal in images using
U\ :math:`^2`-Net and OpenVINO.
For more information about U\ :math:`^2`-Net, including source code and
test data, see the `GitHub
page <https://github.com/xuebinqin/U-2-Net>`__ and the research paper:
`U^2-Net: Going Deeper with Nested U-Structure for Salient Object
Detection <https://arxiv.org/pdf/2005.09007.pdf>`__.
The PyTorch U\ :math:`^2`-Net model is converted to OpenVINO IR format.
The model source is available
`here <https://github.com/xuebinqin/U-2-Net>`__.
Table of contents:
^^^^^^^^^^^^^^^^^^
- `Preparation <#preparation>`__
- `Install requirements <#install-requirements>`__
- `Import the PyTorch Library and U2-Net <#import-the-pytorch-library-and-u2-net>`__
- `Settings <#settings>`__
- `Load the U2-Net Model <#load-the-u2-net-model>`__
- `Convert PyTorch U2-Net model to OpenVINO IR <#convert-pytorch-u2-net-model-to-openvino-ir>`__
- `Load and Pre-Process Input
Image <#load-and-pre-process-input-image>`__
- `Select inference device <#select-inference-device>`__
- `Do Inference on OpenVINO IR
Model <#do-inference-on-openvino-ir-model>`__
- `Visualize Results <#visualize-results>`__
- `Add a Background Image <#add-a-background-image>`__
- `References <#references>`__
Preparation
-----------
Install requirements
~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
%pip install -q "openvino>=2023.1.0"
%pip install -q --extra-index-url https://download.pytorch.org/whl/cpu torch opencv-python matplotlib
%pip install -q "gdown<4.6.4"
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
.. parsed-literal::
Note: you may need to restart the kernel to use updated packages.
Import the PyTorch Library and U\ :math:`^2`-Net
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
import os
import time
import sys
from collections import namedtuple
from pathlib import Path
import cv2
import matplotlib.pyplot as plt
import numpy as np
import openvino as ov
import torch
from IPython.display import HTML, FileLink, display
.. code:: ipython3
# Import local modules
utils_file_path = Path("../utils/notebook_utils.py")
notebook_directory_path = Path(".")
if not utils_file_path.exists():
!git clone --depth 1 https://github.com/openvinotoolkit/openvino_notebooks.git
utils_file_path = Path("./openvino_notebooks/notebooks/utils/notebook_utils.py")
notebook_directory_path = Path("./openvino_notebooks/notebooks/205-vision-background-removal/")
sys.path.append(str(utils_file_path.parent))
sys.path.append(str(notebook_directory_path))
from notebook_utils import load_image
from model.u2net import U2NET, U2NETP
Settings
~~~~~~~~
This tutorial supports using the original U\ :math:`^2`-Net salient
object detection model, as well as the smaller U2NETP version. Two sets
of weights are supported for the original model: salient object
detection and human segmentation.
.. code:: ipython3
model_config = namedtuple("ModelConfig", ["name", "url", "model", "model_args"])
u2net_lite = model_config(
name="u2net_lite",
url="https://drive.google.com/uc?id=1rbSTGKAE-MTxBYHd-51l2hMOQPT_7EPy",
model=U2NETP,
model_args=(),
)
u2net = model_config(
name="u2net",
url="https://drive.google.com/uc?id=1ao1ovG1Qtx4b7EoskHXmi2E9rp5CHLcZ",
model=U2NET,
model_args=(3, 1),
)
u2net_human_seg = model_config(
name="u2net_human_seg",
url="https://drive.google.com/uc?id=1-Yg0cxgrNhHP-016FPdp902BR-kSsA4P",
model=U2NET,
model_args=(3, 1),
)
# Set u2net_model to one of the three configurations listed above.
u2net_model = u2net_lite
.. code:: ipython3
# The filenames of the downloaded and converted models.
MODEL_DIR = "model"
model_path = Path(MODEL_DIR) / u2net_model.name / Path(u2net_model.name).with_suffix(".pth")
Load the U\ :math:`^2`-Net Model
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The U\ :math:`^2`-Net human segmentation model weights are stored on
Google Drive. They will be downloaded if they are not present yet. The
next cell loads the model and the pre-trained weights.
.. code:: ipython3
if not model_path.exists():
import gdown
os.makedirs(name=model_path.parent, exist_ok=True)
print("Start downloading model weights file... ")
with open(model_path, "wb") as model_file:
gdown.download(url=u2net_model.url, output=model_file)
print(f"Model weights have been downloaded to {model_path}")
.. parsed-literal::
Start downloading model weights file...
.. parsed-literal::
Downloading...
From: https://drive.google.com/uc?id=1rbSTGKAE-MTxBYHd-51l2hMOQPT_7EPy
To: <_io.BufferedWriter name='model/u2net_lite/u2net_lite.pth'>
.. parsed-literal::
0%| | 0.00/4.68M [00:00<?, ?B/s]
.. parsed-literal::
11%|█ | 524k/4.68M [00:00<00:01, 3.35MB/s]
.. parsed-literal::
22%|██▏ | 1.05M/4.68M [00:00<00:00, 3.71MB/s]
.. parsed-literal::
34%|███▎ | 1.57M/4.68M [00:00<00:00, 3.78MB/s]
.. parsed-literal::
45%|████▍ | 2.10M/4.68M [00:00<00:00, 3.88MB/s]
.. parsed-literal::
56%|█████▌ | 2.62M/4.68M [00:00<00:00, 3.96MB/s]
.. parsed-literal::
67%|██████▋ | 3.15M/4.68M [00:00<00:00, 3.95MB/s]
.. parsed-literal::
78%|███████▊ | 3.67M/4.68M [00:00<00:00, 3.97MB/s]
.. parsed-literal::
90%|████████▉ | 4.19M/4.68M [00:01<00:00, 4.00MB/s]
.. parsed-literal::
100%|██████████| 4.68M/4.68M [00:01<00:00, 4.15MB/s]
.. parsed-literal::
100%|██████████| 4.68M/4.68M [00:01<00:00, 3.96MB/s]
.. parsed-literal::
Model weights have been downloaded to model/u2net_lite/u2net_lite.pth
.. code:: ipython3
# Load the model.
net = u2net_model.model(*u2net_model.model_args)
net.eval()
# Load the weights.
print(f"Loading model weights from: '{model_path}'")
net.load_state_dict(state_dict=torch.load(model_path, map_location="cpu"))
.. parsed-literal::
Loading model weights from: 'model/u2net_lite/u2net_lite.pth'
.. parsed-literal::
<All keys matched successfully>
Convert PyTorch U\ :math:`^2`-Net model to OpenVINO IR
------------------------------------------------------
We use model conversion Python API to convert the Pytorch model to
OpenVINO IR format. Executing the following command may take a while.
.. code:: ipython3
model_ir = ov.convert_model(net, example_input=torch.zeros((1,3,512,512)), input=([1, 3, 512, 512]))
.. parsed-literal::
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-609/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/functional.py:3769: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.
warnings.warn("nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.")
Load and Pre-Process Input Image
--------------------------------
While OpenCV reads images in ``BGR`` format, the OpenVINO IR model
expects images in ``RGB``. Therefore, convert the images to ``RGB``,
resize them to ``512 x 512``, and transpose the dimensions to the format
the OpenVINO IR model expects.
We add the mean values to the image tensor and scale the input with the
standard deviation. It is called the input data normalization before
propagating it through the network. The mean and standard deviation
values can be found in the
`dataloader <https://github.com/xuebinqin/U-2-Net/blob/master/data_loader.py>`__
file in the `U^2-Net
repository <https://github.com/xuebinqin/U-2-Net/>`__ and multiplied by
255 to support images with pixel values from 0-255.
.. code:: ipython3
IMAGE_URI = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_hollywood.jpg"
input_mean = np.array([123.675, 116.28 , 103.53]).reshape(1, 3, 1, 1)
input_scale = np.array([58.395, 57.12 , 57.375]).reshape(1, 3, 1, 1)
image = cv2.cvtColor(
src=load_image(IMAGE_URI),
code=cv2.COLOR_BGR2RGB,
)
resized_image = cv2.resize(src=image, dsize=(512, 512))
# Convert the image shape to a shape and a data type expected by the network
# for OpenVINO IR model: (1, 3, 512, 512).
input_image = np.expand_dims(np.transpose(resized_image, (2, 0, 1)), 0)
input_image = (input_image - input_mean) / input_scale
Select inference device
-----------------------
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = ov.Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
Do Inference on OpenVINO IR Model
---------------------------------
Load the OpenVINO IR model to OpenVINO Runtime and do inference.
.. code:: ipython3
core = ov.Core()
# Load the network to OpenVINO Runtime.
compiled_model_ir = core.compile_model(model=model_ir, device_name=device.value)
# Get the names of input and output layers.
input_layer_ir = compiled_model_ir.input(0)
output_layer_ir = compiled_model_ir.output(0)
# Do inference on the input image.
start_time = time.perf_counter()
result = compiled_model_ir([input_image])[output_layer_ir]
end_time = time.perf_counter()
print(
f"Inference finished. Inference time: {end_time-start_time:.3f} seconds, "
f"FPS: {1/(end_time-start_time):.2f}."
)
.. parsed-literal::
Inference finished. Inference time: 0.110 seconds, FPS: 9.05.
Visualize Results
-----------------
Show the original image, the segmentation result, and the original image
with the background removed.
.. code:: ipython3
# Resize the network result to the image shape and round the values
# to 0 (background) and 1 (foreground).
# The network result has (1,1,512,512) shape. The `np.squeeze` function converts this to (512, 512).
resized_result = np.rint(
cv2.resize(src=np.squeeze(result), dsize=(image.shape[1], image.shape[0]))
).astype(np.uint8)
# Create a copy of the image and set all background values to 255 (white).
bg_removed_result = image.copy()
bg_removed_result[resized_result == 0] = 255
fig, ax = plt.subplots(nrows=1, ncols=3, figsize=(20, 7))
ax[0].imshow(image)
ax[1].imshow(resized_result, cmap="gray")
ax[2].imshow(bg_removed_result)
for a in ax:
a.axis("off")
.. image:: 205-vision-background-removal-with-output_files/205-vision-background-removal-with-output_22_0.png
Add a Background Image
~~~~~~~~~~~~~~~~~~~~~~
In the segmentation result, all foreground pixels have a value of 1, all
background pixels a value of 0. Replace the background image as follows:
- Load a new ``background_image``.
- Resize the image to the same size as the original image.
- In ``background_image``, set all the pixels, where the resized
segmentation result has a value of 1 - the foreground pixels in the
original image - to 0.
- Add ``bg_removed_result`` from the previous step - the part of the
original image that only contains foreground pixels - to
``background_image``.
.. code:: ipython3
BACKGROUND_FILE = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/wall.jpg"
OUTPUT_DIR = "output"
os.makedirs(name=OUTPUT_DIR, exist_ok=True)
background_image = cv2.cvtColor(src=load_image(BACKGROUND_FILE), code=cv2.COLOR_BGR2RGB)
background_image = cv2.resize(src=background_image, dsize=(image.shape[1], image.shape[0]))
# Set all the foreground pixels from the result to 0
# in the background image and add the image with the background removed.
background_image[resized_result == 1] = 0
new_image = background_image + bg_removed_result
# Save the generated image.
new_image_path = Path(f"{OUTPUT_DIR}/{Path(IMAGE_URI).stem}-{Path(BACKGROUND_FILE).stem}.jpg")
cv2.imwrite(filename=str(new_image_path), img=cv2.cvtColor(new_image, cv2.COLOR_RGB2BGR))
# Display the original image and the image with the new background side by side
fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(18, 7))
ax[0].imshow(image)
ax[1].imshow(new_image)
for a in ax:
a.axis("off")
plt.show()
# Create a link to download the image.
image_link = FileLink(new_image_path)
image_link.html_link_str = "<a href='%s' download>%s</a>"
display(
HTML(
f"The generated image <code>{new_image_path.name}</code> is saved in "
f"the directory <code>{new_image_path.parent}</code>. You can also "
"download the image by clicking on this link: "
f"{image_link._repr_html_()}"
)
)
.. image:: 205-vision-background-removal-with-output_files/205-vision-background-removal-with-output_24_0.png
.. raw:: html
The generated image <code>coco_hollywood-wall.jpg</code> is saved in the directory <code>output</code>. You can also download the image by clicking on this link: output/coco_hollywood-wall.jpg<br>
References
----------
- `PIP install
openvino-dev <https://github.com/openvinotoolkit/openvino/blob/releases/2023/2/docs/install_guides/pypi-openvino-dev.md>`__
- `Model Conversion
API <https://docs.openvino.ai/2024/openvino-workflow/model-preparation.html>`__
- `U^2-Net <https://github.com/xuebinqin/U-2-Net>`__
- U^2-Net research paper: `U^2-Net: Going Deeper with Nested
U-Structure for Salient Object
Detection <https://arxiv.org/pdf/2005.09007.pdf>`__

Some files were not shown because too many files have changed in this diff Show More