From f2017e8c2ef227166d3eaac3b1363218dfd40102 Mon Sep 17 00:00:00 2001 From: OK Date: Wed, 31 May 2023 16:38:57 +0300 Subject: [PATCH] Validation: Test AUTO plugin timetests (#17508) * Validation: Test AUTO plugin * config * Add debug logs against AUTO * Iteration * iteration * iteration * iteration * iter * iteration * iteration * iteration * iteration --------- Co-authored-by: Daria Ilina --- .../.automation/auto_plugin_test_config.yml | 673 ++++++++++++++++++ tests/time_tests/scripts/run_timetest.py | 5 +- .../src/timetests/timetest_infer.cpp | 6 +- .../src/timetests/timetest_infer_api_2.cpp | 7 +- 4 files changed, 685 insertions(+), 6 deletions(-) create mode 100644 tests/time_tests/.automation/auto_plugin_test_config.yml diff --git a/tests/time_tests/.automation/auto_plugin_test_config.yml b/tests/time_tests/.automation/auto_plugin_test_config.yml new file mode 100644 index 00000000000..27cb4f05c74 --- /dev/null +++ b/tests/time_tests/.automation/auto_plugin_test_config.yml @@ -0,0 +1,673 @@ +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/onnx/FP16/resnet-50-pytorch.xml + name: resnet-50-pytorch + precision: FP16 + framework: onnx +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/onnx/FP16/resnet-50-pytorch.xml + name: resnet-50-pytorch + precision: FP16 + framework: onnx +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/onnx/FP16/resnet-50-pytorch.xml + name: resnet-50-pytorch + precision: FP16 + framework: onnx +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/onnx/FP16-INT8/resnet-50-pytorch.xml + name: resnet-50-pytorch + precision: FP16-INT8 + framework: onnx +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/onnx/FP16-INT8/resnet-50-pytorch.xml + name: resnet-50-pytorch + precision: FP16-INT8 + framework: onnx +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/onnx/FP16-INT8/resnet-50-pytorch.xml + name: resnet-50-pytorch + precision: FP16-INT8 + framework: onnx +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe/FP16/mobilenet-v2.xml + name: mobilenet-v2 + precision: FP16 + framework: caffe +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe/FP16/mobilenet-v2.xml + name: mobilenet-v2 + precision: FP16 + framework: caffe +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe/FP16/mobilenet-v2.xml + name: mobilenet-v2 + precision: FP16 + framework: caffe +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe/FP16-INT8/mobilenet-v2.xml + name: mobilenet-v2 + precision: FP16-INT8 + framework: caffe +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe/FP16-INT8/mobilenet-v2.xml + name: mobilenet-v2 + precision: FP16-INT8 + framework: caffe +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe/FP16-INT8/mobilenet-v2.xml + name: mobilenet-v2 + precision: FP16-INT8 + framework: caffe +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/faster-rcnn-resnet101-coco-sparse-60-0001/tf/FP16/faster-rcnn-resnet101-coco-sparse-60-0001.xml + name: faster-rcnn-resnet101-coco-sparse-60-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/faster-rcnn-resnet101-coco-sparse-60-0001/tf/FP16/faster-rcnn-resnet101-coco-sparse-60-0001.xml + name: faster-rcnn-resnet101-coco-sparse-60-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/faster-rcnn-resnet101-coco-sparse-60-0001/tf/FP16/faster-rcnn-resnet101-coco-sparse-60-0001.xml + name: faster-rcnn-resnet101-coco-sparse-60-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/faster-rcnn-resnet101-coco-sparse-60-0001/tf/FP16-INT8/faster-rcnn-resnet101-coco-sparse-60-0001.xml + name: faster-rcnn-resnet101-coco-sparse-60-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/faster-rcnn-resnet101-coco-sparse-60-0001/tf/FP16-INT8/faster-rcnn-resnet101-coco-sparse-60-0001.xml + name: faster-rcnn-resnet101-coco-sparse-60-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/faster-rcnn-resnet101-coco-sparse-60-0001/tf/FP16-INT8/faster-rcnn-resnet101-coco-sparse-60-0001.xml + name: faster-rcnn-resnet101-coco-sparse-60-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/googlenet-v1/tf/FP16/googlenet-v1.xml + name: googlenet-v1 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/googlenet-v1/tf/FP16/googlenet-v1.xml + name: googlenet-v1 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/googlenet-v1/tf/FP16/googlenet-v1.xml + name: googlenet-v1 + precision: FP16 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/googlenet-v1/tf/FP16-INT8/googlenet-v1.xml + name: googlenet-v1 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/googlenet-v1/tf/FP16-INT8/googlenet-v1.xml + name: googlenet-v1 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/googlenet-v1/tf/FP16-INT8/googlenet-v1.xml + name: googlenet-v1 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/googlenet-v3/tf/FP16/googlenet-v3.xml + name: googlenet-v3 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/googlenet-v3/tf/FP16/googlenet-v3.xml + name: googlenet-v3 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/googlenet-v3/tf/FP16/googlenet-v3.xml + name: googlenet-v3 + precision: FP16 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/googlenet-v3/tf/FP16-INT8/googlenet-v3.xml + name: googlenet-v3 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/googlenet-v3/tf/FP16-INT8/googlenet-v3.xml + name: googlenet-v3 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/googlenet-v3/tf/FP16-INT8/googlenet-v3.xml + name: googlenet-v3 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/ssd512/caffe/FP16/ssd512.xml + name: ssd512 + precision: FP16 + framework: caffe +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/ssd512/caffe/FP16/ssd512.xml + name: ssd512 + precision: FP16 + framework: caffe +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/ssd512/caffe/FP16/ssd512.xml + name: ssd512 + precision: FP16 + framework: caffe +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/ssd512/caffe/FP16-INT8/ssd512.xml + name: ssd512 + precision: FP16-INT8 + framework: caffe +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/ssd512/caffe/FP16-INT8/ssd512.xml + name: ssd512 + precision: FP16-INT8 + framework: caffe +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/ssd512/caffe/FP16-INT8/ssd512.xml + name: ssd512 + precision: FP16-INT8 + framework: caffe +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-0001/tf/FP16/yolo-v2-ava-0001.xml + name: yolo-v2-ava-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-0001/tf/FP16/yolo-v2-ava-0001.xml + name: yolo-v2-ava-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-0001/tf/FP16/yolo-v2-ava-0001.xml + name: yolo-v2-ava-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-0001/tf/FP16-INT8/yolo-v2-ava-0001.xml + name: yolo-v2-ava-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-0001/tf/FP16-INT8/yolo-v2-ava-0001.xml + name: yolo-v2-ava-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-0001/tf/FP16-INT8/yolo-v2-ava-0001.xml + name: yolo-v2-ava-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-sparse-35-0001/tf/FP16/yolo-v2-ava-sparse-35-0001.xml + name: yolo-v2-ava-sparse-35-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-sparse-35-0001/tf/FP16/yolo-v2-ava-sparse-35-0001.xml + name: yolo-v2-ava-sparse-35-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-sparse-35-0001/tf/FP16/yolo-v2-ava-sparse-35-0001.xml + name: yolo-v2-ava-sparse-35-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-sparse-35-0001/tf/FP16-INT8/yolo-v2-ava-sparse-35-0001.xml + name: yolo-v2-ava-sparse-35-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-sparse-35-0001/tf/FP16-INT8/yolo-v2-ava-sparse-35-0001.xml + name: yolo-v2-ava-sparse-35-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-sparse-35-0001/tf/FP16-INT8/yolo-v2-ava-sparse-35-0001.xml + name: yolo-v2-ava-sparse-35-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-sparse-70-0001/tf/FP16/yolo-v2-ava-sparse-70-0001.xml + name: yolo-v2-ava-sparse-70-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-sparse-70-0001/tf/FP16/yolo-v2-ava-sparse-70-0001.xml + name: yolo-v2-ava-sparse-70-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-sparse-70-0001/tf/FP16/yolo-v2-ava-sparse-70-0001.xml + name: yolo-v2-ava-sparse-70-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-sparse-70-0001/tf/FP16-INT8/yolo-v2-ava-sparse-70-0001.xml + name: yolo-v2-ava-sparse-70-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-sparse-70-0001/tf/FP16-INT8/yolo-v2-ava-sparse-70-0001.xml + name: yolo-v2-ava-sparse-70-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-ava-sparse-70-0001/tf/FP16-INT8/yolo-v2-ava-sparse-70-0001.xml + name: yolo-v2-ava-sparse-70-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-0001/tf/FP16/yolo-v2-tiny-ava-0001.xml + name: yolo-v2-tiny-ava-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-0001/tf/FP16/yolo-v2-tiny-ava-0001.xml + name: yolo-v2-tiny-ava-0001 + precision: FP16 + framework: tf +- device: + name: AUTO + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-0001/tf/FP16/yolo-v2-tiny-ava-0001.xml + name: yolo-v2-tiny-ava-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-0001/tf/FP16-INT8/yolo-v2-tiny-ava-0001.xml + name: yolo-v2-tiny-ava-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-0001/tf/FP16-INT8/yolo-v2-tiny-ava-0001.xml + name: yolo-v2-tiny-ava-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-0001/tf/FP16-INT8/yolo-v2-tiny-ava-0001.xml + name: yolo-v2-tiny-ava-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-sparse-30-0001/tf/FP16/yolo-v2-tiny-ava-sparse-30-0001.xml + name: yolo-v2-tiny-ava-sparse-30-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-sparse-30-0001/tf/FP16/yolo-v2-tiny-ava-sparse-30-0001.xml + name: yolo-v2-tiny-ava-sparse-30-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-sparse-30-0001/tf/FP16/yolo-v2-tiny-ava-sparse-30-0001.xml + name: yolo-v2-tiny-ava-sparse-30-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-sparse-30-0001/tf/FP16-INT8/yolo-v2-tiny-ava-sparse-30-0001.xml + name: yolo-v2-tiny-ava-sparse-30-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-sparse-30-0001/tf/FP16-INT8/yolo-v2-tiny-ava-sparse-30-0001.xml + name: yolo-v2-tiny-ava-sparse-30-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-sparse-30-0001/tf/FP16-INT8/yolo-v2-tiny-ava-sparse-30-0001.xml + name: yolo-v2-tiny-ava-sparse-30-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-sparse-60-0001/tf/FP16/yolo-v2-tiny-ava-sparse-60-0001.xml + name: yolo-v2-tiny-ava-sparse-60-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-sparse-60-0001/tf/FP16/yolo-v2-tiny-ava-sparse-60-0001.xml + name: yolo-v2-tiny-ava-sparse-60-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-sparse-60-0001/tf/FP16/yolo-v2-tiny-ava-sparse-60-0001.xml + name: yolo-v2-tiny-ava-sparse-60-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-sparse-60-0001/tf/FP16-INT8/yolo-v2-tiny-ava-sparse-60-0001.xml + name: yolo-v2-tiny-ava-sparse-60-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-sparse-60-0001/tf/FP16-INT8/yolo-v2-tiny-ava-sparse-60-0001.xml + name: yolo-v2-tiny-ava-sparse-60-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/yolo-v2-tiny-ava-sparse-60-0001/tf/FP16-INT8/yolo-v2-tiny-ava-sparse-60-0001.xml + name: yolo-v2-tiny-ava-sparse-60-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/squeezenet1.1/caffe/FP16/squeezenet1.1.xml + name: squeezenet1.1 + precision: FP16 + framework: caffe +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/squeezenet1.1/caffe/FP16/squeezenet1.1.xml + name: squeezenet1.1 + precision: FP16 + framework: caffe +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/squeezenet1.1/caffe/FP16/squeezenet1.1.xml + name: squeezenet1.1 + precision: FP16 + framework: caffe +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/squeezenet1.1/caffe/FP16-INT8/squeezenet1.1.xml + name: squeezenet1.1 + precision: FP16-INT8 + framework: caffe +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/squeezenet1.1/caffe/FP16-INT8/squeezenet1.1.xml + name: squeezenet1.1 + precision: FP16-INT8 + framework: caffe +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/squeezenet1.1/caffe/FP16-INT8/squeezenet1.1.xml + name: squeezenet1.1 + precision: FP16-INT8 + framework: caffe +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-0001/tf/FP16/icnet-camvid-ava-0001.xml + name: icnet-camvid-ava-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-0001/tf/FP16/icnet-camvid-ava-0001.xml + name: icnet-camvid-ava-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-0001/tf/FP16/icnet-camvid-ava-0001.xml + name: icnet-camvid-ava-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-0001/tf/FP16-INT8/icnet-camvid-ava-0001.xml + name: icnet-camvid-ava-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-0001/tf/FP16-INT8/icnet-camvid-ava-0001.xml + name: icnet-camvid-ava-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-0001/tf/FP16-INT8/icnet-camvid-ava-0001.xml + name: icnet-camvid-ava-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-sparse-30-0001/tf/FP16/icnet-camvid-ava-sparse-30-0001.xml + name: icnet-camvid-ava-sparse-30-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-sparse-30-0001/tf/FP16/icnet-camvid-ava-sparse-30-0001.xml + name: icnet-camvid-ava-sparse-30-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-sparse-30-0001/tf/FP16/icnet-camvid-ava-sparse-30-0001.xml + name: icnet-camvid-ava-sparse-30-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-sparse-30-0001/tf/FP16-INT8/icnet-camvid-ava-sparse-30-0001.xml + name: icnet-camvid-ava-sparse-30-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-sparse-30-0001/tf/FP16-INT8/icnet-camvid-ava-sparse-30-0001.xml + name: icnet-camvid-ava-sparse-30-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-sparse-30-0001/tf/FP16-INT8/icnet-camvid-ava-sparse-30-0001.xml + name: icnet-camvid-ava-sparse-30-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-sparse-60-0001/tf/FP16/icnet-camvid-ava-sparse-60-0001.xml + name: icnet-camvid-ava-sparse-60-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-sparse-60-0001/tf/FP16/icnet-camvid-ava-sparse-60-0001.xml + name: icnet-camvid-ava-sparse-60-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-sparse-60-0001/tf/FP16/icnet-camvid-ava-sparse-60-0001.xml + name: icnet-camvid-ava-sparse-60-0001 + precision: FP16 + framework: tf +- device: + name: AUTO:CPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-sparse-60-0001/tf/FP16-INT8/icnet-camvid-ava-sparse-60-0001.xml + name: icnet-camvid-ava-sparse-60-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-sparse-60-0001/tf/FP16-INT8/icnet-camvid-ava-sparse-60-0001.xml + name: icnet-camvid-ava-sparse-60-0001 + precision: FP16-INT8 + framework: tf +- device: + name: AUTO:GPU,CPU + model: + path: ${VPUX_MODELS_PKG}/icnet-camvid-ava-sparse-60-0001/tf/FP16-INT8/icnet-camvid-ava-sparse-60-0001.xml + name: icnet-camvid-ava-sparse-60-0001 + precision: FP16-INT8 + framework: tf diff --git a/tests/time_tests/scripts/run_timetest.py b/tests/time_tests/scripts/run_timetest.py index 24795f790e4..92a809a4d69 100644 --- a/tests/time_tests/scripts/run_timetest.py +++ b/tests/time_tests/scripts/run_timetest.py @@ -73,13 +73,14 @@ def run_timetest(args: dict, log=None): log = logging.getLogger("run_timetest") cmd_common = prepare_executable_cmd(args) - + ov_env = os.environ + ov_env['OPENVINO_LOG_LEVEL'] = '4' # Run executable and collect statistics stats = {} logs = [] for run_iter in range(args["niter"]): tmp_stats_path = tempfile.NamedTemporaryFile().name - retcode, msg = cmd_exec(cmd_common + ["-s", str(tmp_stats_path)], log=log) + retcode, msg = cmd_exec(cmd_common + ["-s", str(tmp_stats_path)], log=log, env=ov_env) if os.path.exists(tmp_stats_path): with open(tmp_stats_path, "r") as file: diff --git a/tests/time_tests/src/timetests/timetest_infer.cpp b/tests/time_tests/src/timetests/timetest_infer.cpp index 6bd5ae77a9b..a0c879133d7 100644 --- a/tests/time_tests/src/timetests/timetest_infer.cpp +++ b/tests/time_tests/src/timetests/timetest_infer.cpp @@ -27,6 +27,8 @@ int runPipeline(const std::string &model, const std::string &device, const bool InferenceEngine::InferRequest inferRequest; size_t batchSize = 0; + std::string device_prefix = device.substr(0, device.find(':')); + // first_inference_latency = time_to_inference + first_inference { SCOPED_TIMER(first_inference_latency); @@ -34,8 +36,8 @@ int runPipeline(const std::string &model, const std::string &device, const bool SCOPED_TIMER(time_to_inference); { SCOPED_TIMER(load_plugin); - TimeTest::setPerformanceConfig(ie, device); - ie.GetVersions(device); + TimeTest::setPerformanceConfig(ie, device_prefix); + ie.GetVersions(device_prefix); if (isCacheEnabled) ie.SetConfig({ {CONFIG_KEY(CACHE_DIR), "models_cache"} }); diff --git a/tests/time_tests/src/timetests/timetest_infer_api_2.cpp b/tests/time_tests/src/timetests/timetest_infer_api_2.cpp index ee63b35365a..77b997e0d6a 100644 --- a/tests/time_tests/src/timetests/timetest_infer_api_2.cpp +++ b/tests/time_tests/src/timetests/timetest_infer_api_2.cpp @@ -31,6 +31,9 @@ int runPipeline(const std::string &model, const std::string &device, const bool std::vector> defaultInputs; + ie.set_property("AUTO", ov::log::level(ov::log::Level::DEBUG)); + std::string device_prefix = device.substr(0, device.find(':')); + bool reshape = false; if (!reshapeShapes.empty()) { reshape = true; @@ -51,8 +54,8 @@ int runPipeline(const std::string &model, const std::string &device, const bool SCOPED_TIMER(time_to_inference); { SCOPED_TIMER(load_plugin); - TimeTest::setPerformanceConfig(ie, device); - ie.get_versions(device); + TimeTest::setPerformanceConfig(ie, device_prefix); + ie.get_versions(device_prefix); if (isCacheEnabled) ie.set_property({{CONFIG_KEY(CACHE_DIR), "models_cache"}});