natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Test acutal model (#22841) 

Ticket 25942
This commit is contained in:
Zlobin Vladimir 2024-03-06 15:12:39 +04:00 committed by GitHub
parent 2007b7b1b0
commit 2d3805f6db
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 101 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/job_samples_tests.yml vendored
View File

@ -124,7 +124,7 @@ jobs:
source ${INSTALL_DIR}/setupvars.sh
PYTHONCOERCECLOCALE=warn python3 -bb -W error -X dev -X warn_default_encoding -m pytest $INSTALL_TEST_DIR/smoke_tests \
PYTHONCOERCECLOCALE=warn python3 -bb -W error -X dev -m pytest $INSTALL_TEST_DIR/smoke_tests \
--junitxml=$INSTALL_TEST_DIR/TEST-SamplesSmokeTests.xml
- name: Upload Test Results

13
samples/cpp/classification_sample_async/main.cpp
View File

@ -30,16 +30,9 @@
#include "classification_sample_async.h"
// clang-format on
constexpr auto N_TOP_RESULTS = 10;
using namespace ov::preprocess;
/**
* @brief Checks input args
* @param argc number of args
* @param argv list of input arguments
* @return bool status true(Success) or false(Fail)
*/
namespace {
bool parse_and_check_command_line(int argc, char* argv[]) {
gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true);
if (FLAGS_h) {
@ -61,6 +54,7 @@ bool parse_and_check_command_line(int argc, char* argv[]) {
return true;
}
} // namespace
int main(int argc, char* argv[]) {
try {
@ -100,7 +94,7 @@ int main(int argc, char* argv[]) {
// - precision of tensor is supposed to be 'u8'
// - layout of data is 'NHWC'
input_info.tensor().set_element_type(ov::element::u8).set_layout(tensor_layout);
// 3) Here we suppose model has 'NCHW' layout for input
// 3) Suppose model has 'NCHW' layout for input
input_info.model().set_layout("NCHW");
// 4) output() with no args assumes a model has a single result
// - output() with no args assumes a model has a single result
@ -222,6 +216,7 @@ int main(int argc, char* argv[]) {
}
// Prints formatted classification results
constexpr size_t N_TOP_RESULTS = 10;
ClassificationResult classificationResult(output, valid_image_names, batchSize, N_TOP_RESULTS, labels);
classificationResult.show();
} catch (const std::exception& ex) {

2
samples/cpp/hello_classification/main.cpp
View File

@ -82,7 +82,7 @@ int tmain(int argc, tchar* argv[]) {
// - convert layout to 'NCHW' (from 'NHWC' specified above at tensor layout)
// - apply linear resize from tensor spatial dims to model spatial dims
ppp.input().preprocess().resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR);
// 4) Here we suppose model has 'NCHW' layout for input
// 4) Suppose model has 'NCHW' layout for input
ppp.input().model().set_layout("NCHW");
// 5) Set output tensor information:
// - precision of tensor is supposed to be 'f32'

26
samples/python/classification_sample_async/classification_sample_async.py
View File

@ -75,18 +75,7 @@ def main() -> int:
log.error('Sample supports only single output topologies')
return -1
# --------------------------- Step 3. Set up input --------------------------------------------------------------------
# Read input images
images = [cv2.imread(image_path) for image_path in args.input]
# Resize images to model input dims
_, _, h, w = model.input().shape
resized_images = [cv2.resize(image, (w, h)) for image in images]
# Add N dimension
input_tensors = [np.expand_dims(image, 0) for image in resized_images]
# --------------------------- Step 4. Apply preprocessing -------------------------------------------------------------
# --------------------------- Step 3. Apply preprocessing -------------------------------------------------------------
ppp = ov.preprocess.PrePostProcessor(model)
# 1) Set input tensor information:
@ -97,7 +86,7 @@ def main() -> int:
.set_element_type(ov.Type.u8) \
.set_layout(ov.Layout('NHWC')) # noqa: N400
# 2) Here we suppose model has 'NCHW' layout for input
# 2) Suppose model has 'NCHW' layout for input
ppp.input().model().set_layout(ov.Layout('NCHW'))
# 3) Set output tensor information:
@ -107,6 +96,17 @@ def main() -> int:
# 4) Apply preprocessing modifing the original 'model'
model = ppp.build()
# --------------------------- Step 4. Set up input --------------------------------------------------------------------
# Read input images
images = (cv2.imread(image_path) for image_path in args.input)
# Resize images to model input dims
_, h, w, _ = model.input().shape
resized_images = (cv2.resize(image, (w, h)) for image in images)
# Add N dimension
input_tensors = (np.expand_dims(image, 0) for image in resized_images)
# --------------------------- Step 5. Loading model to the device -----------------------------------------------------
log.info('Loading the model to the plugin')
compiled_model = core.compile_model(model, args.device)

2
tests/samples_tests/smoke_tests/common/common_utils.py
View File

@ -27,7 +27,7 @@ def shell(cmd, env=None, cwd=None, out_format="plain"):
else:
cmd = " ".join(cmd)
sys.stdout.write("Running command:\n" + "".join(cmd) + "\n")
sys.stdout.write("Running command:\n" + " ".join(cmd) + "\n")
p = subprocess.Popen(cmd, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(stdout, stderr) = p.communicate()
stdout = str(stdout.decode('utf-8'))

40
tests/samples_tests/smoke_tests/common/samples_common_test_class.py
View File

@ -10,6 +10,7 @@
See the License for the specific language governing permissions and
limitations under the License.
"""
import cv2
import contextlib
import io
import os
@ -55,9 +56,19 @@ def download(test_data_dir, file_path):
with contextlib.suppress(FileExistsError, PermissionError):
with lock_path.open('bx'):
if not file_path.exists():
response = requests.get("https://storage.openvinotoolkit.org/repositories/openvino/ci_dependencies/test/2021.4/samples_smoke_tests_data_2021.4.zip")
with zipfile.ZipFile(io.BytesIO(response.content)) as zfile:
zfile.extractall(test_data_dir)
if test_data_dir / 'bvlcalexnet-12.onnx' == file_path:
response = requests.get("https://github.com/onnx/models/raw/main/validated/vision/classification/alexnet/model/bvlcalexnet-12.onnx?download=")
with file_path.open('wb') as nfnet:
nfnet.write(response.content)
elif test_data_dir / 'efficientnet-lite4-11-qdq.onnx' == file_path:
response = requests.get("https://github.com/onnx/models/raw/main/validated/vision/classification/efficientnet-lite4/model/efficientnet-lite4-11-qdq.onnx?download=")
with file_path.open('wb') as nfnet:
nfnet.write(response.content)
else:
response = requests.get("https://storage.openvinotoolkit.org/repositories/openvino/ci_dependencies/test/2021.4/samples_smoke_tests_data_2021.4.zip")
with zipfile.ZipFile(io.BytesIO(response.content)) as zfile:
zfile.extractall(test_data_dir)
cv2.imwrite(str(test_data_dir / 'dog-224x224.bmp'), cv2.resize(cv2.imread(str(test_data_dir / 'samples_smoke_tests_data_2021.4/validation_set/227x227/dog.bmp')), (224, 224)))
lock_path.unlink(missing_ok=True)
assert file_path.exists()
return file_path
@ -65,25 +76,14 @@ def download(test_data_dir, file_path):
def prepend(cache, inp='', model=''):
test_data_dir = cache.mkdir('test_data_dir')
unpacked = test_data_dir / 'samples_smoke_tests_data_2021.4'
test_data_dir = cache.mkdir('test_data')
if inp:
inp = '-i', download(test_data_dir, unpacked / 'validation_set' / inp)
inp = '-i', download(test_data_dir, test_data_dir / inp)
if model:
model = '-m', download(test_data_dir, unpacked / 'models' / 'public' / model)
model = '-m', download(test_data_dir, test_data_dir / model)
return *inp, *model
class Environment:
"""
Environment used by tests.
:attr env: environment dictionary. populated dynamically from environment
configuration file.
"""
env = {}
def get_tests(cmd_params, use_device=True, use_batch=False):
# Several keys:
# use_device
@ -171,7 +171,7 @@ class SamplesCommonTestClass():
@staticmethod
def join_env_path(param, cache, executable_path, complete_path=True):
test_data_dir = cache.mkdir('test_data_dir')
test_data_dir = cache.mkdir('test_data')
unpacked = test_data_dir / 'samples_smoke_tests_data_2021.4'
if 'i' in param:
# If batch > 1, then concatenate images
@ -181,10 +181,10 @@ class SamplesCommonTestClass():
param['i'] = list([param['i']])
for k in param.keys():
if ('i' == k) and complete_path:
param['i'] = [str(download(test_data_dir, unpacked / 'validation_set' / e)) for e in param['i']]
param['i'] = [str(download(test_data_dir, test_data_dir / e)) for e in param['i']]
param['i'] = ' '.join(map(str, param['i']))
elif 'm' == k and not param['m'].endswith('/samples/cpp/model_creation_sample/lenet.bin"'):
param['m'] = download(test_data_dir, unpacked / 'models' / 'public' / param['m'])
param['m'] = download(test_data_dir, test_data_dir / param['m'])
@staticmethod
def get_cmd_line(param, use_preffix=True, long_hyphen=None):

8
tests/samples_tests/smoke_tests/test_benchmark_app.py
View File

@ -26,10 +26,10 @@ def get_executable(sample_language):
return 'benchmark_app'
def verify(sample_language, device, api=None, nireq=None, shape=None, data_shape=None, nstreams=None, layout=None, pin=None, cache=None, tmp_path=None):
def verify(sample_language, device, api=None, nireq=None, shape=None, data_shape=None, nstreams=None, layout=None, pin=None, cache=None, tmp_path=None, model='bvlcalexnet-12.onnx'):
output = get_cmd_output(
get_executable(sample_language),
*prepend(cache, '227x227/dog.bmp', 'squeezenet1.1/FP32/squeezenet1.1.xml'),
*prepend(cache, 'dog-224x224.bmp', model),
*('-nstreams', nstreams) if nstreams else '',
*('-layout', layout) if layout else '',
*('-nireq', nireq) if nireq else '',
@ -92,10 +92,10 @@ def test_api(sample_language, api, device, cache, tmp_path):
@pytest.mark.parametrize('sample_language', ['C++', 'Python'])
@pytest.mark.parametrize('device', get_devices())
def test_reshape(sample_language, device, cache):
verify(sample_language, device, shape='data[2,3,227,227]', cache=cache)
verify(sample_language, device, shape='data_0[2,3,224,224]', cache=cache)
@pytest.mark.parametrize('sample_language', ['C++', 'Python'])
@pytest.mark.parametrize('device', get_devices())
def test_dynamic_shape(sample_language, device, cache):
verify(sample_language, device, shape='[?,3,?,?]', data_shape='[1,3,227,227][1,3,227,227]', layout='[NCHW]', cache=cache)
verify(sample_language, device, model='efficientnet-lite4-11-qdq.onnx', shape='[?,224,224,3]', data_shape='[1,224,224,3][2,224,224,3]', layout='[NHWC]', cache=cache)

23
tests/samples_tests/smoke_tests/test_classification_sample_async.py
View File

@ -10,7 +10,6 @@
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import pytest
import re
import sys
@ -20,19 +19,17 @@ from common.samples_common_test_class import get_tests
log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
test_data_fp32 = get_tests(cmd_params={'i': [os.path.join('227x227', 'dog.bmp')],
'm': [os.path.join('squeezenet1.1', 'FP32', 'squeezenet1.1.xml')],
'sample_type': ['C++','Python'],
'batch': [1, 2, 4],
},
)
test_data_fp32 = get_tests({
'i': ['dog-224x224.bmp'],
'm': ['bvlcalexnet-12.onnx'], # Remove the model forom .md and .rst if removed from here
'sample_type': ['C++','Python'],
})
test_data_fp16 = get_tests(cmd_params={'i': [os.path.join('227x227', 'dog.bmp')],
'm': [os.path.join('squeezenet1.1', 'FP16', 'squeezenet1.1.xml')],
'sample_type': ['C++','Python'],
'batch': [1, 2, 4],
},
)
test_data_fp16 = get_tests({
'i': ['dog-224x224.bmp'],
'm': ['bvlcalexnet-12.onnx'],
'sample_type': ['C++','Python'],
})
class TestClassification(SamplesCommonTestClass):

57
tests/samples_tests/smoke_tests/test_hello_classification.py
View File

@ -11,30 +11,41 @@
limitations under the License.
"""
import os
import pathlib
import pytest
import re
import sys
import logging as log
from common.samples_common_test_class import get_cmd_output, get_tests
from common.samples_common_test_class import get_cmd_output, get_tests, download
from common.samples_common_test_class import SamplesCommonTestClass
from common.common_utils import shell
from pathlib import Path
import shutil
log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
test_data_fp32 = get_tests(cmd_params={'i': [os.path.join('227x227', 'dog.bmp')],
'm': [os.path.join('squeezenet1.1', 'FP32', 'squeezenet1.1.xml')],
'sample_type': ['C++', 'C']},
)
test_data_fp32_unicode = get_tests(cmd_params={'i': [os.path.join('227x227', 'dog.bmp')],
'm': [os.path.join('squeezenet1.1', 'FP32', 'squeezenet1.1.xml')],
'sample_type': ['C++']},
)
def get_executable(sample_language):
return pathlib.Path(os.environ['IE_APP_PATH'], 'hello_classification' + ('' if sample_language == 'C++' else '_c')).with_suffix('.exe' if os.name == 'nt' else '')
class TestHello(SamplesCommonTestClass):
def prepend(cache, model, inp):
test_data_dir = cache.mkdir('test_data')
return download(test_data_dir, test_data_dir / model), download(test_data_dir, test_data_dir / inp)
test_data_fp32 = get_tests({
'i': ['dog-224x224.bmp'],
'm': ['bvlcalexnet-12.onnx'], # Remove the model forom .md and .rst if removed from here
'sample_type': ['C++', 'C'],
})
test_data_fp32_unicode = get_tests({
'i': ['dog-224x224.bmp'],
'm': ['bvlcalexnet-12.onnx'],
'sample_type': ['C++'],
})
class Test_hello_classification(SamplesCommonTestClass):
sample_name = 'hello_classification'
@pytest.mark.parametrize("param", test_data_fp32)
@ -46,20 +57,18 @@ class TestHello(SamplesCommonTestClass):
"""
# Run _test function, that returns stdout or 0.
stdout = self._test(param, cache, use_preffix=False, get_cmd_func=self.get_hello_cmd_line)
if not stdout:
return 0
output = get_cmd_output(get_executable(param['sample_type']), *prepend(cache, param['m'], param['i']), param['d'])
stdout = stdout.split('\n')
output = output.split('\n')
is_ok = True
for line in range(len(stdout)):
if re.match('\\d+ +\\d+.\\d+$', stdout[line].replace('[ INFO ]', '').strip()) is not None:
top1 = stdout[line].replace('[ INFO ]', '').strip().split(' ')[0]
for line in range(len(output)):
if re.match('\\d+ +\\d+.\\d+$', output[line].replace('[ INFO ]', '').strip()) is not None:
top1 = output[line].replace('[ INFO ]', '').strip().split(' ')[0]
top1 = re.sub('\\D', '', top1)
if '215' not in top1:
is_ok = False
log.error('Expected class 215, Detected class {}'.format(top1))
log.error('Expected class 262, Detected class {}'.format(top1))
break
assert is_ok, 'Wrong top1 class'
log.info('Accuracy passed')
@ -79,11 +88,10 @@ class TestHello(SamplesCommonTestClass):
tmp_image_dir.mkdir()
tmp_model_dir.mkdir()
test_data_dir = cache.makedir('test_data_dir') / 'samples_smoke_tests_data_2021.4'
test_data_dir = cache.makedir('test_data')
# Copy files
shutil.copy(test_data_dir / 'validation_set' / param['i'], tmp_image_dir)
shutil.copy(test_data_dir / 'models' / 'public' / param['m'], tmp_model_dir)
shutil.copy(test_data_dir / 'models' / 'public' / Path(param['m'].replace('.xml', '.bin')), tmp_model_dir)
shutil.copy(test_data_dir / param['i'], tmp_image_dir)
shutil.copy(test_data_dir / param['m'], tmp_model_dir)
image_path = tmp_image_dir / Path(param['i']).name
original_image_name = image_path.name.split(sep='.')[0]
@ -130,7 +138,6 @@ class TestHello(SamplesCommonTestClass):
new_model_path = tmp_model_dir / (original_model_name + f"_{model_name.decode('utf-8')}.xml")
model_path.rename(new_model_path)
Path(str(model_path).replace('.xml', '.bin')).rename(Path(str(new_model_path).replace('.xml', '.bin')))
stdout = get_cmd_output(executable_path, new_model_path, image_path, param['d'])
probs = []
for line in stdout.split(sep='\n'):

13
tests/samples_tests/smoke_tests/test_hello_nv12_input_classification.py
View File

@ -10,7 +10,6 @@
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import pytest
import re
import sys
@ -20,12 +19,12 @@ from common.samples_common_test_class import SamplesCommonTestClass
log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
test_data_fp32 = get_tests(cmd_params={'i': [os.path.join('224x224', 'dog6.yuv')],
'm': [os.path.join('squeezenet1.1', 'FP32', 'squeezenet1.1.xml')],
'size': ['224x224'],
'sample_type': ['C++', 'C'],
},
)
test_data_fp32 = get_tests({
'i': ['samples_smoke_tests_data_2021.4/validation_set/224x224/dog6.yuv'],
'm': ['bvlcalexnet-12.onnx'], # Remove the model forom .md and .rst if removed from here
'size': ['224x224'],
'sample_type': ['C++', 'C'],
})
class TestHelloNV12Input(SamplesCommonTestClass):
sample_name = 'hello_nv12_input_classification'

12
tests/samples_tests/smoke_tests/test_hello_reshape_ssd.py
View File

@ -10,19 +10,17 @@
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import pytest
from common.samples_common_test_class import get_tests
from common.samples_common_test_class import SamplesCommonTestClass
from common.specific_samples_parsers import parse_hello_reshape_ssd
test_data_fp32 = get_tests(cmd_params={'i': [os.path.join('500x500', 'cat.bmp')],
'm': [os.path.join('ssd512', 'FP32', 'ssd512.xml')],
'sample_type': ['C++','Python'],
'd': ['CPU']},
use_device=['d'], use_batch=False
)
test_data_fp32 = get_tests({
'i': ['samples_smoke_tests_data_2021.4/validation_set/500x500/cat.bmp'],
'm': ['samples_smoke_tests_data_2021.4/models/public/ssd512/FP16/ssd512.xml'],
'sample_type': ['C++','Python'],
'd': ['CPU']}, use_device=['d'], use_batch=False)
class TestHelloShape(SamplesCommonTestClass):

4
tests/samples_tests/smoke_tests/test_sync_benchmark.py
View File

@ -19,7 +19,7 @@ class Test_sync_benchmark_cpp(SamplesCommonTestClass):
sample_name = 'sync_benchmark'
def test(self, cache):
self._test({'m': os.path.join('squeezenet1.1', 'FP32', 'squeezenet1.1.xml')}, cache, use_preffix=False)
self._test({'m': 'bvlcalexnet-12.onnx'}, cache, use_preffix=False)
class Test_sync_benchmark_py(SamplesCommonTestClass):
@ -28,4 +28,4 @@ class Test_sync_benchmark_py(SamplesCommonTestClass):
def test(self, monkeypatch, cache):
monkeypatch.setenv('PYTHONCOERCECLOCALE', 'warn')
self._test({'m': os.path.join('squeezenet1.1', 'FP32', 'squeezenet1.1.xml')}, cache, use_preffix=False)
self._test({'m': 'bvlcalexnet-12.onnx'}, cache, use_preffix=False)

4
tests/samples_tests/smoke_tests/test_throughput_benchmark.py
View File

@ -19,7 +19,7 @@ class Test_throughput_benchmark_cpp(SamplesCommonTestClass):
sample_name = 'throughput_benchmark'
def test(self, cache):
self._test({'m': os.path.join('ssd512', 'FP16', 'ssd512.xml')}, cache, use_preffix=False)
self._test({'m': 'bvlcalexnet-12.onnx'}, cache, use_preffix=False)
class Test_throughput_benchmark_py(SamplesCommonTestClass):
@ -28,4 +28,4 @@ class Test_throughput_benchmark_py(SamplesCommonTestClass):
def test(self, monkeypatch, cache):
monkeypatch.setenv('PYTHONCOERCECLOCALE', 'warn')
self._test({'m': os.path.join('ssd512', 'FP16', 'ssd512.xml')}, cache, use_preffix=False)
self._test({'m': 'bvlcalexnet-12.onnx'}, cache, use_preffix=False)