From e922edf299b1d9bc0843b1e5cf0cafdb2389ee3e Mon Sep 17 00:00:00 2001 From: Xiping Yan Date: Thu, 6 Jun 2024 11:23:26 +0800 Subject: [PATCH] Add test case to verify 4bit precision input model. (#24774) ### Details: - *Complete the remaining work after the [PR](https://github.com/openvinotoolkit/openvino/pull/24297) is merged.* - *Add test case.* ### Tickets: - *141542* --------- Signed-off-by: xipingya Co-authored-by: Zlobin Vladimir --- .../src/nodes/common/cpu_convert.cpp | 6 +-- .../common/samples_common_test_class.py | 22 ++++++-- .../smoke_tests/test_benchmark_app.py | 50 +++++++++++++++---- .../tools/benchmark/utils/inputs_filling.py | 14 +++--- 4 files changed, 67 insertions(+), 25 deletions(-) diff --git a/src/plugins/intel_cpu/src/nodes/common/cpu_convert.cpp b/src/plugins/intel_cpu/src/nodes/common/cpu_convert.cpp index fa7afcc4bcf..0887806517a 100644 --- a/src/plugins/intel_cpu/src/nodes/common/cpu_convert.cpp +++ b/src/plugins/intel_cpu/src/nodes/common/cpu_convert.cpp @@ -589,9 +589,9 @@ struct ConvertFromBinPrecision { #define INTEL_CPU_CVT_FROM_4BIT(DT) OV_CASE(ov::element::DT, PrecisionInfo::value_type) -#define INTEL_CPU_CVT_FROM_4BIT_LIST \ - INTEL_CPU_CVT_FROM_4BIT(f32), INTEL_CPU_CVT_FROM_4BIT(bf16), INTEL_CPU_CVT_FROM_4BIT(i8), \ - INTEL_CPU_CVT_FROM_4BIT(u8) +#define INTEL_CPU_CVT_FROM_4BIT_LIST \ + INTEL_CPU_CVT_FROM_4BIT(f32), INTEL_CPU_CVT_FROM_4BIT(bf16), INTEL_CPU_CVT_FROM_4BIT(f16), \ + INTEL_CPU_CVT_FROM_4BIT(i8), INTEL_CPU_CVT_FROM_4BIT(u8) struct ConvertFrom4BitContext { ov::element::Type_t inType; diff --git a/tests/samples_tests/smoke_tests/common/samples_common_test_class.py b/tests/samples_tests/smoke_tests/common/samples_common_test_class.py index 08ec0c5eb8f..e0fedf64966 100644 --- a/tests/samples_tests/smoke_tests/common/samples_common_test_class.py +++ b/tests/samples_tests/smoke_tests/common/samples_common_test_class.py @@ -26,6 +26,7 @@ import zipfile import logging as log from common.common_utils import shell from shutil import which +import openvino.runtime as ov log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) @@ -75,13 +76,24 @@ def download(test_data_dir, file_path): time.sleep(1.0) -def prepend(cache, inp='', model=''): +def prepend(cache, inp='', model='', tmp_path=None): test_data_dir = cache.mkdir('test_data') - if inp: - inp = '-i', download(test_data_dir, test_data_dir / inp) if model: - model = '-m', download(test_data_dir, test_data_dir / model) - return *inp, *model + if type(model) is ov.ie_api.Model: + model_sv_path = tmp_path / "model_with_4bit_input.xml" + ov.save_model(model, model_sv_path) + model = '-m', model_sv_path + else: + model = '-m', download(test_data_dir, test_data_dir / model) + + if inp: + if os.path.exists(tmp_path / inp): + inp = '-i', tmp_path / inp + else: + inp = '-i', download(test_data_dir, test_data_dir / inp) + return *inp, *model + else: + return *model, def get_tests(cmd_params, use_device=True, use_batch=False): diff --git a/tests/samples_tests/smoke_tests/test_benchmark_app.py b/tests/samples_tests/smoke_tests/test_benchmark_app.py index ecc09fff316..860ccd10641 100644 --- a/tests/samples_tests/smoke_tests/test_benchmark_app.py +++ b/tests/samples_tests/smoke_tests/test_benchmark_app.py @@ -12,10 +12,12 @@ """ import json import os +import numpy as np import pathlib import pytest from common.samples_common_test_class import get_devices, get_cmd_output, prepend - +from openvino.runtime import opset8 as opset +import openvino.runtime as ov def get_executable(sample_language): executable = 'benchmark_app' @@ -25,11 +27,21 @@ def get_executable(sample_language): return executable return 'benchmark_app' +def create_random_4bit_bin_file(tmp_path, shape, name): + fullname = tmp_path / name + rs = np.random.RandomState(np.random.MT19937(np.random.SeedSequence(0))) + pack_shape = [x for x in shape] + pack_shape[-1] = pack_shape[-1]*4 + rand_data = (rs.uniform(0, 15, list(pack_shape)) >= 7).astype(int).flatten() + raw_data = np.packbits(rand_data) + with open(fullname, "wb") as f: + f.write(raw_data) -def verify(sample_language, device, api=None, nireq=None, shape=None, data_shape=None, nstreams=None, layout=None, pin=None, cache=None, tmp_path=None, model='bvlcalexnet-12.onnx'): + +def verify(sample_language, device, api=None, nireq=None, shape=None, data_shape=None, nstreams=None, layout=None, pin=None, cache=None, tmp_path=None, model='bvlcalexnet-12.onnx', inp='dog-224x224.bmp', batch='1', niter='10', tm=None): output = get_cmd_output( get_executable(sample_language), - *prepend(cache, 'dog-224x224.bmp', model), + *prepend(cache, inp, model, tmp_path), *('-nstreams', nstreams) if nstreams else '', *('-layout', layout) if layout else '', *('-nireq', nireq) if nireq else '', @@ -40,7 +52,10 @@ def verify(sample_language, device, api=None, nireq=None, shape=None, data_shape *('-api', api) if api else '', *('-dump_config', tmp_path / 'conf.json') if tmp_path else '', *('-exec_graph_path', tmp_path / 'exec_graph.xml') if tmp_path else '', - '-d', device, '-b', '1', '-niter', '10' + *('-b', batch) if batch else '', + *('-niter', niter) if niter else '10', + *('-t', tm) if tm else '', + '-d', device ) assert 'FPS' in output if tmp_path: @@ -65,8 +80,8 @@ def test_benchmark_app_help(sample_language): @pytest.mark.parametrize('api', ['sync', 'async']) @pytest.mark.parametrize('nireq', ['4', '']) @pytest.mark.parametrize('device', get_devices()) -def test_nireq(sample_language, api, nireq, device, cache): - verify(sample_language, device, api=api, nireq=nireq, cache=cache) +def test_nireq(sample_language, api, nireq, device, cache, tmp_path): + verify(sample_language, device, api=api, nireq=nireq, cache=cache, tmp_path=tmp_path) @pytest.mark.skipif('CPU' not in get_devices(), reason='affinity is a CPU property') @@ -91,11 +106,26 @@ def test_api(sample_language, api, device, cache, tmp_path): @pytest.mark.parametrize('sample_language', ['C++', 'Python']) @pytest.mark.parametrize('device', get_devices()) -def test_reshape(sample_language, device, cache): - verify(sample_language, device, shape='data_0[2,3,224,224]', cache=cache) +def test_reshape(sample_language, device, cache, tmp_path): + verify(sample_language, device, shape='data_0[2,3,224,224]', cache=cache, tmp_path=tmp_path) @pytest.mark.parametrize('sample_language', ['C++', 'Python']) @pytest.mark.parametrize('device', get_devices()) -def test_dynamic_shape(sample_language, device, cache): - verify(sample_language, device, model='efficientnet-lite4-11-qdq.onnx', shape='[?,224,224,3]', data_shape='[1,224,224,3][2,224,224,3]', layout='[NHWC]', cache=cache) +def test_dynamic_shape(sample_language, device, cache, tmp_path): + verify(sample_language, device, model='efficientnet-lite4-11-qdq.onnx', + shape='[?,224,224,3]', data_shape='[1,224,224,3][2,224,224,3]', layout='[NHWC]', cache=cache, tmp_path=tmp_path) + +@pytest.mark.parametrize('sample_language', ['C++', 'Python']) +@pytest.mark.parametrize('device', ['CPU']) +@pytest.mark.parametrize('inp', [None, 'random_4bit_data.bin']) +def test_4bit_precision_input(sample_language, device, inp, cache, tmp_path): + inp_type = ov.Type.i4 + inp_shape = [128] # only pass scalar. + input = opset.parameter(inp_shape, inp_type, name='in') + cvt = opset.convert(input, ov.Type.f32) + result = opset.result(cvt, name='cvt') + model_4bit = ov.Model([result], [input], 'model_with_4bit_input') + if inp != None and inp.endswith(".bin"): + create_random_4bit_bin_file(tmp_path, inp_shape, inp) + verify(sample_language, device, model=model_4bit, inp=inp, cache=cache, tmp_path=tmp_path, batch=None, tm='1') diff --git a/tools/benchmark_tool/openvino/tools/benchmark/utils/inputs_filling.py b/tools/benchmark_tool/openvino/tools/benchmark/utils/inputs_filling.py index d2bb32b346a..24be4c65679 100644 --- a/tools/benchmark_tool/openvino/tools/benchmark/utils/inputs_filling.py +++ b/tools/benchmark_tool/openvino/tools/benchmark/utils/inputs_filling.py @@ -291,7 +291,7 @@ def get_binary_tensors(binary_paths: List[str], info: AppInputInfo, batch_sizes: tensors = [] for i in range(niter): shape_id = i % num_shapes - dtype = get_dtype(info.element_type) + dtype = np.uint8() if info.element_type.bitwidth < 8 else get_dtype(info.element_type) shape = list(info.shapes[shape_id]) binaries = np.ndarray(shape=shape, dtype=dtype) binary_index = processed_frames @@ -301,12 +301,12 @@ def get_binary_tensors(binary_paths: List[str], info: AppInputInfo, batch_sizes: binary_filename: str = binary_paths[binary_index] extension = binary_filename.lower().split('.')[-1] if extension == "bin": - binary_file_size = os.path.getsize(binary_filename) - blob_size = dtype.itemsize * int(np.prod(shape)) - if blob_size != binary_file_size: + binary_file_bit_size = os.path.getsize(binary_filename) * 8 + blob_bit_size = info.element_type.bitwidth * int(np.prod(shape)) + if blob_bit_size != binary_file_bit_size: raise Exception( - f"File {binary_filename} contains {binary_file_size} bytes but model expects {blob_size}") - from_file = np.reshape(np.fromfile(binary_filename, dtype), shape) + f"File {binary_filename} contains {binary_file_bit_size} bites but model expects {blob_bit_size}") + from_file = np.fromfile(binary_filename, dtype) if info.layout.has_name("N"): binaries[[None] * info.layout.get_index_by_name("N") + [b]] = from_file else: @@ -317,7 +317,7 @@ def get_binary_tensors(binary_paths: List[str], info: AppInputInfo, batch_sizes: binary_index += 1 processed_frames += current_batch_size - tensors.append(Tensor(binaries)) + tensors.append(Tensor(binaries, shape, info.element_type)) return tensors