natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Stress, Memory tests update to comply with new OMZ tools layout (#7886) 

* work with omz: change hardcoded paths for memory_tests, fix paths for memcheck get_testdata.py script

* change paths to omz tools

* revert mo changes

* remove mo from arguments

* remove extra var from cmd string

* remove is_db_used check

* add timeout to proc_exec

* merge master
This commit is contained in:
Victor Kuznetsov 2021-10-13 10:42:00 +03:00 committed by GitHub
parent db527fff41
commit 6d9a7a108e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 63 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
tests/memory_tests/test_runner/conftest.py
View File

@ -78,7 +78,7 @@ def pytest_addoption(parser):
help="number of iterations to run executable and aggregate results",
default=3
)
omz_args_parser = parser.getgroup("test with omz models")
omz_args_parser = parser.getgroup("Test with OMZ models")
omz_args_parser.addoption(
"--omz",
type=Path,
@ -103,12 +103,6 @@ def pytest_addoption(parser):
default=abs_path('../_omz_out/irs'),
help='Directory to put test data into. Required for OMZ converter.py only.'
)
omz_args_parser.addoption(
"--mo",
type=Path,
default=abs_path("../../../model-optimizer/mo.py"),
help="Path to model-optimizer mo.py. Required for OMZ converter.py only",
)
helpers_args_parser = parser.getgroup("test helpers")
helpers_args_parser.addoption(
"--dump_refs",
@ -120,9 +114,9 @@ def pytest_addoption(parser):
'--db_submit',
metavar="RUN_ID",
type=str,
help='submit results to the database. ' \
'`RUN_ID` should be a string uniquely identifying the run' \
' (like Jenkins URL or time)'
help='submit results to the database. '
'`RUN_ID` should be a string uniquely identifying the run '
'(like Jenkins URL or time)'
)
is_db_used = db_args_parser.parser.parse_known_args(sys.argv).db_submit
db_args_parser.addoption(
@ -196,19 +190,19 @@ def omz_models_conversion(instance, request):
# Check Open Model Zoo key
omz_path = request.config.getoption("omz")
if omz_path:
cache_dir = request.config.getoption("omz_cache_dir")
omz_models_out_dir = request.config.getoption("omz_models_out_dir")
omz_irs_out_dir = request.config.getoption("omz_irs_out_dir")
mo_path = request.config.getoption("mo")
downloader_path = omz_path / "tools" / "downloader" / "downloader.py"
converter_path = omz_path / "tools" / "downloader" / "converter.py"
info_dumper_path = omz_path / "tools" / "downloader" / "info_dumper.py"
# TODO: After switch to wheel OV installation, omz tools should be accessible through command line
downloader_path = omz_path / "tools" / "model_tools" / "downloader.py"
converter_path = omz_path / "tools" / "model_tools" / "converter.py"
info_dumper_path = omz_path / "tools" / "model_tools" / "info_dumper.py"
if instance["instance"]["model"]["source"] == "omz":
model_name = instance["instance"]["model"]["name"]
model_precision = instance["instance"]["model"]["precision"]
cache_dir = request.config.getoption("omz_cache_dir")
omz_models_out_dir = request.config.getoption("omz_models_out_dir")
omz_irs_out_dir = request.config.getoption("omz_irs_out_dir")
# get full model info
cmd = [f'{sys.executable}', f'{info_dumper_path}', '--name', f'{model_name}']
return_code, info = cmd_exec(cmd, log=logging)
@ -234,7 +228,7 @@ def omz_models_conversion(instance, request):
cmd = [f'{sys.executable}', f'{converter_path}', '--name', f'{model_name}', '-p', f'{sys.executable}',
'--precisions', f'{model_precision}', '--output_dir', f'{omz_irs_out_dir}',
'--download_dir', f'{omz_models_out_dir}', '--mo', f'{mo_path}']
'--download_dir', f'{omz_models_out_dir}']
return_code, _ = cmd_exec(cmd, log=logging)
assert return_code == 0, "Converting OMZ models has failed!"
@ -445,6 +439,8 @@ def prepare_timeline_report(pytestconfig):
template = env.get_template('timeline_report.html')
template.stream(records=records, timelines=timelines).dump(report_path)
logging.info(f"Save html timeline_report to {report_path}")
@pytest.fixture(scope="session", autouse=True)
def prepare_tconf_with_refs(pytestconfig):
@ -454,7 +450,7 @@ def prepare_tconf_with_refs(pytestconfig):
yield
new_tconf_path = pytestconfig.getoption('dump_refs')
if new_tconf_path:
logging.info("Save new test config with test results as references to {}".format(new_tconf_path))
logging.info(f"Save new test config with test results as references to {new_tconf_path}")
upd_cases = []
steps_to_dump = {"create_exenetwork", "first_inference"}

30
tests/stress_tests/scripts/get_testdata.py
View File

@ -55,8 +55,7 @@ class VirtualEnv:
def create(self):
"""Creates virtual environment."""
cmd = '{executable} -m venv {venv}'.format(executable=sys.executable,
venv=self.get_venv_dir())
cmd = '{executable} -m venv {venv}'.format(executable=sys.executable, venv=self.get_venv_dir())
run_in_subprocess(cmd)
self.is_created = True
@ -113,7 +112,6 @@ def main():
parser.add_argument('--omz_repo', required=False,
help='Path to Open Model Zoo (OMZ) repository. It will be used to skip cloning step.')
parser.add_argument('--mo_tool', type=Path,
default=Path(abs_path('../../../model-optimizer/mo.py')).resolve(),
help='Path to Model Optimizer (MO) runner. Required for OMZ converter.py only.')
parser.add_argument('--omz_models_out_dir', type=Path,
default=abs_path('../_omz_out/models'),
@ -147,11 +145,11 @@ def main():
if not args.no_venv:
Venv = VirtualEnv("./.stress_venv")
requirements = [
omz_path / "tools" / "downloader" / "requirements.in",
args.mo_tool.parent / "requirements.txt",
args.mo_tool.parent / "requirements_dev.txt",
omz_path / "tools" / "downloader" / "requirements-caffe2.in",
omz_path / "tools" / "downloader" / "requirements-pytorch.in"
omz_path / "tools" / "model_tools" / "requirements.in",
omz_path / "tools" / "model_tools" / "requirements-caffe2.in",
omz_path / "tools" / "model_tools" / "requirements-pytorch.in"
]
Venv.create_n_install_requirements(*requirements)
python_executable = Venv.get_venv_executable()
@ -166,7 +164,7 @@ def main():
model_name = model_rec.attrib["name"]
precision = model_rec.attrib["precision"]
info_dumper_path = omz_path / "tools" / "downloader" / "info_dumper.py"
info_dumper_path = omz_path / "tools" / "model_tools" / "info_dumper.py"
cmd = '"{executable}" "{info_dumper_path}" --name {model_name}'.format(executable=sys.executable,
info_dumper_path=info_dumper_path,
model_name=model_name)
@ -194,28 +192,28 @@ def main():
args.omz_irs_out_dir / model_rec.attrib["subdirectory"] / precision / (model_rec.attrib["name"] + ".xml"))
# prepare models
downloader_path = omz_path / "tools" / "downloader" / "downloader.py"
downloader_path = omz_path / "tools" / "model_tools" / "downloader.py"
cmd = '{downloader_path} --name {model_name}' \
' --precisions={precision}' \
' --num_attempts {num_attempts}' \
' --output_dir {models_dir}' \
' --cache_dir {cache_dir}'.format(downloader_path=downloader_path, model_name=model_name,
precision=precision, num_attempts=OMZ_NUM_ATTEMPTS,
models_dir=args.omz_models_out_dir, cache_dir=args.omz_cache_dir)
' --cache_dir {cache_dir}'.format(downloader_path=downloader_path, precision=precision,
models_dir=args.omz_models_out_dir,
num_attempts=OMZ_NUM_ATTEMPTS, model_name=model_name,
cache_dir=args.omz_cache_dir)
run_in_subprocess(cmd, check_call=not args.skip_omz_errors)
# convert models to IRs
converter_path = omz_path / "tools" / "downloader" / "converter.py"
converter_path = omz_path / "tools" / "model_tools" / "converter.py"
# NOTE: remove --precisions if both precisions (FP32 & FP16) required
cmd = '{executable} {converter_path} --name {model_name}' \
' -p {executable}' \
' -p "{executable}"' \
' --precisions={precision}' \
' --output_dir {irs_dir}' \
' --download_dir {models_dir}' \
' --mo {mo_tool}'.format(executable=python_executable, precision=precision,
converter_path=converter_path,
model_name=model_name, irs_dir=args.omz_irs_out_dir,
' --mo {mo_tool}'.format(executable=python_executable, converter_path=converter_path,
precision=precision, model_name=model_name, irs_dir=args.omz_irs_out_dir,
models_dir=args.omz_models_out_dir, mo_tool=args.mo_tool)
run_in_subprocess(cmd, check_call=not args.skip_omz_errors)

44
tests/stress_tests/scripts/run_memcheck.py
View File

@ -16,15 +16,11 @@ import subprocess
import sys
from glob import glob
from compare_memcheck_2_runs import compare_memcheck_2_runs, \
get_memcheck_records, get_db_memcheck_records
from compare_memcheck_2_runs import compare_memcheck_2_runs, get_memcheck_records, get_db_memcheck_records
# Database arguments
from memcheck_upload import DATABASE, DB_COLLECTIONS
from memcheck_upload import create_memcheck_records, \
upload_memcheck_records, \
create_memcheck_report, \
metadata_from_manifest, \
info_from_test_config
from memcheck_upload import create_memcheck_records, upload_memcheck_records, create_memcheck_report, \
metadata_from_manifest, info_from_test_config
def run(args, log=None, verbose=True):
@ -74,7 +70,7 @@ def main():
binary_args = []
for idx, arg in enumerate(sys.argv):
if arg == '--':
binary_args = sys.argv[idx+1:]
binary_args = sys.argv[idx + 1:]
sys.argv = sys.argv[:idx]
break
@ -94,15 +90,21 @@ def main():
description='Run memcheck tests',
usage='%(prog)s [options] binary -- [additional args]',
parents=[init_parser])
parser.add_argument('binary', help='test binary to execute')
parser.add_argument('--gtest_parallel', help='path to gtest-parallel to use',
parser.add_argument('binary',
help='test binary to execute')
parser.add_argument('--gtest_parallel',
help='path to gtest-parallel to use',
default='gtest_parallel')
parser.add_argument('--timeout', help='timeout for tests run within gtest-parallel')
parser.add_argument('--timeout',
help='timeout for tests run within gtest-parallel',
default='')
parser.add_argument('-d', '--output_dir',
required=args.timeline_report or args.upload or args.compare,
help='output directory for test logs')
parser.add_argument('-w', '--workers', help='number of gtest-parallel workers to spawn')
help='output directory for test logs',
default='')
parser.add_argument('-w', '--workers',
help='number of gtest-parallel workers to spawn',
default='')
parser.add_argument('--db_url',
required=args.timeline_report or args.upload or
(args.compare and not os.path.isdir(args.compare)),
@ -122,8 +124,7 @@ def main():
parser.add_argument('--ref_db_collection',
required=args.compare and not os.path.isdir(args.compare),
help=f'use collection name in {DATABASE} database to query'
f' reference data',
help=f'use collection name in {DATABASE} database to query reference data',
choices=DB_COLLECTIONS)
parser.add_argument('--comparison_report',
required=args.compare,
@ -145,12 +146,11 @@ def main():
args.output_dir)
sys.exit(1)
return_code, _ = run([sys.executable, args.gtest_parallel] +
(['--output_dir', f'{args.output_dir}'] if args.output_dir else []) +
(['--workers', f'{args.workers}'] if args.workers else []) +
(['--timeout', f'{args.timeout}'] if args.timeout else []) +
[args.binary] +
['--'] + binary_args)
return_code, _ = run([sys.executable, args.gtest_parallel,
'--output_dir', f'{args.output_dir}',
'--workers', f'{args.workers}',
'--timeout', f'{args.timeout}',
args.binary, '--'] + binary_args)
if args.upload or args.timeline_report or args.compare:
# prepare commit information

16
tests/time_tests/scripts/run_timetest.py
View File

@ -65,7 +65,6 @@ def prepare_executable_cmd(args: dict):
def run_timetest(args: dict, log=None):
"""Run provided executable several times and aggregate collected statistics"""
if log is None:
log = logging.getLogger('run_timetest')
@ -77,8 +76,8 @@ def run_timetest(args: dict, log=None):
tmp_stats_path = tempfile.NamedTemporaryFile().name
retcode, msg = cmd_exec(cmd_common + ["-s", str(tmp_stats_path)], log=log)
if retcode != 0:
log.error("Run of executable '{}' failed with return code '{}'. Error: {}\n"
"Statistics aggregation is skipped.".format(args["executable"], retcode, msg))
log.error(f"Run of executable '{args['executable']}' failed with return code '{retcode}'. Error: {msg}\n"
f"Statistics aggregation is skipped.")
return retcode, msg, {}, {}
# Read raw statistics
@ -91,7 +90,7 @@ def run_timetest(args: dict, log=None):
flatten_data = {}
parse_stats(raw_data[0], flatten_data)
log.debug("Statistics after run of executable #{}: {}".format(run_iter, flatten_data))
log.debug(f"Statistics after run of executable #{run_iter}: {flatten_data}")
# Combine statistics from several runs
stats = dict((step_name, stats.get(step_name, []) + [duration])
@ -102,7 +101,7 @@ def run_timetest(args: dict, log=None):
# Aggregate results
aggregated_stats = aggregate_stats(filtered_stats)
log.debug("Aggregated statistics after full run: {}".format(aggregated_stats))
log.debug(f"Aggregated statistics after full run: {aggregated_stats}")
return 0, "", aggregated_stats, stats
@ -144,14 +143,13 @@ if __name__ == "__main__":
logging.basicConfig(format="[ %(levelname)s ] %(message)s",
level=logging.DEBUG, stream=sys.stdout)
exit_code, _, aggr_stats, _ = run_timetest(dict(args._get_kwargs()), log=logging) # pylint: disable=protected-access
exit_code, _, aggr_stats, _ = run_timetest(
dict(args._get_kwargs()), log=logging) # pylint: disable=protected-access
if args.stats_path:
# Save aggregated results to a file
with open(args.stats_path, "w") as file:
yaml.safe_dump(aggr_stats, file)
logging.info("Aggregated statistics saved to a file: '{}'".format(
args.stats_path.resolve()))
logging.info(f"Aggregated statistics saved to a file: '{args.stats_path.resolve()}'")
else:
logging.info("Aggregated statistics:")
pprint(aggr_stats)

7
tests/time_tests/test_runner/conftest.py
View File

@ -139,7 +139,7 @@ def cl_cache_dir(pytestconfig, instance):
if cl_cache_dir.exists():
shutil.rmtree(cl_cache_dir)
cl_cache_dir.mkdir()
logging.info("cl_cache will be created in {}".format(cl_cache_dir))
logging.info(f"cl_cache will be created in {cl_cache_dir}")
yield cl_cache_dir
shutil.rmtree(cl_cache_dir)
else:
@ -156,7 +156,7 @@ def model_cache_dir(pytestconfig, instance):
if model_cache_dir.exists():
shutil.rmtree(model_cache_dir)
model_cache_dir.mkdir()
logging.info("model_cache will be created in {}".format(model_cache_dir))
logging.info(f"model_cache will be created in {model_cache_dir}")
yield model_cache_dir
shutil.rmtree(model_cache_dir)
else:
@ -393,5 +393,6 @@ def pytest_runtest_makereport(item, call):
db_url = item.config.getoption("db_url")
db_collection = item.config.getoption("db_collection")
logging.info("Upload data to {}/{}.{}. Data: {}".format(db_url, 'timetests', db_collection, data))
logging.info(f"Upload data to {db_url}/{'timetests'}.{db_collection}. "
f"Data: {data}")
upload_data(data, db_url, 'timetests', db_collection)