From 6d9a7a108ec8eaed4663e7de155ee06f222f4a9b Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Wed, 13 Oct 2021 10:42:00 +0300 Subject: [PATCH] Stress, Memory tests update to comply with new OMZ tools layout (#7886) * work with omz: change hardcoded paths for memory_tests, fix paths for memcheck get_testdata.py script * change paths to omz tools * revert mo changes * remove mo from arguments * remove extra var from cmd string * remove is_db_used check * add timeout to proc_exec * merge master --- tests/memory_tests/test_runner/conftest.py | 36 ++++++++---------- tests/stress_tests/scripts/get_testdata.py | 30 +++++++-------- tests/stress_tests/scripts/run_memcheck.py | 44 +++++++++++----------- tests/time_tests/scripts/run_timetest.py | 16 ++++---- tests/time_tests/test_runner/conftest.py | 7 ++-- 5 files changed, 63 insertions(+), 70 deletions(-) diff --git a/tests/memory_tests/test_runner/conftest.py b/tests/memory_tests/test_runner/conftest.py index d0ba97059d9..2dc9f419f9c 100644 --- a/tests/memory_tests/test_runner/conftest.py +++ b/tests/memory_tests/test_runner/conftest.py @@ -78,7 +78,7 @@ def pytest_addoption(parser): help="number of iterations to run executable and aggregate results", default=3 ) - omz_args_parser = parser.getgroup("test with omz models") + omz_args_parser = parser.getgroup("Test with OMZ models") omz_args_parser.addoption( "--omz", type=Path, @@ -103,12 +103,6 @@ def pytest_addoption(parser): default=abs_path('../_omz_out/irs'), help='Directory to put test data into. Required for OMZ converter.py only.' ) - omz_args_parser.addoption( - "--mo", - type=Path, - default=abs_path("../../../model-optimizer/mo.py"), - help="Path to model-optimizer mo.py. Required for OMZ converter.py only", - ) helpers_args_parser = parser.getgroup("test helpers") helpers_args_parser.addoption( "--dump_refs", @@ -120,9 +114,9 @@ def pytest_addoption(parser): '--db_submit', metavar="RUN_ID", type=str, - help='submit results to the database. ' \ - '`RUN_ID` should be a string uniquely identifying the run' \ - ' (like Jenkins URL or time)' + help='submit results to the database. ' + '`RUN_ID` should be a string uniquely identifying the run ' + '(like Jenkins URL or time)' ) is_db_used = db_args_parser.parser.parse_known_args(sys.argv).db_submit db_args_parser.addoption( @@ -196,19 +190,19 @@ def omz_models_conversion(instance, request): # Check Open Model Zoo key omz_path = request.config.getoption("omz") if omz_path: - cache_dir = request.config.getoption("omz_cache_dir") - omz_models_out_dir = request.config.getoption("omz_models_out_dir") - omz_irs_out_dir = request.config.getoption("omz_irs_out_dir") - mo_path = request.config.getoption("mo") - - downloader_path = omz_path / "tools" / "downloader" / "downloader.py" - converter_path = omz_path / "tools" / "downloader" / "converter.py" - info_dumper_path = omz_path / "tools" / "downloader" / "info_dumper.py" + # TODO: After switch to wheel OV installation, omz tools should be accessible through command line + downloader_path = omz_path / "tools" / "model_tools" / "downloader.py" + converter_path = omz_path / "tools" / "model_tools" / "converter.py" + info_dumper_path = omz_path / "tools" / "model_tools" / "info_dumper.py" if instance["instance"]["model"]["source"] == "omz": model_name = instance["instance"]["model"]["name"] model_precision = instance["instance"]["model"]["precision"] + cache_dir = request.config.getoption("omz_cache_dir") + omz_models_out_dir = request.config.getoption("omz_models_out_dir") + omz_irs_out_dir = request.config.getoption("omz_irs_out_dir") + # get full model info cmd = [f'{sys.executable}', f'{info_dumper_path}', '--name', f'{model_name}'] return_code, info = cmd_exec(cmd, log=logging) @@ -234,7 +228,7 @@ def omz_models_conversion(instance, request): cmd = [f'{sys.executable}', f'{converter_path}', '--name', f'{model_name}', '-p', f'{sys.executable}', '--precisions', f'{model_precision}', '--output_dir', f'{omz_irs_out_dir}', - '--download_dir', f'{omz_models_out_dir}', '--mo', f'{mo_path}'] + '--download_dir', f'{omz_models_out_dir}'] return_code, _ = cmd_exec(cmd, log=logging) assert return_code == 0, "Converting OMZ models has failed!" @@ -445,6 +439,8 @@ def prepare_timeline_report(pytestconfig): template = env.get_template('timeline_report.html') template.stream(records=records, timelines=timelines).dump(report_path) + logging.info(f"Save html timeline_report to {report_path}") + @pytest.fixture(scope="session", autouse=True) def prepare_tconf_with_refs(pytestconfig): @@ -454,7 +450,7 @@ def prepare_tconf_with_refs(pytestconfig): yield new_tconf_path = pytestconfig.getoption('dump_refs') if new_tconf_path: - logging.info("Save new test config with test results as references to {}".format(new_tconf_path)) + logging.info(f"Save new test config with test results as references to {new_tconf_path}") upd_cases = [] steps_to_dump = {"create_exenetwork", "first_inference"} diff --git a/tests/stress_tests/scripts/get_testdata.py b/tests/stress_tests/scripts/get_testdata.py index a18c6045d02..1cf6d8571fe 100755 --- a/tests/stress_tests/scripts/get_testdata.py +++ b/tests/stress_tests/scripts/get_testdata.py @@ -55,8 +55,7 @@ class VirtualEnv: def create(self): """Creates virtual environment.""" - cmd = '{executable} -m venv {venv}'.format(executable=sys.executable, - venv=self.get_venv_dir()) + cmd = '{executable} -m venv {venv}'.format(executable=sys.executable, venv=self.get_venv_dir()) run_in_subprocess(cmd) self.is_created = True @@ -113,7 +112,6 @@ def main(): parser.add_argument('--omz_repo', required=False, help='Path to Open Model Zoo (OMZ) repository. It will be used to skip cloning step.') parser.add_argument('--mo_tool', type=Path, - default=Path(abs_path('../../../model-optimizer/mo.py')).resolve(), help='Path to Model Optimizer (MO) runner. Required for OMZ converter.py only.') parser.add_argument('--omz_models_out_dir', type=Path, default=abs_path('../_omz_out/models'), @@ -147,11 +145,11 @@ def main(): if not args.no_venv: Venv = VirtualEnv("./.stress_venv") requirements = [ - omz_path / "tools" / "downloader" / "requirements.in", args.mo_tool.parent / "requirements.txt", args.mo_tool.parent / "requirements_dev.txt", - omz_path / "tools" / "downloader" / "requirements-caffe2.in", - omz_path / "tools" / "downloader" / "requirements-pytorch.in" + omz_path / "tools" / "model_tools" / "requirements.in", + omz_path / "tools" / "model_tools" / "requirements-caffe2.in", + omz_path / "tools" / "model_tools" / "requirements-pytorch.in" ] Venv.create_n_install_requirements(*requirements) python_executable = Venv.get_venv_executable() @@ -166,7 +164,7 @@ def main(): model_name = model_rec.attrib["name"] precision = model_rec.attrib["precision"] - info_dumper_path = omz_path / "tools" / "downloader" / "info_dumper.py" + info_dumper_path = omz_path / "tools" / "model_tools" / "info_dumper.py" cmd = '"{executable}" "{info_dumper_path}" --name {model_name}'.format(executable=sys.executable, info_dumper_path=info_dumper_path, model_name=model_name) @@ -194,28 +192,28 @@ def main(): args.omz_irs_out_dir / model_rec.attrib["subdirectory"] / precision / (model_rec.attrib["name"] + ".xml")) # prepare models - downloader_path = omz_path / "tools" / "downloader" / "downloader.py" + downloader_path = omz_path / "tools" / "model_tools" / "downloader.py" cmd = '{downloader_path} --name {model_name}' \ ' --precisions={precision}' \ ' --num_attempts {num_attempts}' \ ' --output_dir {models_dir}' \ - ' --cache_dir {cache_dir}'.format(downloader_path=downloader_path, model_name=model_name, - precision=precision, num_attempts=OMZ_NUM_ATTEMPTS, - models_dir=args.omz_models_out_dir, cache_dir=args.omz_cache_dir) + ' --cache_dir {cache_dir}'.format(downloader_path=downloader_path, precision=precision, + models_dir=args.omz_models_out_dir, + num_attempts=OMZ_NUM_ATTEMPTS, model_name=model_name, + cache_dir=args.omz_cache_dir) run_in_subprocess(cmd, check_call=not args.skip_omz_errors) # convert models to IRs - converter_path = omz_path / "tools" / "downloader" / "converter.py" + converter_path = omz_path / "tools" / "model_tools" / "converter.py" # NOTE: remove --precisions if both precisions (FP32 & FP16) required cmd = '{executable} {converter_path} --name {model_name}' \ - ' -p {executable}' \ + ' -p "{executable}"' \ ' --precisions={precision}' \ ' --output_dir {irs_dir}' \ ' --download_dir {models_dir}' \ - ' --mo {mo_tool}'.format(executable=python_executable, precision=precision, - converter_path=converter_path, - model_name=model_name, irs_dir=args.omz_irs_out_dir, + ' --mo {mo_tool}'.format(executable=python_executable, converter_path=converter_path, + precision=precision, model_name=model_name, irs_dir=args.omz_irs_out_dir, models_dir=args.omz_models_out_dir, mo_tool=args.mo_tool) run_in_subprocess(cmd, check_call=not args.skip_omz_errors) diff --git a/tests/stress_tests/scripts/run_memcheck.py b/tests/stress_tests/scripts/run_memcheck.py index 7ca5cef018f..85cba56901a 100755 --- a/tests/stress_tests/scripts/run_memcheck.py +++ b/tests/stress_tests/scripts/run_memcheck.py @@ -16,15 +16,11 @@ import subprocess import sys from glob import glob -from compare_memcheck_2_runs import compare_memcheck_2_runs, \ - get_memcheck_records, get_db_memcheck_records +from compare_memcheck_2_runs import compare_memcheck_2_runs, get_memcheck_records, get_db_memcheck_records # Database arguments from memcheck_upload import DATABASE, DB_COLLECTIONS -from memcheck_upload import create_memcheck_records, \ - upload_memcheck_records, \ - create_memcheck_report, \ - metadata_from_manifest, \ - info_from_test_config +from memcheck_upload import create_memcheck_records, upload_memcheck_records, create_memcheck_report, \ + metadata_from_manifest, info_from_test_config def run(args, log=None, verbose=True): @@ -74,7 +70,7 @@ def main(): binary_args = [] for idx, arg in enumerate(sys.argv): if arg == '--': - binary_args = sys.argv[idx+1:] + binary_args = sys.argv[idx + 1:] sys.argv = sys.argv[:idx] break @@ -94,15 +90,21 @@ def main(): description='Run memcheck tests', usage='%(prog)s [options] binary -- [additional args]', parents=[init_parser]) - parser.add_argument('binary', help='test binary to execute') - parser.add_argument('--gtest_parallel', help='path to gtest-parallel to use', + parser.add_argument('binary', + help='test binary to execute') + parser.add_argument('--gtest_parallel', + help='path to gtest-parallel to use', default='gtest_parallel') - parser.add_argument('--timeout', help='timeout for tests run within gtest-parallel') + parser.add_argument('--timeout', + help='timeout for tests run within gtest-parallel', + default='') parser.add_argument('-d', '--output_dir', required=args.timeline_report or args.upload or args.compare, - help='output directory for test logs') - parser.add_argument('-w', '--workers', help='number of gtest-parallel workers to spawn') - + help='output directory for test logs', + default='') + parser.add_argument('-w', '--workers', + help='number of gtest-parallel workers to spawn', + default='') parser.add_argument('--db_url', required=args.timeline_report or args.upload or (args.compare and not os.path.isdir(args.compare)), @@ -122,8 +124,7 @@ def main(): parser.add_argument('--ref_db_collection', required=args.compare and not os.path.isdir(args.compare), - help=f'use collection name in {DATABASE} database to query' - f' reference data', + help=f'use collection name in {DATABASE} database to query reference data', choices=DB_COLLECTIONS) parser.add_argument('--comparison_report', required=args.compare, @@ -145,12 +146,11 @@ def main(): args.output_dir) sys.exit(1) - return_code, _ = run([sys.executable, args.gtest_parallel] + - (['--output_dir', f'{args.output_dir}'] if args.output_dir else []) + - (['--workers', f'{args.workers}'] if args.workers else []) + - (['--timeout', f'{args.timeout}'] if args.timeout else []) + - [args.binary] + - ['--'] + binary_args) + return_code, _ = run([sys.executable, args.gtest_parallel, + '--output_dir', f'{args.output_dir}', + '--workers', f'{args.workers}', + '--timeout', f'{args.timeout}', + args.binary, '--'] + binary_args) if args.upload or args.timeline_report or args.compare: # prepare commit information diff --git a/tests/time_tests/scripts/run_timetest.py b/tests/time_tests/scripts/run_timetest.py index c97a85f0c36..012b8f8b3e8 100644 --- a/tests/time_tests/scripts/run_timetest.py +++ b/tests/time_tests/scripts/run_timetest.py @@ -65,7 +65,6 @@ def prepare_executable_cmd(args: dict): def run_timetest(args: dict, log=None): """Run provided executable several times and aggregate collected statistics""" - if log is None: log = logging.getLogger('run_timetest') @@ -77,8 +76,8 @@ def run_timetest(args: dict, log=None): tmp_stats_path = tempfile.NamedTemporaryFile().name retcode, msg = cmd_exec(cmd_common + ["-s", str(tmp_stats_path)], log=log) if retcode != 0: - log.error("Run of executable '{}' failed with return code '{}'. Error: {}\n" - "Statistics aggregation is skipped.".format(args["executable"], retcode, msg)) + log.error(f"Run of executable '{args['executable']}' failed with return code '{retcode}'. Error: {msg}\n" + f"Statistics aggregation is skipped.") return retcode, msg, {}, {} # Read raw statistics @@ -91,7 +90,7 @@ def run_timetest(args: dict, log=None): flatten_data = {} parse_stats(raw_data[0], flatten_data) - log.debug("Statistics after run of executable #{}: {}".format(run_iter, flatten_data)) + log.debug(f"Statistics after run of executable #{run_iter}: {flatten_data}") # Combine statistics from several runs stats = dict((step_name, stats.get(step_name, []) + [duration]) @@ -102,7 +101,7 @@ def run_timetest(args: dict, log=None): # Aggregate results aggregated_stats = aggregate_stats(filtered_stats) - log.debug("Aggregated statistics after full run: {}".format(aggregated_stats)) + log.debug(f"Aggregated statistics after full run: {aggregated_stats}") return 0, "", aggregated_stats, stats @@ -144,14 +143,13 @@ if __name__ == "__main__": logging.basicConfig(format="[ %(levelname)s ] %(message)s", level=logging.DEBUG, stream=sys.stdout) - exit_code, _, aggr_stats, _ = run_timetest(dict(args._get_kwargs()), log=logging) # pylint: disable=protected-access - + exit_code, _, aggr_stats, _ = run_timetest( + dict(args._get_kwargs()), log=logging) # pylint: disable=protected-access if args.stats_path: # Save aggregated results to a file with open(args.stats_path, "w") as file: yaml.safe_dump(aggr_stats, file) - logging.info("Aggregated statistics saved to a file: '{}'".format( - args.stats_path.resolve())) + logging.info(f"Aggregated statistics saved to a file: '{args.stats_path.resolve()}'") else: logging.info("Aggregated statistics:") pprint(aggr_stats) diff --git a/tests/time_tests/test_runner/conftest.py b/tests/time_tests/test_runner/conftest.py index 086d6848084..b7cce72c65f 100644 --- a/tests/time_tests/test_runner/conftest.py +++ b/tests/time_tests/test_runner/conftest.py @@ -139,7 +139,7 @@ def cl_cache_dir(pytestconfig, instance): if cl_cache_dir.exists(): shutil.rmtree(cl_cache_dir) cl_cache_dir.mkdir() - logging.info("cl_cache will be created in {}".format(cl_cache_dir)) + logging.info(f"cl_cache will be created in {cl_cache_dir}") yield cl_cache_dir shutil.rmtree(cl_cache_dir) else: @@ -156,7 +156,7 @@ def model_cache_dir(pytestconfig, instance): if model_cache_dir.exists(): shutil.rmtree(model_cache_dir) model_cache_dir.mkdir() - logging.info("model_cache will be created in {}".format(model_cache_dir)) + logging.info(f"model_cache will be created in {model_cache_dir}") yield model_cache_dir shutil.rmtree(model_cache_dir) else: @@ -393,5 +393,6 @@ def pytest_runtest_makereport(item, call): db_url = item.config.getoption("db_url") db_collection = item.config.getoption("db_collection") - logging.info("Upload data to {}/{}.{}. Data: {}".format(db_url, 'timetests', db_collection, data)) + logging.info(f"Upload data to {db_url}/{'timetests'}.{db_collection}. " + f"Data: {data}") upload_data(data, db_url, 'timetests', db_collection)