From a081dfea0f2d08b1adc4b9fd31f0b0f99fe88096 Mon Sep 17 00:00:00 2001 From: Vitaliy Urusovskij Date: Fri, 30 Oct 2020 21:08:25 +0300 Subject: [PATCH] Align `time_tests` with master branch from 4021e144 (#2881) --- tests/time_tests/CMakeLists.txt | 2 +- tests/time_tests/README.md | 31 ++ tests/time_tests/common/CMakeLists.txt | 23 -- tests/time_tests/common/cli.h | 57 ---- tests/time_tests/common/main.cpp | 52 ---- tests/time_tests/common/statistics_writer.h | 55 ---- tests/time_tests/common/timer.h | 46 --- .../time_tests/ftti_pipeline/ftti_pipeline.h | 49 --- .../include/timetests_helper/timer.h | 32 ++ .../include/timetests_helper/utils.h | 20 ++ tests/time_tests/scripts/requirements.txt | 1 + .../run_timetest.py} | 64 ++-- tests/time_tests/src/CMakeLists.txt | 6 + tests/time_tests/src/timetests/CMakeLists.txt | 19 ++ tests/time_tests/src/timetests/common.h | 154 +++++++++ .../src/timetests/timetest_infer.cpp | 83 +++++ .../src/timetests_helper/CMakeLists.txt | 13 + tests/time_tests/src/timetests_helper/cli.h | 65 ++++ .../time_tests/src/timetests_helper/main.cpp | 55 ++++ .../src/timetests_helper/statistics_writer.h | 57 ++++ .../time_tests/src/timetests_helper/timer.cpp | 29 ++ .../.automation/tgl_test_config.yml | 56 ++++ tests/time_tests/test_runner/conftest.py | 292 ++++++++++++++++++ tests/time_tests/test_runner/requirements.txt | 4 + tests/time_tests/test_runner/test_config.yml | 14 + tests/time_tests/test_runner/test_timetest.py | 83 +++++ tests/time_tests/test_runner/utils.py | 33 ++ 27 files changed, 1078 insertions(+), 317 deletions(-) create mode 100644 tests/time_tests/README.md delete mode 100644 tests/time_tests/common/CMakeLists.txt delete mode 100644 tests/time_tests/common/cli.h delete mode 100644 tests/time_tests/common/main.cpp delete mode 100644 tests/time_tests/common/statistics_writer.h delete mode 100644 tests/time_tests/common/timer.h delete mode 100644 tests/time_tests/ftti_pipeline/ftti_pipeline.h create mode 100644 tests/time_tests/include/timetests_helper/timer.h create mode 100644 tests/time_tests/include/timetests_helper/utils.h create mode 100644 tests/time_tests/scripts/requirements.txt rename tests/time_tests/{run_executable.py => scripts/run_timetest.py} (72%) create mode 100644 tests/time_tests/src/CMakeLists.txt create mode 100644 tests/time_tests/src/timetests/CMakeLists.txt create mode 100644 tests/time_tests/src/timetests/common.h create mode 100644 tests/time_tests/src/timetests/timetest_infer.cpp create mode 100644 tests/time_tests/src/timetests_helper/CMakeLists.txt create mode 100644 tests/time_tests/src/timetests_helper/cli.h create mode 100644 tests/time_tests/src/timetests_helper/main.cpp create mode 100644 tests/time_tests/src/timetests_helper/statistics_writer.h create mode 100644 tests/time_tests/src/timetests_helper/timer.cpp create mode 100644 tests/time_tests/test_runner/.automation/tgl_test_config.yml create mode 100644 tests/time_tests/test_runner/conftest.py create mode 100644 tests/time_tests/test_runner/requirements.txt create mode 100644 tests/time_tests/test_runner/test_config.yml create mode 100644 tests/time_tests/test_runner/test_timetest.py create mode 100644 tests/time_tests/test_runner/utils.py diff --git a/tests/time_tests/CMakeLists.txt b/tests/time_tests/CMakeLists.txt index a404715585d..f4b4c670387 100644 --- a/tests/time_tests/CMakeLists.txt +++ b/tests/time_tests/CMakeLists.txt @@ -11,4 +11,4 @@ endif() find_package(InferenceEngineDeveloperPackage REQUIRED) -add_subdirectory(common) +add_subdirectory(src) diff --git a/tests/time_tests/README.md b/tests/time_tests/README.md new file mode 100644 index 00000000000..a68b2ff4814 --- /dev/null +++ b/tests/time_tests/README.md @@ -0,0 +1,31 @@ +# Time Tests + +This test suite contains pipelines, which are executables. The pipelines measure +the time of their execution, both total and partial. A Python runner calls the +pipelines and calcuates the average execution time. + +## Prerequisites + +To build the time tests, you need to have the `build` folder, which is created +when you configure and build OpenVINO™. + +## Measure Time + +To build and run the tests, open a terminal and run the commands below: + +1. Build tests: +``` bash +mkdir build && cd build +cmake .. -DInferenceEngineDeveloperPackage_DIR=$(realpath ../../../build) && make time_tests +``` + +2. Run test: +``` bash +./scripts/run_timetest.py ../../bin/intel64/Release/timetest_infer -m model.xml -d CPU +``` + +2. Run several configurations using `pytest`: +``` bash +export PYTHONPATH=./:$PYTHONPATH +pytest ./test_runner/test_timetest.py --exe ../../bin/intel64/Release/timetest_infer +``` diff --git a/tests/time_tests/common/CMakeLists.txt b/tests/time_tests/common/CMakeLists.txt deleted file mode 100644 index 86c0e302b84..00000000000 --- a/tests/time_tests/common/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) 2018-2019 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -set (TARGET_NAME "TimeTests") - -file (GLOB SRC - *.cpp - ../ftti_pipeline/*.cpp) - -file (GLOB HDR - *.h - ../ftti_pipeline/*.h) - -# Create library file from sources. -add_executable(${TARGET_NAME} ${HDR} ${SRC}) - -find_package(gflags REQUIRED) - -target_link_libraries(${TARGET_NAME} - gflags - ${InferenceEngine_LIBRARIES} - ) diff --git a/tests/time_tests/common/cli.h b/tests/time_tests/common/cli.h deleted file mode 100644 index b21758d80ee..00000000000 --- a/tests/time_tests/common/cli.h +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include - -/// @brief message for help argument -static const char help_message[] = "Print a usage message"; - -/// @brief message for model argument -static const char model_message[] = "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to a .blob files with a trained compiled model."; - -/// @brief message for target device argument -static const char target_device_message[] = "Required. Specify a target device to infer on. " \ -"Use \"-d HETERO:\" format to specify HETERO plugin. " \ -"Use \"-d MULTI:\" format to specify MULTI plugin. " \ -"The application looks for a suitable plugin for the specified device."; - -/// @brief message for statistics path argument -static const char statistics_path_message[] = "Required. Path to a file to write statistics."; - -/// @brief Define flag for showing help message
-DEFINE_bool(h, false, help_message); - -/// @brief Declare flag for showing help message
-DECLARE_bool(help); - -/// @brief Define parameter for set model file
-/// It is a required parameter -DEFINE_string(m, "", model_message); - -/// @brief Define parameter for set target device to infer on
-/// It is a required parameter -DEFINE_string(d, "", target_device_message); - -/// @brief Define parameter for set path to a file to write statistics
-/// It is a required parameter -DEFINE_string(s, "", statistics_path_message); - -/** -* @brief This function show a help message -*/ -static void showUsage() { - std::cout << std::endl; - std::cout << "TimeTests [OPTION]" << std::endl; - std::cout << "Options:" << std::endl; - std::cout << std::endl; - std::cout << " -h, --help " << help_message << std::endl; - std::cout << " -m \"\" " << model_message << std::endl; - std::cout << " -d \"\" " << target_device_message << std::endl; - std::cout << " -s \"\" " << statistics_path_message << std::endl; -} diff --git a/tests/time_tests/common/main.cpp b/tests/time_tests/common/main.cpp deleted file mode 100644 index 164394f0909..00000000000 --- a/tests/time_tests/common/main.cpp +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "cli.h" -#include "statistics_writer.h" -#include "../ftti_pipeline/ftti_pipeline.h" - -#include - -/** -* @brief Parses command line and check required arguments -*/ -bool parseAndCheckCommandLine(int argc, char **argv) { - gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); - if (FLAGS_help || FLAGS_h) { - showUsage(); - return false; - } - - if (FLAGS_m.empty()) - throw std::logic_error("Model is required but not set. Please set -m option."); - - if (FLAGS_d.empty()) - throw std::logic_error("Device is required but not set. Please set -d option."); - - if (FLAGS_s.empty()) - throw std::logic_error("Statistics file path is required but not set. Please set -s option."); - - return true; -} - - -/** -* @brief Function calls `runPipeline` with mandatory time tracking of full run -*/ -int _runPipeline() { - SCOPED_TIMER(full_run); - return runPipeline(FLAGS_m, FLAGS_d); -} - - -/** -* @brief Main entry point -*/ -int main(int argc, char **argv) { - if (!parseAndCheckCommandLine(argc, argv)) - return -1; - - StatisticsWriter::Instance().setFile(FLAGS_s); - return _runPipeline(); -} \ No newline at end of file diff --git a/tests/time_tests/common/statistics_writer.h b/tests/time_tests/common/statistics_writer.h deleted file mode 100644 index f698ff16a13..00000000000 --- a/tests/time_tests/common/statistics_writer.h +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include - - -/** - * @brief Class response for writing provided statistics - * - * Object of the class is writing provided statistics to a specified - * file in YAML format. - */ -class StatisticsWriter { -private: - std::ofstream statistics_file; - - StatisticsWriter() = default; - StatisticsWriter(const StatisticsWriter&) = delete; - StatisticsWriter& operator=(const StatisticsWriter&) = delete; -public: - /** - * @brief Creates StatisticsWriter singleton object - */ - static StatisticsWriter& Instance(){ - static StatisticsWriter writer; - return writer; - } - - /** - * @brief Specifies, opens and validates statistics path for writing - */ - void setFile(const std::string &statistics_path) { - statistics_file.open(statistics_path); - if (!statistics_file.good()) { - std::stringstream err; - err << "Statistic file \"" << statistics_path << "\" can't be used for writing"; - throw std::runtime_error(err.str()); - } - } - - /** - * @brief Writes provided statistics in YAML format. - */ - void write(const std::pair &record) { - if (!statistics_file) - throw std::runtime_error("Statistic file path isn't set"); - statistics_file << record.first << ": " << record.second << "\n"; - } -}; diff --git a/tests/time_tests/common/timer.h b/tests/time_tests/common/timer.h deleted file mode 100644 index dc7a17abebf..00000000000 --- a/tests/time_tests/common/timer.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include - -#include "statistics_writer.h" - -using time_point = std::chrono::high_resolution_clock::time_point; - -/** -* @brief Class response for encapsulating time measurements. -* -* Object of a class measures time at start and finish of object's life cycle. -* When deleting, reports duration. -*/ -class Timer { -private: - std::string name; - time_point start_time; - -public: - /** - * @brief Constructs Timer object and measures start time - */ - Timer(const std::string &timer_name) { - name = timer_name; - start_time = std::chrono::high_resolution_clock::now(); - } - - /** - * @brief Destructs Timer object, measures duration and reports it - */ - ~Timer(){ - float duration = std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - start_time).count(); - StatisticsWriter::Instance().write({name, duration}); - } -}; - -#define SCOPED_TIMER(timer_name) Timer timer_name(#timer_name); diff --git a/tests/time_tests/ftti_pipeline/ftti_pipeline.h b/tests/time_tests/ftti_pipeline/ftti_pipeline.h deleted file mode 100644 index f3fd289f9ba..00000000000 --- a/tests/time_tests/ftti_pipeline/ftti_pipeline.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "../common/timer.h" - -#include -using namespace InferenceEngine; - - -/** -* @brief Function that contain executable pipeline which will be called from main(). -* The function should not throw any exceptions and responsible for handling it by itself. -*/ -int runPipeline(const std::string &model, const std::string &device) { - auto pipeline = [](const std::string &model, const std::string &device){ - SCOPED_TIMER(first_time_to_inference); - - Core ie; - CNNNetwork cnnNetwork; - ExecutableNetwork exeNetwork; - - { - SCOPED_TIMER(read_network); - cnnNetwork = ie.ReadNetwork(model); - } - - { - SCOPED_TIMER(load_network); - ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, device); - } - }; - - try { - pipeline(model, device); - } catch (const InferenceEngine::details::InferenceEngineException& iex) { - std::cerr << "Inference Engine pipeline failed with Inference Engine exception:\n" << iex.what(); - return 1; - } catch (const std::exception& ex) { - std::cerr << "Inference Engine pipeline failed with exception:\n" << ex.what(); - return 2; - } catch (...) { - std::cerr << "Inference Engine pipeline failed\n"; - return 3; - } - return 0; -} \ No newline at end of file diff --git a/tests/time_tests/include/timetests_helper/timer.h b/tests/time_tests/include/timetests_helper/timer.h new file mode 100644 index 00000000000..9cec3ba123c --- /dev/null +++ b/tests/time_tests/include/timetests_helper/timer.h @@ -0,0 +1,32 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +namespace TimeTest { +using time_point = std::chrono::high_resolution_clock::time_point; + +/** Encapsulate time measurements. +Object of a class measures time at start and finish of object's life cycle. +When destroyed, reports duration. +*/ +class Timer { +private: + std::string name; + time_point start_time; + +public: + /// Constructs Timer object and measures start time. + Timer(const std::string &timer_name); + + /// Destructs Timer object, measures duration and reports it. + ~Timer(); +}; + +#define SCOPED_TIMER(timer_name) TimeTest::Timer timer_name(#timer_name); + +} // namespace TimeTest \ No newline at end of file diff --git a/tests/time_tests/include/timetests_helper/utils.h b/tests/time_tests/include/timetests_helper/utils.h new file mode 100644 index 00000000000..5c4370deab6 --- /dev/null +++ b/tests/time_tests/include/timetests_helper/utils.h @@ -0,0 +1,20 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +namespace TimeTest { +/** +* @brief Get extension from filename +* @param filename - name of the file which extension should be extracted +* @return string with extracted file extension +*/ +std::string fileExt(const std::string& filename) { + auto pos = filename.rfind('.'); + if (pos == std::string::npos) return ""; + return filename.substr(pos + 1); +} +} \ No newline at end of file diff --git a/tests/time_tests/scripts/requirements.txt b/tests/time_tests/scripts/requirements.txt new file mode 100644 index 00000000000..3101697065f --- /dev/null +++ b/tests/time_tests/scripts/requirements.txt @@ -0,0 +1 @@ +PyYAML==5.3.1 \ No newline at end of file diff --git a/tests/time_tests/run_executable.py b/tests/time_tests/scripts/run_timetest.py similarity index 72% rename from tests/time_tests/run_executable.py rename to tests/time_tests/scripts/run_timetest.py index 756d172b08c..8dc1d6ed4c4 100644 --- a/tests/time_tests/run_executable.py +++ b/tests/time_tests/scripts/run_timetest.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """ -This script runs TimeTests executable several times to aggregate +This script runs timetest executable several times and aggregate collected statistics. """ @@ -50,71 +50,66 @@ def run_cmd(args: list, log=None, verbose=True): return proc.returncode, ''.join(output) -def read_stats(stats_path, stats: dict): - """Read statistics from a file and extend provided statistics""" - with open(stats_path, "r") as file: - parsed_data = yaml.load(file, Loader=yaml.FullLoader) - return dict((step_name, stats.get(step_name, []) + [duration]) - for step_name, duration in parsed_data.items()) - - def aggregate_stats(stats: dict): """Aggregate provided statistics""" return {step_name: {"avg": statistics.mean(duration_list), - "stdev": statistics.stdev(duration_list)} + "stdev": statistics.stdev(duration_list) if len(duration_list) > 1 else 0} for step_name, duration_list in stats.items()} -def write_aggregated_stats(stats_path, stats: dict): - """Write aggregated statistics to a file in YAML format""" - with open(stats_path, "w") as file: - yaml.dump(stats, file) - - def prepare_executable_cmd(args: dict): """Generate common part of cmd from arguments to execute""" - return [str(args["executable"].resolve()), - "-m", str(args["model"].resolve()), + return [str(args["executable"].resolve(strict=True)), + "-m", str(args["model"].resolve(strict=True)), "-d", args["device"]] -def generate_tmp_path(): - """Generate temporary file path without file's creation""" - tmp_stats_file = tempfile.NamedTemporaryFile() - path = tmp_stats_file.name - tmp_stats_file.close() # remove temp file in order to create it by executable - return path - - -def run_executable(args: dict, log=None): +def run_timetest(args: dict, log=None): """Run provided executable several times and aggregate collected statistics""" if log is None: - log = logging.getLogger('run_executable') + log = logging.getLogger('run_timetest') cmd_common = prepare_executable_cmd(args) # Run executable and collect statistics stats = {} for run_iter in range(args["niter"]): - tmp_stats_path = generate_tmp_path() + tmp_stats_path = tempfile.NamedTemporaryFile().name # create temp file, get path and delete temp file retcode, msg = run_cmd(cmd_common + ["-s", str(tmp_stats_path)], log=log) if retcode != 0: log.error("Run of executable '{}' failed with return code '{}'. Error: {}\n" "Statistics aggregation is skipped.".format(args["executable"], retcode, msg)) return retcode, {} - stats = read_stats(tmp_stats_path, stats) + # Read raw statistics + with open(tmp_stats_path, "r") as file: + raw_data = yaml.safe_load(file) + log.debug("Raw statistics after run of executable #{}: {}".format(run_iter, raw_data)) + + # Combine statistics from several runs + stats = dict((step_name, stats.get(step_name, []) + [duration]) + for step_name, duration in raw_data.items()) # Aggregate results aggregated_stats = aggregate_stats(stats) + log.debug("Aggregated statistics after full run: {}".format(aggregated_stats)) return 0, aggregated_stats +def check_positive_int(val): + """Check argsparse argument is positive integer and return it""" + value = int(val) + if value < 1: + msg = "%r is less than 1" % val + raise argparse.ArgumentTypeError(msg) + return value + + def cli_parser(): """parse command-line arguments""" - parser = argparse.ArgumentParser(description='Run TimeTests executable') + parser = argparse.ArgumentParser(description='Run timetest executable') parser.add_argument('executable', type=Path, help='binary to execute') @@ -131,7 +126,7 @@ def cli_parser(): help='target device to infer on') parser.add_argument('-niter', default=3, - type=int, + type=check_positive_int, help='number of times to execute binary to aggregate statistics of') parser.add_argument('-s', dest="stats_path", @@ -149,11 +144,12 @@ if __name__ == "__main__": logging.basicConfig(format="[ %(levelname)s ] %(message)s", level=logging.DEBUG, stream=sys.stdout) - exit_code, aggr_stats = run_executable(dict(args._get_kwargs()), log=logging) # pylint: disable=protected-access + exit_code, aggr_stats = run_timetest(dict(args._get_kwargs()), log=logging) # pylint: disable=protected-access if args.stats_path: # Save aggregated results to a file - write_aggregated_stats(args.stats_path, aggr_stats) + with open(args.stats_path, "w") as file: + yaml.safe_dump(aggr_stats, file) logging.info("Aggregated statistics saved to a file: '{}'".format( args.stats_path.resolve())) else: diff --git a/tests/time_tests/src/CMakeLists.txt b/tests/time_tests/src/CMakeLists.txt new file mode 100644 index 00000000000..376b490afed --- /dev/null +++ b/tests/time_tests/src/CMakeLists.txt @@ -0,0 +1,6 @@ +# Copyright (C) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +add_subdirectory(timetests) +add_subdirectory(timetests_helper) \ No newline at end of file diff --git a/tests/time_tests/src/timetests/CMakeLists.txt b/tests/time_tests/src/timetests/CMakeLists.txt new file mode 100644 index 00000000000..c83d9ca5299 --- /dev/null +++ b/tests/time_tests/src/timetests/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright (C) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +# add dummy `time_tests` target combines all time tests +add_custom_target(time_tests) + +# Build test from every source file. +# Test target name is source file name without extension. +FILE(GLOB tests "*.cpp") + +foreach(test_source ${tests}) + get_filename_component(test_name ${test_source} NAME_WE) + add_executable(${test_name} ${test_source}) + + target_link_libraries(${test_name} PRIVATE IE::inference_engine timetests_helper) + + add_dependencies(time_tests ${test_name}) +endforeach() diff --git a/tests/time_tests/src/timetests/common.h b/tests/time_tests/src/timetests/common.h new file mode 100644 index 00000000000..555d1757962 --- /dev/null +++ b/tests/time_tests/src/timetests/common.h @@ -0,0 +1,154 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +using namespace InferenceEngine; + +/** + * @brief Determine if InferenceEngine blob means image or not + */ +template +static bool isImage(const T &blob) { + auto descriptor = blob->getTensorDesc(); + if (descriptor.getLayout() != InferenceEngine::NCHW) { + return false; + } + auto channels = descriptor.getDims()[1]; + return channels == 3; +} + + +/** + * @brief Determine if InferenceEngine blob means image information or not + */ +template +static bool isImageInfo(const T &blob) { + auto descriptor = blob->getTensorDesc(); + if (descriptor.getLayout() != InferenceEngine::NC) { + return false; + } + auto channels = descriptor.getDims()[1]; + return (channels >= 2); +} + + +/** + * @brief Return height and width from provided InferenceEngine tensor description + */ +inline std::pair getTensorHeightWidth(const InferenceEngine::TensorDesc& desc) { + const auto& layout = desc.getLayout(); + const auto& dims = desc.getDims(); + const auto& size = dims.size(); + if ((size >= 2) && + (layout == InferenceEngine::Layout::NCHW || + layout == InferenceEngine::Layout::NHWC || + layout == InferenceEngine::Layout::NCDHW || + layout == InferenceEngine::Layout::NDHWC || + layout == InferenceEngine::Layout::OIHW || + layout == InferenceEngine::Layout::GOIHW || + layout == InferenceEngine::Layout::OIDHW || + layout == InferenceEngine::Layout::GOIDHW || + layout == InferenceEngine::Layout::CHW || + layout == InferenceEngine::Layout::HW)) { + // Regardless of layout, dimensions are stored in fixed order + return std::make_pair(dims.back(), dims.at(size - 2)); + } else { + THROW_IE_EXCEPTION << "Tensor does not have height and width dimensions"; + } +} + + +/** + * @brief Fill InferenceEngine blob with random values + */ +template +void fillBlobRandom(Blob::Ptr& inputBlob) { + MemoryBlob::Ptr minput = as(inputBlob); + // locked memory holder should be alive all time while access to its buffer happens + auto minputHolder = minput->wmap(); + + auto inputBlobData = minputHolder.as(); + for (size_t i = 0; i < inputBlob->size(); i++) { + auto rand_max = RAND_MAX; + inputBlobData[i] = (T) rand() / static_cast(rand_max) * 10; + } +} + + +/** + * @brief Fill InferenceEngine blob with image information + */ +template +void fillBlobImInfo(Blob::Ptr& inputBlob, + const size_t& batchSize, + std::pair image_size) { + MemoryBlob::Ptr minput = as(inputBlob); + // locked memory holder should be alive all time while access to its buffer happens + auto minputHolder = minput->wmap(); + + auto inputBlobData = minputHolder.as(); + for (size_t b = 0; b < batchSize; b++) { + size_t iminfoSize = inputBlob->size()/batchSize; + for (size_t i = 0; i < iminfoSize; i++) { + size_t index = b*iminfoSize + i; + if (0 == i) + inputBlobData[index] = static_cast(image_size.first); + else if (1 == i) + inputBlobData[index] = static_cast(image_size.second); + else + inputBlobData[index] = 1; + } + } +} + + +/** + * @brief Fill InferRequest blobs with random values or image information + */ +void fillBlobs(InferenceEngine::InferRequest inferRequest, + const InferenceEngine::ConstInputsDataMap& inputsInfo, + const size_t& batchSize) { + std::vector> input_image_sizes; + for (const ConstInputsDataMap::value_type& item : inputsInfo) { + if (isImage(item.second)) + input_image_sizes.push_back(getTensorHeightWidth(item.second->getTensorDesc())); + } + + for (const ConstInputsDataMap::value_type& item : inputsInfo) { + Blob::Ptr inputBlob = inferRequest.GetBlob(item.first); + if (isImageInfo(inputBlob) && (input_image_sizes.size() == 1)) { + // Fill image information + auto image_size = input_image_sizes.at(0); + if (item.second->getPrecision() == InferenceEngine::Precision::FP32) { + fillBlobImInfo(inputBlob, batchSize, image_size); + } else if (item.second->getPrecision() == InferenceEngine::Precision::FP16) { + fillBlobImInfo(inputBlob, batchSize, image_size); + } else if (item.second->getPrecision() == InferenceEngine::Precision::I32) { + fillBlobImInfo(inputBlob, batchSize, image_size); + } else { + THROW_IE_EXCEPTION << "Input precision is not supported for image info!"; + } + continue; + } + // Fill random + if (item.second->getPrecision() == InferenceEngine::Precision::FP32) { + fillBlobRandom(inputBlob); + } else if (item.second->getPrecision() == InferenceEngine::Precision::FP16) { + fillBlobRandom(inputBlob); + } else if (item.second->getPrecision() == InferenceEngine::Precision::I32) { + fillBlobRandom(inputBlob); + } else if (item.second->getPrecision() == InferenceEngine::Precision::U8) { + fillBlobRandom(inputBlob); + } else if (item.second->getPrecision() == InferenceEngine::Precision::I8) { + fillBlobRandom(inputBlob); + } else if (item.second->getPrecision() == InferenceEngine::Precision::U16) { + fillBlobRandom(inputBlob); + } else if (item.second->getPrecision() == InferenceEngine::Precision::I16) { + fillBlobRandom(inputBlob); + } else { + THROW_IE_EXCEPTION << "Input precision is not supported for " << item.first; + } + } +} \ No newline at end of file diff --git a/tests/time_tests/src/timetests/timetest_infer.cpp b/tests/time_tests/src/timetests/timetest_infer.cpp new file mode 100644 index 00000000000..fedb7349a51 --- /dev/null +++ b/tests/time_tests/src/timetests/timetest_infer.cpp @@ -0,0 +1,83 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include + +#include "common.h" +#include "timetests_helper/timer.h" +#include "timetests_helper/utils.h" +using namespace InferenceEngine; + + +/** + * @brief Function that contain executable pipeline which will be called from + * main(). The function should not throw any exceptions and responsible for + * handling it by itself. + */ +int runPipeline(const std::string &model, const std::string &device) { + auto pipeline = [](const std::string &model, const std::string &device) { + Core ie; + CNNNetwork cnnNetwork; + ExecutableNetwork exeNetwork; + InferRequest inferRequest; + + { + SCOPED_TIMER(first_inference_latency); + { + SCOPED_TIMER(load_plugin); + ie.GetVersions(device); + } + { + SCOPED_TIMER(create_exenetwork); + if (TimeTest::fileExt(model) == "blob") { + SCOPED_TIMER(import_network); + exeNetwork = ie.ImportNetwork(model, device); + } + else { + { + SCOPED_TIMER(read_network); + cnnNetwork = ie.ReadNetwork(model); + } + + { + SCOPED_TIMER(load_network); + exeNetwork = ie.LoadNetwork(cnnNetwork, device); + } + } + } + } + + { + SCOPED_TIMER(first_inference); + inferRequest = exeNetwork.CreateInferRequest(); + + { + SCOPED_TIMER(fill_inputs) + auto batchSize = cnnNetwork.getBatchSize(); + batchSize = batchSize != 0 ? batchSize : 1; + const InferenceEngine::ConstInputsDataMap inputsInfo(exeNetwork.GetInputsInfo()); + fillBlobs(inferRequest, inputsInfo, batchSize); + } + inferRequest.Infer(); + } + }; + + try { + pipeline(model, device); + } catch (const InferenceEngine::details::InferenceEngineException &iex) { + std::cerr + << "Inference Engine pipeline failed with Inference Engine exception:\n" + << iex.what(); + return 1; + } catch (const std::exception &ex) { + std::cerr << "Inference Engine pipeline failed with exception:\n" + << ex.what(); + return 2; + } catch (...) { + std::cerr << "Inference Engine pipeline failed\n"; + return 3; + } + return 0; +} \ No newline at end of file diff --git a/tests/time_tests/src/timetests_helper/CMakeLists.txt b/tests/time_tests/src/timetests_helper/CMakeLists.txt new file mode 100644 index 00000000000..ba5760edecf --- /dev/null +++ b/tests/time_tests/src/timetests_helper/CMakeLists.txt @@ -0,0 +1,13 @@ +# Copyright (C) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set (TARGET_NAME "timetests_helper") + +find_package(gflags REQUIRED) + +file (GLOB SRC *.cpp) +add_library(${TARGET_NAME} STATIC ${SRC}) +target_include_directories(${TARGET_NAME} PUBLIC "${CMAKE_SOURCE_DIR}/include") + +target_link_libraries(${TARGET_NAME} gflags) diff --git a/tests/time_tests/src/timetests_helper/cli.h b/tests/time_tests/src/timetests_helper/cli.h new file mode 100644 index 00000000000..f22e45b5b21 --- /dev/null +++ b/tests/time_tests/src/timetests_helper/cli.h @@ -0,0 +1,65 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include + +/// @brief message for help argument +static const char help_message[] = "Print a usage message"; + +/// @brief message for model argument +static const char model_message[] = + "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to " + "a .blob files with a trained compiled model."; + +/// @brief message for target device argument +static const char target_device_message[] = + "Required. Specify a target device to infer on. " + "Use \"-d HETERO:\" format to specify HETERO " + "plugin. " + "Use \"-d MULTI:\" format to specify MULTI " + "plugin. " + "The application looks for a suitable plugin for the specified device."; + +/// @brief message for statistics path argument +static const char statistics_path_message[] = + "Required. Path to a file to write statistics."; + +/// @brief Define flag for showing help message
+DEFINE_bool(h, false, help_message); + +/// @brief Declare flag for showing help message
+DECLARE_bool(help); + +/// @brief Define parameter for set model file
+/// It is a required parameter +DEFINE_string(m, "", model_message); + +/// @brief Define parameter for set target device to infer on
+/// It is a required parameter +DEFINE_string(d, "", target_device_message); + +/// @brief Define parameter for set path to a file to write statistics
+/// It is a required parameter +DEFINE_string(s, "", statistics_path_message); + +/** + * @brief This function show a help message + */ +static void showUsage() { + std::cout << std::endl; + std::cout << "TimeTests [OPTION]" << std::endl; + std::cout << "Options:" << std::endl; + std::cout << std::endl; + std::cout << " -h, --help " << help_message << std::endl; + std::cout << " -m \"\" " << model_message << std::endl; + std::cout << " -d \"\" " << target_device_message + << std::endl; + std::cout << " -s \"\" " << statistics_path_message + << std::endl; +} diff --git a/tests/time_tests/src/timetests_helper/main.cpp b/tests/time_tests/src/timetests_helper/main.cpp new file mode 100644 index 00000000000..fda85b94aeb --- /dev/null +++ b/tests/time_tests/src/timetests_helper/main.cpp @@ -0,0 +1,55 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "cli.h" +#include "statistics_writer.h" +#include "timetests_helper/timer.h" + +#include + +int runPipeline(const std::string &model, const std::string &device); + +/** + * @brief Parses command line and check required arguments + */ +bool parseAndCheckCommandLine(int argc, char **argv) { + gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); + if (FLAGS_help || FLAGS_h) { + showUsage(); + return false; + } + + if (FLAGS_m.empty()) + throw std::logic_error( + "Model is required but not set. Please set -m option."); + + if (FLAGS_d.empty()) + throw std::logic_error( + "Device is required but not set. Please set -d option."); + + if (FLAGS_s.empty()) + throw std::logic_error( + "Statistics file path is required but not set. Please set -s option."); + + return true; +} + +/** + * @brief Function calls `runPipeline` with mandatory time tracking of full run + */ +int _runPipeline() { + SCOPED_TIMER(full_run); + return runPipeline(FLAGS_m, FLAGS_d); +} + +/** + * @brief Main entry point + */ +int main(int argc, char **argv) { + if (!parseAndCheckCommandLine(argc, argv)) + return -1; + + StatisticsWriter::Instance().setFile(FLAGS_s); + return _runPipeline(); +} \ No newline at end of file diff --git a/tests/time_tests/src/timetests_helper/statistics_writer.h b/tests/time_tests/src/timetests_helper/statistics_writer.h new file mode 100644 index 00000000000..548e1af0fde --- /dev/null +++ b/tests/time_tests/src/timetests_helper/statistics_writer.h @@ -0,0 +1,57 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include + +/** + * @brief Class response for writing provided statistics + * + * Object of the class is writing provided statistics to a specified + * file in YAML format. + */ +class StatisticsWriter { +private: + std::ofstream statistics_file; + + StatisticsWriter() = default; + StatisticsWriter(const StatisticsWriter &) = delete; + StatisticsWriter &operator=(const StatisticsWriter &) = delete; + +public: + /** + * @brief Creates StatisticsWriter singleton object + */ + static StatisticsWriter &Instance() { + static StatisticsWriter writer; + return writer; + } + + /** + * @brief Specifies, opens and validates statistics path for writing + */ + void setFile(const std::string &statistics_path) { + statistics_file.open(statistics_path); + if (!statistics_file.good()) { + std::stringstream err; + err << "Statistic file \"" << statistics_path + << "\" can't be used for writing"; + throw std::runtime_error(err.str()); + } + } + + /** + * @brief Writes provided statistics in YAML format. + */ + void write(const std::pair &record) { + if (!statistics_file) + throw std::runtime_error("Statistic file path isn't set"); + statistics_file << record.first << ": " << record.second << "\n"; + } +}; diff --git a/tests/time_tests/src/timetests_helper/timer.cpp b/tests/time_tests/src/timetests_helper/timer.cpp new file mode 100644 index 00000000000..3bb1576a850 --- /dev/null +++ b/tests/time_tests/src/timetests_helper/timer.cpp @@ -0,0 +1,29 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "timetests_helper/timer.h" +#include +#include +#include +#include + +#include "statistics_writer.h" + +using time_point = std::chrono::high_resolution_clock::time_point; + +namespace TimeTest { + +Timer::Timer(const std::string &timer_name) { + name = timer_name; + start_time = std::chrono::high_resolution_clock::now(); +} + +Timer::~Timer() { + float duration = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - start_time) + .count(); + StatisticsWriter::Instance().write({name, duration}); +} + +} // namespace TimeTest \ No newline at end of file diff --git a/tests/time_tests/test_runner/.automation/tgl_test_config.yml b/tests/time_tests/test_runner/.automation/tgl_test_config.yml new file mode 100644 index 00000000000..4c97c80f21c --- /dev/null +++ b/tests/time_tests/test_runner/.automation/tgl_test_config.yml @@ -0,0 +1,56 @@ +- device: + name: CPU + model: + path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml + name: resnet-50-pytorch + precision: FP16 + framework: caffe2 +- device: + name: GPU + model: + path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml + name: resnet-50-pytorch + precision: FP16 + framework: caffe2 +- device: + name: CPU + model: + path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml + name: resnet-50-pytorch + precision: FP16-INT8 + framework: caffe2 +- device: + name: GPU + model: + path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml + name: resnet-50-pytorch + precision: FP16-INT8 + framework: caffe2 +- device: + name: CPU + model: + path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml + name: mobilenet-v2 + precision: FP16 + framework: caffe2 +- device: + name: GPU + model: + path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml + name: mobilenet-v2 + precision: FP16 + framework: caffe2 +- device: + name: CPU + model: + path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml + name: mobilenet-v2 + precision: FP16-INT8 + framework: caffe2 +- device: + name: GPU + model: + path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml + name: mobilenet-v2 + precision: FP16-INT8 + framework: caffe2 \ No newline at end of file diff --git a/tests/time_tests/test_runner/conftest.py b/tests/time_tests/test_runner/conftest.py new file mode 100644 index 00000000000..a2a0df40312 --- /dev/null +++ b/tests/time_tests/test_runner/conftest.py @@ -0,0 +1,292 @@ +# Copyright (C) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +""" +Basic high-level plugin file for pytest. + +See [Writing plugins](https://docs.pytest.org/en/latest/writing_plugins.html) +for more information. + +This plugin adds the following command-line options: + +* `--test_conf` - Path to test configuration file. Used to parametrize tests. + Format: YAML file. +* `--exe` - Path to a timetest binary to execute. +* `--niter` - Number of times to run executable. +""" + +# pylint:disable=import-error +import os +import sys +import pytest +from pathlib import Path +import yaml +import hashlib +import shutil +import logging +import tempfile +from jsonschema import validate, ValidationError + +from test_runner.utils import upload_timetest_data, \ + DATABASE, DB_COLLECTIONS +from scripts.run_timetest import check_positive_int + + +# -------------------- CLI options -------------------- + + +def pytest_addoption(parser): + """Specify command-line options for all plugins""" + test_args_parser = parser.getgroup("timetest test run") + test_args_parser.addoption( + "--test_conf", + type=Path, + help="path to a test config", + default=Path(__file__).parent / "test_config.yml" + ) + test_args_parser.addoption( + "--exe", + required=True, + dest="executable", + type=Path, + help="path to a timetest binary to execute" + ) + test_args_parser.addoption( + "--niter", + type=check_positive_int, + help="number of iterations to run executable and aggregate results", + default=3 + ) + # TODO: add support of --mo, --omz etc. required for OMZ support + helpers_args_parser = parser.getgroup("test helpers") + helpers_args_parser.addoption( + "--dump_refs", + type=Path, + help="path to dump test config with references updated with statistics collected while run", + ) + db_args_parser = parser.getgroup("timetest database use") + db_args_parser.addoption( + '--db_submit', + metavar="RUN_ID", + type=str, + help='submit results to the database. ' \ + '`RUN_ID` should be a string uniquely identifying the run' \ + ' (like Jenkins URL or time)' + ) + is_db_used = db_args_parser.parser.parse_known_args(sys.argv).db_submit + db_args_parser.addoption( + '--db_url', + type=str, + required=is_db_used, + help='MongoDB URL in a form "mongodb://server:port"' + ) + db_args_parser.addoption( + '--db_collection', + type=str, + required=is_db_used, + help='collection name in "{}" database'.format(DATABASE), + choices=DB_COLLECTIONS + ) + + +@pytest.fixture(scope="session") +def test_conf(request): + """Fixture function for command-line option.""" + return request.config.getoption('test_conf') + + +@pytest.fixture(scope="session") +def executable(request): + """Fixture function for command-line option.""" + return request.config.getoption('executable') + + +@pytest.fixture(scope="session") +def niter(request): + """Fixture function for command-line option.""" + return request.config.getoption('niter') + +# -------------------- CLI options -------------------- + + +@pytest.fixture(scope="function") +def temp_dir(pytestconfig): + """Create temporary directory for test purposes. + It will be cleaned up after every test run. + """ + temp_dir = tempfile.TemporaryDirectory() + yield Path(temp_dir.name) + temp_dir.cleanup() + + +@pytest.fixture(scope="function") +def cl_cache_dir(pytestconfig): + """Generate directory to save OpenCL cache before test run and clean up after run. + + Folder `cl_cache` should be created in a directory where tests were run. In this case + cache will be saved correctly. This behaviour is OS independent. + More: https://github.com/intel/compute-runtime/blob/master/opencl/doc/FAQ.md#how-can-cl_cache-be-enabled + """ + cl_cache_dir = pytestconfig.invocation_dir / "cl_cache" + # if cl_cache generation to a local `cl_cache` folder doesn't work, specify + # `cl_cache_dir` environment variable in an attempt to fix it (Linux specific) + os.environ["cl_cache_dir"] = str(cl_cache_dir) + if cl_cache_dir.exists(): + shutil.rmtree(cl_cache_dir) + cl_cache_dir.mkdir() + yield cl_cache_dir + shutil.rmtree(cl_cache_dir) + + +@pytest.fixture(scope="function") +def test_info(request, pytestconfig): + """Fixture for collecting timetests information. + + Current fixture fills in `request` and `pytestconfig` global + fixtures with timetests information which will be used for + internal purposes. + """ + setattr(request.node._request, "test_info", {"orig_instance": request.node.funcargs["instance"], + "results": {}}) + if not hasattr(pytestconfig, "session_info"): + setattr(pytestconfig, "session_info", []) + + yield request.node._request.test_info + + pytestconfig.session_info.append(request.node._request.test_info) + + +@pytest.fixture(scope="function") +def validate_test_case(request, test_info): + """Fixture for validating test case on correctness. + + Fixture checks current test case contains all fields required for + a correct work. To submit results to a database test case have + contain several additional properties. + """ + schema = { + "type": "object", + "properties": { + "device": { + "type": "object", + "properties": { + "name": {"type": "string"} + }}, + "model": { + "type": "object", + "properties": { + "path": {"type": "string"} + }}, + }, + } + if request.config.getoption("db_submit"): + # For submission data to a database some additional fields are required + schema["properties"]["model"]["properties"].update({ + "name": {"type": "string"}, + "precision": {"type": "string"}, + "framework": {"type": "string"} + }) + test_info["submit_to_db"] = True + try: + validate(instance=request.node.funcargs["instance"], schema=schema) + except ValidationError: + test_info["submit_to_db"] = False + raise + yield + + +@pytest.fixture(scope="session", autouse=True) +def prepare_tconf_with_refs(pytestconfig): + """Fixture for preparing test config based on original test config + with timetests results saved as references. + """ + yield + new_tconf_path = pytestconfig.getoption('dump_refs') + if new_tconf_path: + logging.info("Save new test config with test results as references to {}".format(new_tconf_path)) + upd_cases = pytestconfig.orig_cases.copy() + for record in pytestconfig.session_info: + rec_i = upd_cases.index(record["orig_instance"]) + upd_cases[rec_i]["references"] = record["results"] + with open(new_tconf_path, "w") as tconf: + yaml.safe_dump(upd_cases, tconf) + + +def pytest_generate_tests(metafunc): + """Pytest hook for test generation. + + Generate parameterized tests from discovered modules and test config + parameters. + """ + with open(metafunc.config.getoption('test_conf'), "r") as file: + test_cases = yaml.safe_load(file) + if test_cases: + metafunc.parametrize("instance", test_cases) + setattr(metafunc.config, "orig_cases", test_cases) + + +def pytest_make_parametrize_id(config, val, argname): + """Pytest hook for user-friendly test name representation""" + + def get_dict_values(d): + """Unwrap dictionary to get all values of nested dictionaries""" + if isinstance(d, dict): + for v in d.values(): + yield from get_dict_values(v) + else: + yield d + + keys = ["device", "model"] + values = {key: val[key] for key in keys} + values = list(get_dict_values(values)) + + return "-".join(["_".join([key, str(val)]) for key, val in zip(keys, values)]) + + +@pytest.mark.hookwrapper +def pytest_runtest_makereport(item, call): + """Pytest hook for report preparation. + + Submit tests' data to a database. + """ + + FIELDS_FOR_ID = ['timetest', 'model', 'device', 'niter', 'run_id'] + FIELDS_FOR_SUBMIT = FIELDS_FOR_ID + ['_id', 'test_name', + 'results', 'status', 'error_msg'] + + run_id = item.config.getoption("db_submit") + db_url = item.config.getoption("db_url") + db_collection = item.config.getoption("db_collection") + if not (run_id and db_url and db_collection): + yield + return + if not item._request.test_info["submit_to_db"]: + logging.error("Data won't be uploaded to a database on '{}' step".format(call.when)) + yield + return + + data = item.funcargs.copy() + data["timetest"] = data.pop("executable").stem + data.update(data["instance"]) + + data['run_id'] = run_id + data['_id'] = hashlib.sha256( + ''.join([str(data[key]) for key in FIELDS_FOR_ID]).encode()).hexdigest() + + data["test_name"] = item.name + data["results"] = item._request.test_info["results"] + data["status"] = "not_finished" + data["error_msg"] = "" + + data = {field: data[field] for field in FIELDS_FOR_SUBMIT} + + report = (yield).get_result() + if call.when in ["setup", "call"]: + if call.when == "call": + if not report.passed: + data["status"] = "failed" + data["error_msg"] = report.longrepr.reprcrash.message + else: + data["status"] = "passed" + logging.info("Upload data to {}/{}.{}. Data: {}".format(db_url, DATABASE, db_collection, data)) + upload_timetest_data(data, db_url, db_collection) diff --git a/tests/time_tests/test_runner/requirements.txt b/tests/time_tests/test_runner/requirements.txt new file mode 100644 index 00000000000..ae3ab36a70d --- /dev/null +++ b/tests/time_tests/test_runner/requirements.txt @@ -0,0 +1,4 @@ +pytest==4.0.1 +attrs==19.1.0 # required for pytest==4.0.1 to resolve compatibility issues +PyYAML==5.3.1 +jsonschema==3.2.0 \ No newline at end of file diff --git a/tests/time_tests/test_runner/test_config.yml b/tests/time_tests/test_runner/test_config.yml new file mode 100644 index 00000000000..3f7c82f316f --- /dev/null +++ b/tests/time_tests/test_runner/test_config.yml @@ -0,0 +1,14 @@ +- device: + name: CPU + model: + path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml # TODO: add link to `test_data` repo model + name: alexnet + precision: FP32 + framework: caffe +- device: + name: GPU + model: + path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml + name: alexnet + precision: FP32 + framework: caffe diff --git a/tests/time_tests/test_runner/test_timetest.py b/tests/time_tests/test_runner/test_timetest.py new file mode 100644 index 00000000000..a9aaf747c18 --- /dev/null +++ b/tests/time_tests/test_runner/test_timetest.py @@ -0,0 +1,83 @@ +# Copyright (C) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +"""Main entry-point to run timetests tests. + +Default run: +$ pytest test_timetest.py + +Options[*]: +--test_conf Path to test config +--exe Path to timetest binary to execute +--niter Number of times to run executable + +[*] For more information see conftest.py +""" + +from pathlib import Path +import logging +import os +import shutil + +from scripts.run_timetest import run_timetest +from test_runner.utils import expand_env_vars + +REFS_FACTOR = 1.2 # 120% + + +def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir, validate_test_case): + """Parameterized test. + + :param instance: test instance. Should not be changed during test run + :param executable: timetest executable to run + :param niter: number of times to run executable + :param cl_cache_dir: directory to store OpenCL cache + :param test_info: custom `test_info` field of built-in `request` pytest fixture + :param temp_dir: path to a temporary directory. Will be cleaned up after test run + :param validate_test_case: custom pytest fixture. Should be declared as test argument to be enabled + """ + # Prepare model to get model_path + model_path = instance["model"].get("path") + assert model_path, "Model path is empty" + model_path = Path(expand_env_vars(model_path)) + + # Copy model to a local temporary directory + model_dir = temp_dir / "model" + shutil.copytree(model_path.parent, model_dir) + model_path = model_dir / model_path.name + + # Run executable + exe_args = { + "executable": Path(executable), + "model": Path(model_path), + "device": instance["device"]["name"], + "niter": niter + } + if exe_args["device"] == "GPU": + # Generate cl_cache via additional timetest run + _exe_args = exe_args.copy() + _exe_args["niter"] = 1 + logging.info("Run timetest once to generate cl_cache to {}".format(cl_cache_dir)) + run_timetest(_exe_args, log=logging) + assert os.listdir(cl_cache_dir), "cl_cache isn't generated" + + retcode, aggr_stats = run_timetest(exe_args, log=logging) + assert retcode == 0, "Run of executable failed" + + # Add timetest results to submit to database and save in new test conf as references + test_info["results"] = aggr_stats + + # Compare with references + comparison_status = 0 + for step_name, references in instance["references"].items(): + for metric, reference_val in references.items(): + if aggr_stats[step_name][metric] > reference_val * REFS_FACTOR: + logging.error("Comparison failed for '{}' step for '{}' metric. Reference: {}. Current values: {}" + .format(step_name, metric, reference_val, aggr_stats[step_name][metric])) + comparison_status = 1 + else: + logging.info("Comparison passed for '{}' step for '{}' metric. Reference: {}. Current values: {}" + .format(step_name, metric, reference_val, aggr_stats[step_name][metric])) + + assert comparison_status == 0, "Comparison with references failed" + diff --git a/tests/time_tests/test_runner/utils.py b/tests/time_tests/test_runner/utils.py new file mode 100644 index 00000000000..8cce2e1cd2d --- /dev/null +++ b/tests/time_tests/test_runner/utils.py @@ -0,0 +1,33 @@ +# Copyright (C) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +"""Utility module.""" + +import os +from pymongo import MongoClient + +# constants +DATABASE = 'timetests' # database name for timetests results +DB_COLLECTIONS = ["commit", "nightly", "weekly"] + + +def expand_env_vars(obj): + """Expand environment variables in provided object.""" + + if isinstance(obj, list): + for i, value in enumerate(obj): + obj[i] = expand_env_vars(value) + elif isinstance(obj, dict): + for name, value in obj.items(): + obj[name] = expand_env_vars(value) + else: + obj = os.path.expandvars(obj) + return obj + + +def upload_timetest_data(data, db_url, db_collection): + """ Upload timetest data to database + """ + client = MongoClient(db_url) + collection = client[DATABASE][db_collection] + collection.replace_one({'_id': data['_id']}, data, upsert=True)