natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Align `time_tests` with master branch from 4021e144 (#2881)

This commit is contained in:
Vitaliy Urusovskij 2020-10-30 21:08:25 +03:00 committed by GitHub
parent 2cf8999d23
commit a081dfea0f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 1078 additions and 317 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/time_tests/CMakeLists.txt
View File

@ -11,4 +11,4 @@ endif()
find_package(InferenceEngineDeveloperPackage REQUIRED)
add_subdirectory(common)
add_subdirectory(src)

31
tests/time_tests/README.md Normal file
View File

@ -0,0 +1,31 @@
# Time Tests
This test suite contains pipelines, which are executables. The pipelines measure
the time of their execution, both total and partial. A Python runner calls the
pipelines and calcuates the average execution time.
## Prerequisites
To build the time tests, you need to have the `build` folder, which is created
when you configure and build OpenVINO™.
## Measure Time
To build and run the tests, open a terminal and run the commands below:
1. Build tests:
``` bash
mkdir build && cd build
cmake .. -DInferenceEngineDeveloperPackage_DIR=$(realpath ../../../build) && make time_tests
```
2. Run test:
``` bash
./scripts/run_timetest.py ../../bin/intel64/Release/timetest_infer -m model.xml -d CPU
```
2. Run several configurations using `pytest`:
``` bash
export PYTHONPATH=./:$PYTHONPATH
pytest ./test_runner/test_timetest.py --exe ../../bin/intel64/Release/timetest_infer
```

23
tests/time_tests/common/CMakeLists.txt
View File

@ -1,23 +0,0 @@
# Copyright (C) 2018-2019 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
set (TARGET_NAME "TimeTests")
file (GLOB SRC
*.cpp
../ftti_pipeline/*.cpp)
file (GLOB HDR
*.h
../ftti_pipeline/*.h)
# Create library file from sources.
add_executable(${TARGET_NAME} ${HDR} ${SRC})
find_package(gflags REQUIRED)
target_link_libraries(${TARGET_NAME}
gflags
${InferenceEngine_LIBRARIES}
)

57
tests/time_tests/common/cli.h
View File

@ -1,57 +0,0 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <string>
#include <vector>
#include <gflags/gflags.h>
#include <iostream>
/// @brief message for help argument
static const char help_message[] = "Print a usage message";
/// @brief message for model argument
static const char model_message[] = "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to a .blob files with a trained compiled model.";
/// @brief message for target device argument
static const char target_device_message[] = "Required. Specify a target device to infer on. " \
"Use \"-d HETERO:<comma-separated_devices_list>\" format to specify HETERO plugin. " \
"Use \"-d MULTI:<comma-separated_devices_list>\" format to specify MULTI plugin. " \
"The application looks for a suitable plugin for the specified device.";
/// @brief message for statistics path argument
static const char statistics_path_message[] = "Required. Path to a file to write statistics.";
/// @brief Define flag for showing help message <br>
DEFINE_bool(h, false, help_message);
/// @brief Declare flag for showing help message <br>
DECLARE_bool(help);
/// @brief Define parameter for set model file <br>
/// It is a required parameter
DEFINE_string(m, "", model_message);
/// @brief Define parameter for set target device to infer on <br>
/// It is a required parameter
DEFINE_string(d, "", target_device_message);
/// @brief Define parameter for set path to a file to write statistics <br>
/// It is a required parameter
DEFINE_string(s, "", statistics_path_message);
/**
* @brief This function show a help message
*/
static void showUsage() {
std::cout << std::endl;
std::cout << "TimeTests [OPTION]" << std::endl;
std::cout << "Options:" << std::endl;
std::cout << std::endl;
std::cout << " -h, --help " << help_message << std::endl;
std::cout << " -m \"<path>\" " << model_message << std::endl;
std::cout << " -d \"<device>\" " << target_device_message << std::endl;
std::cout << " -s \"<path>\" " << statistics_path_message << std::endl;
}

52
tests/time_tests/common/main.cpp
View File

@ -1,52 +0,0 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "cli.h"
#include "statistics_writer.h"
#include "../ftti_pipeline/ftti_pipeline.h"
#include <iostream>
/**
* @brief Parses command line and check required arguments
*/
bool parseAndCheckCommandLine(int argc, char **argv) {
gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true);
if (FLAGS_help || FLAGS_h) {
showUsage();
return false;
}
if (FLAGS_m.empty())
throw std::logic_error("Model is required but not set. Please set -m option.");
if (FLAGS_d.empty())
throw std::logic_error("Device is required but not set. Please set -d option.");
if (FLAGS_s.empty())
throw std::logic_error("Statistics file path is required but not set. Please set -s option.");
return true;
}
/**
* @brief Function calls `runPipeline` with mandatory time tracking of full run
*/
int _runPipeline() {
SCOPED_TIMER(full_run);
return runPipeline(FLAGS_m, FLAGS_d);
}
/**
* @brief Main entry point
*/
int main(int argc, char **argv) {
if (!parseAndCheckCommandLine(argc, argv))
return -1;
StatisticsWriter::Instance().setFile(FLAGS_s);
return _runPipeline();
}

55
tests/time_tests/common/statistics_writer.h
View File

@ -1,55 +0,0 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <string>
#include <sstream>
#include <fstream>
#include <cstdio>
/**
* @brief Class response for writing provided statistics
*
* Object of the class is writing provided statistics to a specified
* file in YAML format.
*/
class StatisticsWriter {
private:
std::ofstream statistics_file;
StatisticsWriter() = default;
StatisticsWriter(const StatisticsWriter&) = delete;
StatisticsWriter& operator=(const StatisticsWriter&) = delete;
public:
/**
* @brief Creates StatisticsWriter singleton object
*/
static StatisticsWriter& Instance(){
static StatisticsWriter writer;
return writer;
}
/**
* @brief Specifies, opens and validates statistics path for writing
*/
void setFile(const std::string &statistics_path) {
statistics_file.open(statistics_path);
if (!statistics_file.good()) {
std::stringstream err;
err << "Statistic file \"" << statistics_path << "\" can't be used for writing";
throw std::runtime_error(err.str());
}
}
/**
* @brief Writes provided statistics in YAML format.
*/
void write(const std::pair<std::string, float> &record) {
if (!statistics_file)
throw std::runtime_error("Statistic file path isn't set");
statistics_file << record.first << ": " << record.second << "\n";
}
};

46
tests/time_tests/common/timer.h
View File

@ -1,46 +0,0 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <string>
#include <chrono>
#include <fstream>
#include <memory>
#include "statistics_writer.h"
using time_point = std::chrono::high_resolution_clock::time_point;
/**
* @brief Class response for encapsulating time measurements.
*
* Object of a class measures time at start and finish of object's life cycle.
* When deleting, reports duration.
*/
class Timer {
private:
std::string name;
time_point start_time;
public:
/**
* @brief Constructs Timer object and measures start time
*/
Timer(const std::string &timer_name) {
name = timer_name;
start_time = std::chrono::high_resolution_clock::now();
}
/**
* @brief Destructs Timer object, measures duration and reports it
*/
~Timer(){
float duration = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::high_resolution_clock::now() - start_time).count();
StatisticsWriter::Instance().write({name, duration});
}
};
#define SCOPED_TIMER(timer_name) Timer timer_name(#timer_name);

49
tests/time_tests/ftti_pipeline/ftti_pipeline.h
View File

@ -1,49 +0,0 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "../common/timer.h"
#include <inference_engine.hpp>
using namespace InferenceEngine;
/**
* @brief Function that contain executable pipeline which will be called from main().
* The function should not throw any exceptions and responsible for handling it by itself.
*/
int runPipeline(const std::string &model, const std::string &device) {
auto pipeline = [](const std::string &model, const std::string &device){
SCOPED_TIMER(first_time_to_inference);
Core ie;
CNNNetwork cnnNetwork;
ExecutableNetwork exeNetwork;
{
SCOPED_TIMER(read_network);
cnnNetwork = ie.ReadNetwork(model);
}
{
SCOPED_TIMER(load_network);
ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, device);
}
};
try {
pipeline(model, device);
} catch (const InferenceEngine::details::InferenceEngineException& iex) {
std::cerr << "Inference Engine pipeline failed with Inference Engine exception:\n" << iex.what();
return 1;
} catch (const std::exception& ex) {
std::cerr << "Inference Engine pipeline failed with exception:\n" << ex.what();
return 2;
} catch (...) {
std::cerr << "Inference Engine pipeline failed\n";
return 3;
}
return 0;
}

32
tests/time_tests/include/timetests_helper/timer.h Normal file
View File

@ -0,0 +1,32 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <chrono>
#include <string>
namespace TimeTest {
using time_point = std::chrono::high_resolution_clock::time_point;
/** Encapsulate time measurements.
Object of a class measures time at start and finish of object's life cycle.
When destroyed, reports duration.
*/
class Timer {
private:
std::string name;
time_point start_time;
public:
/// Constructs Timer object and measures start time.
Timer(const std::string &timer_name);
/// Destructs Timer object, measures duration and reports it.
~Timer();
};
#define SCOPED_TIMER(timer_name) TimeTest::Timer timer_name(#timer_name);
} // namespace TimeTest

20
tests/time_tests/include/timetests_helper/utils.h Normal file
View File

@ -0,0 +1,20 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <string>
namespace TimeTest {
/**
* @brief Get extension from filename
* @param filename - name of the file which extension should be extracted
* @return string with extracted file extension
*/
std::string fileExt(const std::string& filename) {
auto pos = filename.rfind('.');
if (pos == std::string::npos) return "";
return filename.substr(pos + 1);
}
}

1
tests/time_tests/scripts/requirements.txt Normal file
View File

@ -0,0 +1 @@
PyYAML==5.3.1

64
tests/time_tests/run_executable.py → tests/time_tests/scripts/run_timetest.py
View File

@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
#
"""
This script runs TimeTests executable several times to aggregate
This script runs timetest executable several times and aggregate
collected statistics.
"""
@ -50,71 +50,66 @@ def run_cmd(args: list, log=None, verbose=True):
return proc.returncode, ''.join(output)
def read_stats(stats_path, stats: dict):
"""Read statistics from a file and extend provided statistics"""
with open(stats_path, "r") as file:
parsed_data = yaml.load(file, Loader=yaml.FullLoader)
return dict((step_name, stats.get(step_name, []) + [duration])
for step_name, duration in parsed_data.items())
def aggregate_stats(stats: dict):
"""Aggregate provided statistics"""
return {step_name: {"avg": statistics.mean(duration_list),
"stdev": statistics.stdev(duration_list)}
"stdev": statistics.stdev(duration_list) if len(duration_list) > 1 else 0}
for step_name, duration_list in stats.items()}
def write_aggregated_stats(stats_path, stats: dict):
"""Write aggregated statistics to a file in YAML format"""
with open(stats_path, "w") as file:
yaml.dump(stats, file)
def prepare_executable_cmd(args: dict):
"""Generate common part of cmd from arguments to execute"""
return [str(args["executable"].resolve()),
"-m", str(args["model"].resolve()),
return [str(args["executable"].resolve(strict=True)),
"-m", str(args["model"].resolve(strict=True)),
"-d", args["device"]]
def generate_tmp_path():
"""Generate temporary file path without file's creation"""
tmp_stats_file = tempfile.NamedTemporaryFile()
path = tmp_stats_file.name
tmp_stats_file.close() # remove temp file in order to create it by executable
return path
def run_executable(args: dict, log=None):
def run_timetest(args: dict, log=None):
"""Run provided executable several times and aggregate collected statistics"""
if log is None:
log = logging.getLogger('run_executable')
log = logging.getLogger('run_timetest')
cmd_common = prepare_executable_cmd(args)
# Run executable and collect statistics
stats = {}
for run_iter in range(args["niter"]):
tmp_stats_path = generate_tmp_path()
tmp_stats_path = tempfile.NamedTemporaryFile().name # create temp file, get path and delete temp file
retcode, msg = run_cmd(cmd_common + ["-s", str(tmp_stats_path)], log=log)
if retcode != 0:
log.error("Run of executable '{}' failed with return code '{}'. Error: {}\n"
"Statistics aggregation is skipped.".format(args["executable"], retcode, msg))
return retcode, {}
stats = read_stats(tmp_stats_path, stats)
# Read raw statistics
with open(tmp_stats_path, "r") as file:
raw_data = yaml.safe_load(file)
log.debug("Raw statistics after run of executable #{}: {}".format(run_iter, raw_data))
# Combine statistics from several runs
stats = dict((step_name, stats.get(step_name, []) + [duration])
for step_name, duration in raw_data.items())
# Aggregate results
aggregated_stats = aggregate_stats(stats)
log.debug("Aggregated statistics after full run: {}".format(aggregated_stats))
return 0, aggregated_stats
def check_positive_int(val):
"""Check argsparse argument is positive integer and return it"""
value = int(val)
if value < 1:
msg = "%r is less than 1" % val
raise argparse.ArgumentTypeError(msg)
return value
def cli_parser():
"""parse command-line arguments"""
parser = argparse.ArgumentParser(description='Run TimeTests executable')
parser = argparse.ArgumentParser(description='Run timetest executable')
parser.add_argument('executable',
type=Path,
help='binary to execute')
@ -131,7 +126,7 @@ def cli_parser():
help='target device to infer on')
parser.add_argument('-niter',
default=3,
type=int,
type=check_positive_int,
help='number of times to execute binary to aggregate statistics of')
parser.add_argument('-s',
dest="stats_path",
@ -149,11 +144,12 @@ if __name__ == "__main__":
logging.basicConfig(format="[ %(levelname)s ] %(message)s",
level=logging.DEBUG, stream=sys.stdout)
exit_code, aggr_stats = run_executable(dict(args._get_kwargs()), log=logging) # pylint: disable=protected-access
exit_code, aggr_stats = run_timetest(dict(args._get_kwargs()), log=logging) # pylint: disable=protected-access
if args.stats_path:
# Save aggregated results to a file
write_aggregated_stats(args.stats_path, aggr_stats)
with open(args.stats_path, "w") as file:
yaml.safe_dump(aggr_stats, file)
logging.info("Aggregated statistics saved to a file: '{}'".format(
args.stats_path.resolve()))
else:

6
tests/time_tests/src/CMakeLists.txt Normal file
View File

@ -0,0 +1,6 @@
# Copyright (C) 2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
add_subdirectory(timetests)
add_subdirectory(timetests_helper)

19
tests/time_tests/src/timetests/CMakeLists.txt Normal file
View File

@ -0,0 +1,19 @@
# Copyright (C) 2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
# add dummy `time_tests` target combines all time tests
add_custom_target(time_tests)
# Build test from every source file.
# Test target name is source file name without extension.
FILE(GLOB tests "*.cpp")
foreach(test_source ${tests})
get_filename_component(test_name ${test_source} NAME_WE)
add_executable(${test_name} ${test_source})
target_link_libraries(${test_name} PRIVATE IE::inference_engine timetests_helper)
add_dependencies(time_tests ${test_name})
endforeach()

154
tests/time_tests/src/timetests/common.h Normal file
View File

@ -0,0 +1,154 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
using namespace InferenceEngine;
/**
* @brief Determine if InferenceEngine blob means image or not
*/
template<typename T>
static bool isImage(const T &blob) {
auto descriptor = blob->getTensorDesc();
if (descriptor.getLayout() != InferenceEngine::NCHW) {
return false;
}
auto channels = descriptor.getDims()[1];
return channels == 3;
}
/**
* @brief Determine if InferenceEngine blob means image information or not
*/
template<typename T>
static bool isImageInfo(const T &blob) {
auto descriptor = blob->getTensorDesc();
if (descriptor.getLayout() != InferenceEngine::NC) {
return false;
}
auto channels = descriptor.getDims()[1];
return (channels >= 2);
}
/**
* @brief Return height and width from provided InferenceEngine tensor description
*/
inline std::pair<size_t, size_t> getTensorHeightWidth(const InferenceEngine::TensorDesc& desc) {
const auto& layout = desc.getLayout();
const auto& dims = desc.getDims();
const auto& size = dims.size();
if ((size >= 2) &&
(layout == InferenceEngine::Layout::NCHW ||
layout == InferenceEngine::Layout::NHWC ||
layout == InferenceEngine::Layout::NCDHW ||
layout == InferenceEngine::Layout::NDHWC ||
layout == InferenceEngine::Layout::OIHW ||
layout == InferenceEngine::Layout::GOIHW ||
layout == InferenceEngine::Layout::OIDHW ||
layout == InferenceEngine::Layout::GOIDHW ||
layout == InferenceEngine::Layout::CHW ||
layout == InferenceEngine::Layout::HW)) {
// Regardless of layout, dimensions are stored in fixed order
return std::make_pair(dims.back(), dims.at(size - 2));
} else {
THROW_IE_EXCEPTION << "Tensor does not have height and width dimensions";
}
}
/**
* @brief Fill InferenceEngine blob with random values
*/
template<typename T>
void fillBlobRandom(Blob::Ptr& inputBlob) {
MemoryBlob::Ptr minput = as<MemoryBlob>(inputBlob);
// locked memory holder should be alive all time while access to its buffer happens
auto minputHolder = minput->wmap();
auto inputBlobData = minputHolder.as<T *>();
for (size_t i = 0; i < inputBlob->size(); i++) {
auto rand_max = RAND_MAX;
inputBlobData[i] = (T) rand() / static_cast<T>(rand_max) * 10;
}
}
/**
* @brief Fill InferenceEngine blob with image information
*/
template<typename T>
void fillBlobImInfo(Blob::Ptr& inputBlob,
const size_t& batchSize,
std::pair<size_t, size_t> image_size) {
MemoryBlob::Ptr minput = as<MemoryBlob>(inputBlob);
// locked memory holder should be alive all time while access to its buffer happens
auto minputHolder = minput->wmap();
auto inputBlobData = minputHolder.as<T *>();
for (size_t b = 0; b < batchSize; b++) {
size_t iminfoSize = inputBlob->size()/batchSize;
for (size_t i = 0; i < iminfoSize; i++) {
size_t index = b*iminfoSize + i;
if (0 == i)
inputBlobData[index] = static_cast<T>(image_size.first);
else if (1 == i)
inputBlobData[index] = static_cast<T>(image_size.second);
else
inputBlobData[index] = 1;
}
}
}
/**
* @brief Fill InferRequest blobs with random values or image information
*/
void fillBlobs(InferenceEngine::InferRequest inferRequest,
const InferenceEngine::ConstInputsDataMap& inputsInfo,
const size_t& batchSize) {
std::vector<std::pair<size_t, size_t>> input_image_sizes;
for (const ConstInputsDataMap::value_type& item : inputsInfo) {
if (isImage(item.second))
input_image_sizes.push_back(getTensorHeightWidth(item.second->getTensorDesc()));
}
for (const ConstInputsDataMap::value_type& item : inputsInfo) {
Blob::Ptr inputBlob = inferRequest.GetBlob(item.first);
if (isImageInfo(inputBlob) && (input_image_sizes.size() == 1)) {
// Fill image information
auto image_size = input_image_sizes.at(0);
if (item.second->getPrecision() == InferenceEngine::Precision::FP32) {
fillBlobImInfo<float>(inputBlob, batchSize, image_size);
} else if (item.second->getPrecision() == InferenceEngine::Precision::FP16) {
fillBlobImInfo<short>(inputBlob, batchSize, image_size);
} else if (item.second->getPrecision() == InferenceEngine::Precision::I32) {
fillBlobImInfo<int32_t>(inputBlob, batchSize, image_size);
} else {
THROW_IE_EXCEPTION << "Input precision is not supported for image info!";
}
continue;
}
// Fill random
if (item.second->getPrecision() == InferenceEngine::Precision::FP32) {
fillBlobRandom<float>(inputBlob);
} else if (item.second->getPrecision() == InferenceEngine::Precision::FP16) {
fillBlobRandom<short>(inputBlob);
} else if (item.second->getPrecision() == InferenceEngine::Precision::I32) {
fillBlobRandom<int32_t>(inputBlob);
} else if (item.second->getPrecision() == InferenceEngine::Precision::U8) {
fillBlobRandom<uint8_t>(inputBlob);
} else if (item.second->getPrecision() == InferenceEngine::Precision::I8) {
fillBlobRandom<int8_t>(inputBlob);
} else if (item.second->getPrecision() == InferenceEngine::Precision::U16) {
fillBlobRandom<uint16_t>(inputBlob);
} else if (item.second->getPrecision() == InferenceEngine::Precision::I16) {
fillBlobRandom<int16_t>(inputBlob);
} else {
THROW_IE_EXCEPTION << "Input precision is not supported for " << item.first;
}
}
}

83
tests/time_tests/src/timetests/timetest_infer.cpp Normal file
View File

@ -0,0 +1,83 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <inference_engine.hpp>
#include <iostream>
#include "common.h"
#include "timetests_helper/timer.h"
#include "timetests_helper/utils.h"
using namespace InferenceEngine;
/**
* @brief Function that contain executable pipeline which will be called from
* main(). The function should not throw any exceptions and responsible for
* handling it by itself.
*/
int runPipeline(const std::string &model, const std::string &device) {
auto pipeline = [](const std::string &model, const std::string &device) {
Core ie;
CNNNetwork cnnNetwork;
ExecutableNetwork exeNetwork;
InferRequest inferRequest;
{
SCOPED_TIMER(first_inference_latency);
{
SCOPED_TIMER(load_plugin);
ie.GetVersions(device);
}
{
SCOPED_TIMER(create_exenetwork);
if (TimeTest::fileExt(model) == "blob") {
SCOPED_TIMER(import_network);
exeNetwork = ie.ImportNetwork(model, device);
}
else {
{
SCOPED_TIMER(read_network);
cnnNetwork = ie.ReadNetwork(model);
}
{
SCOPED_TIMER(load_network);
exeNetwork = ie.LoadNetwork(cnnNetwork, device);
}
}
}
}
{
SCOPED_TIMER(first_inference);
inferRequest = exeNetwork.CreateInferRequest();
{
SCOPED_TIMER(fill_inputs)
auto batchSize = cnnNetwork.getBatchSize();
batchSize = batchSize != 0 ? batchSize : 1;
const InferenceEngine::ConstInputsDataMap inputsInfo(exeNetwork.GetInputsInfo());
fillBlobs(inferRequest, inputsInfo, batchSize);
}
inferRequest.Infer();
}
};
try {
pipeline(model, device);
} catch (const InferenceEngine::details::InferenceEngineException &iex) {
std::cerr
<< "Inference Engine pipeline failed with Inference Engine exception:\n"
<< iex.what();
return 1;
} catch (const std::exception &ex) {
std::cerr << "Inference Engine pipeline failed with exception:\n"
<< ex.what();
return 2;
} catch (...) {
std::cerr << "Inference Engine pipeline failed\n";
return 3;
}
return 0;
}

13
tests/time_tests/src/timetests_helper/CMakeLists.txt Normal file
View File

@ -0,0 +1,13 @@
# Copyright (C) 2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
set (TARGET_NAME "timetests_helper")
find_package(gflags REQUIRED)
file (GLOB SRC *.cpp)
add_library(${TARGET_NAME} STATIC ${SRC})
target_include_directories(${TARGET_NAME} PUBLIC "${CMAKE_SOURCE_DIR}/include")
target_link_libraries(${TARGET_NAME} gflags)

65
tests/time_tests/src/timetests_helper/cli.h Normal file
View File

@ -0,0 +1,65 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <gflags/gflags.h>
#include <iostream>
#include <string>
#include <vector>
/// @brief message for help argument
static const char help_message[] = "Print a usage message";
/// @brief message for model argument
static const char model_message[] =
"Required. Path to an .xml/.onnx/.prototxt file with a trained model or to "
"a .blob files with a trained compiled model.";
/// @brief message for target device argument
static const char target_device_message[] =
"Required. Specify a target device to infer on. "
"Use \"-d HETERO:<comma-separated_devices_list>\" format to specify HETERO "
"plugin. "
"Use \"-d MULTI:<comma-separated_devices_list>\" format to specify MULTI "
"plugin. "
"The application looks for a suitable plugin for the specified device.";
/// @brief message for statistics path argument
static const char statistics_path_message[] =
"Required. Path to a file to write statistics.";
/// @brief Define flag for showing help message <br>
DEFINE_bool(h, false, help_message);
/// @brief Declare flag for showing help message <br>
DECLARE_bool(help);
/// @brief Define parameter for set model file <br>
/// It is a required parameter
DEFINE_string(m, "", model_message);
/// @brief Define parameter for set target device to infer on <br>
/// It is a required parameter
DEFINE_string(d, "", target_device_message);
/// @brief Define parameter for set path to a file to write statistics <br>
/// It is a required parameter
DEFINE_string(s, "", statistics_path_message);
/**
* @brief This function show a help message
*/
static void showUsage() {
std::cout << std::endl;
std::cout << "TimeTests [OPTION]" << std::endl;
std::cout << "Options:" << std::endl;
std::cout << std::endl;
std::cout << " -h, --help " << help_message << std::endl;
std::cout << " -m \"<path>\" " << model_message << std::endl;
std::cout << " -d \"<device>\" " << target_device_message
<< std::endl;
std::cout << " -s \"<path>\" " << statistics_path_message
<< std::endl;
}

55
tests/time_tests/src/timetests_helper/main.cpp Normal file
View File

@ -0,0 +1,55 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "cli.h"
#include "statistics_writer.h"
#include "timetests_helper/timer.h"
#include <iostream>
int runPipeline(const std::string &model, const std::string &device);
/**
* @brief Parses command line and check required arguments
*/
bool parseAndCheckCommandLine(int argc, char **argv) {
gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true);
if (FLAGS_help || FLAGS_h) {
showUsage();
return false;
}
if (FLAGS_m.empty())
throw std::logic_error(
"Model is required but not set. Please set -m option.");
if (FLAGS_d.empty())
throw std::logic_error(
"Device is required but not set. Please set -d option.");
if (FLAGS_s.empty())
throw std::logic_error(
"Statistics file path is required but not set. Please set -s option.");
return true;
}
/**
* @brief Function calls `runPipeline` with mandatory time tracking of full run
*/
int _runPipeline() {
SCOPED_TIMER(full_run);
return runPipeline(FLAGS_m, FLAGS_d);
}
/**
* @brief Main entry point
*/
int main(int argc, char **argv) {
if (!parseAndCheckCommandLine(argc, argv))
return -1;
StatisticsWriter::Instance().setFile(FLAGS_s);
return _runPipeline();
}

57
tests/time_tests/src/timetests_helper/statistics_writer.h Normal file
View File

@ -0,0 +1,57 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <cstdio>
#include <fstream>
#include <sstream>
#include <string>
#include <stdexcept>
/**
* @brief Class response for writing provided statistics
*
* Object of the class is writing provided statistics to a specified
* file in YAML format.
*/
class StatisticsWriter {
private:
std::ofstream statistics_file;
StatisticsWriter() = default;
StatisticsWriter(const StatisticsWriter &) = delete;
StatisticsWriter &operator=(const StatisticsWriter &) = delete;
public:
/**
* @brief Creates StatisticsWriter singleton object
*/
static StatisticsWriter &Instance() {
static StatisticsWriter writer;
return writer;
}
/**
* @brief Specifies, opens and validates statistics path for writing
*/
void setFile(const std::string &statistics_path) {
statistics_file.open(statistics_path);
if (!statistics_file.good()) {
std::stringstream err;
err << "Statistic file \"" << statistics_path
<< "\" can't be used for writing";
throw std::runtime_error(err.str());
}
}
/**
* @brief Writes provided statistics in YAML format.
*/
void write(const std::pair<std::string, float> &record) {
if (!statistics_file)
throw std::runtime_error("Statistic file path isn't set");
statistics_file << record.first << ": " << record.second << "\n";
}
};

29
tests/time_tests/src/timetests_helper/timer.cpp Normal file
View File

@ -0,0 +1,29 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "timetests_helper/timer.h"
#include <chrono>
#include <fstream>
#include <memory>
#include <string>
#include "statistics_writer.h"
using time_point = std::chrono::high_resolution_clock::time_point;
namespace TimeTest {
Timer::Timer(const std::string &timer_name) {
name = timer_name;
start_time = std::chrono::high_resolution_clock::now();
}
Timer::~Timer() {
float duration = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::high_resolution_clock::now() - start_time)
.count();
StatisticsWriter::Instance().write({name, duration});
}
} // namespace TimeTest

56
tests/time_tests/test_runner/.automation/tgl_test_config.yml Normal file
View File

@ -0,0 +1,56 @@
- device:
name: CPU
model:
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml
name: resnet-50-pytorch
precision: FP16
framework: caffe2
- device:
name: GPU
model:
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml
name: resnet-50-pytorch
precision: FP16
framework: caffe2
- device:
name: CPU
model:
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml
name: resnet-50-pytorch
precision: FP16-INT8
framework: caffe2
- device:
name: GPU
model:
path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml
name: resnet-50-pytorch
precision: FP16-INT8
framework: caffe2
- device:
name: CPU
model:
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml
name: mobilenet-v2
precision: FP16
framework: caffe2
- device:
name: GPU
model:
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml
name: mobilenet-v2
precision: FP16
framework: caffe2
- device:
name: CPU
model:
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml
name: mobilenet-v2
precision: FP16-INT8
framework: caffe2
- device:
name: GPU
model:
path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml
name: mobilenet-v2
precision: FP16-INT8
framework: caffe2

292
tests/time_tests/test_runner/conftest.py Normal file
View File

@ -0,0 +1,292 @@
# Copyright (C) 2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
"""
Basic high-level plugin file for pytest.
See [Writing plugins](https://docs.pytest.org/en/latest/writing_plugins.html)
for more information.
This plugin adds the following command-line options:
* `--test_conf` - Path to test configuration file. Used to parametrize tests.
Format: YAML file.
* `--exe` - Path to a timetest binary to execute.
* `--niter` - Number of times to run executable.
"""
# pylint:disable=import-error
import os
import sys
import pytest
from pathlib import Path
import yaml
import hashlib
import shutil
import logging
import tempfile
from jsonschema import validate, ValidationError
from test_runner.utils import upload_timetest_data, \
DATABASE, DB_COLLECTIONS
from scripts.run_timetest import check_positive_int
# -------------------- CLI options --------------------
def pytest_addoption(parser):
"""Specify command-line options for all plugins"""
test_args_parser = parser.getgroup("timetest test run")
test_args_parser.addoption(
"--test_conf",
type=Path,
help="path to a test config",
default=Path(__file__).parent / "test_config.yml"
)
test_args_parser.addoption(
"--exe",
required=True,
dest="executable",
type=Path,
help="path to a timetest binary to execute"
)
test_args_parser.addoption(
"--niter",
type=check_positive_int,
help="number of iterations to run executable and aggregate results",
default=3
)
# TODO: add support of --mo, --omz etc. required for OMZ support
helpers_args_parser = parser.getgroup("test helpers")
helpers_args_parser.addoption(
"--dump_refs",
type=Path,
help="path to dump test config with references updated with statistics collected while run",
)
db_args_parser = parser.getgroup("timetest database use")
db_args_parser.addoption(
'--db_submit',
metavar="RUN_ID",
type=str,
help='submit results to the database. ' \
'`RUN_ID` should be a string uniquely identifying the run' \
' (like Jenkins URL or time)'
)
is_db_used = db_args_parser.parser.parse_known_args(sys.argv).db_submit
db_args_parser.addoption(
'--db_url',
type=str,
required=is_db_used,
help='MongoDB URL in a form "mongodb://server:port"'
)
db_args_parser.addoption(
'--db_collection',
type=str,
required=is_db_used,
help='collection name in "{}" database'.format(DATABASE),
choices=DB_COLLECTIONS
)
@pytest.fixture(scope="session")
def test_conf(request):
"""Fixture function for command-line option."""
return request.config.getoption('test_conf')
@pytest.fixture(scope="session")
def executable(request):
"""Fixture function for command-line option."""
return request.config.getoption('executable')
@pytest.fixture(scope="session")
def niter(request):
"""Fixture function for command-line option."""
return request.config.getoption('niter')
# -------------------- CLI options --------------------
@pytest.fixture(scope="function")
def temp_dir(pytestconfig):
"""Create temporary directory for test purposes.
It will be cleaned up after every test run.
"""
temp_dir = tempfile.TemporaryDirectory()
yield Path(temp_dir.name)
temp_dir.cleanup()
@pytest.fixture(scope="function")
def cl_cache_dir(pytestconfig):
"""Generate directory to save OpenCL cache before test run and clean up after run.
Folder `cl_cache` should be created in a directory where tests were run. In this case
cache will be saved correctly. This behaviour is OS independent.
More: https://github.com/intel/compute-runtime/blob/master/opencl/doc/FAQ.md#how-can-cl_cache-be-enabled
"""
cl_cache_dir = pytestconfig.invocation_dir / "cl_cache"
# if cl_cache generation to a local `cl_cache` folder doesn't work, specify
# `cl_cache_dir` environment variable in an attempt to fix it (Linux specific)
os.environ["cl_cache_dir"] = str(cl_cache_dir)
if cl_cache_dir.exists():
shutil.rmtree(cl_cache_dir)
cl_cache_dir.mkdir()
yield cl_cache_dir
shutil.rmtree(cl_cache_dir)
@pytest.fixture(scope="function")
def test_info(request, pytestconfig):
"""Fixture for collecting timetests information.
Current fixture fills in `request` and `pytestconfig` global
fixtures with timetests information which will be used for
internal purposes.
"""
setattr(request.node._request, "test_info", {"orig_instance": request.node.funcargs["instance"],
"results": {}})
if not hasattr(pytestconfig, "session_info"):
setattr(pytestconfig, "session_info", [])
yield request.node._request.test_info
pytestconfig.session_info.append(request.node._request.test_info)
@pytest.fixture(scope="function")
def validate_test_case(request, test_info):
"""Fixture for validating test case on correctness.
Fixture checks current test case contains all fields required for
a correct work. To submit results to a database test case have
contain several additional properties.
"""
schema = {
"type": "object",
"properties": {
"device": {
"type": "object",
"properties": {
"name": {"type": "string"}
}},
"model": {
"type": "object",
"properties": {
"path": {"type": "string"}
}},
},
}
if request.config.getoption("db_submit"):
# For submission data to a database some additional fields are required
schema["properties"]["model"]["properties"].update({
"name": {"type": "string"},
"precision": {"type": "string"},
"framework": {"type": "string"}
})
test_info["submit_to_db"] = True
try:
validate(instance=request.node.funcargs["instance"], schema=schema)
except ValidationError:
test_info["submit_to_db"] = False
raise
yield
@pytest.fixture(scope="session", autouse=True)
def prepare_tconf_with_refs(pytestconfig):
"""Fixture for preparing test config based on original test config
with timetests results saved as references.
"""
yield
new_tconf_path = pytestconfig.getoption('dump_refs')
if new_tconf_path:
logging.info("Save new test config with test results as references to {}".format(new_tconf_path))
upd_cases = pytestconfig.orig_cases.copy()
for record in pytestconfig.session_info:
rec_i = upd_cases.index(record["orig_instance"])
upd_cases[rec_i]["references"] = record["results"]
with open(new_tconf_path, "w") as tconf:
yaml.safe_dump(upd_cases, tconf)
def pytest_generate_tests(metafunc):
"""Pytest hook for test generation.
Generate parameterized tests from discovered modules and test config
parameters.
"""
with open(metafunc.config.getoption('test_conf'), "r") as file:
test_cases = yaml.safe_load(file)
if test_cases:
metafunc.parametrize("instance", test_cases)
setattr(metafunc.config, "orig_cases", test_cases)
def pytest_make_parametrize_id(config, val, argname):
"""Pytest hook for user-friendly test name representation"""
def get_dict_values(d):
"""Unwrap dictionary to get all values of nested dictionaries"""
if isinstance(d, dict):
for v in d.values():
yield from get_dict_values(v)
else:
yield d
keys = ["device", "model"]
values = {key: val[key] for key in keys}
values = list(get_dict_values(values))
return "-".join(["_".join([key, str(val)]) for key, val in zip(keys, values)])
@pytest.mark.hookwrapper
def pytest_runtest_makereport(item, call):
"""Pytest hook for report preparation.
Submit tests' data to a database.
"""
FIELDS_FOR_ID = ['timetest', 'model', 'device', 'niter', 'run_id']
FIELDS_FOR_SUBMIT = FIELDS_FOR_ID + ['_id', 'test_name',
'results', 'status', 'error_msg']
run_id = item.config.getoption("db_submit")
db_url = item.config.getoption("db_url")
db_collection = item.config.getoption("db_collection")
if not (run_id and db_url and db_collection):
yield
return
if not item._request.test_info["submit_to_db"]:
logging.error("Data won't be uploaded to a database on '{}' step".format(call.when))
yield
return
data = item.funcargs.copy()
data["timetest"] = data.pop("executable").stem
data.update(data["instance"])
data['run_id'] = run_id
data['_id'] = hashlib.sha256(
''.join([str(data[key]) for key in FIELDS_FOR_ID]).encode()).hexdigest()
data["test_name"] = item.name
data["results"] = item._request.test_info["results"]
data["status"] = "not_finished"
data["error_msg"] = ""
data = {field: data[field] for field in FIELDS_FOR_SUBMIT}
report = (yield).get_result()
if call.when in ["setup", "call"]:
if call.when == "call":
if not report.passed:
data["status"] = "failed"
data["error_msg"] = report.longrepr.reprcrash.message
else:
data["status"] = "passed"
logging.info("Upload data to {}/{}.{}. Data: {}".format(db_url, DATABASE, db_collection, data))
upload_timetest_data(data, db_url, db_collection)

4
tests/time_tests/test_runner/requirements.txt Normal file
View File

@ -0,0 +1,4 @@
pytest==4.0.1
attrs==19.1.0 # required for pytest==4.0.1 to resolve compatibility issues
PyYAML==5.3.1
jsonschema==3.2.0

14
tests/time_tests/test_runner/test_config.yml Normal file
View File

@ -0,0 +1,14 @@
- device:
name: CPU
model:
path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml # TODO: add link to `test_data` repo model
name: alexnet
precision: FP32
framework: caffe
- device:
name: GPU
model:
path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml
name: alexnet
precision: FP32
framework: caffe

83
tests/time_tests/test_runner/test_timetest.py Normal file
View File

@ -0,0 +1,83 @@
# Copyright (C) 2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
"""Main entry-point to run timetests tests.
Default run:
$ pytest test_timetest.py
Options[*]:
--test_conf Path to test config
--exe Path to timetest binary to execute
--niter Number of times to run executable
[*] For more information see conftest.py
"""
from pathlib import Path
import logging
import os
import shutil
from scripts.run_timetest import run_timetest
from test_runner.utils import expand_env_vars
REFS_FACTOR = 1.2 # 120%
def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir, validate_test_case):
"""Parameterized test.
:param instance: test instance. Should not be changed during test run
:param executable: timetest executable to run
:param niter: number of times to run executable
:param cl_cache_dir: directory to store OpenCL cache
:param test_info: custom `test_info` field of built-in `request` pytest fixture
:param temp_dir: path to a temporary directory. Will be cleaned up after test run
:param validate_test_case: custom pytest fixture. Should be declared as test argument to be enabled
"""
# Prepare model to get model_path
model_path = instance["model"].get("path")
assert model_path, "Model path is empty"
model_path = Path(expand_env_vars(model_path))
# Copy model to a local temporary directory
model_dir = temp_dir / "model"
shutil.copytree(model_path.parent, model_dir)
model_path = model_dir / model_path.name
# Run executable
exe_args = {
"executable": Path(executable),
"model": Path(model_path),
"device": instance["device"]["name"],
"niter": niter
}
if exe_args["device"] == "GPU":
# Generate cl_cache via additional timetest run
_exe_args = exe_args.copy()
_exe_args["niter"] = 1
logging.info("Run timetest once to generate cl_cache to {}".format(cl_cache_dir))
run_timetest(_exe_args, log=logging)
assert os.listdir(cl_cache_dir), "cl_cache isn't generated"
retcode, aggr_stats = run_timetest(exe_args, log=logging)
assert retcode == 0, "Run of executable failed"
# Add timetest results to submit to database and save in new test conf as references
test_info["results"] = aggr_stats
# Compare with references
comparison_status = 0
for step_name, references in instance["references"].items():
for metric, reference_val in references.items():
if aggr_stats[step_name][metric] > reference_val * REFS_FACTOR:
logging.error("Comparison failed for '{}' step for '{}' metric. Reference: {}. Current values: {}"
.format(step_name, metric, reference_val, aggr_stats[step_name][metric]))
comparison_status = 1
else:
logging.info("Comparison passed for '{}' step for '{}' metric. Reference: {}. Current values: {}"
.format(step_name, metric, reference_val, aggr_stats[step_name][metric]))
assert comparison_status == 0, "Comparison with references failed"

33
tests/time_tests/test_runner/utils.py Normal file
View File

@ -0,0 +1,33 @@
# Copyright (C) 2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
"""Utility module."""
import os
from pymongo import MongoClient
# constants
DATABASE = 'timetests' # database name for timetests results
DB_COLLECTIONS = ["commit", "nightly", "weekly"]
def expand_env_vars(obj):
"""Expand environment variables in provided object."""
if isinstance(obj, list):
for i, value in enumerate(obj):
obj[i] = expand_env_vars(value)
elif isinstance(obj, dict):
for name, value in obj.items():
obj[name] = expand_env_vars(value)
else:
obj = os.path.expandvars(obj)
return obj
def upload_timetest_data(data, db_url, db_collection):
""" Upload timetest data to database
"""
client = MongoClient(db_url)
collection = client[DATABASE][db_collection]
collection.replace_one({'_id': data['_id']}, data, upsert=True)