[Caching] Add caching options to benchmark app (#4909)
* Python API for LoadNetwork by model file name
* BenchmarkApp: Add caching and LoadNetworkFromFile support
2 new options are introduced
- cache_dir <dir> - enables models caching
- load_from_file - use new perform "LoadNetwork" by model file name
Using both parameters will achieve maximum performance of read/load network on startup
Tests:
1) Run "benchmark_app -h". Help will display 2 new options. After available devices there will be list of devices with cache support
2) ./benchmark_app -d CPU -i <model.xml> -load_from_file
Verify that some test steps are skipped (related to ReadNetwork, re-shaping etc)
3) Pre-requisite: support of caching shall be enabled for Template plugin
./benchmark_app -d TEMPLATE -i <model.onnx> -load_from_file -cache_dir someDir
Verify that "someDir" is created and generated blob is available
Run again, verify that loading works as well (should be faster as it will not load onnx model)
4) Run same test as (3), but without -load_from_file option. Verify that cache is properly created
For some devices loadNetwork time shall be improved when cache is available
* Removed additional timing prints
* Correction from old code
* Revert "Removed additional timing prints"
Additional change - when .blob is chosen instead of .xml, it takes priority over caching flags
* Removed new time printings
As discussed, these time measurements like 'total first inference time' will be available in 'timeTests' scripts
* Fix clang-format issues
This commit is contained in:
Mikhail Nosov
GitHub
parent
647acffd1d
commit
d20900e235
|
|
@ -93,6 +93,8 @@ Options:
|
|||
-progress Optional. Show progress bar (can affect performance measurement). Default values is "false".
|
||||
-shape Optional. Set shape for input. For example, "input1[1,3,224,224],input2[1,4]" or "[1,3,224,224]" in case of one input size.
|
||||
-layout Optional. Prompts how network layouts should be treated by application. For example, "input1[NCHW],input2[NC]" or "[NCHW]" in case of one input size.
|
||||
-cache_dir "<path>" Optional. Enables caching of loaded models to specified directory.
|
||||
-load_from_file Optional. Loads model from file directly without ReadNetwork.
|
||||
|
||||
CPU-specific performance options:
|
||||
-nstreams "<integer>" Optional. Number of streams to use for inference on the CPU, GPU or MYRIAD devices
|
||||
|
|
|
|||
|
|
@ -122,6 +122,14 @@ static const char shape_message[] = "Optional. Set shape for input. For example,
|
|||
static const char layout_message[] = "Optional. Prompts how network layouts should be treated by application. "
|
||||
"For example, \"input1[NCHW],input2[NC]\" or \"[NCHW]\" in case of one input size.";
|
||||
|
||||
// @brief message for enabling caching
|
||||
static const char cache_dir_message[] = "Optional. Enables caching of loaded models to specified directory. "
|
||||
"List of devices which support caching is shown at the end of this message.";
|
||||
|
||||
// @brief message for single load network
|
||||
static const char load_from_file_message[] = "Optional. Loads model from file directly without ReadNetwork."
|
||||
"All CNNNetwork options (like re-shape) will be ignored";
|
||||
|
||||
// @brief message for quantization bits
|
||||
static const char gna_qb_message[] = "Optional. Weight bits for quantization: 8 or 16 (default)";
|
||||
|
||||
|
|
@ -238,6 +246,12 @@ DEFINE_string(op, "", outputs_precision_message);
|
|||
/// Overwrites layout from ip and op options for specified layers.";
|
||||
DEFINE_string(iop, "", iop_message);
|
||||
|
||||
/// @brief Define parameter for cache model dir <br>
|
||||
DEFINE_string(cache_dir, "", cache_dir_message);
|
||||
|
||||
/// @brief Define flag for load network from model file by name without ReadNetwork <br>
|
||||
DEFINE_bool(load_from_file, false, load_from_file_message);
|
||||
|
||||
/**
|
||||
* @brief This function show a help message
|
||||
*/
|
||||
|
|
@ -262,6 +276,8 @@ static void showUsage() {
|
|||
std::cout << " -progress " << progress_message << std::endl;
|
||||
std::cout << " -shape " << shape_message << std::endl;
|
||||
std::cout << " -layout " << layout_message << std::endl;
|
||||
std::cout << " -cache_dir \"<path>\" " << cache_dir_message << std::endl;
|
||||
std::cout << " -load_from_file " << load_from_file_message << std::endl;
|
||||
std::cout << std::endl << " device-specific performance options:" << std::endl;
|
||||
std::cout << " -nstreams \"<integer>\" " << infer_num_streams_message << std::endl;
|
||||
std::cout << " -nthreads \"<integer>\" " << infer_num_threads_message << std::endl;
|
||||
|
|
|
|||
|
|
@ -330,7 +330,29 @@ int main(int argc, char* argv[]) {
|
|||
std::string topology_name = "";
|
||||
benchmark_app::InputsInfo app_inputs_info;
|
||||
std::string output_name;
|
||||
if (!isNetworkCompiled) {
|
||||
|
||||
// Takes priority over config from file
|
||||
if (!FLAGS_cache_dir.empty()) {
|
||||
ie.SetConfig({{CONFIG_KEY(CACHE_DIR), FLAGS_cache_dir}});
|
||||
}
|
||||
|
||||
if (FLAGS_load_from_file && !isNetworkCompiled) {
|
||||
next_step();
|
||||
slog::info << "Skipping the step for loading network from file" << slog::endl;
|
||||
next_step();
|
||||
slog::info << "Skipping the step for loading network from file" << slog::endl;
|
||||
next_step();
|
||||
slog::info << "Skipping the step for loading network from file" << slog::endl;
|
||||
auto startTime = Time::now();
|
||||
exeNetwork = ie.LoadNetwork(FLAGS_m, device_name);
|
||||
auto duration_ms = double_to_string(get_total_ms_time(startTime));
|
||||
slog::info << "Load network took " << duration_ms << " ms" << slog::endl;
|
||||
if (statistics)
|
||||
statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"load network time (ms)", duration_ms}});
|
||||
if (batchSize == 0) {
|
||||
batchSize = 1;
|
||||
}
|
||||
} else if (!isNetworkCompiled) {
|
||||
// ----------------- 4. Reading the Intermediate Representation network
|
||||
// ----------------------------------------
|
||||
next_step();
|
||||
|
|
@ -363,7 +385,7 @@ int main(int argc, char* argv[]) {
|
|||
slog::info << "Reshaping network: " << getShapesString(shapes) << slog::endl;
|
||||
startTime = Time::now();
|
||||
cnnNetwork.reshape(shapes);
|
||||
auto duration_ms = double_to_string(get_total_ms_time(startTime));
|
||||
duration_ms = double_to_string(get_total_ms_time(startTime));
|
||||
slog::info << "Reshape network took " << duration_ms << " ms" << slog::endl;
|
||||
if (statistics)
|
||||
statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"reshape network time (ms)", duration_ms}});
|
||||
|
|
|
|||
|
|
@ -142,6 +142,11 @@ Options:
|
|||
parameters. Please note, command line parameters have
|
||||
higher priority then parameters from configuration
|
||||
file.
|
||||
-cdir CACHE_DIR, --cache_dir CACHE_DIR
|
||||
Optional. Enable model caching to specified directory
|
||||
-lfile [LOAD_FROM_FILE], --load_from_file [LOAD_FROM_FILE]
|
||||
Optional. Loads model from file directly without
|
||||
read_network.
|
||||
```
|
||||
|
||||
Running the application with the empty list of options yields the usage message given above and an error message.
|
||||
|
|
|
|||
|
|
@ -46,6 +46,9 @@ class Benchmark:
|
|||
for device in config.keys():
|
||||
self.ie.set_config(config[device], device)
|
||||
|
||||
def set_cache_dir(self, cache_dir: str):
|
||||
self.ie.set_config({'CACHE_DIR': cache_dir}, '')
|
||||
|
||||
def read_network(self, path_to_model: str):
|
||||
model_filename = os.path.abspath(path_to_model)
|
||||
head, ext = os.path.splitext(model_filename)
|
||||
|
|
@ -63,6 +66,16 @@ class Benchmark:
|
|||
|
||||
return exe_network
|
||||
|
||||
def load_network_from_file(self, path_to_model: str, config = {}):
|
||||
exe_network = self.ie.load_network(path_to_model,
|
||||
self.device,
|
||||
config=config,
|
||||
num_requests=1 if self.api_type == 'sync' else self.nireq or 0)
|
||||
# Number of requests
|
||||
self.nireq = len(exe_network.requests)
|
||||
|
||||
return exe_network
|
||||
|
||||
def import_network(self, path_to_file : str, config = {}):
|
||||
exe_network = self.ie.import_network(model_file=path_to_file,
|
||||
device_name=self.device,
|
||||
|
|
|
|||
|
|
@ -176,12 +176,41 @@ def run(args):
|
|||
|
||||
benchmark.set_config(config)
|
||||
batch_size = args.batch_size
|
||||
if not is_network_compiled:
|
||||
if args.cache_dir:
|
||||
benchmark.set_cache_dir(args.cache_dir)
|
||||
|
||||
topology_name = ""
|
||||
load_from_file_enabled = is_flag_set_in_command_line('load_from_file') or is_flag_set_in_command_line('lfile')
|
||||
if load_from_file_enabled and not is_network_compiled:
|
||||
next_step()
|
||||
print("Skipping the step for loading network from file")
|
||||
next_step()
|
||||
print("Skipping the step for loading network from file")
|
||||
next_step()
|
||||
print("Skipping the step for loading network from file")
|
||||
|
||||
# --------------------- 7. Loading the model to the device -------------------------------------------------
|
||||
next_step()
|
||||
|
||||
start_time = datetime.utcnow()
|
||||
exe_network = benchmark.load_network(args.path_to_model)
|
||||
duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}"
|
||||
logger.info(f"Load network took {duration_ms} ms")
|
||||
if statistics:
|
||||
statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
|
||||
[
|
||||
('load network time (ms)', duration_ms)
|
||||
])
|
||||
app_inputs_info, _ = get_inputs_info(args.shape, args.layout, args.batch_size, exe_network.input_info)
|
||||
if batch_size == 0:
|
||||
batch_size = 1
|
||||
elif not is_network_compiled:
|
||||
# --------------------- 4. Read the Intermediate Representation of the network -----------------------------
|
||||
next_step()
|
||||
|
||||
start_time = datetime.utcnow()
|
||||
ie_network = benchmark.read_network(args.path_to_model)
|
||||
topology_name = ie_network.name
|
||||
duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}"
|
||||
logger.info(f"Read network took {duration_ms} ms")
|
||||
if statistics:
|
||||
|
|
@ -281,7 +310,7 @@ def run(args):
|
|||
if statistics:
|
||||
statistics.add_parameters(StatisticsReport.Category.RUNTIME_CONFIG,
|
||||
[
|
||||
('topology', ie_network.name),
|
||||
('topology', topology_name),
|
||||
('target device', device_name),
|
||||
('API', args.api_type),
|
||||
('precision', "UNSPECIFIED"),
|
||||
|
|
@ -308,7 +337,7 @@ def run(args):
|
|||
|
||||
progress_bar = ProgressBar(progress_bar_total_count, args.stream_output, args.progress) if args.progress else None
|
||||
|
||||
duration_ms = f"{benchmark.first_infer(exe_network):.2f}"
|
||||
duration_ms = f"{benchmark.first_infer(exe_network):.2f}"
|
||||
logger.info(f"First inference took {duration_ms} ms")
|
||||
if statistics:
|
||||
statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
|
||||
|
|
|
|||
|
|
@ -124,6 +124,10 @@ def parse_args():
|
|||
help='Optional. Specifies precision for all output layers of the network.')
|
||||
args.add_argument('-iop', '--input_output_precision', type=str, required=False,
|
||||
help='Optional. Specifies precision for input and output layers by name. Example: -iop "input:FP16, output:FP16". Notice that quotes are required. Overwrites precision from ip and op options for specified layers.')
|
||||
args.add_argument('-cdir', '--cache_dir', type=str, required=False, default='',
|
||||
help="Optional. Enable model caching to specified directory")
|
||||
args.add_argument('-lfile', '--load_from_file', required=False, nargs='?', default=argparse.SUPPRESS,
|
||||
help="Optional. Loads model from file directly without read_network.")
|
||||
parsed_args = parser.parse_args()
|
||||
|
||||
return parsed_args
|
||||
|
|
|
|||
Loading…
Reference in New Issue