diff --git a/inference-engine/samples/benchmark_app/README.md b/inference-engine/samples/benchmark_app/README.md index 432e8b28ec0..7c61bc570d5 100644 --- a/inference-engine/samples/benchmark_app/README.md +++ b/inference-engine/samples/benchmark_app/README.md @@ -93,6 +93,8 @@ Options: -progress Optional. Show progress bar (can affect performance measurement). Default values is "false". -shape Optional. Set shape for input. For example, "input1[1,3,224,224],input2[1,4]" or "[1,3,224,224]" in case of one input size. -layout Optional. Prompts how network layouts should be treated by application. For example, "input1[NCHW],input2[NC]" or "[NCHW]" in case of one input size. + -cache_dir "" Optional. Enables caching of loaded models to specified directory. + -load_from_file Optional. Loads model from file directly without ReadNetwork. CPU-specific performance options: -nstreams "" Optional. Number of streams to use for inference on the CPU, GPU or MYRIAD devices diff --git a/inference-engine/samples/benchmark_app/benchmark_app.hpp b/inference-engine/samples/benchmark_app/benchmark_app.hpp index 8cc5d19c781..af18c908e31 100644 --- a/inference-engine/samples/benchmark_app/benchmark_app.hpp +++ b/inference-engine/samples/benchmark_app/benchmark_app.hpp @@ -122,6 +122,14 @@ static const char shape_message[] = "Optional. Set shape for input. For example, static const char layout_message[] = "Optional. Prompts how network layouts should be treated by application. " "For example, \"input1[NCHW],input2[NC]\" or \"[NCHW]\" in case of one input size."; +// @brief message for enabling caching +static const char cache_dir_message[] = "Optional. Enables caching of loaded models to specified directory. " + "List of devices which support caching is shown at the end of this message."; + +// @brief message for single load network +static const char load_from_file_message[] = "Optional. Loads model from file directly without ReadNetwork." + "All CNNNetwork options (like re-shape) will be ignored"; + // @brief message for quantization bits static const char gna_qb_message[] = "Optional. Weight bits for quantization: 8 or 16 (default)"; @@ -238,6 +246,12 @@ DEFINE_string(op, "", outputs_precision_message); /// Overwrites layout from ip and op options for specified layers."; DEFINE_string(iop, "", iop_message); +/// @brief Define parameter for cache model dir
+DEFINE_string(cache_dir, "", cache_dir_message); + +/// @brief Define flag for load network from model file by name without ReadNetwork
+DEFINE_bool(load_from_file, false, load_from_file_message); + /** * @brief This function show a help message */ @@ -262,6 +276,8 @@ static void showUsage() { std::cout << " -progress " << progress_message << std::endl; std::cout << " -shape " << shape_message << std::endl; std::cout << " -layout " << layout_message << std::endl; + std::cout << " -cache_dir \"\" " << cache_dir_message << std::endl; + std::cout << " -load_from_file " << load_from_file_message << std::endl; std::cout << std::endl << " device-specific performance options:" << std::endl; std::cout << " -nstreams \"\" " << infer_num_streams_message << std::endl; std::cout << " -nthreads \"\" " << infer_num_threads_message << std::endl; diff --git a/inference-engine/samples/benchmark_app/main.cpp b/inference-engine/samples/benchmark_app/main.cpp index 5a34e558cd3..849dc05ad33 100644 --- a/inference-engine/samples/benchmark_app/main.cpp +++ b/inference-engine/samples/benchmark_app/main.cpp @@ -330,7 +330,29 @@ int main(int argc, char* argv[]) { std::string topology_name = ""; benchmark_app::InputsInfo app_inputs_info; std::string output_name; - if (!isNetworkCompiled) { + + // Takes priority over config from file + if (!FLAGS_cache_dir.empty()) { + ie.SetConfig({{CONFIG_KEY(CACHE_DIR), FLAGS_cache_dir}}); + } + + if (FLAGS_load_from_file && !isNetworkCompiled) { + next_step(); + slog::info << "Skipping the step for loading network from file" << slog::endl; + next_step(); + slog::info << "Skipping the step for loading network from file" << slog::endl; + next_step(); + slog::info << "Skipping the step for loading network from file" << slog::endl; + auto startTime = Time::now(); + exeNetwork = ie.LoadNetwork(FLAGS_m, device_name); + auto duration_ms = double_to_string(get_total_ms_time(startTime)); + slog::info << "Load network took " << duration_ms << " ms" << slog::endl; + if (statistics) + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"load network time (ms)", duration_ms}}); + if (batchSize == 0) { + batchSize = 1; + } + } else if (!isNetworkCompiled) { // ----------------- 4. Reading the Intermediate Representation network // ---------------------------------------- next_step(); @@ -363,7 +385,7 @@ int main(int argc, char* argv[]) { slog::info << "Reshaping network: " << getShapesString(shapes) << slog::endl; startTime = Time::now(); cnnNetwork.reshape(shapes); - auto duration_ms = double_to_string(get_total_ms_time(startTime)); + duration_ms = double_to_string(get_total_ms_time(startTime)); slog::info << "Reshape network took " << duration_ms << " ms" << slog::endl; if (statistics) statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"reshape network time (ms)", duration_ms}}); diff --git a/tools/benchmark/README.md b/tools/benchmark/README.md index 280b7a0ef53..50caa430fd4 100644 --- a/tools/benchmark/README.md +++ b/tools/benchmark/README.md @@ -142,6 +142,11 @@ Options: parameters. Please note, command line parameters have higher priority then parameters from configuration file. + -cdir CACHE_DIR, --cache_dir CACHE_DIR + Optional. Enable model caching to specified directory + -lfile [LOAD_FROM_FILE], --load_from_file [LOAD_FROM_FILE] + Optional. Loads model from file directly without + read_network. ``` Running the application with the empty list of options yields the usage message given above and an error message. diff --git a/tools/benchmark/benchmark.py b/tools/benchmark/benchmark.py index 754bdd194e7..c73477acfc2 100644 --- a/tools/benchmark/benchmark.py +++ b/tools/benchmark/benchmark.py @@ -46,6 +46,9 @@ class Benchmark: for device in config.keys(): self.ie.set_config(config[device], device) + def set_cache_dir(self, cache_dir: str): + self.ie.set_config({'CACHE_DIR': cache_dir}, '') + def read_network(self, path_to_model: str): model_filename = os.path.abspath(path_to_model) head, ext = os.path.splitext(model_filename) @@ -63,6 +66,16 @@ class Benchmark: return exe_network + def load_network_from_file(self, path_to_model: str, config = {}): + exe_network = self.ie.load_network(path_to_model, + self.device, + config=config, + num_requests=1 if self.api_type == 'sync' else self.nireq or 0) + # Number of requests + self.nireq = len(exe_network.requests) + + return exe_network + def import_network(self, path_to_file : str, config = {}): exe_network = self.ie.import_network(model_file=path_to_file, device_name=self.device, diff --git a/tools/benchmark/main.py b/tools/benchmark/main.py index 998e95638fb..29aff45742e 100644 --- a/tools/benchmark/main.py +++ b/tools/benchmark/main.py @@ -176,12 +176,41 @@ def run(args): benchmark.set_config(config) batch_size = args.batch_size - if not is_network_compiled: + if args.cache_dir: + benchmark.set_cache_dir(args.cache_dir) + + topology_name = "" + load_from_file_enabled = is_flag_set_in_command_line('load_from_file') or is_flag_set_in_command_line('lfile') + if load_from_file_enabled and not is_network_compiled: + next_step() + print("Skipping the step for loading network from file") + next_step() + print("Skipping the step for loading network from file") + next_step() + print("Skipping the step for loading network from file") + + # --------------------- 7. Loading the model to the device ------------------------------------------------- + next_step() + + start_time = datetime.utcnow() + exe_network = benchmark.load_network(args.path_to_model) + duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}" + logger.info(f"Load network took {duration_ms} ms") + if statistics: + statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, + [ + ('load network time (ms)', duration_ms) + ]) + app_inputs_info, _ = get_inputs_info(args.shape, args.layout, args.batch_size, exe_network.input_info) + if batch_size == 0: + batch_size = 1 + elif not is_network_compiled: # --------------------- 4. Read the Intermediate Representation of the network ----------------------------- next_step() start_time = datetime.utcnow() ie_network = benchmark.read_network(args.path_to_model) + topology_name = ie_network.name duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}" logger.info(f"Read network took {duration_ms} ms") if statistics: @@ -281,7 +310,7 @@ def run(args): if statistics: statistics.add_parameters(StatisticsReport.Category.RUNTIME_CONFIG, [ - ('topology', ie_network.name), + ('topology', topology_name), ('target device', device_name), ('API', args.api_type), ('precision', "UNSPECIFIED"), @@ -308,7 +337,7 @@ def run(args): progress_bar = ProgressBar(progress_bar_total_count, args.stream_output, args.progress) if args.progress else None - duration_ms = f"{benchmark.first_infer(exe_network):.2f}" + duration_ms = f"{benchmark.first_infer(exe_network):.2f}" logger.info(f"First inference took {duration_ms} ms") if statistics: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, diff --git a/tools/benchmark/parameters.py b/tools/benchmark/parameters.py index ca29fb5158c..73e1a931928 100644 --- a/tools/benchmark/parameters.py +++ b/tools/benchmark/parameters.py @@ -124,6 +124,10 @@ def parse_args(): help='Optional. Specifies precision for all output layers of the network.') args.add_argument('-iop', '--input_output_precision', type=str, required=False, help='Optional. Specifies precision for input and output layers by name. Example: -iop "input:FP16, output:FP16". Notice that quotes are required. Overwrites precision from ip and op options for specified layers.') + args.add_argument('-cdir', '--cache_dir', type=str, required=False, default='', + help="Optional. Enable model caching to specified directory") + args.add_argument('-lfile', '--load_from_file', required=False, nargs='?', default=argparse.SUPPRESS, + help="Optional. Loads model from file directly without read_network.") parsed_args = parser.parse_args() return parsed_args