diff --git a/docs/articles_en/openvino_workflow/running_inference_with_openvino/dldt_deployment_optimization_guide/performance_hints.rst b/docs/articles_en/openvino_workflow/running_inference_with_openvino/dldt_deployment_optimization_guide/performance_hints.rst index 8c152dba228..0b6e6d14e37 100644 --- a/docs/articles_en/openvino_workflow/running_inference_with_openvino/dldt_deployment_optimization_guide/performance_hints.rst +++ b/docs/articles_en/openvino_workflow/running_inference_with_openvino/dldt_deployment_optimization_guide/performance_hints.rst @@ -23,7 +23,7 @@ Performance Hints: Latency and Throughput As discussed in the :doc:`Optimization Guide ` there are a few different metrics associated with inference speed. Throughput and latency are some of the most widely used metrics that measure the overall performance of an application. -Therefore, in order to ease the configuration of the device, OpenVINO offers two dedicated hints, namely ``ov::hint::PerformanceMode::THROUGHPUT`` and ``ov::hint::PerformanceMode::LATENCY``. A special ``ov::hint::PerformanceMode::UNDEFINED`` hint acts the same as specifying no hint. +Therefore, in order to ease the configuration of the device, OpenVINO offers two dedicated hints, namely ``ov::hint::PerformanceMode::THROUGHPUT`` and ``ov::hint::PerformanceMode::LATENCY``. For more information on conducting performance measurements with the ``benchmark_app``, refer to the last section in this document. diff --git a/docs/notebooks/108-gpu-device-with-output.rst b/docs/notebooks/108-gpu-device-with-output.rst index d5c7910c38d..f484250f820 100644 --- a/docs/notebooks/108-gpu-device-with-output.rst +++ b/docs/notebooks/108-gpu-device-with-output.rst @@ -246,7 +246,7 @@ for that property. GPU_QUEUE_THROTTLE : Priority.MEDIUM GPU_ENABLE_LOOP_UNROLLING : True CACHE_DIR : - PERFORMANCE_HINT : PerformanceMode.UNDEFINED + PERFORMANCE_HINT : PerformanceMode.LATENCY COMPILATION_NUM_THREADS : 20 NUM_STREAMS : 1 PERFORMANCE_HINT_NUM_REQUESTS : 0 diff --git a/samples/cpp/benchmark_app/main.cpp b/samples/cpp/benchmark_app/main.cpp index d8b5f14bfa4..1178545e233 100644 --- a/samples/cpp/benchmark_app/main.cpp +++ b/samples/cpp/benchmark_app/main.cpp @@ -125,43 +125,52 @@ void next_step(const std::string additional_info = "") { << (additional_info.empty() ? "" : " (" + additional_info + ")") << std::endl; } -ov::hint::PerformanceMode get_performance_hint(const std::string& device, const ov::Core& core) { - OPENVINO_SUPPRESS_DEPRECATED_START - ov::hint::PerformanceMode ov_perf_hint = ov::hint::PerformanceMode::UNDEFINED; - OPENVINO_SUPPRESS_DEPRECATED_END +void handle_performance_hint(const std::string& device, const ov::Core& core, ov::AnyMap& config) { + ov::hint::PerformanceMode ov_perf_hint = ov::hint::PerformanceMode::THROUGHPUT; auto supported_properties = core.get_property(device, ov::supported_properties); if (std::find(supported_properties.begin(), supported_properties.end(), ov::hint::performance_mode) != supported_properties.end()) { - if (FLAGS_hint != "") { + // Use FLAGS_hint to decide performance mode: + // + // "throughput" or "tput": THROUGHPUT mode + // "cumulative_throughput" or "ctput": CUMULATIVE_THROUGHPUT mode + // "latency": LATENCY mode + // "none": not set ov::hint::performance_mode, let plugin use its default performance mode + // "" : use default THROUGHPUT mode, if FLAG_api="sync" then set LATENCY mode + if (FLAGS_hint != "" && FLAGS_hint != "none") { if (FLAGS_hint == "throughput" || FLAGS_hint == "tput") { ov_perf_hint = ov::hint::PerformanceMode::THROUGHPUT; } else if (FLAGS_hint == "latency") { ov_perf_hint = ov::hint::PerformanceMode::LATENCY; } else if (FLAGS_hint == "cumulative_throughput" || FLAGS_hint == "ctput") { ov_perf_hint = ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT; - } else if (FLAGS_hint == "none") { - OPENVINO_SUPPRESS_DEPRECATED_START - ov_perf_hint = ov::hint::PerformanceMode::UNDEFINED; - OPENVINO_SUPPRESS_DEPRECATED_END } else { throw std::logic_error( "Incorrect performance hint. Please set -hint option to" "`throughput`(tput), `latency', 'cumulative_throughput'(ctput) value or 'none'."); } - } else { - ov_perf_hint = - FLAGS_api == "async" ? ov::hint::PerformanceMode::THROUGHPUT : ov::hint::PerformanceMode::LATENCY; - + } else if (FLAGS_hint == "") { + ov_perf_hint = ov::hint::PerformanceMode::THROUGHPUT; + if (FLAGS_api == "sync") { + ov_perf_hint = ov::hint::PerformanceMode::LATENCY; + } slog::warn << "Performance hint was not explicitly specified in command line. " "Device(" << device << ") performance hint will be set to " << ov_perf_hint << "." << slog::endl; } + + if (FLAGS_hint != "none") { + // apply command line hint setting and override if hint exists + config[ov::hint::performance_mode.name()] = ov_perf_hint; + } else { + config.erase(ov::hint::performance_mode.name()); + } } else { - if (FLAGS_hint != "") { + if (FLAGS_hint != "none" || FLAGS_hint != "") { slog::warn << "Device(" << device << ") does not support performance hint property(-hint)." << slog::endl; } } - return ov_perf_hint; + return; } void setDeviceProperty(ov::Core& core, @@ -367,20 +376,7 @@ int main(int argc, char* argv[]) { // Update config per device according to command line parameters for (auto& device : devices) { auto& device_config = config[device]; - auto ov_perf_hint = get_performance_hint(device, core); - OPENVINO_SUPPRESS_DEPRECATED_START - if (isFlagSetInCommandLine("hint")) { - if (ov_perf_hint != ov::hint::PerformanceMode::UNDEFINED) { - // apply command line hint setting and override if hint exists - device_config[ov::hint::performance_mode.name()] = ov_perf_hint; - } else { - device_config.erase(ov::hint::performance_mode.name()); - } - } else if (ov_perf_hint != ov::hint::PerformanceMode::UNDEFINED) { - // keep hint setting in the config if no hint setting from command line - device_config.emplace(ov::hint::performance_mode(ov_perf_hint)); - } - OPENVINO_SUPPRESS_DEPRECATED_END + handle_performance_hint(device, core, device_config); if (FLAGS_nireq != 0) device_config[ov::hint::num_requests.name()] = unsigned(FLAGS_nireq); @@ -443,8 +439,7 @@ int main(int argc, char* argv[]) { ":,:" + " or via configuration file."); } - } else if (ov_perf_hint == ov::hint::PerformanceMode::UNDEFINED && !device_config.count(key) && - (FLAGS_api == "async")) { + } else if (FLAGS_api == "none" && !device_config.count(key) && (FLAGS_api == "async")) { slog::warn << "-nstreams default value is determined automatically for " << device << " device. " "Although the automatic selection usually provides a " diff --git a/src/bindings/python/src/pyopenvino/core/properties/properties.cpp b/src/bindings/python/src/pyopenvino/core/properties/properties.cpp index e8fdc7ceeb8..cf72607ae48 100644 --- a/src/bindings/python/src/pyopenvino/core/properties/properties.cpp +++ b/src/bindings/python/src/pyopenvino/core/properties/properties.cpp @@ -57,7 +57,6 @@ void regmodule_properties(py::module m) { OPENVINO_SUPPRESS_DEPRECATED_START py::enum_(m_hint, "PerformanceMode", py::arithmetic()) - .value("UNDEFINED", ov::hint::PerformanceMode::UNDEFINED) .value("LATENCY", ov::hint::PerformanceMode::LATENCY) .value("THROUGHPUT", ov::hint::PerformanceMode::THROUGHPUT) .value("CUMULATIVE_THROUGHPUT", ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT); diff --git a/src/bindings/python/tests/test_runtime/test_properties.py b/src/bindings/python/tests/test_runtime/test_properties.py index 44d38fb4afe..460a4257f6d 100644 --- a/src/bindings/python/tests/test_runtime/test_properties.py +++ b/src/bindings/python/tests/test_runtime/test_properties.py @@ -37,13 +37,6 @@ def test_properties_rw_base(): assert "incompatible function arguments" in str(e.value) -def test_deprecation(): - with pytest.warns(DeprecationWarning) as w: - _ = hints.PerformanceMode.UNDEFINED - assert issubclass(w[0].category, DeprecationWarning) - assert "PerformanceMode.UNDEFINED is deprecated and will be removed" in str(w[0].message) - - ### # Enum-like values ### @@ -71,7 +64,6 @@ def test_deprecation(): ( hints.PerformanceMode, ( - (hints.PerformanceMode.UNDEFINED, "PerformanceMode.UNDEFINED", -1), (hints.PerformanceMode.LATENCY, "PerformanceMode.LATENCY", 1), (hints.PerformanceMode.THROUGHPUT, "PerformanceMode.THROUGHPUT", 2), (hints.PerformanceMode.CUMULATIVE_THROUGHPUT, "PerformanceMode.CUMULATIVE_THROUGHPUT", 3), @@ -253,7 +245,7 @@ def test_properties_ro(ov_property_ro, expected_value): ( hints.performance_mode, "PERFORMANCE_HINT", - ((hints.PerformanceMode.UNDEFINED, hints.PerformanceMode.UNDEFINED),), + ((hints.PerformanceMode.THROUGHPUT, hints.PerformanceMode.THROUGHPUT),), ), ( hints.enable_cpu_pinning, diff --git a/src/inference/include/openvino/runtime/properties.hpp b/src/inference/include/openvino/runtime/properties.hpp index aa7f8b2acde..6fdb12752cf 100644 --- a/src/inference/include/openvino/runtime/properties.hpp +++ b/src/inference/include/openvino/runtime/properties.hpp @@ -300,8 +300,6 @@ static constexpr Property model_priority{"MODEL_PRIORITY"}; * @ingroup ov_runtime_cpp_prop_api */ enum class PerformanceMode { - UNDEFINED OPENVINO_ENUM_DEPRECATED("Please use actual value instead. Will be removed in 2024.0") = - -1, //!< Undefined value, performance setting may vary from device to device LATENCY = 1, //!< Optimize for latency THROUGHPUT = 2, //!< Optimize for throughput CUMULATIVE_THROUGHPUT = 3, //!< Optimize for cumulative throughput @@ -310,10 +308,6 @@ enum class PerformanceMode { /** @cond INTERNAL */ inline std::ostream& operator<<(std::ostream& os, const PerformanceMode& performance_mode) { switch (performance_mode) { - OPENVINO_SUPPRESS_DEPRECATED_START - case PerformanceMode::UNDEFINED: - return os << "UNDEFINED"; - OPENVINO_SUPPRESS_DEPRECATED_END case PerformanceMode::LATENCY: return os << "LATENCY"; case PerformanceMode::THROUGHPUT: @@ -334,10 +328,6 @@ inline std::istream& operator>>(std::istream& is, PerformanceMode& performance_m performance_mode = PerformanceMode::THROUGHPUT; } else if (str == "CUMULATIVE_THROUGHPUT") { performance_mode = PerformanceMode::CUMULATIVE_THROUGHPUT; - } else if (str == "UNDEFINED") { - OPENVINO_SUPPRESS_DEPRECATED_START - performance_mode = PerformanceMode::UNDEFINED; - OPENVINO_SUPPRESS_DEPRECATED_END } else { OPENVINO_THROW("Unsupported performance mode: ", str); } diff --git a/src/plugins/intel_gpu/src/runtime/execution_config.cpp b/src/plugins/intel_gpu/src/runtime/execution_config.cpp index 55716a0be04..9390ff82802 100644 --- a/src/plugins/intel_gpu/src/runtime/execution_config.cpp +++ b/src/plugins/intel_gpu/src/runtime/execution_config.cpp @@ -30,8 +30,7 @@ public: OPENVINO_SUPPRESS_DEPRECATED_START return mode == ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT || mode == ov::hint::PerformanceMode::THROUGHPUT || - mode == ov::hint::PerformanceMode::LATENCY || - mode == ov::hint::PerformanceMode::UNDEFINED; + mode == ov::hint::PerformanceMode::LATENCY; OPENVINO_SUPPRESS_DEPRECATED_END } }; diff --git a/tools/benchmark_tool/openvino/tools/benchmark/main.py b/tools/benchmark_tool/openvino/tools/benchmark/main.py index 5533b0efa2e..b5eff49d6ba 100644 --- a/tools/benchmark_tool/openvino/tools/benchmark/main.py +++ b/tools/benchmark_tool/openvino/tools/benchmark/main.py @@ -106,7 +106,7 @@ def main(): next_step() def set_performance_hint(device): - perf_hint = properties.hint.PerformanceMode.UNDEFINED + perf_hint = properties.hint.PerformanceMode.THROUGHPUT supported_properties = benchmark.core.get_property(device, properties.supported_properties()) if properties.hint.performance_mode() in supported_properties: if is_flag_set_in_command_line('hint'): @@ -117,16 +117,16 @@ def main(): elif args.perf_hint == "cumulative_throughput" or args.perf_hint == "ctput": perf_hint = properties.hint.PerformanceMode.CUMULATIVE_THROUGHPUT elif args.perf_hint=='none': - perf_hint = properties.hint.PerformanceMode.UNDEFINED + # Not set PerformanceMode, and plugin will apply its internal default PerformanceMode + return else: raise RuntimeError("Incorrect performance hint. Please set -hint option to" "`throughput`(tput), `latency', 'cumulative_throughput'(ctput) value or 'none'.") else: - perf_hint = properties.hint.PerformanceMode.THROUGHPUT if benchmark.api_type == "async" else properties.hint.PerformanceMode.LATENCY + perf_hint = properties.hint.PerformanceMode.LATENCY if benchmark.api_type == "sync" else properties.hint.PerformanceMode.THROUGHPUT logger.warning(f"Performance hint was not explicitly specified in command line. " + f"Device({device}) performance hint will be set to {perf_hint}.") - if perf_hint != properties.hint.PerformanceMode.UNDEFINED: - config[device][properties.hint.performance_mode()] = perf_hint + config[device][properties.hint.performance_mode()] = perf_hint else: logger.warning(f"Device {device} does not support performance hint property(-hint).")