From f4dced2e6e5b14f4ba4260571bbb3d9fa8094659 Mon Sep 17 00:00:00 2001
From: "Wang, Yang" <yang4.wang@intel.com>
Date: Fri, 26 Apr 2024 00:39:51 +0800
Subject: [PATCH] [AUTO] Update the CPU resources for GPU compilation when
 using CPU as accelerator (#23974)

### Details:
- Update the usage of CPU resources for GPU compilation when CPU as
acceleration device
- Don't prioritize to load dynamic model to CPU, but follow the
device::priorities
 -  Add logs for compilation time and first inference time

### Tickets:
 - CVS-138196

---------

Co-authored-by: Chen Peter <peter.chen@intel.com>
---
 src/plugins/auto/src/auto_schedule.cpp        | 64 +++++++++++--------
 src/plugins/auto/src/plugin.cpp               | 22 +++++--
 src/plugins/auto/src/plugin_config.hpp        |  1 +
 src/plugins/auto/src/schedule.cpp             |  5 ++
 .../auto/tests/unit/dynamic_output_test.cpp   | 63 +++++++-----------
 .../auto/tests/unit/stateful_model_test.cpp   |  8 +--
 6 files changed, 84 insertions(+), 79 deletions(-)

diff --git a/src/plugins/auto/src/auto_schedule.cpp b/src/plugins/auto/src/auto_schedule.cpp
index b154bdf1b2a..0bf1acbbd17 100644
--- a/src/plugins/auto/src/auto_schedule.cpp
+++ b/src/plugins/auto/src/auto_schedule.cpp
@@ -18,7 +18,7 @@ bool AutoSchedule::select_other_device(const std::string& cur_dev_name) {
         get_execution_devices = [&](const std::string& device_name) {
             std::string real_device_name;
             bool is_cpuhelp = false;
-                m_compile_context[FALLBACKDEVICE].m_model_precision = m_context->m_model_precision;
+            m_compile_context[FALLBACKDEVICE].m_model_precision = m_context->m_model_precision;
             if (device_name == "CPU_HELP") {
                 // if infer failed in CPU_HELP, we will remove CPU from m_device_priorities
                 // and re-run infer request when m_compile_context[ACTUALDEVICE] is ready
@@ -49,8 +49,8 @@ bool AutoSchedule::select_other_device(const std::string& cur_dev_name) {
             m_compile_context[FALLBACKDEVICE].m_is_reload_success = false;
             m_compile_context[FALLBACKDEVICE].m_device_info =
                 m_plugin->select_device(m_context->m_device_priorities,
-                                                        m_compile_context[FALLBACKDEVICE].m_model_precision,
-                                                        m_context->m_model_priority);
+                                        m_compile_context[FALLBACKDEVICE].m_model_precision,
+                                        m_context->m_model_priority);
             try {
                 m_compile_context[FALLBACKDEVICE].m_task();
                 // FALLBACKDEVICE need to be load again if infer failed, so reset promise here
@@ -165,9 +165,9 @@ void AutoSchedule::init() {
         // so use executor as a member of AutoSchedule.
         m_executor = m_plugin->get_executor_manager()->get_idle_cpu_streams_executor(
                     ov::threading::IStreamsExecutor::Config{"AutoDeviceAsyncCompile",
-                    static_cast<int>(std::thread::hardware_concurrency()) /* max possible #streams*/,
-                    0 /*default threads per stream, workaround for ticket 62376*/,
-                    ov::threading::IStreamsExecutor::ThreadBindingType::NONE});
+                static_cast<int>(std::thread::hardware_concurrency()) /* max possible #streams*/,
+                0 /*default threads per stream, workaround for ticket 62376*/,
+                ov::threading::IStreamsExecutor::ThreadBindingType::NONE});
         for (auto&& device : m_context->m_device_priorities) {
             // initialize containers before run async task
             m_idle_worker_requests[device.device_name];
@@ -206,6 +206,8 @@ void AutoSchedule::init() {
                         cpuhelp_all_end_times.splice(cpuhelp_all_end_times.end(), worker.second->m_end_times);
                     });
                 }
+                std::chrono::duration<double, std::milli> first_infer_time =
+                    cpuhelp_all_end_times.front() - cpuhelp_all_start_times.front();
                 INFO_RUN([this, &cpuhelp_all_start_times, &cpuhelp_all_end_times]() {
                     cpuhelp_all_start_times.sort(std::less<Time>());
                     cpuhelp_all_end_times.sort(std::less<Time>());
@@ -214,15 +216,16 @@ void AutoSchedule::init() {
                 });
                 if (destroynum == m_worker_requests["CPU_HELP"].size()) {
                     std::lock_guard<std::mutex> lock(m_context->m_mutex);
-                    INFO_RUN([this, &cpuhelp_all_start_times, &cpuhelp_all_end_times, &destroynum]() {
+                    INFO_RUN([this, first_infer_time, &cpuhelp_all_start_times, &cpuhelp_all_end_times, &destroynum]() {
                         m_cpuhelp_release_time = std::chrono::steady_clock::now();
                         if (cpuhelp_all_start_times.size() >= destroynum + 1) {
-                            //remove last worksize num requests, so the fps will be more accuracy
+                            // remove last worksize num requests, so the fps will be more accuracy
                             cpuhelp_all_start_times.resize(m_cpuhelp_infer_count - destroynum);
                             cpuhelp_all_end_times.resize(m_cpuhelp_infer_count - destroynum);
                             std::chrono::duration<double, std::milli> durtation =
                                 cpuhelp_all_end_times.back() - cpuhelp_all_start_times.front();
                             m_cpuhelp_fps = cpuhelp_all_start_times.size() * 1000 / durtation.count();
+                            LOG_INFO_TAG("CPU_HELP: first inference time:%lf ms", first_infer_time.count());
                             LOG_INFO_TAG("CPU_HELP:infer:%ld", m_cpuhelp_infer_count);
                             LOG_INFO_TAG("CPU_HELP:fps:%lf", m_cpuhelp_fps);
                         }
@@ -263,33 +266,38 @@ void AutoSchedule::try_to_compile_model(AutoCompileContext& context, const std::
     bool cur_dev_is_gpu = (device.find("GPU") != std::string::npos);
     {
         std::lock_guard<std::mutex> lock(m_context->m_mutex);
-        if (cur_dev_is_gpu && m_compile_context[CPU].m_is_enabled) {
-            // user does not set the compiling threads
-            // limit the threads num for compiling
-            int max_threads = 0;
-            try {
-                max_threads = m_context->m_ov_core->get_property(device, ov::compilation_num_threads);
-            } catch (const ov::Exception&) {
-                LOG_DEBUG_TAG("cannot get MAX_NUM_THREADS from GPU");
-            }
-            if (max_threads == static_cast<int>(std::thread::hardware_concurrency())) {
-                int thread_num = max_threads / 2;
-                device_config.insert(ov::compilation_num_threads(thread_num));
-                LOG_DEBUG_TAG("gpu streams number for compiling: %d", thread_num);
-            } else {
-                // user set the compiling threads num
-                // use the user's val anyway
-                LOG_DEBUG_TAG("user defined compiling threads: %d", max_threads);
-            }
+        // user does not set the compiling threads
+        // limit the threads num for compiling
+        bool is_already_set_gpu =
+            (device_config.find(ov::intel_gpu::hint::host_task_priority.name()) != device_config.end() ||
+             device_config.find(ov::compilation_num_threads.name()) != device_config.end());
+        if (cur_dev_is_gpu && m_compile_context[CPU].m_is_enabled && !is_already_set_gpu) {
+            device_config.insert(ov::intel_gpu::hint::host_task_priority(ov::hint::Priority::HIGH));
+            auto proc_type_table = get_org_proc_type_table();
+            int compilation_num_threads = proc_type_table[0][MAIN_CORE_PROC] != 0
+                                              ? proc_type_table[0][MAIN_CORE_PROC]
+                                              : proc_type_table[0][EFFICIENT_CORE_PROC];
+            if (device_config.insert(ov::compilation_num_threads(compilation_num_threads)).second)
+                LOG_DEBUG_TAG("gpu streams number for compiling: %d", compilation_num_threads);
+            else
+                LOG_DEBUG_TAG("user defined compiling threads: %d",
+                              device_config[ov::compilation_num_threads.name()].as<int32_t>());
         }
     }
     try {
+        auto compile_start_time = std::chrono::high_resolution_clock::now();
         if (!(m_context->m_model_path.empty())) {
-            context.m_compiled_model = m_context->m_ov_core->compile_model(m_context->m_model_path, device, device_config);
+            context.m_compiled_model =
+                m_context->m_ov_core->compile_model(m_context->m_model_path, device, device_config);
         } else {
             context.m_compiled_model = m_context->m_ov_core->compile_model(model, device, device_config);
         }
         context.m_is_load_success = true;
+        auto compile_end_time = std::chrono::high_resolution_clock::now();
+        auto compiled_time =
+            std::chrono::duration_cast<std::chrono::nanoseconds>(compile_end_time - compile_start_time).count() *
+            0.000001;
+        LOG_INFO_TAG("Device: [%s]: Compile model took %lf ms", device.c_str(), compiled_time);
     } catch (const ov::Exception& e) {
         context.m_err_message += device + ":" + e.what();
         context.m_is_load_success = false;
@@ -455,7 +463,7 @@ AutoSchedule::~AutoSchedule() {
     }
     if (m_plugin)
         m_plugin->unregister_priority(m_context->m_model_priority,
-            m_compile_context[ACTUALDEVICE].m_device_info.unique_name);
+                                      m_compile_context[ACTUALDEVICE].m_device_info.unique_name);
     if (m_context) {
         std::lock_guard<std::mutex> lock(m_context->m_fallback_mutex);
         m_context->m_device_priorities.clear();
diff --git a/src/plugins/auto/src/plugin.cpp b/src/plugins/auto/src/plugin.cpp
index 98bc48b06c4..8c426b6d98b 100644
--- a/src/plugins/auto/src/plugin.cpp
+++ b/src/plugins/auto/src/plugin.cpp
@@ -819,16 +819,26 @@ std::vector<DeviceInformation> Plugin::filter_device_by_model(const std::vector<
 
     std::vector<DeviceInformation> filter_device;
     std::vector<std::string> stateful_node_names;
+    auto support_dynamic_devices_info = [&]() -> std::vector<DeviceInformation> {
+        std::vector<DeviceInformation> ret;
+        std::vector<std::string> devices_support_dynamic = {"CPU", "GPU"};
+        for (auto& item : meta_devices) {
+            auto dev_iter = std::find_if(devices_support_dynamic.begin(),
+                                         devices_support_dynamic.end(),
+                                         [item](const std::string& device) {
+                                             return item.device_name.find(device) != std::string::npos;
+                                         });
+            if (dev_iter != devices_support_dynamic.end())
+                ret.push_back(item);
+        }
+        return ret;
+    };
 
-    // Check if CPU is in candidate list
-    auto cpuiter = std::find_if(meta_devices.begin(), meta_devices.end(), [](const DeviceInformation& device_info) {
-        return device_info.device_name.find("CPU") != std::string::npos;
-    });
     // If CPU is in candidate list, load dynamic model to CPU first
     // For MULTI do not only load stateful model to CPU
     // For AUTO CTPUT only load stateful model to CPU
-    if (model->is_dynamic() && cpuiter != meta_devices.end()) {
-        filter_device.push_back(*cpuiter);
+    if (model->is_dynamic()) {
+        filter_device = support_dynamic_devices_info();
         return filter_device;
     }
     // If CPU is not in candidate list, continue to run selection logic regardless of whether the input model is a
diff --git a/src/plugins/auto/src/plugin_config.hpp b/src/plugins/auto/src/plugin_config.hpp
index e5de5a315e5..cc3134d03c3 100644
--- a/src/plugins/auto/src/plugin_config.hpp
+++ b/src/plugins/auto/src/plugin_config.hpp
@@ -6,6 +6,7 @@
 #pragma once
 
 #include "openvino/runtime/auto/properties.hpp"
+#include "openvino/runtime/intel_gpu/properties.hpp"
 #include "utils/log.hpp"
 #include "utils/log_util.hpp"
 #include "openvino/runtime/device_id_parser.hpp"
diff --git a/src/plugins/auto/src/schedule.cpp b/src/plugins/auto/src/schedule.cpp
index 2ad3e95400b..96a815cc21c 100644
--- a/src/plugins/auto/src/schedule.cpp
+++ b/src/plugins/auto/src/schedule.cpp
@@ -275,6 +275,8 @@ Schedule::~Schedule() {
             }
             size_t count = req_all_start_times.size();
             OPENVINO_ASSERT(count == req_all_end_times.size());
+            std::chrono::duration<double, std::milli> first_infer_durtation =
+                req_all_end_times.front() - req_all_start_times.front();
             req_all_start_times.sort(std::less<Time>());
             req_all_end_times.sort(std::less<Time>());
             {
@@ -290,6 +292,9 @@ Schedule::~Schedule() {
                     }
                 }
                 if (n >= 1) {
+                    LOG_INFO_TAG("%s: first inference time:%lf ms",
+                                 worker_request.first.c_str(),
+                                 first_infer_durtation.count());
                     LOG_INFO_TAG("%s:infer:%ld", worker_request.first.c_str(), count);
                     std::chrono::duration<double, std::milli> durtation =
                         req_all_end_times.back() - time;
diff --git a/src/plugins/auto/tests/unit/dynamic_output_test.cpp b/src/plugins/auto/tests/unit/dynamic_output_test.cpp
index 7d3ae031cde..dae7fdaf1a3 100644
--- a/src/plugins/auto/tests/unit/dynamic_output_test.cpp
+++ b/src/plugins/auto/tests/unit/dynamic_output_test.cpp
@@ -58,37 +58,25 @@ void DynamicOutputInferenceTest::SetUp() {
                                                                                            nullptr,
                                                                                            false);
     std::tie(priorityList, targetList) = GetParam();
-    ON_CALL(*core,
-            compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
-                          ::testing::Matcher<const std::string&>(StrEq(ov::test::utils::DEVICE_GPU)),
-                          _))
-        .WillByDefault(InvokeWithoutArgs([this]() {
-            std::this_thread::sleep_for(std::chrono::milliseconds(200));
-            return mockExeNetworkActual;
-        }));
-    ON_CALL(
-        *core,
-        compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
-                      ::testing::Matcher<const std::string&>(StrEq(std::string(ov::test::utils::DEVICE_GPU) + ".0")),
-                      _))
-        .WillByDefault(InvokeWithoutArgs([this]() {
-            std::this_thread::sleep_for(std::chrono::milliseconds(200));
-            return mockExeNetwork;
-        }));
-    ON_CALL(
-        *core,
-        compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
-                      ::testing::Matcher<const std::string&>(StrEq(std::string(ov::test::utils::DEVICE_GPU) + ".1")),
-                      _))
-        .WillByDefault(InvokeWithoutArgs([this]() {
-            std::this_thread::sleep_for(std::chrono::milliseconds(200));
-            return mockExeNetworkActual;
-        }));
-    ON_CALL(*core,
-            compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
-                          ::testing::Matcher<const std::string&>(StrEq(ov::test::utils::DEVICE_CPU)),
-                          (_)))
-        .WillByDefault(Return(mockExeNetwork));
+    auto targets = targetList.as<std::vector<std::string>>();
+    ON_CALL(*core, get_available_devices()).WillByDefault(Return(targets));
+    for (auto device : targets) {
+        ON_CALL(*core,
+                compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
+                              ::testing::Matcher<const std::string&>(StrEq(device)),
+                              _))
+            .WillByDefault(InvokeWithoutArgs([this, device]() {
+                if (device.find("GPU") != std::string::npos) {
+                    if (device == "GPU.1") {
+                        return mockExeNetwork;
+                    } else {
+                        std::this_thread::sleep_for(std::chrono::milliseconds(200));
+                        return mockExeNetworkActual;
+                    }
+                }
+                return mockExeNetwork;
+            }));
+    }
 }
 
 TEST_P(DynamicOutputInferenceTest, CanSelectCorrectTargetDeviceandInitizeBlobWithCorrectSize) {
@@ -99,17 +87,10 @@ TEST_P(DynamicOutputInferenceTest, CanSelectCorrectTargetDeviceandInitizeBlobWit
     for (auto& iter : targets) {
         EXPECT_CALL(*core,
                     compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
-                                  ::testing::Matcher<const std::string&>(HasSubstr(iter)),
+                                  ::testing::Matcher<const std::string&>(StrEq(iter)),
                                   ::testing::Matcher<const ov::AnyMap&>(_)))
             .Times(1);
     }
-    if (priorityList.as<std::string>().find("CPU") != std::string::npos) {
-        EXPECT_CALL(*core,
-                    compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
-                                  ::testing::Matcher<const std::string&>(StrEq("GPU")),
-                                  ::testing::Matcher<const ov::AnyMap&>(_)))
-            .Times(0);
-    }
     ASSERT_NO_THROW(exeNetwork = plugin->compile_model(model, config));
 }
 
@@ -137,8 +118,8 @@ TEST_P(DynamicOutputInferenceTest, CanInferWithOutputChangedFromDynamicOnAutoToS
 }
 
 const std::vector<DynamicOutputConfigParams> testConfigs = {
-    DynamicOutputConfigParams{"CPU,GPU", std::vector<std::string>{"CPU"}},
-    DynamicOutputConfigParams{"GPU,CPU", std::vector<std::string>{"CPU"}},
+    DynamicOutputConfigParams{"CPU,GPU", std::vector<std::string>{"CPU", "GPU"}},
+    DynamicOutputConfigParams{"GPU,CPU", std::vector<std::string>{"CPU", "GPU"}},
     DynamicOutputConfigParams{"GPU.0,GPU.1", std::vector<std::string>{"GPU.0", "GPU.1"}},
 };
 
diff --git a/src/plugins/auto/tests/unit/stateful_model_test.cpp b/src/plugins/auto/tests/unit/stateful_model_test.cpp
index 11b8f49590e..4f2ab35c240 100644
--- a/src/plugins/auto/tests/unit/stateful_model_test.cpp
+++ b/src/plugins/auto/tests/unit/stateful_model_test.cpp
@@ -255,25 +255,25 @@ const std::vector<StatefulModelConfigParams> testConfigs = {
                               true,
                               std::map<std::string, bool>{{"CPU", true}, {"GPU", true}},
                               true,
-                              std::vector<std::pair<std::string, int>>{{"CPU", 1}, {"GPU", 0}}},
+                              std::vector<std::pair<std::string, int>>{{"CPU", 1}, {"GPU", 1}}},
     StatefulModelConfigParams{"GPU,CPU",
                               true,
                               true,
                               std::map<std::string, bool>{{"CPU", true}, {"GPU", true}},
                               true,
-                              std::vector<std::pair<std::string, int>>{{"GPU", 0}, {"CPU", 1}}},
+                              std::vector<std::pair<std::string, int>>{{"GPU", 1}, {"CPU", 1}}},
     StatefulModelConfigParams{"CPU,GPU",
                               true,
                               false,
                               std::map<std::string, bool>{{"CPU", true}, {"GPU", true}},
                               true,
-                              std::vector<std::pair<std::string, int>>{{"CPU", 1}, {"GPU", 0}}},
+                              std::vector<std::pair<std::string, int>>{{"CPU", 1}, {"GPU", 1}}},
     StatefulModelConfigParams{"GPU,CPU",
                               true,
                               false,
                               std::map<std::string, bool>{{"CPU", true}, {"GPU", true}},
                               true,
-                              std::vector<std::pair<std::string, int>>{{"GPU", 0}, {"CPU", 1}}},
+                              std::vector<std::pair<std::string, int>>{{"GPU", 1}, {"CPU", 1}}},
     StatefulModelConfigParams{"CPU",
                               false,
                               false,