[AUTO] Update the CPU resources for GPU compilation when using CPU as accelerator (#23974)
### Details: - Update the usage of CPU resources for GPU compilation when CPU as acceleration device - Don't prioritize to load dynamic model to CPU, but follow the device::priorities - Add logs for compilation time and first inference time ### Tickets: - CVS-138196 --------- Co-authored-by: Chen Peter <peter.chen@intel.com>
This commit is contained in:
Wang, Yang
GitHub
parent
1320977d89
commit
f4dced2e6e
|
|
@ -18,7 +18,7 @@ bool AutoSchedule::select_other_device(const std::string& cur_dev_name) {
|
|||
get_execution_devices = [&](const std::string& device_name) {
|
||||
std::string real_device_name;
|
||||
bool is_cpuhelp = false;
|
||||
m_compile_context[FALLBACKDEVICE].m_model_precision = m_context->m_model_precision;
|
||||
m_compile_context[FALLBACKDEVICE].m_model_precision = m_context->m_model_precision;
|
||||
if (device_name == "CPU_HELP") {
|
||||
// if infer failed in CPU_HELP, we will remove CPU from m_device_priorities
|
||||
// and re-run infer request when m_compile_context[ACTUALDEVICE] is ready
|
||||
|
|
@ -49,8 +49,8 @@ bool AutoSchedule::select_other_device(const std::string& cur_dev_name) {
|
|||
m_compile_context[FALLBACKDEVICE].m_is_reload_success = false;
|
||||
m_compile_context[FALLBACKDEVICE].m_device_info =
|
||||
m_plugin->select_device(m_context->m_device_priorities,
|
||||
m_compile_context[FALLBACKDEVICE].m_model_precision,
|
||||
m_context->m_model_priority);
|
||||
m_compile_context[FALLBACKDEVICE].m_model_precision,
|
||||
m_context->m_model_priority);
|
||||
try {
|
||||
m_compile_context[FALLBACKDEVICE].m_task();
|
||||
// FALLBACKDEVICE need to be load again if infer failed, so reset promise here
|
||||
|
|
@ -165,9 +165,9 @@ void AutoSchedule::init() {
|
|||
// so use executor as a member of AutoSchedule.
|
||||
m_executor = m_plugin->get_executor_manager()->get_idle_cpu_streams_executor(
|
||||
ov::threading::IStreamsExecutor::Config{"AutoDeviceAsyncCompile",
|
||||
static_cast<int>(std::thread::hardware_concurrency()) /* max possible #streams*/,
|
||||
0 /*default threads per stream, workaround for ticket 62376*/,
|
||||
ov::threading::IStreamsExecutor::ThreadBindingType::NONE});
|
||||
static_cast<int>(std::thread::hardware_concurrency()) /* max possible #streams*/,
|
||||
0 /*default threads per stream, workaround for ticket 62376*/,
|
||||
ov::threading::IStreamsExecutor::ThreadBindingType::NONE});
|
||||
for (auto&& device : m_context->m_device_priorities) {
|
||||
// initialize containers before run async task
|
||||
m_idle_worker_requests[device.device_name];
|
||||
|
|
@ -206,6 +206,8 @@ void AutoSchedule::init() {
|
|||
cpuhelp_all_end_times.splice(cpuhelp_all_end_times.end(), worker.second->m_end_times);
|
||||
});
|
||||
}
|
||||
std::chrono::duration<double, std::milli> first_infer_time =
|
||||
cpuhelp_all_end_times.front() - cpuhelp_all_start_times.front();
|
||||
INFO_RUN([this, &cpuhelp_all_start_times, &cpuhelp_all_end_times]() {
|
||||
cpuhelp_all_start_times.sort(std::less<Time>());
|
||||
cpuhelp_all_end_times.sort(std::less<Time>());
|
||||
|
|
@ -214,15 +216,16 @@ void AutoSchedule::init() {
|
|||
});
|
||||
if (destroynum == m_worker_requests["CPU_HELP"].size()) {
|
||||
std::lock_guard<std::mutex> lock(m_context->m_mutex);
|
||||
INFO_RUN([this, &cpuhelp_all_start_times, &cpuhelp_all_end_times, &destroynum]() {
|
||||
INFO_RUN([this, first_infer_time, &cpuhelp_all_start_times, &cpuhelp_all_end_times, &destroynum]() {
|
||||
m_cpuhelp_release_time = std::chrono::steady_clock::now();
|
||||
if (cpuhelp_all_start_times.size() >= destroynum + 1) {
|
||||
//remove last worksize num requests, so the fps will be more accuracy
|
||||
// remove last worksize num requests, so the fps will be more accuracy
|
||||
cpuhelp_all_start_times.resize(m_cpuhelp_infer_count - destroynum);
|
||||
cpuhelp_all_end_times.resize(m_cpuhelp_infer_count - destroynum);
|
||||
std::chrono::duration<double, std::milli> durtation =
|
||||
cpuhelp_all_end_times.back() - cpuhelp_all_start_times.front();
|
||||
m_cpuhelp_fps = cpuhelp_all_start_times.size() * 1000 / durtation.count();
|
||||
LOG_INFO_TAG("CPU_HELP: first inference time:%lf ms", first_infer_time.count());
|
||||
LOG_INFO_TAG("CPU_HELP:infer:%ld", m_cpuhelp_infer_count);
|
||||
LOG_INFO_TAG("CPU_HELP:fps:%lf", m_cpuhelp_fps);
|
||||
}
|
||||
|
|
@ -263,33 +266,38 @@ void AutoSchedule::try_to_compile_model(AutoCompileContext& context, const std::
|
|||
bool cur_dev_is_gpu = (device.find("GPU") != std::string::npos);
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(m_context->m_mutex);
|
||||
if (cur_dev_is_gpu && m_compile_context[CPU].m_is_enabled) {
|
||||
// user does not set the compiling threads
|
||||
// limit the threads num for compiling
|
||||
int max_threads = 0;
|
||||
try {
|
||||
max_threads = m_context->m_ov_core->get_property(device, ov::compilation_num_threads);
|
||||
} catch (const ov::Exception&) {
|
||||
LOG_DEBUG_TAG("cannot get MAX_NUM_THREADS from GPU");
|
||||
}
|
||||
if (max_threads == static_cast<int>(std::thread::hardware_concurrency())) {
|
||||
int thread_num = max_threads / 2;
|
||||
device_config.insert(ov::compilation_num_threads(thread_num));
|
||||
LOG_DEBUG_TAG("gpu streams number for compiling: %d", thread_num);
|
||||
} else {
|
||||
// user set the compiling threads num
|
||||
// use the user's val anyway
|
||||
LOG_DEBUG_TAG("user defined compiling threads: %d", max_threads);
|
||||
}
|
||||
// user does not set the compiling threads
|
||||
// limit the threads num for compiling
|
||||
bool is_already_set_gpu =
|
||||
(device_config.find(ov::intel_gpu::hint::host_task_priority.name()) != device_config.end() ||
|
||||
device_config.find(ov::compilation_num_threads.name()) != device_config.end());
|
||||
if (cur_dev_is_gpu && m_compile_context[CPU].m_is_enabled && !is_already_set_gpu) {
|
||||
device_config.insert(ov::intel_gpu::hint::host_task_priority(ov::hint::Priority::HIGH));
|
||||
auto proc_type_table = get_org_proc_type_table();
|
||||
int compilation_num_threads = proc_type_table[0][MAIN_CORE_PROC] != 0
|
||||
? proc_type_table[0][MAIN_CORE_PROC]
|
||||
: proc_type_table[0][EFFICIENT_CORE_PROC];
|
||||
if (device_config.insert(ov::compilation_num_threads(compilation_num_threads)).second)
|
||||
LOG_DEBUG_TAG("gpu streams number for compiling: %d", compilation_num_threads);
|
||||
else
|
||||
LOG_DEBUG_TAG("user defined compiling threads: %d",
|
||||
device_config[ov::compilation_num_threads.name()].as<int32_t>());
|
||||
}
|
||||
}
|
||||
try {
|
||||
auto compile_start_time = std::chrono::high_resolution_clock::now();
|
||||
if (!(m_context->m_model_path.empty())) {
|
||||
context.m_compiled_model = m_context->m_ov_core->compile_model(m_context->m_model_path, device, device_config);
|
||||
context.m_compiled_model =
|
||||
m_context->m_ov_core->compile_model(m_context->m_model_path, device, device_config);
|
||||
} else {
|
||||
context.m_compiled_model = m_context->m_ov_core->compile_model(model, device, device_config);
|
||||
}
|
||||
context.m_is_load_success = true;
|
||||
auto compile_end_time = std::chrono::high_resolution_clock::now();
|
||||
auto compiled_time =
|
||||
std::chrono::duration_cast<std::chrono::nanoseconds>(compile_end_time - compile_start_time).count() *
|
||||
0.000001;
|
||||
LOG_INFO_TAG("Device: [%s]: Compile model took %lf ms", device.c_str(), compiled_time);
|
||||
} catch (const ov::Exception& e) {
|
||||
context.m_err_message += device + ":" + e.what();
|
||||
context.m_is_load_success = false;
|
||||
|
|
@ -455,7 +463,7 @@ AutoSchedule::~AutoSchedule() {
|
|||
}
|
||||
if (m_plugin)
|
||||
m_plugin->unregister_priority(m_context->m_model_priority,
|
||||
m_compile_context[ACTUALDEVICE].m_device_info.unique_name);
|
||||
m_compile_context[ACTUALDEVICE].m_device_info.unique_name);
|
||||
if (m_context) {
|
||||
std::lock_guard<std::mutex> lock(m_context->m_fallback_mutex);
|
||||
m_context->m_device_priorities.clear();
|
||||
|
|
|
|||
|
|
@ -819,16 +819,26 @@ std::vector<DeviceInformation> Plugin::filter_device_by_model(const std::vector<
|
|||
|
||||
std::vector<DeviceInformation> filter_device;
|
||||
std::vector<std::string> stateful_node_names;
|
||||
auto support_dynamic_devices_info = [&]() -> std::vector<DeviceInformation> {
|
||||
std::vector<DeviceInformation> ret;
|
||||
std::vector<std::string> devices_support_dynamic = {"CPU", "GPU"};
|
||||
for (auto& item : meta_devices) {
|
||||
auto dev_iter = std::find_if(devices_support_dynamic.begin(),
|
||||
devices_support_dynamic.end(),
|
||||
[item](const std::string& device) {
|
||||
return item.device_name.find(device) != std::string::npos;
|
||||
});
|
||||
if (dev_iter != devices_support_dynamic.end())
|
||||
ret.push_back(item);
|
||||
}
|
||||
return ret;
|
||||
};
|
||||
|
||||
// Check if CPU is in candidate list
|
||||
auto cpuiter = std::find_if(meta_devices.begin(), meta_devices.end(), [](const DeviceInformation& device_info) {
|
||||
return device_info.device_name.find("CPU") != std::string::npos;
|
||||
});
|
||||
// If CPU is in candidate list, load dynamic model to CPU first
|
||||
// For MULTI do not only load stateful model to CPU
|
||||
// For AUTO CTPUT only load stateful model to CPU
|
||||
if (model->is_dynamic() && cpuiter != meta_devices.end()) {
|
||||
filter_device.push_back(*cpuiter);
|
||||
if (model->is_dynamic()) {
|
||||
filter_device = support_dynamic_devices_info();
|
||||
return filter_device;
|
||||
}
|
||||
// If CPU is not in candidate list, continue to run selection logic regardless of whether the input model is a
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#pragma once
|
||||
|
||||
#include "openvino/runtime/auto/properties.hpp"
|
||||
#include "openvino/runtime/intel_gpu/properties.hpp"
|
||||
#include "utils/log.hpp"
|
||||
#include "utils/log_util.hpp"
|
||||
#include "openvino/runtime/device_id_parser.hpp"
|
||||
|
|
|
|||
|
|
@ -275,6 +275,8 @@ Schedule::~Schedule() {
|
|||
}
|
||||
size_t count = req_all_start_times.size();
|
||||
OPENVINO_ASSERT(count == req_all_end_times.size());
|
||||
std::chrono::duration<double, std::milli> first_infer_durtation =
|
||||
req_all_end_times.front() - req_all_start_times.front();
|
||||
req_all_start_times.sort(std::less<Time>());
|
||||
req_all_end_times.sort(std::less<Time>());
|
||||
{
|
||||
|
|
@ -290,6 +292,9 @@ Schedule::~Schedule() {
|
|||
}
|
||||
}
|
||||
if (n >= 1) {
|
||||
LOG_INFO_TAG("%s: first inference time:%lf ms",
|
||||
worker_request.first.c_str(),
|
||||
first_infer_durtation.count());
|
||||
LOG_INFO_TAG("%s:infer:%ld", worker_request.first.c_str(), count);
|
||||
std::chrono::duration<double, std::milli> durtation =
|
||||
req_all_end_times.back() - time;
|
||||
|
|
|
|||
|
|
@ -58,37 +58,25 @@ void DynamicOutputInferenceTest::SetUp() {
|
|||
nullptr,
|
||||
false);
|
||||
std::tie(priorityList, targetList) = GetParam();
|
||||
ON_CALL(*core,
|
||||
compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
|
||||
::testing::Matcher<const std::string&>(StrEq(ov::test::utils::DEVICE_GPU)),
|
||||
_))
|
||||
.WillByDefault(InvokeWithoutArgs([this]() {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(200));
|
||||
return mockExeNetworkActual;
|
||||
}));
|
||||
ON_CALL(
|
||||
*core,
|
||||
compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
|
||||
::testing::Matcher<const std::string&>(StrEq(std::string(ov::test::utils::DEVICE_GPU) + ".0")),
|
||||
_))
|
||||
.WillByDefault(InvokeWithoutArgs([this]() {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(200));
|
||||
return mockExeNetwork;
|
||||
}));
|
||||
ON_CALL(
|
||||
*core,
|
||||
compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
|
||||
::testing::Matcher<const std::string&>(StrEq(std::string(ov::test::utils::DEVICE_GPU) + ".1")),
|
||||
_))
|
||||
.WillByDefault(InvokeWithoutArgs([this]() {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(200));
|
||||
return mockExeNetworkActual;
|
||||
}));
|
||||
ON_CALL(*core,
|
||||
compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
|
||||
::testing::Matcher<const std::string&>(StrEq(ov::test::utils::DEVICE_CPU)),
|
||||
(_)))
|
||||
.WillByDefault(Return(mockExeNetwork));
|
||||
auto targets = targetList.as<std::vector<std::string>>();
|
||||
ON_CALL(*core, get_available_devices()).WillByDefault(Return(targets));
|
||||
for (auto device : targets) {
|
||||
ON_CALL(*core,
|
||||
compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
|
||||
::testing::Matcher<const std::string&>(StrEq(device)),
|
||||
_))
|
||||
.WillByDefault(InvokeWithoutArgs([this, device]() {
|
||||
if (device.find("GPU") != std::string::npos) {
|
||||
if (device == "GPU.1") {
|
||||
return mockExeNetwork;
|
||||
} else {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(200));
|
||||
return mockExeNetworkActual;
|
||||
}
|
||||
}
|
||||
return mockExeNetwork;
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(DynamicOutputInferenceTest, CanSelectCorrectTargetDeviceandInitizeBlobWithCorrectSize) {
|
||||
|
|
@ -99,17 +87,10 @@ TEST_P(DynamicOutputInferenceTest, CanSelectCorrectTargetDeviceandInitizeBlobWit
|
|||
for (auto& iter : targets) {
|
||||
EXPECT_CALL(*core,
|
||||
compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
|
||||
::testing::Matcher<const std::string&>(HasSubstr(iter)),
|
||||
::testing::Matcher<const std::string&>(StrEq(iter)),
|
||||
::testing::Matcher<const ov::AnyMap&>(_)))
|
||||
.Times(1);
|
||||
}
|
||||
if (priorityList.as<std::string>().find("CPU") != std::string::npos) {
|
||||
EXPECT_CALL(*core,
|
||||
compile_model(::testing::Matcher<const std::shared_ptr<const ov::Model>&>(_),
|
||||
::testing::Matcher<const std::string&>(StrEq("GPU")),
|
||||
::testing::Matcher<const ov::AnyMap&>(_)))
|
||||
.Times(0);
|
||||
}
|
||||
ASSERT_NO_THROW(exeNetwork = plugin->compile_model(model, config));
|
||||
}
|
||||
|
||||
|
|
@ -137,8 +118,8 @@ TEST_P(DynamicOutputInferenceTest, CanInferWithOutputChangedFromDynamicOnAutoToS
|
|||
}
|
||||
|
||||
const std::vector<DynamicOutputConfigParams> testConfigs = {
|
||||
DynamicOutputConfigParams{"CPU,GPU", std::vector<std::string>{"CPU"}},
|
||||
DynamicOutputConfigParams{"GPU,CPU", std::vector<std::string>{"CPU"}},
|
||||
DynamicOutputConfigParams{"CPU,GPU", std::vector<std::string>{"CPU", "GPU"}},
|
||||
DynamicOutputConfigParams{"GPU,CPU", std::vector<std::string>{"CPU", "GPU"}},
|
||||
DynamicOutputConfigParams{"GPU.0,GPU.1", std::vector<std::string>{"GPU.0", "GPU.1"}},
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -255,25 +255,25 @@ const std::vector<StatefulModelConfigParams> testConfigs = {
|
|||
true,
|
||||
std::map<std::string, bool>{{"CPU", true}, {"GPU", true}},
|
||||
true,
|
||||
std::vector<std::pair<std::string, int>>{{"CPU", 1}, {"GPU", 0}}},
|
||||
std::vector<std::pair<std::string, int>>{{"CPU", 1}, {"GPU", 1}}},
|
||||
StatefulModelConfigParams{"GPU,CPU",
|
||||
true,
|
||||
true,
|
||||
std::map<std::string, bool>{{"CPU", true}, {"GPU", true}},
|
||||
true,
|
||||
std::vector<std::pair<std::string, int>>{{"GPU", 0}, {"CPU", 1}}},
|
||||
std::vector<std::pair<std::string, int>>{{"GPU", 1}, {"CPU", 1}}},
|
||||
StatefulModelConfigParams{"CPU,GPU",
|
||||
true,
|
||||
false,
|
||||
std::map<std::string, bool>{{"CPU", true}, {"GPU", true}},
|
||||
true,
|
||||
std::vector<std::pair<std::string, int>>{{"CPU", 1}, {"GPU", 0}}},
|
||||
std::vector<std::pair<std::string, int>>{{"CPU", 1}, {"GPU", 1}}},
|
||||
StatefulModelConfigParams{"GPU,CPU",
|
||||
true,
|
||||
false,
|
||||
std::map<std::string, bool>{{"CPU", true}, {"GPU", true}},
|
||||
true,
|
||||
std::vector<std::pair<std::string, int>>{{"GPU", 0}, {"CPU", 1}}},
|
||||
std::vector<std::pair<std::string, int>>{{"GPU", 1}, {"CPU", 1}}},
|
||||
StatefulModelConfigParams{"CPU",
|
||||
false,
|
||||
false,
|
||||
|
|
|
|||
Loading…
Reference in New Issue