natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[PyOV] Fix hanging on infer request destruction (#24722) 

### Details:
- Initial problem: `test_custom_op` hanged on destruction because it was
waiting for a thread which tried to acquire GIL.
- The second problem is that pybind11 doesn't allow to work with GIL
besides of current scope and it's impossible to release GIL for
destructors. https://github.com/pybind/pybind11/issues/1446
- Current solution allows to release GIL for InferRequest and all called
by chain destructors.

### Tickets:
 - CVS-141744
This commit is contained in:
Anastasia Kuporosova 2024-06-07 13:56:36 +02:00 committed by GitHub
parent bb179c69d1
commit ba5c45a4ee
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 95 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp
View File

@ -15,6 +15,7 @@
#include "pyopenvino/core/common.hpp" #include "pyopenvino/core/common.hpp"
#include "pyopenvino/core/infer_request.hpp" #include "pyopenvino/core/infer_request.hpp"
#include "pyopenvino/utils/utils.hpp"
namespace py = pybind11; namespace py = pybind11;
@ -64,7 +65,7 @@ public:
}); });
size_t idle_handle = m_idle_handles.front(); size_t idle_handle = m_idle_handles.front();
// wait for request to make sure it returned from callback // wait for request to make sure it returned from callback
m_requests[idle_handle].m_request.wait(); m_requests[idle_handle].m_request->wait();
if (m_errors.size() > 0) if (m_errors.size() > 0)
throw m_errors.front(); throw m_errors.front();
return idle_handle; return idle_handle;
@ -75,7 +76,7 @@ public:
// release GIL to avoid deadlock on python callback // release GIL to avoid deadlock on python callback
py::gil_scoped_release release; py::gil_scoped_release release;
for (auto&& request : m_requests) { for (auto&& request : m_requests) {
request.m_request.wait(); request.m_request->wait();
} }
// acquire the mutex to access m_errors // acquire the mutex to access m_errors
std::lock_guard<std::mutex> lock(m_mutex); std::lock_guard<std::mutex> lock(m_mutex);
@ -87,7 +88,7 @@ public:
for (size_t handle = 0; handle < m_requests.size(); handle++) { for (size_t handle = 0; handle < m_requests.size(); handle++) {
// auto end_time = m_requests[handle].m_end_time; // TODO: pass it bellow? like in InferRequestWrapper // auto end_time = m_requests[handle].m_end_time; // TODO: pass it bellow? like in InferRequestWrapper
m_requests[handle].m_request.set_callback([this, handle /* ... */](std::exception_ptr exception_ptr) { m_requests[handle].m_request->set_callback([this, handle /* ... */](std::exception_ptr exception_ptr) {
*m_requests[handle].m_end_time = Time::now(); *m_requests[handle].m_end_time = Time::now();
{ {
// acquire the mutex to access m_idle_handles // acquire the mutex to access m_idle_handles
@ -110,14 +111,17 @@ public:
} }
void set_custom_callbacks(py::function f_callback) { void set_custom_callbacks(py::function f_callback) {
// need to acquire GIL before py::function deletion
auto callback_sp = Common::utils::wrap_pyfunction(std::move(f_callback));
for (size_t handle = 0; handle < m_requests.size(); handle++) { for (size_t handle = 0; handle < m_requests.size(); handle++) {
m_requests[handle].m_request.set_callback([this, f_callback, handle](std::exception_ptr exception_ptr) { m_requests[handle].m_request->set_callback([this, callback_sp, handle](std::exception_ptr exception_ptr) {
*m_requests[handle].m_end_time = Time::now(); *m_requests[handle].m_end_time = Time::now();
if (exception_ptr == nullptr) { if (exception_ptr == nullptr) {
// Acquire GIL, execute Python function // Acquire GIL, execute Python function
py::gil_scoped_acquire acquire; py::gil_scoped_acquire acquire;
try { try {
f_callback(m_requests[handle], m_user_ids[handle]); (*callback_sp)(m_requests[handle], m_user_ids[handle]);
} catch (const py::error_already_set& py_error) { } catch (const py::error_already_set& py_error) {
// This should behave the same as assert(!PyErr_Occurred()) // This should behave the same as assert(!PyErr_Occurred())
// since constructor for pybind11's error_already_set is // since constructor for pybind11's error_already_set is
@ -193,13 +197,13 @@ void regclass_AsyncInferQueue(py::module m) {
// Set new inputs label/id from user // Set new inputs label/id from user
self.m_user_ids[handle] = userdata; self.m_user_ids[handle] = userdata;
// Update inputs if there are any // Update inputs if there are any
self.m_requests[handle].m_request.set_input_tensor(inputs); self.m_requests[handle].m_request->set_input_tensor(inputs);
// Now GIL can be released - we are NOT working with Python objects in this block // Now GIL can be released - we are NOT working with Python objects in this block
{ {
py::gil_scoped_release release; py::gil_scoped_release release;
*self.m_requests[handle].m_start_time = Time::now(); *self.m_requests[handle].m_start_time = Time::now();
// Start InferRequest in asynchronus mode // Start InferRequest in asynchronus mode
self.m_requests[handle].m_request.start_async(); self.m_requests[handle].m_request->start_async();
} }
}, },
py::arg("inputs"), py::arg("inputs"),
@ -239,13 +243,13 @@ void regclass_AsyncInferQueue(py::module m) {
// Set new inputs label/id from user // Set new inputs label/id from user
self.m_user_ids[handle] = userdata; self.m_user_ids[handle] = userdata;
// Update inputs if there are any // Update inputs if there are any
Common::set_request_tensors(self.m_requests[handle].m_request, inputs); Common::set_request_tensors(*self.m_requests[handle].m_request, inputs);
// Now GIL can be released - we are NOT working with Python objects in this block // Now GIL can be released - we are NOT working with Python objects in this block
{ {
py::gil_scoped_release release; py::gil_scoped_release release;
*self.m_requests[handle].m_start_time = Time::now(); *self.m_requests[handle].m_start_time = Time::now();
// Start InferRequest in asynchronus mode // Start InferRequest in asynchronus mode
self.m_requests[handle].m_request.start_async(); self.m_requests[handle].m_request->start_async();
} }
}, },
py::arg("inputs"), py::arg("inputs"),

8
src/bindings/python/src/pyopenvino/core/common.cpp
View File

@ -433,10 +433,14 @@ ov::op::v0::Constant create_shared(py::array& array) {
// If ndim is equal to 0, creates scalar Constant. // If ndim is equal to 0, creates scalar Constant.
// If size is equal to 0, creates empty Constant. // If size is equal to 0, creates empty Constant.
if (array_helpers::is_contiguous(array)) { if (array_helpers::is_contiguous(array)) {
auto memory = std::make_shared<ov::SharedBuffer<py::array>>( auto buffer = new ov::SharedBuffer<py::array>(
static_cast<char*>((array.ndim() == 0 || array.size() == 0) ? array.mutable_data() : array.mutable_data(0)), static_cast<char*>((array.ndim() == 0 || array.size() == 0) ? array.mutable_data() : array.mutable_data(0)),
array.ndim() == 0 ? array.itemsize() : array.nbytes(), array.ndim() == 0 ? array.itemsize() : array.nbytes(),
array); array);
std::shared_ptr<ov::SharedBuffer<py::array>> memory(buffer, [](ov::SharedBuffer<py::array>* buffer) {
py::gil_scoped_acquire acquire;
delete buffer;
});
return ov::op::v0::Constant(type_helpers::get_ov_type(array), array_helpers::get_shape(array), memory); return ov::op::v0::Constant(type_helpers::get_ov_type(array), array_helpers::get_shape(array), memory);
} }
// If passed array is not C-style, throw an error. // If passed array is not C-style, throw an error.
@ -614,7 +618,7 @@ uint32_t get_optimal_number_of_requests(const ov::CompiledModel& actual) {
py::dict outputs_to_dict(InferRequestWrapper& request, bool share_outputs, bool decode_strings) { py::dict outputs_to_dict(InferRequestWrapper& request, bool share_outputs, bool decode_strings) {
py::dict res; py::dict res;
for (const auto& out : request.m_outputs) { for (const auto& out : request.m_outputs) {
auto t = request.m_request.get_tensor(out); auto t = request.m_request->get_tensor(out);
if (t.get_element_type() == ov::element::string) { if (t.get_element_type() == ov::element::string) {
if (share_outputs) { if (share_outputs) {
PyErr_WarnEx(PyExc_RuntimeWarning, "Result of a string type will be copied to OVDict!", 1); PyErr_WarnEx(PyExc_RuntimeWarning, "Result of a string type will be copied to OVDict!", 1);

80
src/bindings/python/src/pyopenvino/core/infer_request.cpp
View File

@ -18,7 +18,7 @@ inline py::object run_sync_infer(InferRequestWrapper& self, bool share_outputs,
{ {
py::gil_scoped_release release; py::gil_scoped_release release;
*self.m_start_time = Time::now(); *self.m_start_time = Time::now();
self.m_request.infer(); self.m_request->infer();
*self.m_end_time = Time::now(); *self.m_end_time = Time::now();
} }
return Common::outputs_to_dict(self, share_outputs, decode_strings); return Common::outputs_to_dict(self, share_outputs, decode_strings);
@ -38,7 +38,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"set_tensors", "set_tensors",
[](InferRequestWrapper& self, const py::dict& inputs) { [](InferRequestWrapper& self, const py::dict& inputs) {
Common::set_request_tensors(self.m_request, inputs); Common::set_request_tensors(*self.m_request, inputs);
}, },
py::arg("inputs"), py::arg("inputs"),
R"( R"(
@ -51,7 +51,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"set_tensors", "set_tensors",
[](InferRequestWrapper& self, const std::string& tensor_name, const std::vector<ov::Tensor>& tensors) { [](InferRequestWrapper& self, const std::string& tensor_name, const std::vector<ov::Tensor>& tensors) {
self.m_request.set_tensors(tensor_name, tensors); self.m_request->set_tensors(tensor_name, tensors);
}, },
py::arg("tensor_name"), py::arg("tensor_name"),
py::arg("tensors"), py::arg("tensors"),
@ -73,7 +73,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"set_tensors", "set_tensors",
[](InferRequestWrapper& self, const ov::Output<const ov::Node>& port, const std::vector<ov::Tensor>& tensors) { [](InferRequestWrapper& self, const ov::Output<const ov::Node>& port, const std::vector<ov::Tensor>& tensors) {
self.m_request.set_tensors(port, tensors); self.m_request->set_tensors(port, tensors);
}, },
py::arg("port"), py::arg("port"),
py::arg("tensors"), py::arg("tensors"),
@ -100,7 +100,7 @@ void regclass_InferRequest(py::module m) {
[](InferRequestWrapper& self, const py::dict& outputs) { [](InferRequestWrapper& self, const py::dict& outputs) {
auto outputs_map = Common::containers::cast_to_tensor_index_map(outputs); auto outputs_map = Common::containers::cast_to_tensor_index_map(outputs);
for (auto&& output : outputs_map) { for (auto&& output : outputs_map) {
self.m_request.set_output_tensor(output.first, output.second); self.m_request->set_output_tensor(output.first, output.second);
} }
}, },
py::arg("outputs"), py::arg("outputs"),
@ -117,7 +117,7 @@ void regclass_InferRequest(py::module m) {
[](InferRequestWrapper& self, const py::dict& inputs) { [](InferRequestWrapper& self, const py::dict& inputs) {
auto inputs_map = Common::containers::cast_to_tensor_index_map(inputs); auto inputs_map = Common::containers::cast_to_tensor_index_map(inputs);
for (auto&& input : inputs_map) { for (auto&& input : inputs_map) {
self.m_request.set_input_tensor(input.first, input.second); self.m_request->set_input_tensor(input.first, input.second);
} }
}, },
py::arg("inputs"), py::arg("inputs"),
@ -131,7 +131,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"set_input_tensors", "set_input_tensors",
[](InferRequestWrapper& self, const std::vector<ov::Tensor>& tensors) { [](InferRequestWrapper& self, const std::vector<ov::Tensor>& tensors) {
self.m_request.set_input_tensors(tensors); self.m_request->set_input_tensors(tensors);
}, },
py::arg("tensors"), py::arg("tensors"),
R"( R"(
@ -148,7 +148,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"set_input_tensors", "set_input_tensors",
[](InferRequestWrapper& self, size_t idx, const std::vector<ov::Tensor>& tensors) { [](InferRequestWrapper& self, size_t idx, const std::vector<ov::Tensor>& tensors) {
self.m_request.set_input_tensors(idx, tensors); self.m_request->set_input_tensors(idx, tensors);
}, },
py::arg("idx"), py::arg("idx"),
py::arg("tensors"), py::arg("tensors"),
@ -168,7 +168,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"infer", "infer",
[](InferRequestWrapper& self, const ov::Tensor& inputs, bool share_outputs, bool decode_strings) { [](InferRequestWrapper& self, const ov::Tensor& inputs, bool share_outputs, bool decode_strings) {
self.m_request.set_input_tensor(inputs); self.m_request->set_input_tensor(inputs);
return run_sync_infer(self, share_outputs, decode_strings); return run_sync_infer(self, share_outputs, decode_strings);
}, },
py::arg("inputs"), py::arg("inputs"),
@ -197,7 +197,7 @@ void regclass_InferRequest(py::module m) {
"infer", "infer",
[](InferRequestWrapper& self, const py::dict& inputs, bool share_outputs, bool decode_strings) { [](InferRequestWrapper& self, const py::dict& inputs, bool share_outputs, bool decode_strings) {
// Update inputs if there are any // Update inputs if there are any
Common::set_request_tensors(self.m_request, inputs); Common::set_request_tensors(*self.m_request, inputs);
// Call Infer function // Call Infer function
return run_sync_infer(self, share_outputs, decode_strings); return run_sync_infer(self, share_outputs, decode_strings);
}, },
@ -222,7 +222,7 @@ void regclass_InferRequest(py::module m) {
"start_async", "start_async",
[](InferRequestWrapper& self, const ov::Tensor& inputs, py::object& userdata) { [](InferRequestWrapper& self, const ov::Tensor& inputs, py::object& userdata) {
// Update inputs if there are any // Update inputs if there are any
self.m_request.set_input_tensor(inputs); self.m_request->set_input_tensor(inputs);
if (!userdata.is(py::none())) { if (!userdata.is(py::none())) {
if (self.m_user_callback_defined) { if (self.m_user_callback_defined) {
self.m_userdata = userdata; self.m_userdata = userdata;
@ -232,7 +232,7 @@ void regclass_InferRequest(py::module m) {
} }
py::gil_scoped_release release; py::gil_scoped_release release;
*self.m_start_time = Time::now(); *self.m_start_time = Time::now();
self.m_request.start_async(); self.m_request->start_async();
}, },
py::arg("inputs"), py::arg("inputs"),
py::arg("userdata"), py::arg("userdata"),
@ -261,7 +261,7 @@ void regclass_InferRequest(py::module m) {
"start_async", "start_async",
[](InferRequestWrapper& self, const py::dict& inputs, py::object& userdata) { [](InferRequestWrapper& self, const py::dict& inputs, py::object& userdata) {
// Update inputs if there are any // Update inputs if there are any
Common::set_request_tensors(self.m_request, inputs); Common::set_request_tensors(*self.m_request, inputs);
if (!userdata.is(py::none())) { if (!userdata.is(py::none())) {
if (self.m_user_callback_defined) { if (self.m_user_callback_defined) {
self.m_userdata = userdata; self.m_userdata = userdata;
@ -271,7 +271,7 @@ void regclass_InferRequest(py::module m) {
} }
py::gil_scoped_release release; py::gil_scoped_release release;
*self.m_start_time = Time::now(); *self.m_start_time = Time::now();
self.m_request.start_async(); self.m_request->start_async();
}, },
py::arg("inputs"), py::arg("inputs"),
py::arg("userdata"), py::arg("userdata"),
@ -293,7 +293,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"cancel", "cancel",
[](InferRequestWrapper& self) { [](InferRequestWrapper& self) {
self.m_request.cancel(); self.m_request->cancel();
}, },
R"( R"(
Cancels inference request. Cancels inference request.
@ -303,7 +303,7 @@ void regclass_InferRequest(py::module m) {
"wait", "wait",
[](InferRequestWrapper& self) { [](InferRequestWrapper& self) {
py::gil_scoped_release release; py::gil_scoped_release release;
self.m_request.wait(); self.m_request->wait();
}, },
R"( R"(
Waits for the result to become available. Waits for the result to become available.
@ -316,7 +316,7 @@ void regclass_InferRequest(py::module m) {
"wait_for", "wait_for",
[](InferRequestWrapper& self, const int timeout) { [](InferRequestWrapper& self, const int timeout) {
py::gil_scoped_release release; py::gil_scoped_release release;
return self.m_request.wait_for(std::chrono::milliseconds(timeout)); return self.m_request->wait_for(std::chrono::milliseconds(timeout));
}, },
py::arg("timeout"), py::arg("timeout"),
R"( R"(
@ -337,7 +337,11 @@ void regclass_InferRequest(py::module m) {
[](InferRequestWrapper& self, py::function callback, py::object& userdata) { [](InferRequestWrapper& self, py::function callback, py::object& userdata) {
self.m_userdata = userdata; self.m_userdata = userdata;
self.m_user_callback_defined = true; self.m_user_callback_defined = true;
self.m_request.set_callback([&self, callback](std::exception_ptr exception_ptr) {
// need to acquire GIL before py::function deletion
auto callback_sp = Common::utils::wrap_pyfunction(std::move(callback));
self.m_request->set_callback([&self, callback_sp](std::exception_ptr exception_ptr) {
*self.m_end_time = Time::now(); *self.m_end_time = Time::now();
try { try {
if (exception_ptr) { if (exception_ptr) {
@ -348,7 +352,7 @@ void regclass_InferRequest(py::module m) {
} }
// Acquire GIL, execute Python function // Acquire GIL, execute Python function
py::gil_scoped_acquire acquire; py::gil_scoped_acquire acquire;
callback(self.m_userdata); (*callback_sp)(self.m_userdata);
}); });
}, },
py::arg("callback"), py::arg("callback"),
@ -365,7 +369,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"get_tensor", "get_tensor",
[](InferRequestWrapper& self, const std::string& name) { [](InferRequestWrapper& self, const std::string& name) {
return self.m_request.get_tensor(name); return self.m_request->get_tensor(name);
}, },
py::arg("name"), py::arg("name"),
R"( R"(
@ -380,7 +384,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"get_tensor", "get_tensor",
[](InferRequestWrapper& self, const ov::Output<const ov::Node>& port) { [](InferRequestWrapper& self, const ov::Output<const ov::Node>& port) {
return self.m_request.get_tensor(port); return self.m_request->get_tensor(port);
}, },
py::arg("port"), py::arg("port"),
R"( R"(
@ -395,7 +399,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"get_tensor", "get_tensor",
[](InferRequestWrapper& self, const ov::Output<ov::Node>& port) { [](InferRequestWrapper& self, const ov::Output<ov::Node>& port) {
return self.m_request.get_tensor(port); return self.m_request->get_tensor(port);
}, },
py::arg("port"), py::arg("port"),
R"( R"(
@ -410,7 +414,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"get_input_tensor", "get_input_tensor",
[](InferRequestWrapper& self, size_t idx) { [](InferRequestWrapper& self, size_t idx) {
return self.m_request.get_input_tensor(idx); return self.m_request->get_input_tensor(idx);
}, },
py::arg("index"), py::arg("index"),
R"( R"(
@ -427,7 +431,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"get_input_tensor", "get_input_tensor",
[](InferRequestWrapper& self) { [](InferRequestWrapper& self) {
return self.m_request.get_input_tensor(); return self.m_request->get_input_tensor();
}, },
R"( R"(
Gets input tensor of InferRequest. Gets input tensor of InferRequest.
@ -440,7 +444,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"get_output_tensor", "get_output_tensor",
[](InferRequestWrapper& self, size_t idx) { [](InferRequestWrapper& self, size_t idx) {
return self.m_request.get_output_tensor(idx); return self.m_request->get_output_tensor(idx);
}, },
py::arg("index"), py::arg("index"),
R"( R"(
@ -456,7 +460,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"get_output_tensor", "get_output_tensor",
[](InferRequestWrapper& self) { [](InferRequestWrapper& self) {
return self.m_request.get_output_tensor(); return self.m_request->get_output_tensor();
}, },
R"( R"(
Gets output tensor of InferRequest. Gets output tensor of InferRequest.
@ -469,7 +473,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"set_tensor", "set_tensor",
[](InferRequestWrapper& self, const std::string& name, const ov::Tensor& tensor) { [](InferRequestWrapper& self, const std::string& name, const ov::Tensor& tensor) {
self.m_request.set_tensor(name, tensor); self.m_request->set_tensor(name, tensor);
}, },
py::arg("name"), py::arg("name"),
py::arg("tensor"), py::arg("tensor"),
@ -486,7 +490,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"set_tensor", "set_tensor",
[](InferRequestWrapper& self, const ov::Output<const ov::Node>& port, const ov::Tensor& tensor) { [](InferRequestWrapper& self, const ov::Output<const ov::Node>& port, const ov::Tensor& tensor) {
self.m_request.set_tensor(port, tensor); self.m_request->set_tensor(port, tensor);
}, },
py::arg("port"), py::arg("port"),
py::arg("tensor"), py::arg("tensor"),
@ -503,7 +507,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"set_tensor", "set_tensor",
[](InferRequestWrapper& self, const ov::Output<ov::Node>& port, const ov::Tensor& tensor) { [](InferRequestWrapper& self, const ov::Output<ov::Node>& port, const ov::Tensor& tensor) {
self.m_request.set_tensor(port, tensor); self.m_request->set_tensor(port, tensor);
}, },
py::arg("port"), py::arg("port"),
py::arg("tensor"), py::arg("tensor"),
@ -520,7 +524,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"set_input_tensor", "set_input_tensor",
[](InferRequestWrapper& self, size_t idx, const ov::Tensor& tensor) { [](InferRequestWrapper& self, size_t idx, const ov::Tensor& tensor) {
self.m_request.set_input_tensor(idx, tensor); self.m_request->set_input_tensor(idx, tensor);
}, },
py::arg("index"), py::arg("index"),
py::arg("tensor"), py::arg("tensor"),
@ -538,7 +542,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"set_input_tensor", "set_input_tensor",
[](InferRequestWrapper& self, const ov::Tensor& tensor) { [](InferRequestWrapper& self, const ov::Tensor& tensor) {
self.m_request.set_input_tensor(tensor); self.m_request->set_input_tensor(tensor);
}, },
py::arg("tensor"), py::arg("tensor"),
R"( R"(
@ -553,7 +557,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"set_output_tensor", "set_output_tensor",
[](InferRequestWrapper& self, size_t idx, const ov::Tensor& tensor) { [](InferRequestWrapper& self, size_t idx, const ov::Tensor& tensor) {
self.m_request.set_output_tensor(idx, tensor); self.m_request->set_output_tensor(idx, tensor);
}, },
py::arg("index"), py::arg("index"),
py::arg("tensor"), py::arg("tensor"),
@ -570,7 +574,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"set_output_tensor", "set_output_tensor",
[](InferRequestWrapper& self, const ov::Tensor& tensor) { [](InferRequestWrapper& self, const ov::Tensor& tensor) {
self.m_request.set_output_tensor(tensor); self.m_request->set_output_tensor(tensor);
}, },
py::arg("tensor"), py::arg("tensor"),
R"( R"(
@ -585,7 +589,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"get_profiling_info", "get_profiling_info",
[](InferRequestWrapper& self) { [](InferRequestWrapper& self) {
return self.m_request.get_profiling_info(); return self.m_request->get_profiling_info();
}, },
py::call_guard<py::gil_scoped_release>(), py::call_guard<py::gil_scoped_release>(),
R"( R"(
@ -602,7 +606,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"query_state", "query_state",
[](InferRequestWrapper& self) { [](InferRequestWrapper& self) {
return self.m_request.query_state(); return self.m_request->query_state();
}, },
py::call_guard<py::gil_scoped_release>(), py::call_guard<py::gil_scoped_release>(),
R"( R"(
@ -617,7 +621,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"reset_state", "reset_state",
[](InferRequestWrapper& self) { [](InferRequestWrapper& self) {
return self.m_request.reset_state(); return self.m_request->reset_state();
}, },
R"( R"(
Resets all internal variable states for relevant infer request to Resets all internal variable states for relevant infer request to
@ -627,7 +631,7 @@ void regclass_InferRequest(py::module m) {
cls.def( cls.def(
"get_compiled_model", "get_compiled_model",
[](InferRequestWrapper& self) { [](InferRequestWrapper& self) {
return self.m_request.get_compiled_model(); return self.m_request->get_compiled_model();
}, },
R"( R"(
Returns the compiled model. Returns the compiled model.
@ -700,7 +704,7 @@ void regclass_InferRequest(py::module m) {
cls.def_property_readonly( cls.def_property_readonly(
"profiling_info", "profiling_info",
[](InferRequestWrapper& self) { [](InferRequestWrapper& self) {
return self.m_request.get_profiling_info(); return self.m_request->get_profiling_info();
}, },
py::call_guard<py::gil_scoped_release>(), py::call_guard<py::gil_scoped_release>(),
R"( R"(

15
src/bindings/python/src/pyopenvino/core/infer_request.hpp
View File

@ -32,7 +32,7 @@ public:
const std::vector<ov::Output<const ov::Node>>& outputs, const std::vector<ov::Output<const ov::Node>>& outputs,
bool set_default_callback = true, bool set_default_callback = true,
py::object userdata = py::none()) py::object userdata = py::none())
: m_request{std::move(request)}, : m_request{InferRequestWrapper::wrap_infer_request_to_sp(std::move(request))},
m_inputs{inputs}, m_inputs{inputs},
m_outputs{outputs}, m_outputs{outputs},
m_userdata{userdata} { m_userdata{userdata} {
@ -44,7 +44,7 @@ public:
// Bump reference counter // Bump reference counter
auto end_time = m_end_time; auto end_time = m_end_time;
// Set standard callback which saves "end-time" for inference call // Set standard callback which saves "end-time" for inference call
m_request.set_callback([end_time](std::exception_ptr exception_ptr) { m_request->set_callback([end_time](std::exception_ptr exception_ptr) {
*end_time = Time::now(); *end_time = Time::now();
try { try {
if (exception_ptr) { if (exception_ptr) {
@ -73,7 +73,7 @@ public:
} }
// Original ov::InferRequest class that is held by this wrapper // Original ov::InferRequest class that is held by this wrapper
ov::InferRequest m_request; std::shared_ptr<ov::InferRequest> m_request;
// Inputs and Outputs inherrited from ov::CompiledModel // Inputs and Outputs inherrited from ov::CompiledModel
std::vector<ov::Output<const ov::Node>> m_inputs; std::vector<ov::Output<const ov::Node>> m_inputs;
std::vector<ov::Output<const ov::Node>> m_outputs; std::vector<ov::Output<const ov::Node>> m_outputs;
@ -91,11 +91,18 @@ private:
tensors.reserve(v.size()); tensors.reserve(v.size());
for (auto&& node : v) { for (auto&& node : v) {
tensors.push_back(m_request.get_tensor(node)); tensors.push_back(m_request->get_tensor(node));
} }
return tensors; return tensors;
} }
static std::shared_ptr<ov::InferRequest> wrap_infer_request_to_sp(ov::InferRequest request) {
return std::shared_ptr<ov::InferRequest>(new ov::InferRequest(std::move(request)), [](ov::InferRequest* request) {
py::gil_scoped_release release;
delete request;
});
}
}; };
void regclass_InferRequest(py::module m); void regclass_InferRequest(py::module m);

19
src/bindings/python/src/pyopenvino/frontend/extension.cpp
View File

@ -30,19 +30,26 @@ void regclass_frontend_TelemetryExtension(py::module m) {
py::function& send_event, py::function& send_event,
py::function& send_error, py::function& send_error,
py::function& send_stack_trace) { py::function& send_stack_trace) {
auto send_event_sp = Common::utils::wrap_pyfunction(send_event);
auto send_error_sp = Common::utils::wrap_pyfunction(send_error);
auto send_stack_trace_sp = Common::utils::wrap_pyfunction(send_stack_trace);
return std::make_shared<TelemetryExtension>( return std::make_shared<TelemetryExtension>(
event_category, event_category,
[send_event](const std::string& category, const std::string& action, const std::string& label, int value) { [send_event_sp](const std::string& category,
const std::string& action,
const std::string& label,
int value) {
py::gil_scoped_acquire acquire; py::gil_scoped_acquire acquire;
send_event(category, action, label, value); (*send_event_sp)(category, action, label, value);
}, },
[send_error](const std::string& category, const std::string& error_message) { [send_error_sp](const std::string& category, const std::string& error_message) {
py::gil_scoped_acquire acquire; py::gil_scoped_acquire acquire;
send_error(category, error_message); (*send_error_sp)(category, error_message);
}, },
[send_stack_trace](const std::string& category, const std::string& error_message) { [send_stack_trace_sp](const std::string& category, const std::string& error_message) {
py::gil_scoped_acquire acquire; py::gil_scoped_acquire acquire;
send_stack_trace(category, error_message); (*send_stack_trace_sp)(category, error_message);
}); });
})); }));

7
src/bindings/python/src/pyopenvino/utils/utils.cpp
View File

@ -419,5 +419,12 @@ ov::Any py_object_to_any(const py::object& py_obj) {
} }
OPENVINO_ASSERT(false, "Unsupported attribute type."); OPENVINO_ASSERT(false, "Unsupported attribute type.");
} }
std::shared_ptr<py::function> wrap_pyfunction(py::function f_callback) {
auto callback_sp = std::shared_ptr<py::function>(new py::function(std::move(f_callback)), [](py::function* c) {
py::gil_scoped_acquire acquire;
delete c;
});
return callback_sp;
}
}; // namespace utils }; // namespace utils
}; // namespace Common }; // namespace Common

2
src/bindings/python/src/pyopenvino/utils/utils.hpp
View File

@ -58,5 +58,7 @@ namespace utils {
ov::pass::Serialize::Version convert_to_version(const std::string& version); ov::pass::Serialize::Version convert_to_version(const std::string& version);
std::shared_ptr<py::function> wrap_pyfunction(py::function f_callback);
}; // namespace utils }; // namespace utils
}; // namespace Common }; // namespace Common

4
src/bindings/python/tests/test_graph/test_op.py
View File

@ -107,9 +107,7 @@ def test_custom_add_model():
def test_custom_op(): def test_custom_op():
model = create_snake_model() model = create_snake_model()
# todo: CVS-141744 compiled_model = compile_model(model)
# it hangs with AUTO plugin, but works well with CPU
compiled_model = compile_model(model, "CPU")
assert isinstance(compiled_model, CompiledModel) assert isinstance(compiled_model, CompiledModel)
request = compiled_model.create_infer_request() request = compiled_model.create_infer_request()