natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[GPU] Disable predict_preallocation_shape for blocked format (#23796) 

### Details:
 - *Disable predict_preallocation_shape for blocked format*
 - *Fix to use layout instead of shape and data_type*

### Tickets:
 - *132334*
This commit is contained in:
Steve Yoo 2024-04-12 07:41:42 +09:00 committed by GitHub
parent 2a86e64f1a
commit b2b5d3efb0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 83 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/plugins/intel_gpu/include/intel_gpu/plugin/common_utils.hpp
View File

@ -95,16 +95,17 @@ inline ov::Shape get_tensor_shape(const ov::PartialShape& pshape) {
return res;
}
inline ov::Shape predict_shape(const std::string& name, const ov::Shape current_shape, ov::element::Type element_type, cldnn::ShapePredictor& shape_predictor) {
auto prealloc_info = shape_predictor.predict_preallocation_shape(name, current_shape, element_type.bitwidth(), false);
inline ov::Shape predict_shape(const std::string& name, const cldnn::layout layout, cldnn::ShapePredictor& shape_predictor) {
auto prealloc_info = shape_predictor.predict_preallocation_shape(name, layout, false);
const auto& preallocation_shape = prealloc_info.second;
auto can_preallocate_buffer = prealloc_info.first &&
shape_predictor.can_preallocate(cldnn::ceil_div(ov::shape_size(preallocation_shape) * element_type.bitwidth(), 8));
shape_predictor.can_preallocate(cldnn::ceil_div(ov::shape_size(preallocation_shape) *
ov::element::Type(layout.data_type).bitwidth(), 8));
if (can_preallocate_buffer) {
return preallocation_shape;
}
return current_shape;
return layout.get_shape();
}
/// WA: Force exit. Any opencl api call can be hang after CL_OUT_OF_RESOURCES.

6
src/plugins/intel_gpu/include/intel_gpu/runtime/shape_predictor.hpp
View File

@ -44,14 +44,12 @@ public:
/// ov::intel_gpu::buffers_preallocation_ratio property, it increases buffer size by
/// `_buffers_preallocation_ratio` value unconditionally.
/// \param id Primitive id.
/// \param current_shape Primitive's shape on current iteration.
/// \param dt_size Primitive's data_type size.
/// \param layout Primitive's layout on current iteration.
/// \param can_reuse_buffer Specifies if current memory buffer is enough to store data.
/// \return The result of shape size prediction as std::pair<bool, ov::Shape>, where the first element
/// says if shape is successfully predicted and can be preallocated, and the second element is ov::Shape itself.
std::pair<bool, ov::Shape> predict_preallocation_shape(const std::string& id,
const ov::Shape& current_shape,
size_t dt_bitwidth,
const cldnn::layout& layout,
bool can_reuse_buffer,
int32_t next_iters_prealloc_count = -1);

4
src/plugins/intel_gpu/src/graph/primitive_inst.cpp
View File

@ -496,7 +496,7 @@ event::ptr primitive_inst::realloc_if_needed() {
_outputs[0] = variable.get_memory();
// To record shape predictor
auto prealloc_info = sp.predict_preallocation_shape(id(), _impl_params->output_layouts[0].get_shape(), dt_size, true);
auto prealloc_info = sp.predict_preallocation_shape(id(), _impl_params->output_layouts[0], true);
return ev;
} else if (_outputs[0] && variable.get_memory() && get_network().get_engine().is_the_same_buffer(*_outputs[0], *variable.get_memory())) {
GPU_DEBUG_TRACE_DETAIL << id() << " : realloc_if_needed: Reset output mem" << std::endl;
@ -586,7 +586,7 @@ event::ptr primitive_inst::realloc_if_needed() {
// If debug config is set, repsect the config most
tmp_prealloc_count = -1;
}
prealloc_info = sp.predict_preallocation_shape(id(), current_shape, dt_size, can_reuse_buffer, tmp_prealloc_count);
prealloc_info = sp.predict_preallocation_shape(id(), updated_layout, can_reuse_buffer, tmp_prealloc_count);
if (prealloc_info.first && sp.can_preallocate(ov::shape_size(prealloc_info.second) * dt_size)) {
auto new_layout = updated_layout;

15
src/plugins/intel_gpu/src/plugin/sync_infer_request.cpp
View File

@ -359,7 +359,10 @@ void SyncInferRequest::wait() {
if (need_reallocate) {
std::string internal_name = m_output_names_map.at(port_idx);
auto actual_memory_shape = predict_shape(internal_name, mem_shape, output_tensor->get_element_type(), *m_shape_predictor);
auto actual_memory_shape = predict_shape(internal_name, cldnn::layout(mem_shape,
output_tensor->get_element_type(),
cldnn::format::get_default_format(mem_shape.size())),
*m_shape_predictor);
output_tensor->set_shape(actual_memory_shape);
}
}
@ -480,7 +483,10 @@ TensorWrapper SyncInferRequest::create_or_share_device_tensor(const TensorWrappe
auto actual_memory_shape = tensor_shape;
if (is_dynamic) {
actual_memory_shape = predict_shape(name, tensor_shape, element_type, *m_shape_predictor);
actual_memory_shape = predict_shape(name, cldnn::layout(tensor_shape,
element_type,
cldnn::format::get_default_format(tensor_shape.size())),
*m_shape_predictor);
}
return { create_device_tensor(actual_memory_shape, element_type, need_lockable_mem), TensorOwner::PLUGIN };
@ -714,7 +720,10 @@ std::vector<cldnn::event::ptr> SyncInferRequest::prepare_input(const std::string
auto device_tensor = std::dynamic_pointer_cast<RemoteTensorImpl>(device_tensor_wrapper.ptr);
if (is_dynamic) {
if (device_tensor->get_original_memory()->size() < user_tensor->get_byte_size()) {
auto actual_shape = predict_shape(internal_name, user_tensor->get_shape(), device_tensor_et, *m_shape_predictor);
auto actual_shape = predict_shape(internal_name, cldnn::layout(user_tensor->get_shape(),
element_type,
cldnn::format::get_default_format(user_tensor->get_shape().size())),
*m_shape_predictor);
GPU_DEBUG_TRACE_DETAIL << " actual memory shape: " << actual_shape.to_string() << std::endl;
auto new_tensor = create_device_tensor(actual_shape, device_tensor_et, false);
new_tensor->set_shape(user_tensor->get_shape());

2
src/plugins/intel_gpu/src/plugin/variable_state.cpp
View File

@ -77,7 +77,7 @@ void VariableState::update_device_buffer() {
const auto alloc_type = m_context->get_engine().use_unified_shared_memory() ? cldnn::allocation_type::usm_device : cldnn::allocation_type::cl_mem;
const auto current_buf_size = m_layout.get_buffer_size().sizes();
ov::Shape current_shape(current_buf_size.begin(), current_buf_size.end());
const auto alloc_shape = predict_shape(m_name, current_shape, m_layout.data_type, *m_shape_predictor);
const auto alloc_shape = predict_shape(m_name, cldnn::layout(current_shape, m_layout.data_type, m_layout.format), *m_shape_predictor);
const auto alloc_layout = cldnn::layout(alloc_shape, m_layout.data_type, m_layout.format);
m_memory = m_context->get_engine().allocate_memory(alloc_layout, alloc_type, false);
actual_size = std::max(actual_size, alloc_layout.bytes_count());

8
src/plugins/intel_gpu/src/runtime/shape_predictor.cpp
View File

@ -57,13 +57,15 @@ bool ShapePredictor::can_preallocate(size_t desired_buffer_size) {
}
std::pair<bool, ov::Shape> ShapePredictor::predict_preallocation_shape(const std::string& id,
const ov::Shape& current_shape,
size_t dt_bitwidth,
const cldnn::layout& layout,
bool can_reuse_buffer,
int32_t custom_next_iters_prealloc_count) {
size_t next_iters_prealloc_count = custom_next_iters_prealloc_count > 0
? static_cast<size_t>(custom_next_iters_prealloc_count)
: _next_iters_preallocation_count;
auto current_shape = layout.get_shape();
auto dt_bitwidth = ov::element::Type(layout.data_type).bitwidth();
add_shape(id, current_shape);
// Save shape information and exit without pre-allocation suggestion if current
@ -124,6 +126,8 @@ std::pair<bool, ov::Shape> ShapePredictor::predict_preallocation_shape(const std
auto new_shape = current_shape + preallocation_shape;
return {true, new_shape};
} else if (_buffers_preallocation_ratio > 1.0f) {
if (format::is_blocked(layout.format))
return {false, {}};
// Apply percentage buffer preallocation
auto current_shape_size = ov::shape_size(current_shape);
ov::Shape new_shape_size(current_shape.size(), 1);

57
src/plugins/intel_gpu/tests/unit/module_tests/shape_predictor_test.cpp
View File

@ -25,10 +25,12 @@ TEST_P(shape_predictor_tests, prediction) {
ShapePredictor sp(&engine, p.buffers_preallocation_ratio);
std::pair<bool, ov::Shape> result;
const auto dt_bitwidth = ov::element::f32.bitwidth();
for (auto& shape : in_shapes)
result = sp.predict_preallocation_shape("dummy_name", shape, dt_bitwidth, p.can_reuse_buffer);
result = sp.predict_preallocation_shape("dummy_name", cldnn::layout(shape,
ov::element::f32,
cldnn::format::get_default_format(shape.size())),
p.can_reuse_buffer);
ASSERT_TRUE(result.first == !expected_predicted_shape.empty());
ASSERT_EQ(result.second, expected_predicted_shape);
@ -64,3 +66,54 @@ INSTANTIATE_TEST_SUITE_P(smoke, shape_predictor_tests,
{{{1,1}, {1,128}, {1,256}}, {}, 1.1f, true},
{{{1,3,128}, {1,3,112}, {1,3,418}, {1,3,512}}, {}, 1.1f, true},
}));
class shape_predictor_tests_b_fs_yx_fsv16 : public testing::TestWithParam<shape_predictor_test_params> {};
TEST_P(shape_predictor_tests_b_fs_yx_fsv16, prediction) {
auto p = GetParam();
auto& in_shapes = p.in_shapes;
auto& expected_predicted_shape = p.expected_predicted_shape;
auto& engine = get_test_engine();
ShapePredictor sp(&engine, p.buffers_preallocation_ratio);
std::pair<bool, ov::Shape> result;
for (auto& shape : in_shapes)
result = sp.predict_preallocation_shape("dummy_name", cldnn::layout(shape,
ov::element::f32,
cldnn::format::b_fs_yx_fsv16),
p.can_reuse_buffer);
ASSERT_TRUE(result.first == !expected_predicted_shape.empty());
ASSERT_EQ(result.second, expected_predicted_shape);
}
INSTANTIATE_TEST_SUITE_P(smoke, shape_predictor_tests_b_fs_yx_fsv16,
testing::ValuesIn(std::vector<shape_predictor_test_params>{
// Preallocation for next N iterations tests
{{{1,1}, {1,1}, {1,1}}, {}, 1.0f, false},
{{{1,1}, {1,21}, {1,31}}, {}, 1.0f, false},
{{{1,3}, {1,2}, {1,1}}, {}, 1.0f, false},
{{{1,1}, {1,2}, {1,3}}, {1,13}, 1.0f, false},
{{{1,1}, {1,2}, {1,3}}, {1,13}, 1.1f, false},
{{{1,1,1}, {1,2,2}, {1,3,3}}, {1,13,13}, 1.0f, false},
{{{1,1,1}, {1,2,2}, {1,3,3}}, {1,13,13}, 1.1f, false},
{{{1,1,1}, {1,3,2}, {1,7,3}}, {}, 1.0f, false},
{{{1,1,1}, {1,1,3}, {1,1,5}}, {1,1,25}, 1.0f, false},
{{{1,1,1}, {1,1,3}, {1,1,5}}, {1,1,25}, 1.1f, false},
{{{1,1,1}, {1,1,3}, {1,1,5}}, {1,1,25}, 1.0f, false},
{{{1,1,1}, {1,1,3}, {1,1,5}}, {1,1,25}, 1.1f, false},
{{{1,1}, {1,1}, {1,1}, {1,1}, {1,1}, {1,1}}, {}, 1.0f, false},
{{{1,10}, {1,1}, {1,2}, {1,3}}, {1,13}, 1.0f, false},
{{{1,10}, {1,1}, {1,2}, {1,3}}, {1,13}, 1.1f, false},
{{{1,3,480,720}, {3,3,480,720}, {5,3,480,720}}, {}, 1.0f, false},
{{{1,1}, {1,1}, {1,1}}, {}, 1.0f, true},
{{{1,1}, {1,2}, {1,3}}, {}, 1.0f, true},
// Percentage preallocation tests
{{{1,1}, {1,1}, {1,1}}, {}, 1.1f, false},
{{{1,1}, {1,128}, {1,256}}, {}, 1.1f, false},
{{{1,3,128}, {1,3,112}, {1,3,418}, {1,3,512}}, {}, 1.1f, false},
{{{1,1}, {1,1}, {1,1}}, {}, 1.1f, true},
{{{1,1}, {1,128}, {1,256}}, {}, 1.1f, true},
{{{1,3,128}, {1,3,112}, {1,3,418}, {1,3,512}}, {}, 1.1f, true},
}));