From b2b5d3efb0b7f1dd3a0a827c2be4589b36927e10 Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Fri, 12 Apr 2024 07:41:42 +0900 Subject: [PATCH] [GPU] Disable predict_preallocation_shape for blocked format (#23796) ### Details: - *Disable predict_preallocation_shape for blocked format* - *Fix to use layout instead of shape and data_type* ### Tickets: - *132334* --- .../include/intel_gpu/plugin/common_utils.hpp | 9 +-- .../intel_gpu/runtime/shape_predictor.hpp | 6 +- .../intel_gpu/src/graph/primitive_inst.cpp | 4 +- .../src/plugin/sync_infer_request.cpp | 15 ++++- .../intel_gpu/src/plugin/variable_state.cpp | 2 +- .../intel_gpu/src/runtime/shape_predictor.cpp | 8 ++- .../module_tests/shape_predictor_test.cpp | 57 ++++++++++++++++++- 7 files changed, 83 insertions(+), 18 deletions(-) diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/common_utils.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/common_utils.hpp index 80c19bf2f37..688ab6a72c0 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/common_utils.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/common_utils.hpp @@ -95,16 +95,17 @@ inline ov::Shape get_tensor_shape(const ov::PartialShape& pshape) { return res; } -inline ov::Shape predict_shape(const std::string& name, const ov::Shape current_shape, ov::element::Type element_type, cldnn::ShapePredictor& shape_predictor) { - auto prealloc_info = shape_predictor.predict_preallocation_shape(name, current_shape, element_type.bitwidth(), false); +inline ov::Shape predict_shape(const std::string& name, const cldnn::layout layout, cldnn::ShapePredictor& shape_predictor) { + auto prealloc_info = shape_predictor.predict_preallocation_shape(name, layout, false); const auto& preallocation_shape = prealloc_info.second; auto can_preallocate_buffer = prealloc_info.first && - shape_predictor.can_preallocate(cldnn::ceil_div(ov::shape_size(preallocation_shape) * element_type.bitwidth(), 8)); + shape_predictor.can_preallocate(cldnn::ceil_div(ov::shape_size(preallocation_shape) * + ov::element::Type(layout.data_type).bitwidth(), 8)); if (can_preallocate_buffer) { return preallocation_shape; } - return current_shape; + return layout.get_shape(); } /// WA: Force exit. Any opencl api call can be hang after CL_OUT_OF_RESOURCES. diff --git a/src/plugins/intel_gpu/include/intel_gpu/runtime/shape_predictor.hpp b/src/plugins/intel_gpu/include/intel_gpu/runtime/shape_predictor.hpp index 01ae5e1a5b6..320e0b40f96 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/runtime/shape_predictor.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/runtime/shape_predictor.hpp @@ -44,14 +44,12 @@ public: /// ov::intel_gpu::buffers_preallocation_ratio property, it increases buffer size by /// `_buffers_preallocation_ratio` value unconditionally. /// \param id Primitive id. -/// \param current_shape Primitive's shape on current iteration. -/// \param dt_size Primitive's data_type size. +/// \param layout Primitive's layout on current iteration. /// \param can_reuse_buffer Specifies if current memory buffer is enough to store data. /// \return The result of shape size prediction as std::pair, where the first element /// says if shape is successfully predicted and can be preallocated, and the second element is ov::Shape itself. std::pair predict_preallocation_shape(const std::string& id, - const ov::Shape& current_shape, - size_t dt_bitwidth, + const cldnn::layout& layout, bool can_reuse_buffer, int32_t next_iters_prealloc_count = -1); diff --git a/src/plugins/intel_gpu/src/graph/primitive_inst.cpp b/src/plugins/intel_gpu/src/graph/primitive_inst.cpp index 57256345afd..56231751990 100644 --- a/src/plugins/intel_gpu/src/graph/primitive_inst.cpp +++ b/src/plugins/intel_gpu/src/graph/primitive_inst.cpp @@ -496,7 +496,7 @@ event::ptr primitive_inst::realloc_if_needed() { _outputs[0] = variable.get_memory(); // To record shape predictor - auto prealloc_info = sp.predict_preallocation_shape(id(), _impl_params->output_layouts[0].get_shape(), dt_size, true); + auto prealloc_info = sp.predict_preallocation_shape(id(), _impl_params->output_layouts[0], true); return ev; } else if (_outputs[0] && variable.get_memory() && get_network().get_engine().is_the_same_buffer(*_outputs[0], *variable.get_memory())) { GPU_DEBUG_TRACE_DETAIL << id() << " : realloc_if_needed: Reset output mem" << std::endl; @@ -586,7 +586,7 @@ event::ptr primitive_inst::realloc_if_needed() { // If debug config is set, repsect the config most tmp_prealloc_count = -1; } - prealloc_info = sp.predict_preallocation_shape(id(), current_shape, dt_size, can_reuse_buffer, tmp_prealloc_count); + prealloc_info = sp.predict_preallocation_shape(id(), updated_layout, can_reuse_buffer, tmp_prealloc_count); if (prealloc_info.first && sp.can_preallocate(ov::shape_size(prealloc_info.second) * dt_size)) { auto new_layout = updated_layout; diff --git a/src/plugins/intel_gpu/src/plugin/sync_infer_request.cpp b/src/plugins/intel_gpu/src/plugin/sync_infer_request.cpp index dcdfe046558..3018a4f0b96 100644 --- a/src/plugins/intel_gpu/src/plugin/sync_infer_request.cpp +++ b/src/plugins/intel_gpu/src/plugin/sync_infer_request.cpp @@ -359,7 +359,10 @@ void SyncInferRequest::wait() { if (need_reallocate) { std::string internal_name = m_output_names_map.at(port_idx); - auto actual_memory_shape = predict_shape(internal_name, mem_shape, output_tensor->get_element_type(), *m_shape_predictor); + auto actual_memory_shape = predict_shape(internal_name, cldnn::layout(mem_shape, + output_tensor->get_element_type(), + cldnn::format::get_default_format(mem_shape.size())), + *m_shape_predictor); output_tensor->set_shape(actual_memory_shape); } } @@ -480,7 +483,10 @@ TensorWrapper SyncInferRequest::create_or_share_device_tensor(const TensorWrappe auto actual_memory_shape = tensor_shape; if (is_dynamic) { - actual_memory_shape = predict_shape(name, tensor_shape, element_type, *m_shape_predictor); + actual_memory_shape = predict_shape(name, cldnn::layout(tensor_shape, + element_type, + cldnn::format::get_default_format(tensor_shape.size())), + *m_shape_predictor); } return { create_device_tensor(actual_memory_shape, element_type, need_lockable_mem), TensorOwner::PLUGIN }; @@ -714,7 +720,10 @@ std::vector SyncInferRequest::prepare_input(const std::string auto device_tensor = std::dynamic_pointer_cast(device_tensor_wrapper.ptr); if (is_dynamic) { if (device_tensor->get_original_memory()->size() < user_tensor->get_byte_size()) { - auto actual_shape = predict_shape(internal_name, user_tensor->get_shape(), device_tensor_et, *m_shape_predictor); + auto actual_shape = predict_shape(internal_name, cldnn::layout(user_tensor->get_shape(), + element_type, + cldnn::format::get_default_format(user_tensor->get_shape().size())), + *m_shape_predictor); GPU_DEBUG_TRACE_DETAIL << " actual memory shape: " << actual_shape.to_string() << std::endl; auto new_tensor = create_device_tensor(actual_shape, device_tensor_et, false); new_tensor->set_shape(user_tensor->get_shape()); diff --git a/src/plugins/intel_gpu/src/plugin/variable_state.cpp b/src/plugins/intel_gpu/src/plugin/variable_state.cpp index 19c8c20016b..81871bbf8fa 100644 --- a/src/plugins/intel_gpu/src/plugin/variable_state.cpp +++ b/src/plugins/intel_gpu/src/plugin/variable_state.cpp @@ -77,7 +77,7 @@ void VariableState::update_device_buffer() { const auto alloc_type = m_context->get_engine().use_unified_shared_memory() ? cldnn::allocation_type::usm_device : cldnn::allocation_type::cl_mem; const auto current_buf_size = m_layout.get_buffer_size().sizes(); ov::Shape current_shape(current_buf_size.begin(), current_buf_size.end()); - const auto alloc_shape = predict_shape(m_name, current_shape, m_layout.data_type, *m_shape_predictor); + const auto alloc_shape = predict_shape(m_name, cldnn::layout(current_shape, m_layout.data_type, m_layout.format), *m_shape_predictor); const auto alloc_layout = cldnn::layout(alloc_shape, m_layout.data_type, m_layout.format); m_memory = m_context->get_engine().allocate_memory(alloc_layout, alloc_type, false); actual_size = std::max(actual_size, alloc_layout.bytes_count()); diff --git a/src/plugins/intel_gpu/src/runtime/shape_predictor.cpp b/src/plugins/intel_gpu/src/runtime/shape_predictor.cpp index 599878ca556..feb72d1879d 100644 --- a/src/plugins/intel_gpu/src/runtime/shape_predictor.cpp +++ b/src/plugins/intel_gpu/src/runtime/shape_predictor.cpp @@ -57,13 +57,15 @@ bool ShapePredictor::can_preallocate(size_t desired_buffer_size) { } std::pair ShapePredictor::predict_preallocation_shape(const std::string& id, - const ov::Shape& current_shape, - size_t dt_bitwidth, + const cldnn::layout& layout, bool can_reuse_buffer, int32_t custom_next_iters_prealloc_count) { size_t next_iters_prealloc_count = custom_next_iters_prealloc_count > 0 ? static_cast(custom_next_iters_prealloc_count) : _next_iters_preallocation_count; + auto current_shape = layout.get_shape(); + auto dt_bitwidth = ov::element::Type(layout.data_type).bitwidth(); + add_shape(id, current_shape); // Save shape information and exit without pre-allocation suggestion if current @@ -124,6 +126,8 @@ std::pair ShapePredictor::predict_preallocation_shape(const std auto new_shape = current_shape + preallocation_shape; return {true, new_shape}; } else if (_buffers_preallocation_ratio > 1.0f) { + if (format::is_blocked(layout.format)) + return {false, {}}; // Apply percentage buffer preallocation auto current_shape_size = ov::shape_size(current_shape); ov::Shape new_shape_size(current_shape.size(), 1); diff --git a/src/plugins/intel_gpu/tests/unit/module_tests/shape_predictor_test.cpp b/src/plugins/intel_gpu/tests/unit/module_tests/shape_predictor_test.cpp index 5e5caa91cd7..38a5d9196d4 100644 --- a/src/plugins/intel_gpu/tests/unit/module_tests/shape_predictor_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/module_tests/shape_predictor_test.cpp @@ -25,10 +25,12 @@ TEST_P(shape_predictor_tests, prediction) { ShapePredictor sp(&engine, p.buffers_preallocation_ratio); std::pair result; - const auto dt_bitwidth = ov::element::f32.bitwidth(); for (auto& shape : in_shapes) - result = sp.predict_preallocation_shape("dummy_name", shape, dt_bitwidth, p.can_reuse_buffer); + result = sp.predict_preallocation_shape("dummy_name", cldnn::layout(shape, + ov::element::f32, + cldnn::format::get_default_format(shape.size())), + p.can_reuse_buffer); ASSERT_TRUE(result.first == !expected_predicted_shape.empty()); ASSERT_EQ(result.second, expected_predicted_shape); @@ -64,3 +66,54 @@ INSTANTIATE_TEST_SUITE_P(smoke, shape_predictor_tests, {{{1,1}, {1,128}, {1,256}}, {}, 1.1f, true}, {{{1,3,128}, {1,3,112}, {1,3,418}, {1,3,512}}, {}, 1.1f, true}, })); + +class shape_predictor_tests_b_fs_yx_fsv16 : public testing::TestWithParam {}; +TEST_P(shape_predictor_tests_b_fs_yx_fsv16, prediction) { + auto p = GetParam(); + auto& in_shapes = p.in_shapes; + auto& expected_predicted_shape = p.expected_predicted_shape; + auto& engine = get_test_engine(); + + ShapePredictor sp(&engine, p.buffers_preallocation_ratio); + std::pair result; + + for (auto& shape : in_shapes) + result = sp.predict_preallocation_shape("dummy_name", cldnn::layout(shape, + ov::element::f32, + cldnn::format::b_fs_yx_fsv16), + p.can_reuse_buffer); + + ASSERT_TRUE(result.first == !expected_predicted_shape.empty()); + ASSERT_EQ(result.second, expected_predicted_shape); +} + +INSTANTIATE_TEST_SUITE_P(smoke, shape_predictor_tests_b_fs_yx_fsv16, + testing::ValuesIn(std::vector{ + // Preallocation for next N iterations tests + {{{1,1}, {1,1}, {1,1}}, {}, 1.0f, false}, + {{{1,1}, {1,21}, {1,31}}, {}, 1.0f, false}, + {{{1,3}, {1,2}, {1,1}}, {}, 1.0f, false}, + {{{1,1}, {1,2}, {1,3}}, {1,13}, 1.0f, false}, + {{{1,1}, {1,2}, {1,3}}, {1,13}, 1.1f, false}, + {{{1,1,1}, {1,2,2}, {1,3,3}}, {1,13,13}, 1.0f, false}, + {{{1,1,1}, {1,2,2}, {1,3,3}}, {1,13,13}, 1.1f, false}, + {{{1,1,1}, {1,3,2}, {1,7,3}}, {}, 1.0f, false}, + {{{1,1,1}, {1,1,3}, {1,1,5}}, {1,1,25}, 1.0f, false}, + {{{1,1,1}, {1,1,3}, {1,1,5}}, {1,1,25}, 1.1f, false}, + {{{1,1,1}, {1,1,3}, {1,1,5}}, {1,1,25}, 1.0f, false}, + {{{1,1,1}, {1,1,3}, {1,1,5}}, {1,1,25}, 1.1f, false}, + {{{1,1}, {1,1}, {1,1}, {1,1}, {1,1}, {1,1}}, {}, 1.0f, false}, + {{{1,10}, {1,1}, {1,2}, {1,3}}, {1,13}, 1.0f, false}, + {{{1,10}, {1,1}, {1,2}, {1,3}}, {1,13}, 1.1f, false}, + {{{1,3,480,720}, {3,3,480,720}, {5,3,480,720}}, {}, 1.0f, false}, + {{{1,1}, {1,1}, {1,1}}, {}, 1.0f, true}, + {{{1,1}, {1,2}, {1,3}}, {}, 1.0f, true}, + + // Percentage preallocation tests + {{{1,1}, {1,1}, {1,1}}, {}, 1.1f, false}, + {{{1,1}, {1,128}, {1,256}}, {}, 1.1f, false}, + {{{1,3,128}, {1,3,112}, {1,3,418}, {1,3,512}}, {}, 1.1f, false}, + {{{1,1}, {1,1}, {1,1}}, {}, 1.1f, true}, + {{{1,1}, {1,128}, {1,256}}, {}, 1.1f, true}, + {{{1,3,128}, {1,3,112}, {1,3,418}, {1,3,512}}, {}, 1.1f, true}, + }));