diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/softmax_gpu_bf.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/softmax_gpu_bf.cl index b70fe7a5173..d01f23249be 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/softmax_gpu_bf.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/softmax_gpu_bf.cl @@ -77,6 +77,14 @@ KERNEL (softmax_gpu_continuous_bfyx)( const uint actual_leftovers = leftovers - aligned_offset; const uint leftover_idx = data_set_offset + aligned_offset + workers_per_data_set * items_num + in_data_set_idx; +#if IS_DYNAMIC + // use output buffer as intermediate variable instead of my_chunk when (item_num+2) is bigger than STACK_SIZE + // this happens when data_set_size > 16384 (engineInfo.maxWorkGroupSize=512) or data_set_size > 32768 (engineInfo.maxWorkGroupSize=1024) + const bool use_output_buffer = (items_num + 2) > STACK_SIZE; +#else + const bool use_output_buffer = false; +#endif + INPUT0_TYPE my_chunk[STACK_SIZE]; INPUT0_TYPE my_sum = UNIT_VAL_ZERO; @@ -84,6 +92,18 @@ KERNEL (softmax_gpu_continuous_bfyx)( // Read inputs and Get maximum value from data set uint input_idx=0; + + OUTPUT_TYPE* tmp_out_ptr; + + // Case for my_chunk[] when (items_num + 2) <= STACK_SIZE + if (use_output_buffer) + tmp_out_ptr = output; + else + tmp_out_ptr = &my_chunk[0]; + + uint data_offset = (use_output_buffer) ? aligned_data_offset + get_sub_group_local_id() : 0; + uint idx_stride = (use_output_buffer) ? get_sub_group_size() : 1; + #if IS_DYNAMIC if (workers_per_data_set > SUB_GROUP_SIZE) { @@ -93,14 +113,14 @@ KERNEL (softmax_gpu_continuous_bfyx)( BLOCK_TYPE_OPT vec_tmp = BLOCK_READ_OPT(input, aligned_data_offset + input_idx * get_sub_group_size()); unroll_for (int j = 0; j < OPT_BLOCK_SIZE; j++) { - my_chunk[input_idx+j] = vec_tmp[j]; + tmp_out_ptr[data_offset + idx_stride * (input_idx + j)] = vec_tmp[j]; } } for (; input_idx < items_num; input_idx++) { BLOCK_TYPE vec_tmp = BLOCK_READ(input, aligned_data_offset + input_idx * get_sub_group_size()); - my_chunk[input_idx] = vec_tmp; + tmp_out_ptr[data_offset + idx_stride * input_idx] = vec_tmp; } } #else @@ -110,12 +130,12 @@ KERNEL (softmax_gpu_continuous_bfyx)( { BLOCK_TYPE vec_tmp = BLOCK_READ(input, aligned_data_offset + input_idx * get_sub_group_size()); #if SUBGROUP_BLOCK_SIZE == 1 - my_chunk[input_idx] = vec_tmp; + tmp_out_ptr[input_idx] = vec_tmp; #else unroll_for (int j = 0; j < SUBGROUP_BLOCK_SIZE; j++) { INPUT0_TYPE tmp = vec_tmp[j]; - my_chunk[input_idx+j] = tmp; + tmp_out_ptr[input_idx+j] = tmp; } #endif } @@ -124,27 +144,36 @@ KERNEL (softmax_gpu_continuous_bfyx)( for (; input_idx < items_num; input_idx++) { - my_chunk[input_idx] = input[aligned_data_offset + get_sub_group_local_id() + input_idx * get_sub_group_size()]; + tmp_out_ptr[data_offset + idx_stride * input_idx] = input[aligned_data_offset + get_sub_group_local_id() + input_idx * get_sub_group_size()]; } if (in_data_set_idx < aligned_offset) { INPUT0_TYPE tmp = input[data_set_offset + in_data_set_idx]; - my_chunk[input_idx++] = tmp; + tmp_out_ptr[use_output_buffer ? data_set_offset + in_data_set_idx : input_idx++] = tmp; } if (in_data_set_idx < actual_leftovers) { INPUT0_TYPE tmp = input[leftover_idx]; - my_chunk[input_idx++] = tmp; + tmp_out_ptr[use_output_buffer ? leftover_idx : input_idx++] = tmp; } INPUT0_TYPE my_maximum = -UNIT_VAL_MAX; { const uint num_iters = input_idx; + for (uint j=0; j(model_type, shape)); - const auto softMax = std::make_shared(params.at(0), axis); + const auto softMax = std::make_shared(params.at(0), axis); auto makeFunction = [](ov::ParameterVector ¶ms, const std::shared_ptr &lastNode) { ov::ResultVector results; @@ -74,7 +74,7 @@ TEST_P(SoftMaxLayerGPUTest, Inference) { } const std::vector netPrecisions = { - ov::element::f32, ov::element::f16 + ov::element::f32, ov::element::f16 }; const std::vector axis2D = {0, 1}; @@ -90,6 +90,15 @@ const std::vector inputShapes2D = { } }; +const std::vector axis3D = {-1}; + +const std::vector inputShapes3D = { + { + {-1, -1, -1}, + {{16, 64, 64}, {4, 6, 27200}} + }, +}; + const std::vector axis4D = {1, 2, 3}; const std::vector inputShapes4D = { @@ -127,6 +136,13 @@ INSTANTIATE_TEST_SUITE_P(softMaxGPUDynamicTest2D, testing::ValuesIn(axis2D)), SoftMaxLayerGPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(softMaxGPUDynamicTest3D, + SoftMaxLayerGPUTest, + ::testing::Combine(testing::ValuesIn(netPrecisions), + testing::ValuesIn(inputShapes3D), + testing::ValuesIn(axis3D)), + SoftMaxLayerGPUTest::getTestCaseName); + INSTANTIATE_TEST_SUITE_P(softMaxGPUDynamicTest4D, SoftMaxLayerGPUTest, ::testing::Combine(testing::ValuesIn(netPrecisions),