diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/softmax_gpu_bf.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/softmax_gpu_bf.cl index c2ff40c796d..b70fe7a5173 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/softmax_gpu_bf.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/softmax_gpu_bf.cl @@ -7,6 +7,22 @@ #include "include/batch_headers/sub_group_block_read.cl" #include "include/batch_headers/sub_group_block_write.cl" +#if IS_DYNAMIC + +#define CALC_POWER(n) ({uint pos = 0; uint i = n; do { i >>= 1; ++pos; } while (i); --pos;}) + +#define BLOCK_READ(ptr, offset) DT_INPUT_BLOCK_READ(ptr, offset) +#define BLOCK_WRITE(ptr, offset, val) DT_OUTPUT_BLOCK_WRITE(ptr, offset, val) +#define BLOCK_TYPE INPUT0_TYPE + +#define OPT_BLOCK_SIZE 8 + +#define BLOCK_READ_OPT(ptr, offset) CAT(DT_INPUT_BLOCK_READ, OPT_BLOCK_SIZE)(ptr, offset) +#define BLOCK_WRITE_OPT(ptr, offset, val) CAT(DT_OUTPUT_BLOCK_WRITE, OPT_BLOCK_SIZE)(ptr, offset, val) +#define BLOCK_TYPE_OPT MAKE_VECTOR_TYPE(INPUT0_TYPE, OPT_BLOCK_SIZE) + +#else + #if SUBGROUP_BLOCK_SIZE == 1 #define BLOCK_READ(ptr, offset) DT_INPUT_BLOCK_READ(ptr, offset) #define BLOCK_WRITE(ptr, offset, val) DT_OUTPUT_BLOCK_WRITE(ptr, offset, val) @@ -17,8 +33,6 @@ #define BLOCK_TYPE MAKE_VECTOR_TYPE(INPUT0_TYPE, SUBGROUP_BLOCK_SIZE) #endif -#if IS_DYNAMIC -#define CALC_POWER(n) ({uint pos = 0; uint i = n; do { i >>= 1; ++pos; } while (i); --pos;}) #endif REQD_SUB_GROUP_SIZE(SUB_GROUP_SIZE) @@ -64,53 +78,76 @@ KERNEL (softmax_gpu_continuous_bfyx)( const uint leftover_idx = data_set_offset + aligned_offset + workers_per_data_set * items_num + in_data_set_idx; INPUT0_TYPE my_chunk[STACK_SIZE]; - INPUT0_TYPE my_maximum = -UNIT_VAL_MAX; INPUT0_TYPE my_sum = UNIT_VAL_ZERO; __local INPUT0_TYPE lg_storage[SLM_SIZE]; // Read inputs and Get maximum value from data set uint input_idx=0; +#if IS_DYNAMIC + if (workers_per_data_set > SUB_GROUP_SIZE) + { + const uint num_iters = items_num - (items_num % OPT_BLOCK_SIZE); + for (; input_idx < num_iters; input_idx += OPT_BLOCK_SIZE) + { + BLOCK_TYPE_OPT vec_tmp = BLOCK_READ_OPT(input, aligned_data_offset + input_idx * get_sub_group_size()); + unroll_for (int j = 0; j < OPT_BLOCK_SIZE; j++) + { + my_chunk[input_idx+j] = vec_tmp[j]; + } + } + + for (; input_idx < items_num; input_idx++) + { + BLOCK_TYPE vec_tmp = BLOCK_READ(input, aligned_data_offset + input_idx * get_sub_group_size()); + my_chunk[input_idx] = vec_tmp; + } + } +#else if (workers_per_data_set > SUB_GROUP_SIZE) { for (; input_idx SUB_GROUP_SIZE) + { + const uint num_iters = items_num - (items_num % OPT_BLOCK_SIZE); + for (; output_idx < num_iters; output_idx += OPT_BLOCK_SIZE) + { + BLOCK_TYPE_OPT vec_tmp; + unroll_for (int j = 0; j < OPT_BLOCK_SIZE; j++) + { + ACTIVATION_TYPE dequantized = my_chunk[output_idx + j] / my_sum; + FUSED_OPS_MAIN; + vec_tmp[j] = FUSED_OPS_RESULT_MAIN; + } + BLOCK_WRITE_OPT(output, aligned_data_offset + output_idx * get_sub_group_size(), vec_tmp); + } + + for (; output_idx SUB_GROUP_SIZE) { for (; output_idx < items_num - (items_num % SUBGROUP_BLOCK_SIZE); output_idx+=SUBGROUP_BLOCK_SIZE) @@ -180,6 +242,7 @@ KERNEL (softmax_gpu_continuous_bfyx)( BLOCK_WRITE(output, aligned_data_offset + output_idx * get_sub_group_size(), vec_tmp); } } +#endif for (; output_idx < items_num; output_idx++) { ACTIVATION_TYPE dequantized = my_chunk[output_idx] / my_sum; @@ -200,6 +263,26 @@ KERNEL (softmax_gpu_continuous_bfyx)( FUSED_OPS_LEFTOVERS; output[leftover_idx] = FUSED_OPS_RESULT_LEFTOVERS; } +#else +#if IS_DYNAMIC + if (workers_per_data_set > SUB_GROUP_SIZE) + { + const uint num_iters = items_num - (items_num % OPT_BLOCK_SIZE); + for (; output_idx < num_iters; output_idx += OPT_BLOCK_SIZE) + { + BLOCK_TYPE_OPT vec_tmp; + unroll_for (int j = 0; j < OPT_BLOCK_SIZE; j++) + vec_tmp[j] = ACTIVATION(my_chunk[output_idx + j] / my_sum, ACTIVATION_PARAMS); + BLOCK_WRITE_OPT(output, aligned_data_offset + output_idx * get_sub_group_size(), vec_tmp); + } + + for (; output_idx < items_num; output_idx++) + { + BLOCK_TYPE vec_tmp; + vec_tmp = ACTIVATION(my_chunk[output_idx] / my_sum, ACTIVATION_PARAMS); + BLOCK_WRITE(output, aligned_data_offset + output_idx * get_sub_group_size(), vec_tmp); + } + } #else if (workers_per_data_set > SUB_GROUP_SIZE) { @@ -215,7 +298,7 @@ KERNEL (softmax_gpu_continuous_bfyx)( BLOCK_WRITE(output, aligned_data_offset + output_idx * get_sub_group_size(), vec_tmp); } } - +#endif for (; output_idx < items_num; output_idx++) { output[aligned_data_offset + get_sub_group_local_id() + output_idx * get_sub_group_size()] = ACTIVATION(my_chunk[output_idx] / my_sum, ACTIVATION_PARAMS);