natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[GPU] Use 8 subgroup block read /write functions on dynamic model (#23927) 

### Details:
- *Use 8 block size for subgroup read / write on dynamic model at
runtime*

### Tickets:
 - *133443*
This commit is contained in:
Paul Youngsoo Ahn 2024-04-11 11:58:15 +09:00 committed by GitHub
parent 7a31382f96
commit 2e518e8384
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 95 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
src/plugins/intel_gpu/src/kernel_selector/cl_kernels/softmax_gpu_bf.cl
View File

@ -7,6 +7,22 @@
#include "include/batch_headers/sub_group_block_read.cl"
#include "include/batch_headers/sub_group_block_write.cl"
#if IS_DYNAMIC
#define CALC_POWER(n) ({uint pos = 0; uint i = n; do { i >>= 1; ++pos; } while (i); --pos;})
#define BLOCK_READ(ptr, offset) DT_INPUT_BLOCK_READ(ptr, offset)
#define BLOCK_WRITE(ptr, offset, val) DT_OUTPUT_BLOCK_WRITE(ptr, offset, val)
#define BLOCK_TYPE INPUT0_TYPE
#define OPT_BLOCK_SIZE 8
#define BLOCK_READ_OPT(ptr, offset) CAT(DT_INPUT_BLOCK_READ, OPT_BLOCK_SIZE)(ptr, offset)
#define BLOCK_WRITE_OPT(ptr, offset, val) CAT(DT_OUTPUT_BLOCK_WRITE, OPT_BLOCK_SIZE)(ptr, offset, val)
#define BLOCK_TYPE_OPT MAKE_VECTOR_TYPE(INPUT0_TYPE, OPT_BLOCK_SIZE)
#else
#if SUBGROUP_BLOCK_SIZE == 1
#define BLOCK_READ(ptr, offset) DT_INPUT_BLOCK_READ(ptr, offset)
#define BLOCK_WRITE(ptr, offset, val) DT_OUTPUT_BLOCK_WRITE(ptr, offset, val)
@ -17,8 +33,6 @@
#define BLOCK_TYPE MAKE_VECTOR_TYPE(INPUT0_TYPE, SUBGROUP_BLOCK_SIZE)
#endif
#if IS_DYNAMIC
#define CALC_POWER(n) ({uint pos = 0; uint i = n; do { i >>= 1; ++pos; } while (i); --pos;})
#endif
REQD_SUB_GROUP_SIZE(SUB_GROUP_SIZE)
@ -64,53 +78,76 @@ KERNEL (softmax_gpu_continuous_bfyx)(
const uint leftover_idx = data_set_offset + aligned_offset + workers_per_data_set * items_num + in_data_set_idx;
INPUT0_TYPE my_chunk[STACK_SIZE];
INPUT0_TYPE my_maximum = -UNIT_VAL_MAX;
INPUT0_TYPE my_sum = UNIT_VAL_ZERO;
__local INPUT0_TYPE lg_storage[SLM_SIZE];
// Read inputs and Get maximum value from data set
uint input_idx=0;
#if IS_DYNAMIC
if (workers_per_data_set > SUB_GROUP_SIZE)
{
const uint num_iters = items_num - (items_num % OPT_BLOCK_SIZE);
for (; input_idx < num_iters; input_idx += OPT_BLOCK_SIZE)
{
BLOCK_TYPE_OPT vec_tmp = BLOCK_READ_OPT(input, aligned_data_offset + input_idx * get_sub_group_size());
unroll_for (int j = 0; j < OPT_BLOCK_SIZE; j++)
{
my_chunk[input_idx+j] = vec_tmp[j];
}
}
for (; input_idx < items_num; input_idx++)
{
BLOCK_TYPE vec_tmp = BLOCK_READ(input, aligned_data_offset + input_idx * get_sub_group_size());
my_chunk[input_idx] = vec_tmp;
}
}
#else
if (workers_per_data_set > SUB_GROUP_SIZE)
{
for (; input_idx<items_num - (items_num % SUBGROUP_BLOCK_SIZE); input_idx+=SUBGROUP_BLOCK_SIZE)
{
BLOCK_TYPE vec_tmp = BLOCK_READ(input, aligned_data_offset + input_idx * get_sub_group_size());
#if SUBGROUP_BLOCK_SIZE == 1
my_maximum = max(my_maximum, vec_tmp);
my_chunk[input_idx] = vec_tmp;
#else
for (int j = 0; j < SUBGROUP_BLOCK_SIZE; j++)
unroll_for (int j = 0; j < SUBGROUP_BLOCK_SIZE; j++)
{
INPUT0_TYPE tmp = vec_tmp[j];
my_maximum = max(my_maximum, tmp);
my_chunk[input_idx+j] = tmp;
}
#endif
}
}
#endif
for (; input_idx < items_num; input_idx++)
{
INPUT0_TYPE tmp = input[aligned_data_offset + get_sub_group_local_id() + input_idx * get_sub_group_size()];
my_maximum = max(my_maximum, tmp);
my_chunk[input_idx] = tmp;
my_chunk[input_idx] = input[aligned_data_offset + get_sub_group_local_id() + input_idx * get_sub_group_size()];
}
if (in_data_set_idx < aligned_offset)
{
INPUT0_TYPE tmp = input[data_set_offset + in_data_set_idx];
my_maximum = max(my_maximum, tmp);
my_chunk[input_idx++] = tmp;
}
if (in_data_set_idx < actual_leftovers)
{
INPUT0_TYPE tmp = input[leftover_idx];
my_maximum = max(my_maximum, tmp);
my_chunk[input_idx++] = tmp;
}
INPUT0_TYPE my_maximum = -UNIT_VAL_MAX;
{
const uint num_iters = input_idx;
for (uint j=0; j<num_iters; ++j)
{
my_maximum = max(my_maximum, my_chunk[j]);
}
}
my_maximum = sub_group_reduce_max(my_maximum);
if (get_sub_group_local_id() == 0)
@ -160,6 +197,31 @@ KERNEL (softmax_gpu_continuous_bfyx)(
// Write outputs
uint output_idx = 0;
#if HAS_FUSED_OPS
#if IS_DYNAMIC
if (workers_per_data_set > SUB_GROUP_SIZE)
{
const uint num_iters = items_num - (items_num % OPT_BLOCK_SIZE);
for (; output_idx < num_iters; output_idx += OPT_BLOCK_SIZE)
{
BLOCK_TYPE_OPT vec_tmp;
unroll_for (int j = 0; j < OPT_BLOCK_SIZE; j++)
{
ACTIVATION_TYPE dequantized = my_chunk[output_idx + j] / my_sum;
FUSED_OPS_MAIN;
vec_tmp[j] = FUSED_OPS_RESULT_MAIN;
}
BLOCK_WRITE_OPT(output, aligned_data_offset + output_idx * get_sub_group_size(), vec_tmp);
}
for (; output_idx<items_num; output_idx++)
{
ACTIVATION_TYPE dequantized = my_chunk[output_idx] / my_sum;
FUSED_OPS_MAIN;
vec_tmp = FUSED_OPS_RESULT_MAIN;
BLOCK_WRITE(output, aligned_data_offset + output_idx * get_sub_group_size(), vec_tmp);
}
}
#else
if (workers_per_data_set > SUB_GROUP_SIZE)
{
for (; output_idx < items_num - (items_num % SUBGROUP_BLOCK_SIZE); output_idx+=SUBGROUP_BLOCK_SIZE)
@ -180,6 +242,7 @@ KERNEL (softmax_gpu_continuous_bfyx)(
BLOCK_WRITE(output, aligned_data_offset + output_idx * get_sub_group_size(), vec_tmp);
}
}
#endif
for (; output_idx < items_num; output_idx++)
{
ACTIVATION_TYPE dequantized = my_chunk[output_idx] / my_sum;
@ -200,6 +263,26 @@ KERNEL (softmax_gpu_continuous_bfyx)(
FUSED_OPS_LEFTOVERS;
output[leftover_idx] = FUSED_OPS_RESULT_LEFTOVERS;
}
#else
#if IS_DYNAMIC
if (workers_per_data_set > SUB_GROUP_SIZE)
{
const uint num_iters = items_num - (items_num % OPT_BLOCK_SIZE);
for (; output_idx < num_iters; output_idx += OPT_BLOCK_SIZE)
{
BLOCK_TYPE_OPT vec_tmp;
unroll_for (int j = 0; j < OPT_BLOCK_SIZE; j++)
vec_tmp[j] = ACTIVATION(my_chunk[output_idx + j] / my_sum, ACTIVATION_PARAMS);
BLOCK_WRITE_OPT(output, aligned_data_offset + output_idx * get_sub_group_size(), vec_tmp);
}
for (; output_idx < items_num; output_idx++)
{
BLOCK_TYPE vec_tmp;
vec_tmp = ACTIVATION(my_chunk[output_idx] / my_sum, ACTIVATION_PARAMS);
BLOCK_WRITE(output, aligned_data_offset + output_idx * get_sub_group_size(), vec_tmp);
}
}
#else
if (workers_per_data_set > SUB_GROUP_SIZE)
{
@ -215,7 +298,7 @@ KERNEL (softmax_gpu_continuous_bfyx)(
BLOCK_WRITE(output, aligned_data_offset + output_idx * get_sub_group_size(), vec_tmp);
}
}
#endif
for (; output_idx < items_num; output_idx++)
{
output[aligned_data_offset + get_sub_group_local_id() + output_idx * get_sub_group_size()] = ACTIVATION(my_chunk[output_idx] / my_sum, ACTIVATION_PARAMS);