From beb37eb4e14006c96e26fe85b5c5763863660486 Mon Sep 17 00:00:00 2001 From: Sergey Shlyapnikov Date: Fri, 7 Jun 2024 17:04:12 +0400 Subject: [PATCH] [GPU] Move scaling from QK dot product to Q (#24896) ### Details: - Move scaling from QK dot product to Q input for better SDPA accuracy with high-value ranges ### Tickets: - [CVS-143256](https://jira.devtools.intel.com/browse/CVS-143256) --- .../intel_gpu/src/kernel_selector/cl_kernels/sdpa_opt.cl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_opt.cl index 635aa4d796d..b1aaded5ad7 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_opt.cl @@ -202,8 +202,7 @@ KERNEL(sdpa_opt)( #define QUERY_BLOCK_SIZE 1 INPUT0_TYPE val = BLOCK_READN(INPUT0_TYPE, QUERY_BLOCK_SIZE, query_input, query_offset); - - query_local[query_local_offset] = val; + query_local[query_local_offset] = val * scale_val; query_local_offset += QUERY_STEP_LOCAL; query_offset += query_pitch; } @@ -338,7 +337,6 @@ KERNEL(sdpa_opt)( for (uint seq_len = sgid * SUBGROUP_SIZE + sglid; seq_len < partition_seq_len; seq_len += (HEAD_SIZE)) { // Read value from SLM and apply scale qk_val[seq_idx] = qk_local[seq_idx * SEQ_LEN_PARTITION_SIZE + seq_len]; - qk_val[seq_idx] *= scale_val; // Apply attention mask #if IS_CAUSAL