hyf152
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!26678 optimize matmul broadcast 

Merge pull request !26678 from wangyanling/r1.5
This commit is contained in:
i-robot 2021-11-24 02:51:05 +00:00 committed by Gitee
parent f87c6f5ff7 d78e557011
commit 8a6b1c086a
10 changed files with 65 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/infer/matmul_infer.c
View File

@ -15,6 +15,7 @@
*/
#include "nnacl/infer/matmul_infer.h"
#include <math.h>
#include "nnacl/infer/infer_register.h"
#define MIN_SHAPE_SIZE 2
@ -24,6 +25,9 @@ int CheckMatmulInputShape(int *a_shape, size_t a_shape_size, int *b_shape, size_
if (a_shape_size < MIN_SHAPE_SIZE || b_shape_size < MIN_SHAPE_SIZE) {
return NNACL_PARAM_INVALID;
}
if (b_shape_size < 1) {
return NNACL_ERR;
}
for (size_t i = 0; i < (a_shape_size - 2) && i < (b_shape_size - 2); ++i) {
int min_value = MSMIN(a_shape[i], b_shape[i]);
int max_value = MSMAX(a_shape[i], b_shape[i]);
@ -52,29 +56,6 @@ int CheckMatmulInputShape(int *a_shape, size_t a_shape_size, int *b_shape, size_
return NNACL_OK;
}
bool BroadcastInfer(int *a_shape, size_t a_shape_size, int *b_shape, size_t b_shape_size) {
for (size_t i = 0; i < (a_shape_size - 2) && i < (b_shape_size - 2); ++i) {
int min_value = MSMIN(a_shape[i], b_shape[i]);
int max_value = MSMAX(a_shape[i], b_shape[i]);
if (a_shape[i] != b_shape[i] && max_value % min_value == 0) {
return true;
}
}
return false;
}
void AlignsDims(const TensorC *const *inputs) {
TensorC *input0 = (TensorC *)inputs[0];
TensorC *input1 = (TensorC *)inputs[1];
TensorC *in_1 = input0->shape_size_ > input1->shape_size_ ? input0 : input1;
TensorC *in_2 = input0->shape_size_ > input1->shape_size_ ? input1 : input0;
size_t diff = in_1->shape_size_ - in_2->shape_size_;
for (size_t i = 0; i < diff; ++i) {
ShapeInsert(in_2->shape_, &in_2->shape_size_, 0, 1);
}
return;
}
int MatmulInferShape(const TensorC *const *inputs, size_t inputs_size, TensorC **outputs, size_t outputs_size,
OpParameter *parameter) {
int check_ret = CheckAugmentNullSizeInputTwo(inputs, inputs_size, outputs, outputs_size, parameter, 2, 3, 1);
@ -85,7 +66,12 @@ int MatmulInferShape(const TensorC *const *inputs, size_t inputs_size, TensorC *
TensorC *input0 = (TensorC *)inputs[0];
TensorC *input1 = (TensorC *)inputs[1];
TensorC *output = outputs[0];
AlignsDims(inputs);
int diff = abs((int)input0->shape_size_ - (int)input1->shape_size_);
TensorC *in = input0->shape_size_ > input1->shape_size_ ? input1 : input0;
for (int i = 0; i < diff; ++i) {
ShapeInsert(in->shape_, &in->shape_size_, 0, 1);
}
SetDataTypeFormat(output, input0);
MatMulParameter *param = (MatMulParameter *)parameter;
if (!InferFlag(inputs, inputs_size)) {
@ -132,9 +118,6 @@ int MatmulInferShape(const TensorC *const *inputs, size_t inputs_size, TensorC *
int c_shape[MAX_SHAPE_SIZE];
size_t c_shape_size = 0;
ShapeSet(c_shape, &c_shape_size, a_shape, a_shape_size);
if (c_shape_size < 1 || b_shape_size < 1) {
return NNACL_ERR;
}
c_shape[c_shape_size - 1] = b_shape[b_shape_size - 1];
if (del_start) {
int erase_ret = ShapeErase(c_shape, &c_shape_size, 0);
@ -145,11 +128,10 @@ int MatmulInferShape(const TensorC *const *inputs, size_t inputs_size, TensorC *
if (del_end) {
c_shape_size--;
}
if (BroadcastInfer(a_shape, a_shape_size, b_shape, b_shape_size)) {
for (size_t i = 0; i < (a_shape_size - 2) && i < (b_shape_size - 2); ++i) {
c_shape[i] = MSMAX(a_shape[i], b_shape[i]);
}
for (size_t i = 0; i < (a_shape_size - 2) && i < (b_shape_size - 2); ++i) {
c_shape[i] = MSMAX(a_shape[i], b_shape[i]);
}
SetShapeArray(output, c_shape, c_shape_size);
return NNACL_OK;
}

2
mindspore/lite/src/runtime/kernel/arm/fp16/fullconnection_fp16.cc
View File

@ -56,6 +56,8 @@ int FullconnectionFP16CPUKernel::Init() {
row_tile_ = C12NUM;
#endif
params_->batch = 1;
a_offset_.resize(params_->batch, 0);
b_offset_.resize(params_->batch, 0);
a_batch_ = 1;
b_batch_ = 1;
params_->a_transpose_ = false;

80
mindspore/lite/src/runtime/kernel/arm/fp16/matmul_base_fp16.cc
View File

@ -314,62 +314,6 @@ int MatmulBaseFP16CPUKernel::RunImpl(int task_id) {
return RET_OK;
}
int MatmulBaseFP16CPUKernel::BroadcastMatmulRun() {
auto c_ptr = reinterpret_cast<float16_t *>(out_tensors_[0]->data());
CHECK_NULL_RETURN(c_ptr);
for (int i = 0; i < params_->batch; ++i) {
if (vec_matmul_) {
batch_a_ptr_ = a_pack_ptr_ + a_offset_[i] * params_->deep_;
#ifdef ENABLE_ARM64
batch_b_ptr_ = b_pack_ptr_ + b_offset_[i] * params_->deep_ * params_->col_align_;
#else
batch_b_ptr_ = b_pack_ptr_ + b_offset_[i] * params_->deep_ * params_->col_;
#endif
batch_c_ptr_ = c_ptr + i * params_->row_ * params_->col_;
} else {
batch_a_ptr_ = a_pack_ptr_ + a_offset_[i] * params_->row_align_ * params_->deep_;
batch_b_ptr_ = b_pack_ptr_ + b_offset_[i] * params_->deep_ * params_->col_align_;
batch_c_ptr_ = c_ptr + i * params_->row_ * params_->col_;
}
auto ret = ParallelLaunch(this->ms_context_, MatmulBaseFP16Run, this, thread_count_);
if (ret != RET_OK) {
MS_LOG(ERROR) << "MatmulBaseFloatRun failed";
return RET_ERROR;
}
InitMatrixB(in_tensors_.at(1)->data(), in_tensors_.at(1)->data_type());
InitBias();
}
return RET_OK;
}
int MatmulBaseFP16CPUKernel::NormalMatmulRun() {
auto c_ptr = reinterpret_cast<float16_t *>(out_tensors_.at(0)->data());
CHECK_NULL_RETURN(c_ptr);
for (int i = 0; i < params_->batch; ++i) {
if (vec_matmul_) {
batch_a_ptr_ = a_pack_ptr_ + i * params_->deep_;
#ifdef ENABLE_ARM64
batch_b_ptr_ = b_pack_ptr_ + i * params_->deep_ * params_->col_align_;
#else
batch_b_ptr_ = b_pack_ptr_ + i * params_->deep_ * params_->col_;
#endif
batch_c_ptr_ = c_ptr + i * params_->row_ * params_->col_;
} else {
batch_a_ptr_ = a_pack_ptr_ + i * params_->row_align_ * params_->deep_;
batch_b_ptr_ = b_pack_ptr_ + i * params_->deep_ * params_->col_align_;
batch_c_ptr_ = c_ptr + i * params_->row_ * params_->col_;
}
auto ret = ParallelLaunch(this->ms_context_, MatmulBaseFP16Run, this, thread_count_);
if (ret != RET_OK) {
MS_LOG(ERROR) << "MatmulBaseFloatRun failed";
return ret;
}
}
return RET_OK;
}
int MatmulBaseFP16CPUKernel::Run() {
auto c_ptr = reinterpret_cast<float16_t *>(out_tensors_[0]->data());
CHECK_NULL_RETURN(c_ptr);
@ -388,16 +332,24 @@ int MatmulBaseFP16CPUKernel::Run() {
InitMatrixB(in_tensors_[1]->data(), in_tensors_[1]->data_type());
InitBias();
}
int ret = RET_OK;
if (!a_broadcast_ && !b_broadcast_) {
ret = NormalMatmulRun();
if (ret != RET_OK) {
MS_LOG(ERROR) << "NormalMatmulRun failed";
for (int i = 0; i < params_->batch; ++i) {
if (vec_matmul_) {
batch_a_ptr_ = a_pack_ptr_ + a_offset_[i] * params_->deep_;
#ifdef ENABLE_ARM64
batch_b_ptr_ = b_pack_ptr_ + b_offset_[i] * params_->deep_ * params_->col_align_;
#else
batch_b_ptr_ = b_pack_ptr_ + b_offset_[i] * params_->deep_ * params_->col_;
#endif
batch_c_ptr_ = c_ptr + i * params_->row_ * params_->col_;
} else {
batch_a_ptr_ = a_pack_ptr_ + a_offset_[i] * params_->row_align_ * params_->deep_;
batch_b_ptr_ = b_pack_ptr_ + b_offset_[i] * params_->deep_ * params_->col_align_;
batch_c_ptr_ = c_ptr + i * params_->row_ * params_->col_;
}
} else {
ret = BroadcastMatmulRun();
auto ret = ParallelLaunch(this->ms_context_, MatmulBaseFP16Run, this, thread_count_);
if (ret != RET_OK) {
MS_LOG(ERROR) << "BroadcastMatmulRun failed";
MS_LOG(ERROR) << "MatmulBaseFloatRun failed";
return ret;
}
}

4
mindspore/lite/src/runtime/kernel/arm/fp16/matmul_base_fp16.h
View File

@ -65,9 +65,7 @@ class MatmulBaseFP16CPUKernel : public InnerKernel {
bool b_broadcast_ = false;
int a_batch_ = 1;
int b_batch_ = 1;
int batch_sizes_[MAX_SHAPE_SIZE] = {0};
int a_batch_sizes_[MAX_SHAPE_SIZE] = {0};
int b_batch_sizes_[MAX_SHAPE_SIZE] = {0};
std::vector<int> a_offset_;
std::vector<int> b_offset_;

23
mindspore/lite/src/runtime/kernel/arm/fp16/matmul_fp16.cc
View File

@ -105,15 +105,18 @@ int MatmulFP16CPUKernel::InitBroadcastParams() {
}
}
int batch_sizes[MAX_SHAPE_SIZE] = {0};
int a_batch_sizes[MAX_SHAPE_SIZE] = {0};
int b_batch_sizes[MAX_SHAPE_SIZE] = {0};
for (int i = a_shape.size() - kCHWDimNumber; i >= 0; --i) {
if (static_cast<int>(a_shape.size() - kCHWDimNumber) == i) {
batch_sizes_[i] = std::max(a_shape[i], b_shape[i]);
a_batch_sizes_[i] = a_shape[i];
b_batch_sizes_[i] = b_shape[i];
batch_sizes[i] = std::max(a_shape[i], b_shape[i]);
a_batch_sizes[i] = a_shape[i];
b_batch_sizes[i] = b_shape[i];
} else {
batch_sizes_[i] = batch_sizes_[i + 1] * std::max(a_shape[i], b_shape[i]);
a_batch_sizes_[i] = a_batch_sizes_[i + 1] * a_shape[i];
b_batch_sizes_[i] = b_batch_sizes_[i + 1] * b_shape[i];
batch_sizes[i] = batch_sizes[i + 1] * std::max(a_shape[i], b_shape[i]);
a_batch_sizes[i] = a_batch_sizes[i + 1] * a_shape[i];
b_batch_sizes[i] = b_batch_sizes[i + 1] * b_shape[i];
}
}
@ -139,13 +142,11 @@ int MatmulFP16CPUKernel::InitBroadcastParams() {
int b_offset = 0;
for (size_t j = 0; j < a_shape.size() - kHWDimNumber; ++j) {
if (j > 0) {
delta = delta % batch_sizes_[j];
delta = delta % batch_sizes[j];
}
if (j < (a_shape.size() - kCHWDimNumber)) {
a_offset +=
(delta / batch_sizes_[j + 1] * a_shape[j] / std::max(a_shape[j], b_shape[j])) * a_batch_sizes_[j + 1];
b_offset +=
(delta / batch_sizes_[j + 1] * b_shape[j] / std::max(a_shape[j], b_shape[j])) * b_batch_sizes_[j + 1];
a_offset += (delta / batch_sizes[j + 1] * a_shape[j] / std::max(a_shape[j], b_shape[j])) * a_batch_sizes[j + 1];
b_offset += (delta / batch_sizes[j + 1] * b_shape[j] / std::max(a_shape[j], b_shape[j])) * b_batch_sizes[j + 1];
} else {
a_offset += (delta * a_shape[j] / std::max(a_shape[j], b_shape[j]));
b_offset += (delta * b_shape[j] / std::max(a_shape[j], b_shape[j]));

2
mindspore/lite/src/runtime/kernel/arm/fp32/fullconnection_fp32.cc
View File

@ -45,6 +45,8 @@ int FullconnectionCPUKernel::Init() {
}
params_->batch = 1;
a_offset_.resize(params_->batch, 0);
b_offset_.resize(params_->batch, 0);
a_batch_ = 1;
b_batch_ = 1;
params_->a_transpose_ = false;

33
mindspore/lite/src/runtime/kernel/arm/fp32/matmul_fp32.cc
View File

@ -94,26 +94,27 @@ int MatmulCPUKernel::InitBroadcastParams() {
}
}
int batch_sizes[MAX_SHAPE_SIZE] = {0};
int a_batch_sizes[MAX_SHAPE_SIZE] = {0};
int b_batch_sizes[MAX_SHAPE_SIZE] = {0};
for (int i = a_shape.size() - kCHWDimNumber; i >= 0; --i) {
if (static_cast<int>(a_shape.size() - kCHWDimNumber) == i) {
batch_sizes_[i] = std::max(a_shape[i], b_shape[i]);
a_batch_sizes_[i] = a_shape[i];
b_batch_sizes_[i] = b_shape[i];
batch_sizes[i] = std::max(a_shape[i], b_shape[i]);
a_batch_sizes[i] = a_shape[i];
b_batch_sizes[i] = b_shape[i];
} else {
batch_sizes_[i] = batch_sizes_[i + 1] * std::max(a_shape[i], b_shape[i]);
a_batch_sizes_[i] = a_batch_sizes_[i + 1] * a_shape[i];
b_batch_sizes_[i] = b_batch_sizes_[i + 1] * b_shape[i];
batch_sizes[i] = batch_sizes[i + 1] * std::max(a_shape[i], b_shape[i]);
a_batch_sizes[i] = a_batch_sizes[i + 1] * a_shape[i];
b_batch_sizes[i] = b_batch_sizes[i + 1] * b_shape[i];
}
}
int out_batch = 1;
for (size_t i = 0; i < a_shape.size() - kHWDimNumber; ++i) {
out_batch *= MSMAX(a_shape[i], b_shape[i]);
if (a_shape[i] < b_shape[i] && b_shape[i] % a_shape[i] == 0) {
a_broadcast_ = true;
} else if (a_shape[i] > b_shape[i] && a_shape[i] % b_shape[i] == 0) {
b_broadcast_ = true;
} else if (a_shape[i] != b_shape[i]) {
int max_v = MSMAX(a_shape[i], b_shape[i]);
int min_v = MSMIN(a_shape[i], b_shape[i]) > 0 ? MSMIN(a_shape[i], b_shape[i]) : 1;
out_batch *= max_v;
if (max_v != min_v && max_v % min_v != 0) {
MS_LOG(ERROR) << "matmul don't support broadcast for dimension " << a_shape << " and " << b_shape;
return RET_ERROR;
}
@ -128,13 +129,11 @@ int MatmulCPUKernel::InitBroadcastParams() {
int b_offset = 0;
for (size_t j = 0; j < a_shape.size() - kHWDimNumber; ++j) {
if (j > 0) {
delta = delta % batch_sizes_[j];
delta = delta % batch_sizes[j];
}
if (j < (a_shape.size() - kCHWDimNumber)) {
a_offset +=
(delta / batch_sizes_[j + 1] * a_shape[j] / std::max(a_shape[j], b_shape[j])) * a_batch_sizes_[j + 1];
b_offset +=
(delta / batch_sizes_[j + 1] * b_shape[j] / std::max(a_shape[j], b_shape[j])) * b_batch_sizes_[j + 1];
a_offset += (delta / batch_sizes[j + 1] * a_shape[j] / std::max(a_shape[j], b_shape[j])) * a_batch_sizes[j + 1];
b_offset += (delta / batch_sizes[j + 1] * b_shape[j] / std::max(a_shape[j], b_shape[j])) * b_batch_sizes[j + 1];
} else {
a_offset += (delta * a_shape[j] / std::max(a_shape[j], b_shape[j]));
b_offset += (delta * b_shape[j] / std::max(a_shape[j], b_shape[j]));

9
mindspore/lite/src/runtime/kernel/arm/fp32/matmul_fp32_base.cc
View File

@ -472,13 +472,8 @@ int MatmulFp32BaseCPUKernel::Run() {
}
for (int i = 0; i < params_->batch; ++i) {
if (!a_broadcast_ && !b_broadcast_) {
batch_a_ptr_ = a_pack_ptr_ + i * params_->row_align_ * params_->deep_;
batch_b_ptr_ = b_pack_ptr_ + i * params_->deep_ * params_->col_align_;
} else {
batch_a_ptr_ = a_pack_ptr_ + a_offset_[i] * params_->row_align_ * params_->deep_;
batch_b_ptr_ = b_pack_ptr_ + b_offset_[i] * params_->deep_ * params_->col_align_;
}
batch_a_ptr_ = a_pack_ptr_ + a_offset_[i] * params_->row_align_ * params_->deep_;
batch_b_ptr_ = b_pack_ptr_ + b_offset_[i] * params_->deep_ * params_->col_align_;
if (vec_matmul_) {
batch_c_ptr_ = output_data_ + i * params_->row_ * params_->col_align_;
} else {

5
mindspore/lite/src/runtime/kernel/arm/fp32/matmul_fp32_base.h
View File

@ -67,13 +67,8 @@ class MatmulFp32BaseCPUKernel : public InnerKernel {
MatMulParameter *params_ = nullptr;
float *a_pack_ptr_ = nullptr;
float *b_pack_ptr_ = nullptr;
bool a_broadcast_ = false;
bool b_broadcast_ = false;
int a_batch_ = 1;
int b_batch_ = 1;
int batch_sizes_[MAX_SHAPE_SIZE] = {0};
int a_batch_sizes_[MAX_SHAPE_SIZE] = {0};
int b_batch_sizes_[MAX_SHAPE_SIZE] = {0};
std::vector<int> a_offset_;
std::vector<int> b_offset_;

1
mindspore/lite/test/config/models_tf.cfg
View File

@ -108,3 +108,4 @@ fsr_270_mindspore.pb
fsr_360_mindspore.pb
fsr_720_mindspore.pb
hiai_asr_last_e1_cpu_fast_wavenet_batch1_frame1_one_cache.pb;2
matmul_broadcast.pb;12:Placeholder,Placeholder_1,Placeholder_2,Placeholder_3,Placeholder_4,Placeholder_5,Placeholder_6,Placeholder_7,Placeholder_8,Placeholder_9,Placeholder_10,Placeholder_11;12,1,1,8:12,36,8,32:12,1,32,1:12,36,64,32:1,36,32,32:12,1,32,8:1,36,32,64:1,1,1,32,:12,1,32,32:1,36,32,32:32,32,1,1,32,8