diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16/batchnorm_fp16.c b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16/batchnorm_fp16.c index 05f0b8078e3..0395bbaecc6 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16/batchnorm_fp16.c +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16/batchnorm_fp16.c @@ -55,3 +55,33 @@ void FusedBatchNormFp16(const void *input, const void *scale, const void *offset cur_offset += param->channel_; } } + +void FusedBatchNormFp16MeanVar(const float16_t *input, float16_t *run_mean, float16_t *run_var, + const BatchNormParameter *param, float16_t *save_mean, float16_t *save_var) { + const float N = (float)param->unit_; + const float VN = N; + const float VNUB = (N > 1.0f) ? (N - 1.0f) : 1.0f; + const float momentum = (1.0f - param->momentum_); + + for (int i = 0; i < param->unit_; i++) { + for (int c = 0; c < param->channel_; c++) { + int idx = i * param->channel_ + c; + run_mean[c] += input[idx]; + } + } + for (int c = 0; c < param->channel_; c++) { + run_mean[c] /= (float16_t)N; + } + for (int i = 0; i < param->unit_; i++) { + for (int c = 0; c < param->channel_; c++) { + int idx = i * param->channel_ + c; + run_var[c] += (float16_t)((float)(input[idx] - run_mean[c]) * (float)(input[idx] - run_mean[c])); + } + } + for (int c = 0; c < param->channel_; c++) { + float unbiased_var = ((float)run_var[c] / VNUB); + run_var[c] = (float16_t)((float)run_var[c] / VN); + save_mean[c] = (float16_t)(momentum * (float)save_mean[c] + (1.0f - momentum) * (float)run_mean[c]); + save_var[c] = (float16_t)(momentum * (float)save_var[c] + (1.0f - momentum) * unbiased_var); + } +} diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16/batchnorm_fp16.h b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16/batchnorm_fp16.h index e6f459cb572..678a1ae6598 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16/batchnorm_fp16.h +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16/batchnorm_fp16.h @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MINDSPORE_NNACL_FP16_BATCHNORM_FP16_H_ -#define MINDSPORE_NNACL_FP16_BATCHNORM_FP16_H_ +#ifndef MINDSPORE_CCSRC_BACKEND_KERNEL_COMPILER_CPU_NNACL_FP16_BATCHNORM_FP16_H_ +#define MINDSPORE_CCSRC_BACKEND_KERNEL_COMPILER_CPU_NNACL_FP16_BATCHNORM_FP16_H_ #include "nnacl/batchnorm_parameter.h" @@ -26,9 +26,10 @@ void BatchNormFp16(const float16_t *input, const void *mean, const void *varianc int task_id, float16_t *output); void FusedBatchNormFp16(const void *input, const void *scale, const void *offset, const void *mean, const void *variance, BatchNormParameter *param, int task_id, void *output); - +void FusedBatchNormFp16MeanVar(const float16_t *input, float16_t *run_mean, float16_t *run_var, + const BatchNormParameter *param, float16_t *save_mean, float16_t *save_var); #ifdef __cplusplus } #endif -#endif // MINDSPORE_NNACL_FP16_BATCHNORM_FP16_H_ +#endif // MINDSPORE_CCSRC_BACKEND_KERNEL_COMPILER_CPU_NNACL_FP16_BATCHNORM_FP16_H_ diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16/cast_fp16.h b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16/cast_fp16.h index 62608d002bb..78d0a5bcfb3 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16/cast_fp16.h +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16/cast_fp16.h @@ -47,14 +47,35 @@ inline void Float16ToInt64(const float16_t *input, int64_t *output, int number) } #ifdef ENABLE_ARM64 -inline void Float32ToFloat16(const float *input, float16_t *output, int number) { - for (int i = 0; i < number; ++i) { +inline void Float32ToFloat16(const float *__restrict input, float16_t *__restrict output, int number) { + int count = (number & ~(C8NUM - 1)); + int i = 0; + for (; i < count; i += C8NUM) { + float32x4_t in1 = vld1q_f32(input + i); + float16x4_t out1 = vcvt_f16_f32(in1); + float32x4_t in2 = vld1q_f32(input + i + 4); + float16x4_t out2 = vcvt_f16_f32(in2); + float16x8_t out = vcombine_f16(out1, out2); + vst1q_f16(output + i, out); + } + for (; i < number; ++i) { output[i] = (float16_t)input[i]; } } -inline void Float16ToFloat32(const float16_t *input, float *output, int number) { - for (int i = 0; i < number; ++i) { +inline void Float16ToFloat32(const float16_t *__restrict input, float *__restrict output, int number) { + int count = number & ~(C8NUM - 1); + int i = 0; + for (; i < count; i += C8NUM) { + float16x8_t in = vld1q_f16(input + i); + float16x4_t in1 = vget_low_f16(in); + float16x4_t in2 = vget_high_f16(in); + float32x4_t out1 = vcvt_f32_f16(in1); + vst1q_f32(output + i, out1); + float32x4_t out2 = vcvt_f32_f16(in2); + vst1q_f32(output + i + 4, out2); + } + for (; i < number; ++i) { output[i] = (float)input[i]; } } diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16_grad/batch_norm.c b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16_grad/batch_norm.c index 8867ed73fb5..9514c71385e 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16_grad/batch_norm.c +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16_grad/batch_norm.c @@ -27,7 +27,6 @@ void backwardAllFp16(const float16_t *restrict in, const float16_t *restrict yt, const float16_t *restrict invar, const float16_t *restrict scale, int size, int ch, float *restrict dxhat_sum, float *restrict dxhathat_sum, float16_t *restrict dbias, float16_t *restrict dscale, float16_t *restrict dx) { - float16_t N = (float16_t)size; for (int i = 0; i < size; i++) { for (int c = 0; c < ch; c++) { int ix = i * ch + c; @@ -41,13 +40,14 @@ void backwardAllFp16(const float16_t *restrict in, const float16_t *restrict yt, dxhathat_sum[c] += (float)(dx_hat * x_hat); } } + float N = (float)size; for (int i = 0; i < size; i++) { for (int c = 0; c < ch; c++) { // dx_2 int ix = i * ch + c; float16_t x_hat = (in[ix] - mean[c]) * invar[c]; float16_t dx_hat = yt[ix] * scale[c]; - dx[ix] = 1.0f / N * (float)((invar[c]) * (N * dx_hat - dxhat_sum[c] - x_hat * dxhathat_sum[c])); + dx[ix] = (float16_t)((float)((invar[c]) * (N * dx_hat - dxhat_sum[c] - x_hat * dxhathat_sum[c])) / N); } } } @@ -80,7 +80,7 @@ void backwardP2Fp16(const float16_t *restrict in, const float16_t *restrict yt, int ix = i * ch + c; float x_hat = (float)((in[ix] - mean[c]) * invar[c]); float dx_hat = (float)(yt[ix] * scale[c]); - dx[ix] = (float16_t)(1.0f / N * (float)(invar[c]) * (N * dx_hat - dxhat_sum[c] - x_hat * dxhathat_sum[c])); + dx[ix] = (float16_t)(((float)(invar[c]) * (N * dx_hat - dxhat_sum[c] - x_hat * dxhathat_sum[c])) / N); } } } diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16_grad/convolution_grad_filter.c b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16_grad/convolution_grad_filter.c index 6162a88bb90..01499bccdb5 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16_grad/convolution_grad_filter.c +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16_grad/convolution_grad_filter.c @@ -16,6 +16,7 @@ #include "nnacl/fp16_grad/convolution_grad_filter.h" #include "nnacl/intrinsics/ms_simd_instructions_fp16.h" +#include "nnacl/errorcode.h" #ifdef ENABLE_NEON #include #endif @@ -356,5 +357,5 @@ int ConvDwFilterFp16Grad(const float16_t *x, const float16_t *dy, float16_t *dw, dw[i_c * k_spatial + k_idx] = sum; } } - return 0; + return NNACL_OK; } diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16_grad/convolution_grad_input.c b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16_grad/convolution_grad_input.c new file mode 100644 index 00000000000..f15a8b8a033 --- /dev/null +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16_grad/convolution_grad_input.c @@ -0,0 +1,100 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "nnacl/fp16_grad/convolution_grad_input.h" +#include "nnacl/errorcode.h" +#ifdef ENABLE_ARM +#include +#endif + +int ConvDwInputGradFp16(const float16_t *dy, const float16_t *w, float16_t *dx, int start, int count, + const ConvParameter *conv_param) { + int in_h = conv_param->input_h_; + int in_w = conv_param->input_w_; + int out_w = conv_param->output_w_; + int out_ch = conv_param->output_channel_; + int in_ch = conv_param->input_channel_; + int out_spatial = conv_param->output_h_ * conv_param->output_w_; + int k_h = conv_param->kernel_h_; + int k_w = conv_param->kernel_w_; + int k_spatial = k_h * k_w; + int end = start + count; + + int j = start; + for (; j <= (end - C4NUM); j += C4NUM) { + float16_t *c = dx + j; + const float16_t *mat_b_0 = w + (j + 0) * k_spatial; + const float16_t *mat_b_1 = w + (j + 1) * k_spatial; + const float16_t *mat_b_2 = w + (j + 2) * k_spatial; + const float16_t *mat_b_3 = w + (j + 3) * k_spatial; + + for (int si = 0; si < out_spatial; si++) { + const float16_t *a = dy + j + si * out_ch; +#ifdef ENABLE_ARM + float16x4_t mat_a = vld1_f16(a); +#else + float16_t mat_a[4] = {a[0], a[1], a[2], a[3]}; +#endif + int output_row = (si) / out_w; + int output_col = (si) % out_w; + for (int k = 0; k < k_spatial; k++) { + int row_stride_offset = output_row * conv_param->stride_h_; + int col_stride_offset = output_col * conv_param->stride_w_; + int kernel_row = k / k_w; + int kernel_col = k % k_w; + int input_row = -conv_param->pad_u_ + kernel_row * conv_param->dilation_h_ + row_stride_offset; + int input_col = -conv_param->pad_l_ + kernel_col * conv_param->dilation_w_ + col_stride_offset; + if (((unsigned)(input_row) < (unsigned)(in_h)) && ((unsigned)(input_col) < (unsigned)(in_w))) { + int offset = (input_row * in_w + input_col) * in_ch; +#ifdef ENABLE_ARM + float16x4_t mat_b = {mat_b_0[k], mat_b_1[k], mat_b_2[k], mat_b_3[k]}; + float16x4_t mat_c = vld1_f16(c + offset); + mat_c = vfma_f16(mat_c, mat_b, mat_a); + vst1_f16(c + offset, mat_c); +#else + c[offset + 0] += mat_a[0] * mat_b_0[k]; + c[offset + 1] += mat_a[1] * mat_b_1[k]; + c[offset + 2] += mat_a[2] * mat_b_2[k]; + c[offset + 3] += mat_a[3] * mat_b_3[k]; +#endif + } + } + } + } + + for (; j < end; j++) { + float16_t *c = dx + j; + const float16_t *b = w + j * k_spatial; + for (int si = 0; si < out_spatial; si++) { + const float16_t *a = dy + j + si * out_ch; + int output_row = si / out_w; + int output_col = si % out_w; + int row_stride_offset = output_row * conv_param->stride_h_; + int col_stride_offset = output_col * conv_param->stride_w_; + for (int k = 0; k < k_spatial; k++) { + int kernel_row = k / k_w; + int kernel_col = k % k_w; + int input_row = -conv_param->pad_u_ + kernel_row * conv_param->dilation_h_ + row_stride_offset; + int input_col = -conv_param->pad_l_ + kernel_col * conv_param->dilation_w_ + col_stride_offset; + if (((unsigned)(input_row) < (unsigned)(in_h)) && ((unsigned)(input_col) < (unsigned)(in_w))) { + int offset = (input_row * in_w + input_col) * in_ch; + c[offset] += a[0] * b[k]; + } + } + } + } + return NNACL_OK; +} diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16_grad/convolution_grad_input.h b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16_grad/convolution_grad_input.h new file mode 100644 index 00000000000..5cf4e27e686 --- /dev/null +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp16_grad/convolution_grad_input.h @@ -0,0 +1,33 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_NNACL_FP32_GRAD_CONVOLUTION_GRAD_INPUT_H_ +#define MINDSPORE_NNACL_FP32_GRAD_CONVOLUTION_GRAD_INPUT_H_ + +#include +#include "nnacl/conv_parameter.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int ConvDwInputGradFp16(const float16_t *dy, const float16_t *w, float16_t *dx, int start, int count, + const ConvParameter *conv_param); +#ifdef __cplusplus +} +#endif + +#endif // MINDSPORE_NNACL_FP32_GRAD_CONVOLUTION_GRAD_INPUT_H_ diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp32_grad/convolution_grad_filter.c b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp32_grad/convolution_grad_filter.c index 24214089150..8003b88cc62 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp32_grad/convolution_grad_filter.c +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp32_grad/convolution_grad_filter.c @@ -15,6 +15,7 @@ */ #include "nnacl/fp32_grad/convolution_grad_filter.h" +#include "nnacl/errorcode.h" #ifdef ENABLE_ARM #include #endif @@ -375,5 +376,5 @@ int ConvDwFilterGrad(const float *x, const float *dy, float *dw, int start, int dw[i_c * k_spatial + k_idx] = sum; } } - return 0; + return NNACL_OK; } diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp32_grad/convolution_grad_input.c b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp32_grad/convolution_grad_input.c new file mode 100644 index 00000000000..aecaca6eb5f --- /dev/null +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp32_grad/convolution_grad_input.c @@ -0,0 +1,99 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "nnacl/fp32_grad/convolution_grad_input.h" +#include "nnacl/errorcode.h" +#ifdef ENABLE_ARM +#include +#endif + +int ConvDwInputGrad(const float *dy, const float *w, float *dx, int start, int count, const ConvParameter *conv_param) { + int in_h = conv_param->input_h_; + int in_w = conv_param->input_w_; + int out_w = conv_param->output_w_; + int out_ch = conv_param->output_channel_; + int in_ch = conv_param->input_channel_; + int out_spatial = conv_param->output_h_ * conv_param->output_w_; + int k_h = conv_param->kernel_h_; + int k_w = conv_param->kernel_w_; + int k_spatial = k_h * k_w; + int end = start + count; + + int j = start; + for (; j <= (end - C4NUM); j += C4NUM) { + float *c = dx + j; + const float *mat_b_0 = w + (j + 0) * k_spatial; + const float *mat_b_1 = w + (j + 1) * k_spatial; + const float *mat_b_2 = w + (j + 2) * k_spatial; + const float *mat_b_3 = w + (j + 3) * k_spatial; + + for (int si = 0; si < out_spatial; si++) { + const float *a = dy + j + si * out_ch; +#ifdef ENABLE_ARM + float32x4_t mat_a = vld1q_f32(a); +#else + float mat_a[4] = {a[0], a[1], a[2], a[3]}; +#endif + int output_row = (si) / out_w; + int output_col = (si) % out_w; + for (int k = 0; k < k_spatial; k++) { + int row_stride_offset = output_row * conv_param->stride_h_; + int col_stride_offset = output_col * conv_param->stride_w_; + int kernel_row = k / k_w; + int kernel_col = k % k_w; + int input_row = -conv_param->pad_u_ + kernel_row * conv_param->dilation_h_ + row_stride_offset; + int input_col = -conv_param->pad_l_ + kernel_col * conv_param->dilation_w_ + col_stride_offset; + if (((unsigned)(input_row) < (unsigned)(in_h)) && ((unsigned)(input_col) < (unsigned)(in_w))) { + int offset = (input_row * in_w + input_col) * in_ch; +#ifdef ENABLE_ARM + float32x4_t mat_b = {mat_b_0[k], mat_b_1[k], mat_b_2[k], mat_b_3[k]}; + float32x4_t mat_c = vld1q_f32(c + offset); + mat_c = vmlaq_f32(mat_c, mat_b, mat_a); + vst1q_f32(c + offset, mat_c); +#else + c[offset + 0] += mat_a[0] * mat_b_0[k]; + c[offset + 1] += mat_a[1] * mat_b_1[k]; + c[offset + 2] += mat_a[2] * mat_b_2[k]; + c[offset + 3] += mat_a[3] * mat_b_3[k]; +#endif + } + } + } + } + + for (; j < end; j++) { + float *c = dx + j; + const float *b = w + j * k_spatial; + for (int si = 0; si < out_spatial; si++) { + const float *a = dy + j + si * out_ch; + int output_row = si / out_w; + int output_col = si % out_w; + int row_stride_offset = output_row * conv_param->stride_h_; + int col_stride_offset = output_col * conv_param->stride_w_; + for (int k = 0; k < k_spatial; k++) { + int kernel_row = k / k_w; + int kernel_col = k % k_w; + int input_row = -conv_param->pad_u_ + kernel_row * conv_param->dilation_h_ + row_stride_offset; + int input_col = -conv_param->pad_l_ + kernel_col * conv_param->dilation_w_ + col_stride_offset; + if (((unsigned)(input_row) < (unsigned)(in_h)) && ((unsigned)(input_col) < (unsigned)(in_w))) { + int offset = (input_row * in_w + input_col) * in_ch; + c[offset] += a[0] * b[k]; + } + } + } + } + return NNACL_OK; +} diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp32_grad/convolution_grad_input.h b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp32_grad/convolution_grad_input.h new file mode 100644 index 00000000000..7d8b0a63eaa --- /dev/null +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/fp32_grad/convolution_grad_input.h @@ -0,0 +1,32 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_NNACL_FP32_GRAD_CONVOLUTION_GRAD_INPUT_H_ +#define MINDSPORE_NNACL_FP32_GRAD_CONVOLUTION_GRAD_INPUT_H_ + +#include +#include "nnacl/conv_parameter.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int ConvDwInputGrad(const float *dy, const float *w, float *dx, int start, int count, const ConvParameter *conv_param); +#ifdef __cplusplus +} +#endif + +#endif // MINDSPORE_NNACL_FP32_GRAD_CONVOLUTION_GRAD_INPUT_H_ diff --git a/mindspore/lite/CMakeLists.txt b/mindspore/lite/CMakeLists.txt index 31ebc5fa60e..dd5f0e302f9 100644 --- a/mindspore/lite/CMakeLists.txt +++ b/mindspore/lite/CMakeLists.txt @@ -105,10 +105,11 @@ message(STATUS "\tMSLITE_ENABLE_TOOLS = \t${MSLITE_ENABLE_TOOLS}") message(STATUS "\tMSLITE_ENABLE_TESTCASES = \t${MSLITE_ENABLE_TESTCASES}") if(ENABLE_ASAN) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fsanitize-recover=address -fno-omit-frame-pointer") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address -fsanitize-recover=address -fno-omit-frame-pointer") + add_definitions(-fsanitize=address -fno-omit-frame-pointer -mllvm -asan-use-private-alias=1) + add_link_options(-fsanitize=address) endif() + set(PKG_NAME_PREFIX mindspore-lite-${MS_VERSION_MAJOR}.${MS_VERSION_MINOR}.${MS_VERSION_REVISION}) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") set(BUILD_MINDDATA "lite_cv" CACHE STRING "off, lite, lite_cv, wrapper or full") diff --git a/mindspore/lite/examples/export_models/models/mobilenetv2_train_export.py b/mindspore/lite/examples/export_models/models/mobilenetv2_train_export.py index 04330639992..a476b0b89ad 100644 --- a/mindspore/lite/examples/export_models/models/mobilenetv2_train_export.py +++ b/mindspore/lite/examples/export_models/models/mobilenetv2_train_export.py @@ -23,7 +23,7 @@ from mindspore import context, Tensor, nn from mindspore.train.serialization import export context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) -batch = 16 +batch = 8 backbone_net = MobileNetV2Backbone() head_net = MobileNetV2Head(input_channel=backbone_net.out_channels, num_classes=10) diff --git a/mindspore/lite/examples/train_lenet/Makefile b/mindspore/lite/examples/train_lenet/Makefile index 869caa0e14d..c021ac4f56b 100644 --- a/mindspore/lite/examples/train_lenet/Makefile +++ b/mindspore/lite/examples/train_lenet/Makefile @@ -14,8 +14,8 @@ OBJ:=$(SRC:.cc=.o) CFLAGS := -Ofast -std=c++17 \ -I . \ - -I ./msl/train \ - -I ./msl/train/minddata \ + -I ./msl/inference \ + -I ./msl/inference/minddata \ -I ./msl/tools/third_party/flatbuffers/include diff --git a/mindspore/lite/examples/train_lenet/model/lenet_export.py b/mindspore/lite/examples/train_lenet/model/lenet_export.py index c774887e831..43c03106274 100644 --- a/mindspore/lite/examples/train_lenet/model/lenet_export.py +++ b/mindspore/lite/examples/train_lenet/model/lenet_export.py @@ -14,6 +14,7 @@ # ============================================================================ """lenet_export.""" +import sys import numpy as np from mindspore import context, Tensor import mindspore.common.dtype as mstype @@ -21,11 +22,12 @@ from mindspore.train.serialization import export from lenet import LeNet5 from train_utils import train_wrap + n = LeNet5() n.set_train() context.set_context(mode=context.PYNATIVE_MODE, device_target="CPU", save_graphs=False) -BATCH_SIZE = 32 +BATCH_SIZE = int(sys.argv[1]) x = Tensor(np.ones((BATCH_SIZE, 1, 32, 32)), mstype.float32) label = Tensor(np.zeros([BATCH_SIZE]).astype(np.int32)) net = train_wrap(n) diff --git a/mindspore/lite/examples/train_lenet/model/prepare_model.sh b/mindspore/lite/examples/train_lenet/model/prepare_model.sh index 34364aa35fb..28ff1f8a9d1 100755 --- a/mindspore/lite/examples/train_lenet/model/prepare_model.sh +++ b/mindspore/lite/examples/train_lenet/model/prepare_model.sh @@ -1,12 +1,12 @@ #!/bin/bash echo "============Exporting==========" -if [ -n "$1" ]; then - DOCKER_IMG=$1 - docker run -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER --privileged=true ${DOCKER_IMG} /bin/bash -c "PYTHONPATH=../../../../../model_zoo/official/cv/lenet/src python lenet_export.py; chmod 444 lenet_tod.mindir; rm -rf __pycache__" +if [ -n "$2" ]; then + DOCKER_IMG=$2 + docker run -w $PWD --runtime=nvidia -v /home/$USER:/home/$USER --privileged=true ${DOCKER_IMG} /bin/bash -c "PYTHONPATH=../../../../../model_zoo/official/cv/lenet/src python lenet_export.py '$1'; chmod 444 lenet_tod.mindir; rm -rf __pycache__" else echo "MindSpore docker was not provided, attempting to run locally" - PYTHONPATH=../../../../../model_zoo/official/cv/lenet/src python lenet_export.py + PYTHONPATH=../../../../../model_zoo/official/cv/lenet/src python lenet_export.py $1 fi diff --git a/mindspore/lite/examples/train_lenet/prepare_and_run.sh b/mindspore/lite/examples/train_lenet/prepare_and_run.sh index 62e8f349973..650933e300e 100755 --- a/mindspore/lite/examples/train_lenet/prepare_and_run.sh +++ b/mindspore/lite/examples/train_lenet/prepare_and_run.sh @@ -2,7 +2,7 @@ display_usage() { - echo -e "\nUsage: prepare_and_run.sh -D dataset_path [-d mindspore_docker] [-r release.tar.gz] [-t arm64|x86] [-q]\n" + echo -e "\nUsage: prepare_and_run.sh -D dataset_path [-d mindspore_docker] [-r release.tar.gz] [-t arm64|x86] [-q] [-o] [-b virtual_batch]\n" } checkopts() @@ -11,7 +11,9 @@ checkopts() DOCKER="" MNIST_DATA_PATH="" QUANTIZE="" - while getopts 'D:d:r:t:q' opt + ENABLEFP16=false + VIRTUAL_BATCH=-1 + while getopts 'D:d:r:t:qob:' opt do case "${opt}" in D) @@ -35,6 +37,12 @@ checkopts() q) QUANTIZE="QUANTIZE" ;; + o) + ENABLEFP16=true + ;; + b) + VIRTUAL_BATCH=$OPTARG + ;; *) echo "Unknown option ${opt}!" display_usage @@ -65,10 +73,18 @@ if [ "$TARBALL" == "" ]; then fi fi + # Prepare the model +if [[ "${VIRTUAL_BATCH}" == "-1" ]]; then + BATCH=32 +else + BATCH=1 +fi + + cd model/ || exit 1 rm -f *.ms -QUANTIZE=${QUANTIZE} ./prepare_model.sh $DOCKER || exit 1 +QUANTIZE=${QUANTIZE} ./prepare_model.sh $BATCH $DOCKER || exit 1 cd ../ # Copy the .ms model to the package folder @@ -84,11 +100,11 @@ cp scripts/*.sh ${PACKAGE}/ # Copy the shared MindSpore ToD library tar -xzf ${TARBALL} -mv mindspore-*/train/lib ${PACKAGE}/ -mv mindspore-*/train/third_party/libjpeg-turbo/lib/* ${PACKAGE}/lib/ +mv mindspore-*/inference/lib ${PACKAGE}/ +mv mindspore-*/inference/third_party/libjpeg-turbo/lib/* ${PACKAGE}/lib/ if [ "${TARGET}" == "arm64" ]; then tar -xzf ${TARBALL} --wildcards --no-anchored hiai_ddk - mv mindspore-*/train/third_party/hiai_ddk/lib/* ${PACKAGE}/lib/ + mv mindspore-*/inference/third_party/hiai_ddk/lib/* ${PACKAGE}/lib/ fi rm -rf msl @@ -111,11 +127,21 @@ if [ "${TARGET}" == "arm64" ]; then adb push ${PACKAGE} /data/local/tmp/ echo "========Training on Device=====" - adb shell "cd /data/local/tmp/package-arm64 && /system/bin/sh train.sh" + if "$ENABLEFP16"; then + echo "Training fp16.." + adb shell "cd /data/local/tmp/package-arm64 && /system/bin/sh train.sh -o -b ${VIRTUAL_BATCH}" + else + adb shell "cd /data/local/tmp/package-arm64 && /system/bin/sh train.sh -b ${VIRTUAL_BATCH}" + fi echo echo "===Evaluating trained Model=====" - adb shell "cd /data/local/tmp/package-arm64 && /system/bin/sh eval.sh" + if "$ENABLEFP16"; then + echo "Evaluating fp16 Model.." + adb shell "cd /data/local/tmp/package-arm64 && /system/bin/sh eval.sh -o" + else + adb shell "cd /data/local/tmp/package-arm64 && /system/bin/sh eval.sh" + fi echo else cd ${PACKAGE} || exit 1 @@ -124,6 +150,7 @@ else echo "===Evaluating trained Model=====" ./eval.sh + cd .. fi diff --git a/mindspore/lite/examples/train_lenet/scripts/eval.sh b/mindspore/lite/examples/train_lenet/scripts/eval.sh index 9fd82e7af91..1726c296e53 100755 --- a/mindspore/lite/examples/train_lenet/scripts/eval.sh +++ b/mindspore/lite/examples/train_lenet/scripts/eval.sh @@ -15,4 +15,4 @@ # ============================================================================ # an simple tutorial as follows, more parameters can be setting -LD_LIBRARY_PATH=./lib/ bin/net_runner -f model/lenet_tod_trained.ms -e 0 -d dataset +LD_LIBRARY_PATH=./lib/ bin/net_runner -f model/lenet_tod_trained.ms -e 0 -d dataset $1 diff --git a/mindspore/lite/examples/train_lenet/scripts/train.sh b/mindspore/lite/examples/train_lenet/scripts/train.sh index d3927967368..369f429022f 100755 --- a/mindspore/lite/examples/train_lenet/scripts/train.sh +++ b/mindspore/lite/examples/train_lenet/scripts/train.sh @@ -15,4 +15,4 @@ # ============================================================================ # an simple tutorial as follows, more parameters can be setting -LD_LIBRARY_PATH=./lib/ bin/net_runner -f model/lenet_tod.ms -e 5 -d dataset +LD_LIBRARY_PATH=./lib/ bin/net_runner -f model/lenet_tod.ms -e 5 -d dataset $1 $2 $3 diff --git a/mindspore/lite/examples/train_lenet/src/net_runner.cc b/mindspore/lite/examples/train_lenet/src/net_runner.cc index 4045bdbf17e..48ca1438aba 100644 --- a/mindspore/lite/examples/train_lenet/src/net_runner.cc +++ b/mindspore/lite/examples/train_lenet/src/net_runner.cc @@ -51,7 +51,7 @@ constexpr int kNCHWDims = 4; constexpr int kNCHWCDim = 2; constexpr int kPrintTimes = 100; constexpr int kSaveSteps = 1000; -constexpr float kLearningRate = 0.7f; +constexpr float kGammaFactor = 0.7f; class Rescaler : public mindspore::session::TrainLoopCallBack { public: explicit Rescaler(float scale) : scale_(scale) { @@ -70,6 +70,36 @@ class Rescaler : public mindspore::session::TrainLoopCallBack { float scale_ = 1.0; }; +class Measurement : public mindspore::session::TrainLoopCallBack { + public: + explicit Measurement(unsigned int epochs) + : epochs_(epochs), time_avg_(std::chrono::duration(0)) {} + ~Measurement() override = default; + void EpochBegin(const mindspore::session::TrainLoopCallBackData &cb_data) override { + start_time_ = std::chrono::high_resolution_clock::now(); + } + int EpochEnd(const mindspore::session::TrainLoopCallBackData &cb_data) override { + end_time_ = std::chrono::high_resolution_clock::now(); + auto time = std::chrono::duration(end_time_ - start_time_); + time_avg_ += time; + return mindspore::session::RET_CONTINUE; + } + void End(const mindspore::session::TrainLoopCallBackData &cb_data) override { + if (epochs_ > 0) { + std::cout << "AvgRunTime: " << time_avg_.count() / epochs_ << " ms" << std::endl; + } + + struct mallinfo info = mallinfo(); + std::cout << "Total allocation: " << info.arena + info.hblkhd << std::endl; + } + + private: + std::chrono::time_point start_time_; + std::chrono::time_point end_time_; + std::chrono::duration time_avg_; + unsigned int epochs_; +}; + // Definition of verbose callback function after forwarding operator. bool after_callback(const std::vector &after_inputs, const std::vector &after_outputs, @@ -106,12 +136,14 @@ NetRunner::~NetRunner() { void NetRunner::InitAndFigureInputs() { mindspore::lite::Context context; context.device_list_[0].device_info_.cpu_device_info_.cpu_bind_mode_ = mindspore::lite::NO_BIND; - context.device_list_[0].device_info_.cpu_device_info_.enable_float16_ = false; + context.device_list_[0].device_info_.cpu_device_info_.enable_float16_ = enable_fp16_; context.device_list_[0].device_type_ = mindspore::lite::DT_CPU; context.thread_num_ = 2; session_ = mindspore::session::LiteSession::CreateTrainSession(ms_file_, &context, true); MS_ASSERT(session_ != nullptr); + + session_->SetupVirtualBatch(virtual_batch_); loop_ = mindspore::session::TrainLoop::CreateTrainLoop(session_); if (verbose_) { @@ -143,7 +175,7 @@ float NetRunner::CalculateAccuracy(int max_tests) { Rescaler rescale(kScalePoint); loop_->Eval(test_ds_.get(), std::vector{&rescale}); - std::cout << "Eval Accuracy is " << acc_metrics_->Eval() << std::endl; + std::cout << "Accuracy is " << acc_metrics_->Eval() << std::endl; return 0.0; } @@ -171,15 +203,22 @@ int NetRunner::InitDB() { } int NetRunner::TrainLoop() { - struct mindspore::lite::StepLRLambda step_lr_lambda(1, kLearningRate); - mindspore::lite::LRScheduler step_lr_sched(mindspore::lite::StepLRLambda, static_cast(&step_lr_lambda), 1); - mindspore::lite::LossMonitor lm(kPrintTimes); mindspore::lite::ClassificationTrainAccuracyMonitor am(1); + mindspore::lite::CkptSaver cs(kSaveSteps, std::string("lenet")); Rescaler rescale(kScalePoint); + Measurement measure(epochs_); + + if (virtual_batch_ > 0) { + loop_->Train(epochs_, train_ds_.get(), std::vector{&rescale, &lm, &cs, &am, &measure}); + } else { + struct mindspore::lite::StepLRLambda step_lr_lambda(1, kGammaFactor); + mindspore::lite::LRScheduler step_lr_sched(mindspore::lite::StepLRLambda, static_cast(&step_lr_lambda), 1); + loop_->Train(epochs_, train_ds_.get(), + std::vector{&rescale, &lm, &cs, &am, &step_lr_sched, &measure}); + } - loop_->Train(epochs_, train_ds_.get(), std::vector{&rescale, &lm, &cs, &am, &step_lr_sched}); return 0; } @@ -206,7 +245,7 @@ void NetRunner::Usage() { bool NetRunner::ReadArgs(int argc, char *argv[]) { int opt; - while ((opt = getopt(argc, argv, "f:e:d:s:ihc:v")) != -1) { + while ((opt = getopt(argc, argv, "f:e:d:s:ihc:vob:")) != -1) { switch (opt) { case 'f': ms_file_ = std::string(optarg); @@ -223,6 +262,13 @@ bool NetRunner::ReadArgs(int argc, char *argv[]) { case 's': save_checkpoint_ = atoi(optarg); break; + case 'o': + enable_fp16_ = true; + break; + case 'b': + virtual_batch_ = atoi(optarg); + std::cout << "virtual_batch_: " << virtual_batch_ << std::endl; + break; case 'h': default: Usage(); diff --git a/mindspore/lite/examples/train_lenet/src/net_runner.h b/mindspore/lite/examples/train_lenet/src/net_runner.h index c4017e758f9..b02f3620b29 100644 --- a/mindspore/lite/examples/train_lenet/src/net_runner.h +++ b/mindspore/lite/examples/train_lenet/src/net_runner.h @@ -57,6 +57,8 @@ class NetRunner { std::string data_dir_ = ""; unsigned int epochs_ = 10; bool verbose_ = false; + bool enable_fp16_ = false; + int virtual_batch_ = -1; int save_checkpoint_ = 0; int batch_size_ = 32; int h_ = 32; diff --git a/mindspore/lite/include/train/train_cfg.h b/mindspore/lite/include/train/train_cfg.h index 3ebee7e92a5..cc5db943dd2 100644 --- a/mindspore/lite/include/train/train_cfg.h +++ b/mindspore/lite/include/train/train_cfg.h @@ -42,10 +42,10 @@ class MixPrecisionCfg { this->num_of_not_nan_iter_th_ = rhs.num_of_not_nan_iter_th_; return *this; } - bool dynamic_loss_scale_; /**< Enable\disable dynamic loss scale during mix precision training */ - float loss_scale_; /**< Initial loss scale factor */ - bool keep_batchnorm_fp32_; /**< Keep batch norm in FP32 while training */ - int num_of_not_nan_iter_th_; /**< a threshold for modifying loss scale when dynamic loss scale is enabled */ + bool dynamic_loss_scale_ = false; /**< Enable\disable dynamic loss scale during mix precision training */ + float loss_scale_; /**< Initial loss scale factor */ + bool keep_batchnorm_fp32_ = true; /**< Keep batch norm in FP32 while training */ + uint32_t num_of_not_nan_iter_th_; /**< a threshold for modifying loss scale when dynamic loss scale is enabled */ }; /// \brief TrainCfg defined for holding train configuration. diff --git a/mindspore/lite/src/inner_kernel.cc b/mindspore/lite/src/inner_kernel.cc index ab5af65580d..522fd29a374 100644 --- a/mindspore/lite/src/inner_kernel.cc +++ b/mindspore/lite/src/inner_kernel.cc @@ -25,21 +25,21 @@ namespace mindspore::kernel { using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; -void *InnerKernel::workspace_ = nullptr; - -void InnerKernel::AllocWorkspace(size_t size) { - if (size == 0) { +void InnerKernel::AllocWorkspace() { + workspace_ = malloc(workspace_size()); + if (workspace_ == nullptr) { + MS_LOG(ERROR) << "fail to alloc " << workspace_size() << "in kernel" << name(); return; } - workspace_ = malloc(size); - if (workspace_ == nullptr) { - MS_LOG(ERROR) << "fail to alloc " << size; - } + ws_allocated_ = true; } void InnerKernel::FreeWorkspace() { - free(workspace_); + if (ws_allocated_) { + free(workspace_); + } workspace_ = nullptr; + ws_allocated_ = false; } int InnerKernel::PreProcess() { diff --git a/mindspore/lite/src/inner_kernel.h b/mindspore/lite/src/inner_kernel.h index d7be21a01b3..1a061d6087b 100644 --- a/mindspore/lite/src/inner_kernel.h +++ b/mindspore/lite/src/inner_kernel.h @@ -46,6 +46,7 @@ class InnerKernel : public Kernel { if (op_parameter_ != nullptr) { free(op_parameter_); op_parameter_ = nullptr; + FreeWorkspace(); } } @@ -77,7 +78,7 @@ class InnerKernel : public Kernel { ret = PostProcess(); if (lite::RET_OK != ret) { - MS_LOG(ERROR) << "run kernel PreProcess failed, name: " << this->name(); + MS_LOG(ERROR) << "run kernel PostProcess failed, name: " << this->name(); return ret; } return lite::RET_OK; @@ -192,9 +193,15 @@ class InnerKernel : public Kernel { void set_workspace_size(size_t value) { workspace_size_ = value; } size_t workspace_size() { return workspace_size_; } - static void AllocWorkspace(size_t size); - static void FreeWorkspace(); + void AllocWorkspace(); + void FreeWorkspace(); void *workspace() { return workspace_; } + void set_workspace(void *ws) { + if (ws_allocated_ == false) { + workspace_ = ws; + } + } + bool ws_allocated_ = false; protected: OpParameter *op_parameter_ = nullptr; @@ -205,7 +212,7 @@ class InnerKernel : public Kernel { bool trainable_ = false; // parameters of this Kernel are trained in Train Session TypeId registry_data_type_ = kTypeUnknown; size_t workspace_size_ = 0; - static void *workspace_; + void *workspace_ = nullptr; }; } // namespace mindspore::kernel diff --git a/mindspore/lite/src/lite_session.cc b/mindspore/lite/src/lite_session.cc index b0ebaf6a25f..2fdf8a465e3 100644 --- a/mindspore/lite/src/lite_session.cc +++ b/mindspore/lite/src/lite_session.cc @@ -496,6 +496,7 @@ int LiteSession::CompileGraph(Model *model) { #else Scheduler scheduler(context_, model, &tensors_, is_train_session_, delegate_); #endif + scheduler.SetupSchedulerCb(std::move(sched_cb_)); ret = scheduler.Schedule(&kernels_); if (ret != RET_OK) { MS_LOG(ERROR) << "Schedule kernels failed: " << ret; diff --git a/mindspore/lite/src/lite_session.h b/mindspore/lite/src/lite_session.h index e1b71f5be1b..e5852d4c823 100644 --- a/mindspore/lite/src/lite_session.h +++ b/mindspore/lite/src/lite_session.h @@ -41,6 +41,7 @@ #elif GPU_VULKAN #include "src/runtime/gpu/vulkan/vulkan_runtime.h" #endif +#include "src/scheduler_cb.h" namespace mindspore { namespace lite { @@ -151,6 +152,7 @@ class LiteSession : public session::LiteSession { #elif GPU_VULKAN gpu::GpuRuntimeWrapper *vk_runtime_wrap_{nullptr}; #endif + std::unique_ptr sched_cb_; std::shared_ptr delegate_ = nullptr; }; } // namespace lite diff --git a/mindspore/lite/src/runtime/kernel/arm/base/convolution_base.h b/mindspore/lite/src/runtime/kernel/arm/base/convolution_base.h index 80bbd9b25d0..378e7780dae 100644 --- a/mindspore/lite/src/runtime/kernel/arm/base/convolution_base.h +++ b/mindspore/lite/src/runtime/kernel/arm/base/convolution_base.h @@ -58,12 +58,14 @@ class ConvolutionBaseCPUKernel : public InnerKernel { void FreeQuantParam(); protected: + bool is_repack() { return is_repack_; } void *bias_data_ = nullptr; const InnerContext *ctx_ = nullptr; ConvParameter *conv_param_ = nullptr; ConvQuantArg *conv_quant_arg_ = nullptr; int tile_num_ = 0; int thread_count_ = 1; + bool is_repack_ = false; }; } // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/base/reshape_base.cc b/mindspore/lite/src/runtime/kernel/arm/base/reshape_base.cc index 63307f36263..b40235af561 100644 --- a/mindspore/lite/src/runtime/kernel/arm/base/reshape_base.cc +++ b/mindspore/lite/src/runtime/kernel/arm/base/reshape_base.cc @@ -81,6 +81,7 @@ REG_KERNEL(kCPU, kNumberTypeFloat16, PrimitiveType_Reshape, LiteKernelCreator) REG_KERNEL(kCPU, kNumberTypeFloat16, PrimitiveType_Flatten, LiteKernelCreator) REG_KERNEL(kCPU, kNumberTypeFloat32, PrimitiveType_Flatten, LiteKernelCreator) +REG_KERNEL(kCPU, kNumberTypeFloat16, PrimitiveType_FlattenGrad, LiteKernelCreator) REG_KERNEL(kCPU, kNumberTypeFloat32, PrimitiveType_FlattenGrad, LiteKernelCreator) REG_KERNEL(kCPU, kNumberTypeInt32, PrimitiveType_ExpandDims, LiteKernelCreator) REG_KERNEL(kCPU, kNumberTypeFloat16, PrimitiveType_ExpandDims, LiteKernelCreator) diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/activation_fp16.cc b/mindspore/lite/src/runtime/kernel/arm/fp16/activation_fp16.cc index 8bff3a5d232..be69727810f 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/activation_fp16.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/activation_fp16.cc @@ -37,7 +37,8 @@ int ActivationFp16CPUKernel::Init() { if (type_ != schema::ActivationType_RELU && type_ != schema::ActivationType_RELU6 && type_ != schema::ActivationType_LEAKY_RELU && type_ != schema::ActivationType_SIGMOID && type_ != schema::ActivationType_TANH && type_ != schema::ActivationType_HSWISH && - type_ != schema::ActivationType_SWISH && type_ != schema::ActivationType_HARD_TANH) { + type_ != schema::ActivationType_SWISH && type_ != schema::ActivationType_HARD_TANH && + type_ != schema::ActivationType_GELU) { MS_LOG(ERROR) << "Activation fp16 not support type: " << type_; return RET_ERROR; } diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/arithmetic_fp16.cc b/mindspore/lite/src/runtime/kernel/arm/fp16/arithmetic_fp16.cc index d585e741d9f..64bb50be966 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/arithmetic_fp16.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/arithmetic_fp16.cc @@ -39,6 +39,7 @@ using mindspore::schema::PrimitiveType_Maximum; using mindspore::schema::PrimitiveType_Minimum; using mindspore::schema::PrimitiveType_MulFusion; using mindspore::schema::PrimitiveType_NotEqual; +using mindspore::schema::PrimitiveType_RealDiv; using mindspore::schema::PrimitiveType_SquaredDifference; using mindspore::schema::PrimitiveType_SubFusion; @@ -105,6 +106,9 @@ void ArithmeticFP16CPUKernel::InitRunFunction(int primitive_type) { {PrimitiveType_DivFusion, schema::ActivationType_RELU, ElementDivReluFp16, ElementOptDivReluFp16}, {PrimitiveType_DivFusion, schema::ActivationType_RELU6, ElementDivRelu6Fp16, ElementOptDivRelu6Fp16}, {PrimitiveType_DivFusion, schema::ActivationType_NO_ACTIVATION, ElementDivFp16, ElementOptDivFp16}, + {PrimitiveType_RealDiv, schema::ActivationType_RELU, ElementDivReluFp16, ElementOptDivReluFp16}, + {PrimitiveType_RealDiv, schema::ActivationType_RELU6, ElementDivRelu6Fp16, ElementOptDivRelu6Fp16}, + {PrimitiveType_RealDiv, schema::ActivationType_NO_ACTIVATION, ElementDivFp16, ElementOptDivFp16}, {PrimitiveType_FloorMod, schema::ActivationType_NO_ACTIVATION, ElementFloorModFp16, ElementOptFloorModFp16}, {PrimitiveType_FloorDiv, schema::ActivationType_NO_ACTIVATION, ElementFloorDivFp16, ElementOptFloorDivFp16}, {PrimitiveType_LogicalAnd, schema::ActivationType_NO_ACTIVATION, ElementLogicalAndFp16, ElementOptLogicalAndFp16}, @@ -214,5 +218,6 @@ REG_KERNEL(kCPU, kNumberTypeFloat16, PrimitiveType_LogicalOr, LiteKernelCreator< REG_KERNEL(kCPU, kNumberTypeFloat16, PrimitiveType_Maximum, LiteKernelCreator) REG_KERNEL(kCPU, kNumberTypeFloat16, PrimitiveType_Minimum, LiteKernelCreator) REG_KERNEL(kCPU, kNumberTypeFloat16, PrimitiveType_Eltwise, LiteKernelCreator) +REG_KERNEL(kCPU, kNumberTypeFloat16, PrimitiveType_RealDiv, LiteKernelCreator) REG_KERNEL(kCPU, kNumberTypeFloat16, PrimitiveType_SquaredDifference, LiteKernelCreator) } // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/biasadd_fp16.cc b/mindspore/lite/src/runtime/kernel/arm/fp16/biasadd_fp16.cc index 1d819c1611d..66d558ef2bd 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/biasadd_fp16.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/biasadd_fp16.cc @@ -52,6 +52,12 @@ int BiasAddCPUFp16Kernel::Run() { return ret; } } + if (op_parameter_->is_train_session_) { + if ((is_trainable() && (IsTrain() || is_repack())) || (bias_data_type_ == kNumberTypeFloat16)) { + PackWeight(); + is_repack_ = false; + } + } auto in = reinterpret_cast(in_tensors_.at(0)->MutableData()); auto out = reinterpret_cast(out_tensors_.at(0)->MutableData()); size_t data_size = in_tensors_.at(0)->ElementsNum(); @@ -80,10 +86,12 @@ BiasAddCPUFp16Kernel::~BiasAddCPUFp16Kernel() { int BiasAddCPUFp16Kernel::GetBiasData() { bias_data_type_ = bias_tensor_->data_type(); if (bias_data_type_ == kNumberTypeFloat || bias_data_type_ == kNumberTypeFloat32) { - bias_data_ = reinterpret_cast(malloc(bias_tensor_->ElementsNum() * sizeof(float16_t))); if (bias_data_ == nullptr) { - MS_LOG(ERROR) << "bias_data_ is nullptr"; - return RET_NULL_PTR; + bias_data_ = reinterpret_cast(malloc(bias_tensor_->ElementsNum() * sizeof(float16_t))); + if (bias_data_ == nullptr) { + MS_LOG(ERROR) << "bias_data_ is nullptr"; + return RET_NULL_PTR; + } } auto bias = reinterpret_cast(bias_tensor_->MutableData()); if (bias == nullptr) { @@ -91,7 +99,7 @@ int BiasAddCPUFp16Kernel::GetBiasData() { return RET_NULL_PTR; } for (int i = 0; i < bias_tensor_->ElementsNum(); ++i) { - bias_data_[i] = (float16_t)(bias[i]); + bias_data_[i] = static_cast(bias[i]); } } else { bias_data_ = reinterpret_cast(bias_tensor_->MutableData()); @@ -106,18 +114,30 @@ int BiasAddCPUFp16Kernel::GetBiasData() { int BiasAddCPUFp16Kernel::Init() { bias_tensor_ = in_tensors_.at(1); MS_ASSERT(bias_tensor_ != nullptr); - if (bias_tensor_->IsConst()) { - auto ret = GetBiasData(); - if (ret != RET_OK) { - MS_LOG(ERROR) << "GetBiasData is error in Init()!"; - return ret; - } - } if (!InferShapeDone()) { return RET_OK; } return ReSize(); } +void BiasAddCPUFp16Kernel::PackWeight() { + if (bias_data_type_ == kNumberTypeFloat || bias_data_type_ == kNumberTypeFloat32) { + auto bias = reinterpret_cast(bias_tensor_->data_c()); + for (int i = 0; i < bias_tensor_->ElementsNum(); ++i) { + bias_data_[i] = static_cast(bias[i]); + } + } else { + bias_data_ = reinterpret_cast(bias_tensor_->data_c()); + } +} + +int BiasAddCPUFp16Kernel::Eval() { + InnerKernel::Eval(); + if (is_trainable()) { + is_repack_ = true; + } + return RET_OK; +} + REG_KERNEL(kCPU, kNumberTypeFloat16, PrimitiveType_BiasAdd, LiteKernelCreator) } // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/biasadd_fp16.h b/mindspore/lite/src/runtime/kernel/arm/fp16/biasadd_fp16.h index d1d3c5eb3a7..23725f5a04a 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/biasadd_fp16.h +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/biasadd_fp16.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_BIASADD_H_ -#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_BIASADD_H_ +#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_BIASADD_FP16_H_ +#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_BIASADD_FP16_H_ #include #include "src/inner_kernel.h" #include "nnacl/fp16/arithmetic_fp16.h" @@ -33,14 +33,18 @@ class BiasAddCPUFp16Kernel : public InnerKernel { int Init() override; int ReSize() override; int Run() override; + int Eval() override; private: int GetBiasData(); + void PackWeight(); + bool is_repack() { return is_repack_; } ArithmeticParameter *bias_param_ = nullptr; float16_t *bias_data_ = nullptr; lite::Tensor *bias_tensor_ = nullptr; TypeId bias_data_type_; + bool is_repack_ = false; }; } // namespace mindspore::kernel -#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_BIASADD_H_ +#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_BIASADD_FP16_H_ diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_1x1_fp16.cc b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_1x1_fp16.cc index 97063e80d7d..2c74cfef369 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_1x1_fp16.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_1x1_fp16.cc @@ -90,24 +90,31 @@ int Convolution1x1FP16CPUKernel::InitWeightBias() { if (in_tensors_.size() == 3) { size_t size = UP_ROUND(output_channel, col_tile_) * sizeof(float16_t); size_t bias_size = output_channel * sizeof(float16_t); - bias_data_ = malloc(size); if (bias_data_ == nullptr) { - MS_LOG(ERROR) << "Conv1x1 Malloc bias_ptr_ error!"; - return RET_ERROR; + bias_data_ = malloc(size); + if (bias_data_ == nullptr) { + MS_LOG(ERROR) << "Conv1x1 Malloc bias_ptr_ error!"; + return RET_ERROR; + } } - memcpy(bias_data_, origin_bias_, output_channel * sizeof(float16_t)); + void *bias_origin_tmp = is_trainable() ? in_tensors_.at(2)->data_c() : origin_bias_; + memcpy(bias_data_, bias_origin_tmp, output_channel * sizeof(float16_t)); memset(reinterpret_cast(bias_data_) + bias_size, 0, size - bias_size); } size_t size = input_channel * UP_ROUND(output_channel, col_tile_) * sizeof(float16_t); size_t down_size = input_channel * DOWN_DIV(output_channel, col_tile_) * col_tile_ * sizeof(float16_t); - weight_ptr_ = reinterpret_cast(malloc(size)); if (weight_ptr_ == nullptr) { - MS_LOG(ERROR) << "Conv1x1 Malloc weight_ptr_ error!"; - return RET_ERROR; + weight_ptr_ = reinterpret_cast(malloc(size)); + if (weight_ptr_ == nullptr) { + MS_LOG(ERROR) << "Conv1x1 Malloc weight_ptr_ error!"; + return RET_ERROR; + } } + void *weight_origin_tmp = is_trainable() ? weight_tensor->data_c() : origin_weight_; memset(reinterpret_cast(weight_ptr_) + down_size, 0, size - down_size); - ColMajor2Row8MajorFp16(origin_weight_, weight_ptr_, input_channel, output_channel, true); + ColMajor2Row8MajorFp16(weight_origin_tmp, weight_ptr_, input_channel, output_channel, true); + return RET_OK; } @@ -233,6 +240,15 @@ int Convolution1x1FP16CPUKernel::Run() { return RET_MEMORY_FAILED; } + if (is_trainable() && (IsTrain() || is_repack())) { + auto ret = InitWeightBias(); + if (ret != 0) { + MS_LOG(ERROR) << "Convolution 1x1 fp16 repack weight failure"; + return RET_ERROR; + } + is_repack_ = false; + } + for (int batch_index = 0; batch_index < conv_param_->input_batch_; batch_index++) { output_ptr_ = output_data + batch_index * matmul_param_->row_ * matmul_param_->col_; float16_t *batch_in = @@ -263,9 +279,16 @@ int Convolution1x1FP16CPUKernel::Run() { return ret; } } - ctx_->allocator->Free(pack_input_); pack_input_ = nullptr; return RET_OK; } + +int Convolution1x1FP16CPUKernel::Eval() { + if (is_trainable()) { + is_repack_ = true; + } + return InnerKernel::Eval(); +} + } // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_1x1_fp16.h b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_1x1_fp16.h index ef8cb0886d6..822572aba2a 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_1x1_fp16.h +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_1x1_fp16.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_1x1_FP16_H_ -#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_1x1_FP16_H_ +#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_1X1_FP16_H_ +#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_1X1_FP16_H_ #include #include @@ -39,6 +39,7 @@ class Convolution1x1FP16CPUKernel : public ConvolutionBaseCPUKernel { int Init() override; int ReSize() override; int Run() override; + int Eval() override; public: int RunOc(int task_id); @@ -67,4 +68,4 @@ class Convolution1x1FP16CPUKernel : public ConvolutionBaseCPUKernel { }; } // namespace mindspore::kernel -#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_1x1_FP16_H_ +#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_1X1_FP16_H_ diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_delegate_fp16.h b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_delegate_fp16.h index 0507499b0ba..af9c9b7ef07 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_delegate_fp16.h +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_delegate_fp16.h @@ -47,6 +47,18 @@ class ConvolutionDelegateFP16CPUKernel : public InnerKernel { fp16_conv_kernel_->set_name(name_); return fp16_conv_kernel_->Run(); } + int Eval() override { + InnerKernel::Eval(); + return fp16_conv_kernel_->Eval(); + } + int Train() override { + InnerKernel::Train(); + return fp16_conv_kernel_->Train(); + } + void set_trainable(bool trainable) override { + InnerKernel::set_trainable(trainable); + return fp16_conv_kernel_->set_trainable(trainable); + } void set_in_tensor(lite::Tensor *in_tensor, int index) override { MS_ASSERT(index < in_tensors_.size()); diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_fp16.cc b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_fp16.cc index e949e939a05..1c6ad7b9a26 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_fp16.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_fp16.cc @@ -37,19 +37,22 @@ int ConvolutionDepthwiseFp16CPUKernel::InitWeightBias() { int pack_weight_size = channel * weight_tensor->Height() * weight_tensor->Width(); auto origin_weight = reinterpret_cast(weight_tensor->data_c()); MS_ASSERT(origin_weight != nullptr); - - packed_weight_ = reinterpret_cast(malloc(pack_weight_size * sizeof(float16_t))); if (packed_weight_ == nullptr) { - MS_LOG(ERROR) << "Malloc buffer failed."; - return RET_ERROR; + packed_weight_ = reinterpret_cast(malloc(pack_weight_size * sizeof(float16_t))); + if (packed_weight_ == nullptr) { + MS_LOG(ERROR) << "Malloc buffer failed."; + return RET_ERROR; + } } PackNCHWToNHWCFp16(origin_weight, packed_weight_, 1, weight_tensor->Height() * weight_tensor->Width(), weight_tensor->Batch(), 0, 0); - bias_data_ = reinterpret_cast(malloc(channel * sizeof(float16_t))); if (bias_data_ == nullptr) { - MS_LOG(ERROR) << "Malloc buffer failed."; - return RET_ERROR; + bias_data_ = reinterpret_cast(malloc(channel * sizeof(float16_t))); + if (bias_data_ == nullptr) { + MS_LOG(ERROR) << "Malloc buffer failed."; + return RET_ERROR; + } } memset(bias_data_, 0, channel * sizeof(float16_t)); if (in_tensors_.size() == kInputSize2) { @@ -106,6 +109,14 @@ static int ConvDwFp16Run(void *cdata, int task_id, float lhs_scale, float rhs_sc } int ConvolutionDepthwiseFp16CPUKernel::Run() { + if (is_trainable() && (IsTrain() || is_repack())) { + auto ret = InitWeightBias(); + if (ret != 0) { + MS_LOG(ERROR) << "Convolution depthwise fp16 repack weight failure"; + return RET_ERROR; + } + is_repack_ = false; + } auto ret = static_cast(this->context_) ->thread_pool_->ParallelLaunch(ConvDwFp16Run, this, conv_param_->thread_num_); if (ret != RET_OK) { @@ -113,4 +124,11 @@ int ConvolutionDepthwiseFp16CPUKernel::Run() { } return ret; } + +int ConvolutionDepthwiseFp16CPUKernel::Eval() { + if (is_trainable()) { + is_repack_ = true; + } + return InnerKernel::Eval(); +} } // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_fp16.h b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_fp16.h index adb2836f4b4..4255ff18094 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_fp16.h +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_fp16.h @@ -42,6 +42,7 @@ class ConvolutionDepthwiseFp16CPUKernel : public ConvolutionBaseCPUKernel { int Init() override; int ReSize() override; int Run() override; + int Eval() override; int InitWeightBias(); int Execute(int task_id); diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_slidewindow_fp16.cc b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_slidewindow_fp16.cc index bc3c8c7c2d9..8f0218b5375 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_slidewindow_fp16.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_slidewindow_fp16.cc @@ -64,18 +64,22 @@ int ConvolutionDepthwiseSWFp16CPUKernel::InitWeightBias() { auto origin_weight = reinterpret_cast(weight_tensor->data_c()); MS_ASSERT(origin_weight != nullptr); - packed_weight_ = reinterpret_cast(malloc(pack_weight_size * sizeof(float16_t))); if (packed_weight_ == nullptr) { - MS_LOG(ERROR) << "Malloc buffer failed."; - return RET_ERROR; + packed_weight_ = reinterpret_cast(malloc(pack_weight_size * sizeof(float16_t))); + if (packed_weight_ == nullptr) { + MS_LOG(ERROR) << "Malloc buffer failed."; + return RET_ERROR; + } } PackNCHWFp16ToNC8HW8Fp16(origin_weight, packed_weight_, 1, weight_tensor->Height() * weight_tensor->Width(), weight_tensor->Batch()); - bias_data_ = reinterpret_cast(malloc(C8NUM * OC8 * sizeof(float16_t))); if (bias_data_ == nullptr) { - MS_LOG(ERROR) << "Malloc buffer failed."; - return RET_ERROR; + bias_data_ = reinterpret_cast(malloc(C8NUM * OC8 * sizeof(float16_t))); + if (bias_data_ == nullptr) { + MS_LOG(ERROR) << "Malloc buffer failed."; + return RET_ERROR; + } } memset(bias_data_, 0, C8NUM * OC8 * sizeof(float16_t)); if (in_tensors_.size() == kInputSize2) { @@ -86,7 +90,7 @@ int ConvolutionDepthwiseSWFp16CPUKernel::InitWeightBias() { conv_param_->thread_num_ = MSMIN(thread_count_, OC8); return RET_OK; -} +} // namespace mindspore::kernel int ConvolutionDepthwiseSWFp16CPUKernel::Init() { sliding_ = new (std::nothrow) SlidingWindowParam; @@ -157,6 +161,14 @@ int ConvolutionDepthwiseSWFp16CPUKernel::Run() { packed_output_ = output_ptr; } + if (is_trainable() && (IsTrain() || is_repack())) { + ret = InitWeightBias(); + if (ret != 0) { + MS_LOG(ERROR) << "Convolution depthwise fp16 repack weight failure"; + return RET_ERROR; + } + is_repack_ = false; + } ret = static_cast(this->context_) ->thread_pool_->ParallelLaunch(ConvDwSWFp16Run, this, conv_param_->thread_num_); if (ret != RET_OK) { @@ -179,4 +191,11 @@ void ConvolutionDepthwiseSWFp16CPUKernel::FreePackedInputOutput() { packed_output_ = nullptr; } } + +int ConvolutionDepthwiseSWFp16CPUKernel::Eval() { + if (is_trainable()) { + is_repack_ = true; + } + return InnerKernel::Eval(); +} } // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_slidewindow_fp16.h b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_slidewindow_fp16.h index a3af802a860..94a8071bd99 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_slidewindow_fp16.h +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_depthwise_slidewindow_fp16.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_DEPTHWISE_SW_FP16_H_ -#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_DEPTHWISE_SW_FP16_H_ +#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_DEPTHWISE_SLIDEWINDOW_FP16_H_ +#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_DEPTHWISE_SLIDEWINDOW_FP16_H_ #include #include "src/inner_kernel.h" @@ -43,6 +43,7 @@ class ConvolutionDepthwiseSWFp16CPUKernel : public ConvolutionBaseCPUKernel { int Init() override; int ReSize() override; int Run() override; + int Eval() override; int InitPackedInputOutput(); int InitWeightBias(); @@ -58,4 +59,4 @@ class ConvolutionDepthwiseSWFp16CPUKernel : public ConvolutionBaseCPUKernel { }; } // namespace mindspore::kernel -#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_DEPTHWISE_SW_FP16_H_ +#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_DEPTHWISE_SLIDEWINDOW_FP16_H_ diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_fp16.cc b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_fp16.cc index 129df8f2684..f01c9eca977 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_fp16.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_fp16.cc @@ -38,23 +38,30 @@ int ConvolutionFP16CPUKernel::InitWeightBias() { int pack_weight_size = oc8 * in_channel * kernel_plane; // init weight - packed_weight_ = reinterpret_cast(malloc(pack_weight_size * sizeof(float16_t))); if (packed_weight_ == nullptr) { - MS_LOG(ERROR) << "malloc packed_weight_ failed."; - return RET_ERROR; + packed_weight_ = reinterpret_cast(malloc(pack_weight_size * sizeof(float16_t))); + if (packed_weight_ == nullptr) { + MS_LOG(ERROR) << "malloc packed_weight_ failed."; + return RET_ERROR; + } } memset(packed_weight_, 0, pack_weight_size * sizeof(float16_t)); - RowMajor2Col8MajorFp16(origin_weight_, packed_weight_, out_channel, in_channel * kernel_plane, false); + void *weight_origin_tmp = is_trainable() ? filter_tensor->data_c() : origin_weight_; + RowMajor2Col8MajorFp16(weight_origin_tmp, packed_weight_, out_channel, in_channel * kernel_plane, false); // init bias - bias_data_ = malloc(oc8 * sizeof(float16_t)); if (bias_data_ == nullptr) { - MS_LOG(ERROR) << "malloc bias_data_ failed."; - return RET_ERROR; + bias_data_ = malloc(oc8 * sizeof(float16_t)); + if (bias_data_ == nullptr) { + MS_LOG(ERROR) << "malloc bias_data_ failed."; + return RET_ERROR; + } } memset(bias_data_, 0, oc8 * sizeof(float16_t)); if (in_tensors_.size() == kInputSize2) { - memcpy(bias_data_, origin_bias_, out_channel * sizeof(float16_t)); + auto bias_tensor = in_tensors_.at(kBiasIndex); + void *bias_origin_tmp = is_trainable() ? bias_tensor->data_c() : origin_bias_; + memcpy(bias_data_, bias_origin_tmp, out_channel * sizeof(float16_t)); } return RET_OK; } @@ -145,6 +152,14 @@ int ConvolutionFP16CPUKernel::Run() { return RET_ERROR; } + if (is_trainable() && (IsTrain() || is_repack())) { + ret = InitWeightBias(); + if (ret != 0) { + MS_LOG(ERROR) << "Convolution 1x1 fp16 repack weight failure"; + return RET_ERROR; + } + is_repack_ = false; + } ret = static_cast(this->context_) ->thread_pool_->ParallelLaunch(ConvolutionFp16Impl, this, thread_count_); if (ret != RET_OK) { @@ -154,4 +169,11 @@ int ConvolutionFP16CPUKernel::Run() { FreeTmpBuffer(); return ret; } + +int ConvolutionFP16CPUKernel::Eval() { + if (is_trainable()) { + is_repack_ = true; + } + return InnerKernel::Eval(); +} } // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_fp16.h b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_fp16.h index 185270ceb7e..011976a2314 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_fp16.h +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_fp16.h @@ -41,6 +41,7 @@ class ConvolutionFP16CPUKernel : public ConvolutionBaseCPUKernel { int Init() override; int ReSize() override; int Run() override; + int Eval() override; int RunImpl(int task_id); int InitWeightBias(); int InitTmpBuffer(); diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_winograd_fp16.cc b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_winograd_fp16.cc index 1dac5509f5f..2467e44c1f4 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_winograd_fp16.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_winograd_fp16.cc @@ -41,10 +41,12 @@ int ConvolutionWinogradFP16CPUKernel::InitWeightBias() { // init weight // set data auto trans_matrix_data_size = input_unit_ * input_unit_ * in_channel * oc_block_num * col_tile_ * sizeof(float16_t); - trans_weight_ = reinterpret_cast(malloc(trans_matrix_data_size)); if (trans_weight_ == nullptr) { - MS_LOG(ERROR) << "malloc trans_weight_ failed."; - return RET_ERROR; + trans_weight_ = reinterpret_cast(malloc(trans_matrix_data_size)); + if (trans_weight_ == nullptr) { + MS_LOG(ERROR) << "malloc trans_weight_ failed."; + return RET_ERROR; + } } memset(trans_weight_, 0, trans_matrix_data_size); @@ -64,21 +66,26 @@ int ConvolutionWinogradFP16CPUKernel::InitWeightBias() { MS_LOG(ERROR) << "get matrix g from CookToomFilter failed."; return ret; } - ret = WinogradFilterTransformFp16(reinterpret_cast(origin_weight_), matrix_g, matrix_gt, col_tile_); + void *weight_origin_tmp = is_trainable() ? weight_tensor->data_c() : origin_weight_; + ret = WinogradFilterTransformFp16(reinterpret_cast(weight_origin_tmp), matrix_g, matrix_gt, col_tile_); if (ret != RET_OK) { MS_LOG(ERROR) << "winograd filter transform failed."; return ret; } // init bias - bias_data_ = malloc(oc_block_num * col_tile_ * sizeof(float16_t)); if (bias_data_ == nullptr) { - MS_LOG(ERROR) << "malloc bias_data_ failed."; - return RET_ERROR; + bias_data_ = malloc(oc_block_num * col_tile_ * sizeof(float16_t)); + if (bias_data_ == nullptr) { + MS_LOG(ERROR) << "malloc bias_data_ failed."; + return RET_ERROR; + } } memset(bias_data_, 0, oc_block_num * col_tile_ * sizeof(float16_t)); if (in_tensors_.size() == kInputSize2) { - memcpy(bias_data_, origin_bias_, out_channel * sizeof(float16_t)); + auto bias_tensor = in_tensors_.at(kBiasIndex); + void *bias_origin_tmp = is_trainable() ? bias_tensor->data_c() : origin_bias_; + memcpy(bias_data_, bias_origin_tmp, out_channel * sizeof(float16_t)); } return RET_OK; } @@ -222,14 +229,27 @@ int ConvolutionWinogradFP16CPUKernel::Run() { FreeTmpBuffer(); return RET_ERROR; } - + if (is_trainable() && (IsTrain() || is_repack())) { + ret = InitWeightBias(); + if (ret != 0) { + MS_LOG(ERROR) << "ConvolutionWinogradFP16 repack weight failure"; + return RET_ERROR; + } + is_repack_ = false; + } ret = static_cast(this->context_) ->thread_pool_->ParallelLaunch(ConvolutionWinogradFp16Impl, this, thread_count_); if (ret != RET_OK) { MS_LOG(ERROR) << "conv winograd error error_code[" << ret << "]"; } - FreeTmpBuffer(); return ret; } + +int ConvolutionWinogradFP16CPUKernel::Eval() { + if (is_trainable()) { + is_repack_ = true; + } + return InnerKernel::Eval(); +} } // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_winograd_fp16.h b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_winograd_fp16.h index ff013be3103..c41cea67f0d 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_winograd_fp16.h +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/convolution_winograd_fp16.h @@ -46,6 +46,7 @@ class ConvolutionWinogradFP16CPUKernel : public ConvolutionBaseCPUKernel { int Init() override; int ReSize() override; int Run() override; + int Eval() override; int RunImpl(int task_id); int InitWeightBias(); int InitTmpBuffer(); diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/fused_batchnorm_fp16.cc b/mindspore/lite/src/runtime/kernel/arm/fp16/fused_batchnorm_fp16.cc index 013023fe705..259513e5d3f 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/fused_batchnorm_fp16.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/fused_batchnorm_fp16.cc @@ -25,6 +25,29 @@ using mindspore::lite::RET_OK; using mindspore::schema::PrimitiveType_FusedBatchNorm; namespace mindspore::kernel { + +void FusedBatchnormFp16CPUKernel::CalcMeanVar(float16_t *in, float16_t *scale, float16_t *offset, float16_t *save_mean, + float16_t *save_variance) { + auto param = reinterpret_cast(op_parameter_); + float16_t *current_mean = static_cast(mean_); + float16_t *current_var = static_cast(variance_); + + std::fill(current_mean, current_mean + in_tensors_.at(3)->ElementsNum(), 0.f); + std::fill(current_var, current_var + in_tensors_.at(4)->ElementsNum(), 0.f); + FusedBatchNormFp16MeanVar(in, current_mean, current_var, param, save_mean, save_variance); + + memcpy(out_tensors_.at(1)->data_c(), scale, out_tensors_.at(1)->Size()); + memcpy(out_tensors_.at(2)->data_c(), offset, out_tensors_.at(2)->Size()); + memcpy(out_tensors_.at(3)->data_c(), current_mean, out_tensors_.at(3)->Size()); + memcpy(out_tensors_.at(4)->data_c(), current_var, out_tensors_.at(4)->Size()); + + // Copy to local variables + memcpy(scale_, scale, in_tensors_[1]->Size()); + memcpy(offset_, offset, in_tensors_[2]->Size()); + + trained_ = true; // trained at least once +} + int FusedBatchnormFp16CPUKernel::DoExecute(int task_id) { auto param = reinterpret_cast(op_parameter_); MS_ASSERT(param); @@ -54,18 +77,25 @@ int FusedBatchnormFp16CPUKernel::DoExecute(int task_id) { context_->allocator->Free(output_fp16); return RET_ERROR; } - Float32ToFloat16(reinterpret_cast(input->MutableData()), reinterpret_cast(input_fp16), + Float32ToFloat16(reinterpret_cast(input->data_c()), reinterpret_cast(input_fp16), input->ElementsNum()); - Float32ToFloat16(reinterpret_cast(scale->MutableData()), reinterpret_cast(scale_fp16), + Float32ToFloat16(reinterpret_cast(scale->data_c()), reinterpret_cast(scale_fp16), scale->ElementsNum()); - Float32ToFloat16(reinterpret_cast(offset->MutableData()), reinterpret_cast(offset_fp16), + Float32ToFloat16(reinterpret_cast(offset->data_c()), reinterpret_cast(offset_fp16), offset->ElementsNum()); - Float32ToFloat16(reinterpret_cast(mean->MutableData()), reinterpret_cast(mean_fp16), + Float32ToFloat16(reinterpret_cast(mean->data_c()), reinterpret_cast(mean_fp16), mean->ElementsNum()); - Float32ToFloat16(reinterpret_cast(variance->MutableData()), reinterpret_cast(variance_fp16), + Float32ToFloat16(reinterpret_cast(variance->data_c()), reinterpret_cast(variance_fp16), variance->ElementsNum()); - FusedBatchNormFp16(input_fp16, scale_fp16, offset_fp16, mean_fp16, variance_fp16, param, task_id, output_fp16); + if (IsTrain() && is_trainable() && in_tensors_.size() >= 5) { + CalcMeanVar(reinterpret_cast(input_fp16), reinterpret_cast(scale_fp16), + reinterpret_cast(offset_fp16), reinterpret_cast(mean_fp16), + reinterpret_cast(variance_fp16)); + } + FusedBatchNormFp16(reinterpret_cast(input_fp16), reinterpret_cast(scale_fp16), + reinterpret_cast(offset_fp16), reinterpret_cast(mean_fp16), + reinterpret_cast(variance_fp16), param, task_id, output_fp16); Float16ToFloat32(reinterpret_cast(output_fp16), reinterpret_cast(output), output->ElementsNum()); @@ -77,8 +107,32 @@ int FusedBatchnormFp16CPUKernel::DoExecute(int task_id) { context_->allocator->Free(output_fp16); return RET_OK; } - FusedBatchNormFp16(in_tensors_.at(0)->MutableData(), scale_, offset_, mean_, variance_, param, task_id, - out_tensors_.at(0)->MutableData()); + + if (IsTrain() && is_trainable() && in_tensors_.size() >= 5) { + CalcMeanVar( + static_cast(in_tensors_.at(0)->data_c()), static_cast(in_tensors_.at(1)->data_c()), + static_cast(in_tensors_.at(2)->data_c()), static_cast(in_tensors_.at(3)->data_c()), + static_cast(in_tensors_.at(4)->data_c())); + } + FusedBatchNormFp16(in_tensors_.at(0)->data_c(), scale_, offset_, mean_, variance_, param, task_id, + out_tensors_.at(0)->data_c()); + return RET_OK; +} + +int FusedBatchnormFp16CPUKernel::Eval() { + InnerKernel::Eval(); + if (trained_) { + float16_t *save_mean = static_cast(in_tensors_.at(3)->data_c()); + float16_t *save_var = static_cast(in_tensors_.at(4)->data_c()); + float16_t *scale = static_cast(in_tensors_.at(1)->data_c()); + float16_t *bias = static_cast(in_tensors_.at(2)->data_c()); + + // Copy to local variables + memcpy(scale_, scale, in_tensors_.at(1)->Size()); + memcpy(offset_, bias, in_tensors_.at(2)->Size()); + memcpy(mean_, save_mean, in_tensors_.at(3)->Size()); + memcpy(variance_, save_var, in_tensors_.at(4)->Size()); + } return RET_OK; } } // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/fused_batchnorm_fp16.h b/mindspore/lite/src/runtime/kernel/arm/fp16/fused_batchnorm_fp16.h index dc77f74daf8..193c6104e51 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/fused_batchnorm_fp16.h +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/fused_batchnorm_fp16.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,7 +28,11 @@ class FusedBatchnormFp16CPUKernel : public FusedBatchnormCPUKernel { : FusedBatchnormCPUKernel(parameter, inputs, outputs, ctx) {} virtual ~FusedBatchnormFp16CPUKernel() {} - virtual int DoExecute(int task_id); + int DoExecute(int task_id) override; + int Eval() override; + + protected: + void CalcMeanVar(float16_t *in, float16_t *scale, float16_t *offset, float16_t *save_mean, float16_t *save_variance); }; } // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_base_fp16.cc b/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_base_fp16.cc index f6020aec3a2..175ee9587b9 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_base_fp16.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_base_fp16.cc @@ -69,10 +69,12 @@ int MatmulBaseFP16CPUKernel::InitBias() { if (in_tensors_.size() == 3) { auto bias_tensor = in_tensors_[2]; int max_bias_data = UP_ROUND(bias_tensor->ElementsNum(), C8NUM); - bias_ptr_ = reinterpret_cast(malloc(max_bias_data * sizeof(float16_t))); if (bias_ptr_ == nullptr) { - MS_LOG(ERROR) << "malloc bias_ptr_ failed"; - return RET_ERROR; + bias_ptr_ = reinterpret_cast(malloc(max_bias_data * sizeof(float16_t))); + if (bias_ptr_ == nullptr) { + MS_LOG(ERROR) << "malloc bias_ptr_ failed"; + return RET_ERROR; + } } memset(bias_ptr_, 0, max_bias_data * sizeof(float16_t)); if (bias_tensor->data_type() == kNumberTypeFloat32) { @@ -268,18 +270,19 @@ int MatmulBaseFP16CPUKernel::RunImpl(int task_id) { int MatmulBaseFP16CPUKernel::Run() { auto c_ptr = reinterpret_cast(out_tensors_.at(0)->data_c()); - if (params_->a_const_ == false) { + if ((params_->a_const_ == false) || is_repack()) { if (RET_OK != InitBufferA()) { return RET_ERROR; } InitMatrixA(in_tensors_.at(0)->data_c()); } - if (params_->b_const_ == false) { + if ((params_->b_const_ == false) || is_repack()) { if (RET_OK != InitBufferB()) { FreeResizeBufA(); return RET_ERROR; } InitMatrixB(in_tensors_.at(1)->data_c(), in_tensors_.at(1)->data_type()); + InitBias(); } for (int i = 0; i < params_->batch; ++i) { diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_base_fp16.h b/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_base_fp16.h index 78e3eb4e1de..89735a7558e 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_base_fp16.h +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_base_fp16.h @@ -42,6 +42,8 @@ class MatmulBaseFP16CPUKernel : public InnerKernel { protected: void InitParameter(); + bool is_repack() { return is_repack_; } + bool is_repack_ = false; private: int InitBias(); diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_fp16.cc b/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_fp16.cc index 9423e2ad31e..e046211b6f0 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_fp16.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_fp16.cc @@ -86,12 +86,24 @@ int MatmulFP16CPUKernel::ReSize() { } int MatmulFP16CPUKernel::Run() { + if (is_trainable() && (IsTrain())) { + is_repack_ = true; + } auto ret = MatmulBaseFP16CPUKernel::Run(); if (ret != RET_OK) { MS_LOG(ERROR) << "MatmulFP16CPUKernel run failed"; } + is_repack_ = false; return ret; } +int MatmulFP16CPUKernel::Eval() { + InnerKernel::Eval(); + if (is_trainable()) { + is_repack_ = true; + } + return RET_OK; +} + REG_KERNEL(kCPU, kNumberTypeFloat16, PrimitiveType_MatMul, LiteKernelCreator) } // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_fp16.h b/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_fp16.h index a8e7d5b8a7d..b0499dff12a 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_fp16.h +++ b/mindspore/lite/src/runtime/kernel/arm/fp16/matmul_fp16.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_MATMUL_H_ -#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_MATMUL_H_ +#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_MATMUL_FP16_H_ +#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_MATMUL_FP16_H_ #include #include "src/runtime/kernel/arm/fp16/matmul_base_fp16.h" @@ -30,6 +30,7 @@ class MatmulFP16CPUKernel : public MatmulBaseFP16CPUKernel { int Init() override; int ReSize() override; int Run() override; + int Eval() override; private: void InitAShape(); @@ -37,4 +38,4 @@ class MatmulFP16CPUKernel : public MatmulBaseFP16CPUKernel { }; } // namespace mindspore::kernel -#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_MATMUL_H_ +#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_MATMUL_FP16_H_ diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16_grad/convolution_fp16_grad_input.cc b/mindspore/lite/src/runtime/kernel/arm/fp16_grad/convolution_fp16_grad_input.cc index 41a52d4c3cc..213c92955b2 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16_grad/convolution_fp16_grad_input.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp16_grad/convolution_fp16_grad_input.cc @@ -19,6 +19,7 @@ #include "nnacl/pack.h" #include "nnacl/fp16_grad/pack_fp16_ext.h" #include "nnacl/fp16_grad/gemm_fp16.h" +#include "nnacl/fp16_grad/convolution_grad_input.h" #include "include/errorcode.h" using mindspore::kernel::KERNEL_ARCH::kCPU; @@ -64,6 +65,11 @@ int ConvolutionGradInputCPUKernelFp16::ReSize() { ? false : true; + do_dw_ = (conv_param->output_channel_ == conv_param->group_) && + (conv_param->input_channel_ == conv_param->output_channel_) && (conv_param->dilation_h_ == 1) && + (conv_param->dilation_w_ == 1) + ? true + : false; return RET_OK; } @@ -103,6 +109,18 @@ int ConvolutionGradInputCPUKernelFp16::Execute(int task_id) { int start = stride * task_id; int end = start + count; + if (do_dw_) { + stride = UP_DIV(groups, thread_num); + count = MSMIN(stride, groups - stride * task_id); + count = (count < 0) ? 0 : count; + start = stride * task_id; + for (i = 0; i < batch; ++i) { + ConvDwInputGradFp16(dy_addr + (i * groups) * m * k, w_addr, dx_addr + (i * groups) * in_h * in_w, start, count, + conv_param); + } + return RET_OK; + } + for (i = start; i < end; ++i) { for (j = 0; j < groups; ++j) { GemmCbFp16 gcb; diff --git a/mindspore/lite/src/runtime/kernel/arm/fp16_grad/convolution_fp16_grad_input.h b/mindspore/lite/src/runtime/kernel/arm/fp16_grad/convolution_fp16_grad_input.h index fe8f57ecb3f..56ea427f090 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp16_grad/convolution_fp16_grad_input.h +++ b/mindspore/lite/src/runtime/kernel/arm/fp16_grad/convolution_fp16_grad_input.h @@ -37,6 +37,7 @@ class ConvolutionGradInputCPUKernelFp16 : public InnerKernel { size_t ws_size_ = 0; size_t mat_alloc_ = 0; bool do_img2col_ = true; + bool do_dw_ = false; const int chunk_ = C12NUM; }; } // namespace mindspore::kernel diff --git a/mindspore/lite/src/runtime/kernel/arm/fp32_grad/convolution_grad_input.cc b/mindspore/lite/src/runtime/kernel/arm/fp32_grad/convolution_grad_input.cc index e92b32b465a..1da08eb6387 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp32_grad/convolution_grad_input.cc +++ b/mindspore/lite/src/runtime/kernel/arm/fp32_grad/convolution_grad_input.cc @@ -19,6 +19,7 @@ #include "nnacl/pack.h" #include "nnacl/fp32_grad/pack_ext.h" #include "nnacl/fp32_grad/gemm.h" +#include "nnacl/fp32_grad/convolution_grad_input.h" #include "include/errorcode.h" using mindspore::kernel::KERNEL_ARCH; @@ -56,6 +57,20 @@ int ConvolutionGradInputCPUKernel::ReSize() { int thread_num = op_parameter_->thread_num_; mat_alloc_ = MatSizeTotal(chunk_, n, k, 0); set_workspace_size((ws_size_ + mat_alloc_) * sizeof(float) * thread_num); + + do_img2col_ = (conv_param->kernel_h_ == 1) && (conv_param->kernel_w_ == 1) && (conv_param->pad_d_ == 0) && + (conv_param->pad_u_ == 0) && (conv_param->pad_l_ == 0) && (conv_param->pad_r_ == 0) && + (conv_param->dilation_h_ == 1) && (conv_param->dilation_w_ == 1) && (conv_param->stride_h_ == 1) && + (conv_param->stride_w_ == 1) && (conv_param->group_ == 1) + ? false + : true; + + do_dw_ = (conv_param->output_channel_ == conv_param->group_) && + (conv_param->input_channel_ == conv_param->output_channel_) && (conv_param->dilation_h_ == 1) && + (conv_param->dilation_w_ == 1) + ? true + : false; + return RET_OK; } @@ -95,6 +110,18 @@ int ConvolutionGradInputCPUKernel::Execute(int task_id) { int start = stride * task_id; int end = start + count; + if (do_dw_) { + stride = UP_DIV(groups, thread_num); + count = MSMIN(stride, groups - stride * task_id); + count = (count < 0) ? 0 : count; + start = stride * task_id; + for (i = 0; i < batch; ++i) { + ConvDwInputGrad(dy_addr + (i * groups) * m * k, w_addr, dx_addr + (i * groups) * in_h * in_w, start, count, + conv_param); + } + return RET_OK; + } + for (i = start; i < end; ++i) { for (j = 0; j < groups; ++j) { GemmCb gcb; @@ -112,10 +139,16 @@ int ConvolutionGradInputCPUKernel::Execute(int task_id) { } int real_chunk = MSMIN(m - ci, chunk_); float *mat_a = dy_addr + (i * groups) * m * k + j * (out_ch / groups) + ci * out_ch; - float *mat_c = workspace_temp; - GemmMatmulPlus(0, 0, real_chunk, n, k, 1, mat_a, out_ch, mat_b, n, 0, mat_c, n, mat_workspace, &gcb); - rolling_col2im_hwc(mat_c, dx_addr + (i * groups) * (in_ch / groups) * in_h * in_w + j * (in_ch / groups), - conv_param, real_chunk, ci); + if (do_img2col_) { + float *mat_c = workspace_temp; + GemmMatmulPlus(0, 0, real_chunk, n, k, 1, mat_a, out_ch, mat_b, n, 0, mat_c, n, mat_workspace, &gcb); + rolling_col2im_hwc(mat_c, dx_addr + (i * groups) * (in_ch / groups) * in_h * in_w + j * (in_ch / groups), + conv_param, real_chunk, ci); + } else { + float *mat_c = + dx_addr + (i * groups) * (in_ch / groups) * in_h * in_w + j * (in_ch / groups) + ci * (in_ch / groups); + GemmMatmulPlus(0, 0, real_chunk, n, k, 1, mat_a, out_ch, mat_b, n, 0, mat_c, n, mat_workspace, &gcb); + } } } } diff --git a/mindspore/lite/src/runtime/kernel/arm/fp32_grad/convolution_grad_input.h b/mindspore/lite/src/runtime/kernel/arm/fp32_grad/convolution_grad_input.h index b6db92fc504..b2eee1494b9 100644 --- a/mindspore/lite/src/runtime/kernel/arm/fp32_grad/convolution_grad_input.h +++ b/mindspore/lite/src/runtime/kernel/arm/fp32_grad/convolution_grad_input.h @@ -36,6 +36,8 @@ class ConvolutionGradInputCPUKernel : public InnerKernel { private: size_t ws_size_ = 0; size_t mat_alloc_ = 0; + bool do_img2col_ = true; + bool do_dw_ = false; #ifdef ENABLE_ARM32 const int chunk_ = C4NUM; #else diff --git a/mindspore/lite/src/scheduler.cc b/mindspore/lite/src/scheduler.cc index 70ec73a1147..744a803630d 100644 --- a/mindspore/lite/src/scheduler.cc +++ b/mindspore/lite/src/scheduler.cc @@ -428,12 +428,12 @@ int Scheduler::FindCpuKernel(const std::vector &in_tensors, const std: } std::map restored_origin_tensors; + ret = CastConstTensorsData(in_tensors, &restored_origin_tensors, kernel_data_type); + if (ret != RET_OK) { + MS_LOG(DEBUG) << "CastConstTensorsData failed: " << ret; + return RET_NOT_SUPPORT; + } if (!is_train_session_) { - ret = CastConstTensorsData(in_tensors, &restored_origin_tensors, kernel_data_type); - if (ret != RET_OK) { - MS_LOG(DEBUG) << "CastConstTensorsData failed: " << ret; - return RET_NOT_SUPPORT; - } // we don't need to restore tensor for copy data ret = CopyConstTensorData(in_tensors, op_type); if (ret != RET_OK) { @@ -444,7 +444,11 @@ int Scheduler::FindCpuKernel(const std::vector &in_tensors, const std: ret = KernelRegistry::GetInstance()->GetKernel(in_tensors, out_tensors, context_, cpu_desc, op_parameter, kernel); if (ret == RET_OK) { MS_LOG(DEBUG) << "Get TypeId(" << kernel_data_type << ") op success: " << PrimitiveCurVersionTypeName(op_type); - FreeRestoreTensors(&restored_origin_tensors); + if (is_train_session_) { + RestoreTensorData(&restored_origin_tensors); + } else { + FreeRestoreTensors(&restored_origin_tensors); + } } else { RestoreTensorData(&restored_origin_tensors); } @@ -612,7 +616,8 @@ kernel::LiteKernel *Scheduler::FindBackendKernel(const std::vector &in } } #endif - if (prefer_data_type == kNumberTypeFloat16 || prefer_data_type == kTypeUnknown) { + if ((prefer_data_type == kNumberTypeFloat16 || prefer_data_type == kTypeUnknown) && + ((is_train_session_ == false) || (sched_cb_ && sched_cb_->SchedFp16Kernel(node)))) { status = FindCpuKernel(in_tensors, out_tensors, op_parameter, desc, kNumberTypeFloat16, &kernel); if (status == RET_OK) { return kernel; diff --git a/mindspore/lite/src/scheduler.h b/mindspore/lite/src/scheduler.h index 2c6649d439e..1e1d29b484c 100644 --- a/mindspore/lite/src/scheduler.h +++ b/mindspore/lite/src/scheduler.h @@ -24,6 +24,8 @@ #include "src/sub_graph_kernel.h" #include "src/inner_context.h" #include "include/model.h" +#include "src/scheduler_cb.h" + #if SUPPORT_NPU #include "src/runtime/agent/npu/optimizer/npu_pass_manager.h" #endif @@ -54,6 +56,7 @@ class Scheduler { ~Scheduler() = default; int Schedule(std::vector *dst_kernels); + void SetupSchedulerCb(std::unique_ptr cb) { sched_cb_ = std::move(cb); } private: void FindNodeInoutTensors(const lite::Model::Node &node, std::vector *inputs, @@ -132,6 +135,7 @@ class Scheduler { std::vector graph_output_node_indexes_; std::map op_parameters_; bool is_train_session_ = false; + std::unique_ptr sched_cb_; std::map primitives_; std::shared_ptr delegate_ = nullptr; }; diff --git a/mindspore/lite/src/scheduler_cb.h b/mindspore/lite/src/scheduler_cb.h new file mode 100644 index 00000000000..93c3850d1e4 --- /dev/null +++ b/mindspore/lite/src/scheduler_cb.h @@ -0,0 +1,43 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_SRC_SCHEDELER_CB_H_ +#define MINDSPORE_LITE_SRC_SCHEDELER_CB_H_ + +#include +#include "include/model.h" + +namespace mindspore::lite { +using SchedCallBack = std::function; + +class SchedulerCb { + public: + explicit SchedulerCb(SchedCallBack fp16_cb) : fp16_cb_(fp16_cb) {} + + ~SchedulerCb() = default; + bool SchedFp16Kernel(const Model::Node *node) { + if (fp16_cb_) { + return fp16_cb_(node); + } + return false; + } + + private: + SchedCallBack fp16_cb_; +}; +} // namespace mindspore::lite + +#endif // MINDSPORE_LITE_SRC_SCHEDELER_CB_H_ diff --git a/mindspore/lite/src/tensor.h b/mindspore/lite/src/tensor.h index 88069088388..b7e859fca03 100644 --- a/mindspore/lite/src/tensor.h +++ b/mindspore/lite/src/tensor.h @@ -197,6 +197,27 @@ class Tensor : public mindspore::tensor::MSTensor { void set_own_data(bool own_data) { this->own_data_ = own_data; } + template + int Scale(float scale) { + T cast_scale = static_cast(scale); + auto data = reinterpret_cast(data_c()); + if (data == nullptr) { + return RET_ERROR; + } + int length = ElementsNum(); + for (int i = 0; i < length; i++) { + data[i] *= cast_scale; + } + scale_ *= scale; + return RET_OK; + } + + float get_scale() const { return this->scale_; } + + void set_scale(float scale) { this->scale_ = scale; } + + bool IsScale() const { return (std::abs(this->scale_ - 1.0f) > 1.0e-05); } + private: template std::string DataToString(void *data, size_t data_number) const { @@ -225,6 +246,7 @@ class Tensor : public mindspore::tensor::MSTensor { AllocatorPtr allocator_ = nullptr; Tensor *root_tensor_ = nullptr; bool own_data_{false}; + float scale_ = 1.0f; }; inline size_t DataTypeSize(const TypeId type) { diff --git a/mindspore/lite/src/train/optimizer_kernel.h b/mindspore/lite/src/train/optimizer_kernel.h index 56ec62450e7..938ba059258 100644 --- a/mindspore/lite/src/train/optimizer_kernel.h +++ b/mindspore/lite/src/train/optimizer_kernel.h @@ -16,10 +16,18 @@ #ifndef MINDSPORE_LITE_SRC_TRAIN_OPTIMIZER_KERNEL_H_ #define MINDSPORE_LITE_SRC_TRAIN_OPTIMIZER_KERNEL_H_ #include +#include +#include #include "src/lite_kernel.h" #include "include/errorcode.h" using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; +using mindspore::lite::RET_OUT_OF_TENSOR_RANGE; + +static __attribute__((always_inline)) inline bool MS_ISNAN(float var) { + volatile float d = var; + return d != d; +} namespace mindspore::kernel { @@ -103,6 +111,37 @@ class OptimizerKernel : public InnerKernel { return InnerKernel::Eval(); } + int PreProcess() override { + auto ret = InnerKernel::PreProcess(); + if (ret != RET_OK) { + return ret; + } + + auto ctx = static_cast(this->context_); + if (ctx->IsCpuFloat16Enabled()) { + auto t = in_tensors_.at(grad_idx_); + auto gradient = reinterpret_cast(t->data_c()); + int length = in_tensors_.at(grad_idx_)->ElementsNum(); + + for (int i = 0; i < length; ++i) { + if (MS_ISNAN(gradient[i]) || std::isinf(gradient[i])) { + MS_LOG(INFO) << "optimizer grad is nan or inf"; + return RET_OUT_OF_TENSOR_RANGE; + } + } + + auto is_scale = t->IsScale(); + auto scale = t->get_scale(); + if (is_scale) { + t->set_scale(1.0f / scale); + for (int i = 0; i < length; ++i) { + gradient[i] *= (1.0f / scale); + } + } + } + return RET_OK; + } + protected: float default_lr_ = 0.0f; float lr_ = 0.0f; diff --git a/mindspore/lite/src/train/train_session.cc b/mindspore/lite/src/train/train_session.cc index e520f457d65..b4c947f2367 100644 --- a/mindspore/lite/src/train/train_session.cc +++ b/mindspore/lite/src/train/train_session.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include "include/errorcode.h" #include "src/common/utils.h" #include "src/tensor.h" @@ -38,12 +39,16 @@ #include "src/common/tensor_util.h" #include "src/train/train_utils.h" #include "src/train/train_export.h" +#include "src/common/prim_util.h" namespace mindspore { namespace lite { +const char *kGradName = "Gradients"; +const char *kOptimizerName = "optimizer"; TrainSession::TrainSession() { is_train_session_ = true; + InitCallBack(); #ifdef ENABLE_V0 if (VersionManager::GetInstance()->CheckV0Schema()) { kernel::PopulateTrainV0Parameters(); @@ -56,7 +61,7 @@ TrainSession::TrainSession() { int TrainSession::Init(const Context *context, const TrainCfg *train_cfg) { if (train_cfg != nullptr) { - train_cfg_ = *train_cfg; + cfg_ = *train_cfg; } return lite::LiteSession::Init(context); } @@ -82,22 +87,88 @@ void TrainSession::RestoreOps(const std::vector &restore) { } } -void TrainSession::AllocWorkSpace() { +int TrainSession::AllocWorkSpace() { size_t workspace_size = 0; + for (auto kernel : this->train_kernels_) { if (workspace_size < static_cast(kernel->kernel())->workspace_size()) { workspace_size = static_cast(kernel->kernel())->workspace_size(); } } - mindspore::kernel::InnerKernel::AllocWorkspace(workspace_size); + workspace_ = malloc(workspace_size); + if (workspace_ == nullptr) { + MS_LOG(ERROR) << "cannot allocate " << workspace_size << " for workspace"; + return RET_ERROR; + } + for (auto kernel : this->train_kernels_) { + static_cast(kernel->kernel())->set_workspace(workspace_); + } + return RET_OK; +} + +void TrainSession::FreeWorkSpace() { + free(workspace_); + for (auto kernel : this->train_kernels_) { + static_cast(kernel->kernel())->FreeWorkspace(); + } +} + +int TrainSession::InitCallBack() { + sched_mix_precision_callback_ = [&](const Model::Node *node) { + auto node_type = GetPrimitiveType(node->primitive_); + if (node_type == schema::PrimitiveType_Cast) { + return false; + } + TensorPtrVector inputs; + auto in_size = node->input_indices_.size(); + inputs.reserve(in_size); + for (size_t k = 0; k < in_size; ++k) { + inputs.emplace_back(model_->all_tensors_.at(node->input_indices_[k])); + } + bool force_fp16 = (inputs.size() > 0 && std::any_of(inputs.begin(), inputs.end(), + [&](schema::Tensor *tensor) { + return ((tensor->dataType() == kNumberTypeFloat16) && + (tensor->nodeType() == NodeType_ValueNode)); + })) + ? true + : false; + inputs.clear(); + auto node_name = node->name_; + + bool is_fp16 = true; + if (!force_fp16) { + // optimizer runs in fp32 + if (node_name.find(kOptimizerName) != std::string::npos) { + is_fp16 = false; + } + // loss function runs in fp32 + if ((node_name.find(get_loss_name()) != std::string::npos)) { + is_fp16 = false; + } + // run bn according to user configuration + if ((cfg_.mix_precision_cfg_.keep_batchnorm_fp32_) && + (node_type == schema::PrimitiveType_FusedBatchNorm || node_type == schema::PrimitiveType_BatchNorm || + node_type == schema::PrimitiveType_BatchNormGrad)) { + is_fp16 = false; + } + } + MS_LOG(DEBUG) << "Debug: " << node_name << ((is_fp16) ? " fp16" : " fp32"); + return is_fp16; + }; + return RET_OK; } int TrainSession::CompileGraph(lite::Model *model) { return lite::RET_ERROR; } int TrainSession::CompileTrainGraph(mindspore::lite::Model *model) { model_ = model; - auto restore = ReplaceOps(); + sched_cb_ = std::make_unique(sched_mix_precision_callback_); + if (sched_cb_ == nullptr) { + MS_LOG(ERROR) << "Failed to create SchedulerCb node"; + return RET_ERROR; + } + auto ret = lite::LiteSession::CompileGraph(model); if (ret != RET_OK) { MS_LOG(ERROR) << "failed to compile train model"; @@ -113,26 +184,190 @@ int TrainSession::CompileTrainGraph(mindspore::lite::Model *model) { CompileTrainOutputs(); // prepare outputs in train mode CompileEvalOutputs(); // prepare outputs in eval mode CompileInferenceKernels(); // Prepare a list of eval kernels - AllocWorkSpace(); - + ret = AllocWorkSpace(); + if (ret != RET_OK) { + MS_LOG(ERROR) << "failed to allocate space"; + return RET_ERROR; + } return RET_OK; } TrainSession::~TrainSession() { - mindspore::kernel::InnerKernel::FreeWorkspace(); + FreeWorkSpace(); if (model_ != nullptr) { delete model_; model_ = nullptr; } } +int TrainSession::ExecKernels(const KernelCallBack &before, const KernelCallBack &after, + const std::vector &run_kernels) { + for (auto *kernel : run_kernels) { + MS_ASSERT(nullptr != kernel); + auto ret = kernel->Execute(before, after); + if (RET_OK != ret) { + MS_LOG(ERROR) << "Execute kernel failed, name: " << kernel->name(); + return ret; + } + } + return RET_OK; +} + +void TrainSession::RestoreTensorData() { + for (auto &restored_origin_tensor : restored_origin_tensors_) { + auto *origin_tensor = restored_origin_tensor.first; + auto *restored_tensor = restored_origin_tensor.second; + MS_ASSERT(origin_tensor != nullptr); + MS_ASSERT(restored_tensor != nullptr); + + bool own_data = restored_tensor->own_data(); + if (origin_tensor->data_c() == nullptr) { + restored_tensor->FreeData(); + } else { + origin_tensor->FreeData(); + } + origin_tensor->set_data_type(restored_tensor->data_type()); + origin_tensor->set_data(restored_tensor->data_c()); + origin_tensor->set_own_data(own_data); + } +} + +void TrainSession::FreeRestoreTensors() { + for (auto &restored_origin_tensor : restored_origin_tensors_) { + auto *restored_tensor = restored_origin_tensor.second; + restored_tensor->set_data(nullptr); + delete (restored_tensor); + } + restored_origin_tensors_.clear(); +} + +bool TrainSession::IsLossTensor(Tensor *tensor) { + MS_ASSERT(tensor != nullptr); + auto t_n = tensor->tensor_name(); + return (t_n.find(get_loss_name()) != std::string::npos); +} + +bool TrainSession::AllInputsNeedScale(kernel::LiteKernel *kernel) { + auto type = kernel->type(); + auto is_scale = false; + + for (auto &tensor : kernel->in_tensors()) { + is_scale |= tensor->IsScale(); + } + + switch (type) { + case schema::PrimitiveType_AbsGrad: + case schema::PrimitiveType_AddFusion: + case schema::PrimitiveType_SubFusion: + case schema::PrimitiveType_AddN: + return (true && is_scale); + default: + return false; + } + return false; +} + +int TrainSession::MixPrecisionPreProcess(kernel::LiteKernel *kernel, float scale) { + auto kernel_type = kernel->desc().data_type; + auto all_scale = AllInputsNeedScale(kernel); + + for (auto &tensor : kernel->in_tensors()) { + if ((tensor->IsScale() == false) && ((!IsLossKernel(kernel) && IsLossTensor(tensor)) || (all_scale == true))) { + ScaleTensor(tensor, scale); + } + // adjust tensor data type + if (tensor->data_type() != kernel_type) { + auto restore_tensor = CastTensor(tensor, kernel_type); + if (restore_tensor != nullptr) { + restored_origin_tensors_[tensor] = restore_tensor; + } + } + } + return RET_OK; +} + +int TrainSession::MixPrecisionPostProcess(kernel::LiteKernel *kernel) { + RestoreTensorData(); + FreeRestoreTensors(); + + float scale = 1.0f; + auto all_scale = AllInputsNeedScale(kernel); + for (auto &tensor : kernel->in_tensors()) { + if (tensor->IsScale()) { + scale *= tensor->get_scale(); + if (all_scale) { + break; + } + } + } + for (auto &tensor : kernel->out_tensors()) { + tensor->set_scale(scale); + } + + for (auto &tensor : kernel->in_tensors()) { + if ((tensor->IsScale() == true) && ((!IsLossKernel(kernel) && IsLossTensor(tensor)) || (all_scale == true))) { + ScaleTensor(tensor, 1.0f / scale); + } + } + return RET_OK; +} + +int TrainSession::MixPrecisionExecKernels(const KernelCallBack &before, const KernelCallBack &after, + const std::vector &run_kernels) { + float scale = cfg_.mix_precision_cfg_.loss_scale_; + for (auto *kernel : run_kernels) { + MS_ASSERT(nullptr != kernel); + MixPrecisionPreProcess(kernel, scale); + auto ret = kernel->Execute(before, after); + if (RET_OK != ret) { + MixPrecisionPostProcess(kernel); + // decrease loss scale in case of nan or inf + if (ret == RET_OUT_OF_TENSOR_RANGE) { + bool is_dynamic_scale = cfg_.mix_precision_cfg_.dynamic_loss_scale_; + cfg_.mix_precision_cfg_.loss_scale_ = std::max(((is_dynamic_scale) ? (scale / 2.f) : scale), 1.0f); + num_of_not_nan_iter_ = 0; + return RET_OK; + } + MS_LOG(ERROR) << "Execute kernel failed, name: " << kernel->name(); + return ret; + } + MixPrecisionPostProcess(kernel); + } + // increase dynamic loss scale if pass pass threshold + if (cfg_.mix_precision_cfg_.dynamic_loss_scale_) { + num_of_not_nan_iter_++; + if (num_of_not_nan_iter_ >= cfg_.mix_precision_cfg_.num_of_not_nan_iter_th_) { + cfg_.mix_precision_cfg_.loss_scale_ = std::min((cfg_.mix_precision_cfg_.loss_scale_ * 2.0f), 65536.0f); + num_of_not_nan_iter_ = 0; + } + } + + // cast output to FP32 + if (train_mode_ == false) { + for (auto t : this->outputs_) { + if (t->data_type() == kNumberTypeFloat16) { + auto restore = CastTensor(t, kNumberTypeFloat32); + delete restore; + } + } + } + return RET_OK; +} + int TrainSession::RunGraph(const KernelCallBack &before, const KernelCallBack &after) { - this->outputs_.clear(); + // check inputs + auto ret = CheckTensorsInvalid(inputs_); + if (ret != RET_OK) { + MS_LOG(ERROR) << "CheckInputs failed"; + return ret; + } // build out tensor - for (auto ms_tensors : output_node_map_) { - for (auto ms_tensor : ms_tensors.second) { - this->outputs_.push_back((static_cast(ms_tensor))); + this->outputs_.clear(); + for (auto &ms_tensors : output_node_map_) { + for (auto &ms_tensor : ms_tensors.second) { + auto lite_tensor = static_cast(ms_tensor); + this->outputs_.push_back(lite_tensor); } } @@ -140,21 +375,15 @@ int TrainSession::RunGraph(const KernelCallBack &before, const KernelCallBack &a MS_LOG(ERROR) << "context is null"; return lite::RET_NULL_PTR; } - auto run_kernel = (train_mode_) ? train_kernels_ : inference_kernels_; - - auto ret = CheckTensorsInvalid(inputs_); - if (ret != RET_OK) { - MS_LOG(ERROR) << "CheckInputs failed"; - return ret; + auto &run_kernels = (train_mode_) ? train_kernels_ : inference_kernels_; + if (context_->IsCpuFloat16Enabled()) { + ret = MixPrecisionExecKernels(before, after, run_kernels); + } else { + ret = ExecKernels(before, after, run_kernels); } - - for (auto *kernel : run_kernel) { - MS_ASSERT(nullptr != kernel); - ret = kernel->Execute(before, after); - if (RET_OK != ret) { - MS_LOG(ERROR) << "run kernel failed, name: " << kernel->name(); - return ret; - } + if (ret != RET_OK) { + MS_LOG(ERROR) << "failed to run model kernels"; + return ret; } if (train_mode_ && virtual_batch_multiplier_) { @@ -175,7 +404,7 @@ int TrainSession::Train() { // shift kernels to train mode train_mode_ = true; virtual_batch_idx_ = 0; - for (auto kernel : this->train_kernels_) { + for (auto &kernel : this->train_kernels_) { MS_ASSERT(nullptr != kernel); auto ret = kernel->Train(); if (ret != RET_OK) { @@ -194,7 +423,7 @@ int TrainSession::Eval() { // shift kernels to eval mode train_mode_ = false; virtual_batch_idx_ = 0; - for (auto kernel : this->train_kernels_) { + for (auto &kernel : this->train_kernels_) { MS_ASSERT(kernel != nullptr); auto ret = kernel->Eval(); if (ret != RET_OK) { @@ -427,11 +656,11 @@ bool TrainSession::IsLossKernel(const kernel::LiteKernel *kernel) const { kernel->type() == schema::PrimitiveType_SmoothL1LossGrad || kernel->type() == schema::PrimitiveType_SigmoidCrossEntropyWithLogits || kernel->type() == schema::PrimitiveType_SigmoidCrossEntropyWithLogitsGrad) || - kernel->name().find(train_cfg_.loss_name_) != std::string::npos; + kernel->name().find(cfg_.loss_name_) != std::string::npos; } bool TrainSession::IsGradKernel(const kernel::LiteKernel *kernel) const { - return kernel->name().find("Gradients") != std::string::npos; + return kernel->name().find(kGradName) != std::string::npos; } bool TrainSession::IsOptimizer(kernel::LiteKernel *kernel) const { @@ -478,12 +707,18 @@ int TrainSession::Export(const std::string &file_name, ModelType model_type, Qua if (orig_train_state) Train(); return status; } - } // namespace lite session::LiteSession *session::LiteSession::CreateTrainSession(const std::string &fn, const lite::Context *context, bool train_mode, const lite::TrainCfg *cfg) { - auto session = new (std::nothrow) lite::TrainSession(); + if (cfg != nullptr) { + // test legal configuration + if (cfg->mix_precision_cfg_.loss_scale_ <= 0) { + MS_LOG(ERROR) << "illegal loss scale configuration"; + return nullptr; + } + } + auto session = std::make_unique(); if (session == nullptr) { MS_LOG(ERROR) << "create session failed"; return nullptr; @@ -492,7 +727,6 @@ session::LiteSession *session::LiteSession::CreateTrainSession(const std::string auto ret = session->Init(context, cfg); if (ret != mindspore::lite::RET_OK) { MS_LOG(ERROR) << "init session failed"; - delete session; return nullptr; } @@ -504,14 +738,12 @@ session::LiteSession *session::LiteSession::CreateTrainSession(const std::string auto *model = mindspore::lite::Model::Import(filename.c_str()); if (model == nullptr) { MS_LOG(ERROR) << "create model for train session failed " << filename; - delete session; return nullptr; } ret = session->CompileTrainGraph(model); if (ret != mindspore::lite::RET_OK) { MS_LOG(ERROR) << "Compiling Train Graph session failed"; - delete session; return nullptr; } @@ -522,10 +754,10 @@ session::LiteSession *session::LiteSession::CreateTrainSession(const std::string } if (ret != mindspore::lite::RET_OK) { MS_LOG(ERROR) << "Could not switch to Train Modei " << train_mode; - delete session; return nullptr; } - return session; + return session.release(); } + } // namespace mindspore diff --git a/mindspore/lite/src/train/train_session.h b/mindspore/lite/src/train/train_session.h index bf7e3d19f7c..80e3e6350d6 100644 --- a/mindspore/lite/src/train/train_session.h +++ b/mindspore/lite/src/train/train_session.h @@ -92,7 +92,7 @@ class TrainSession : virtual public lite::LiteSession { int Export(const std::string &fb_name, ModelType model_type, QuantizationType quant_type, FormatType) override; protected: - void AllocWorkSpace(); + int AllocWorkSpace(); bool IsLossKernel(const kernel::LiteKernel *kernel) const; bool IsGradKernel(const kernel::LiteKernel *kernel) const; bool IsOptimizer(kernel::LiteKernel *kernel) const; @@ -106,9 +106,9 @@ class TrainSession : virtual public lite::LiteSession { virtual void CompileOptimizedKernels(); virtual void CompileTrainOutputs(); virtual void CompileEvalOutputs(); - + virtual int InitCallBack(); Model *model_ = nullptr; - TrainCfg train_cfg_; + // TrainCfg train_cfg_; std::unordered_map> orig_output_node_map_; std::unordered_map orig_output_tensor_map_; std::vector orig_output_tensor_names_; @@ -123,21 +123,31 @@ class TrainSession : virtual public lite::LiteSession { std::vector inference_kernels_; std::vector train_kernels_; + TrainCfg cfg_; private: + std::string get_loss_name() const { return cfg_.loss_name_; } void BuildInferenceKernelsRecursive(kernel::LiteKernel *ker, std::vector *req_kernels); int AdminSetupVirtualBatch(int virtual_batch_multiplier, float lr, float momentum); int OptimizerStep(); int ExecKernels(const KernelCallBack &before, const KernelCallBack &after, - std::vector run_kernel); + const std::vector &run_kernel); int MixPrecisionExecKernels(const KernelCallBack &before, const KernelCallBack &after, - std::vector run_kernel); - int CopyTensor(Tensor *tensor, TypeId dst_data_type); + const std::vector &run_kernel); + int MixPrecisionPreProcess(kernel::LiteKernel *kernel, float scale); + int MixPrecisionPostProcess(kernel::LiteKernel *kernel); + bool IsLossTensor(Tensor *tensor); void RestoreTensorData(); void FreeRestoreTensors(); + bool AllInputsNeedScale(kernel::LiteKernel *kernel); + void FreeWorkSpace(); + std::map restored_origin_tensors_; int virtual_batch_idx_ = 0; int virtual_batch_multiplier_ = 0; + uint32_t num_of_not_nan_iter_ = 0; + void *workspace_ = nullptr; + SchedCallBack sched_mix_precision_callback_; bool train_mode_ = false; }; diff --git a/mindspore/lite/src/train/train_utils.cc b/mindspore/lite/src/train/train_utils.cc index 8eafb27f39f..3af1abe52c9 100644 --- a/mindspore/lite/src/train/train_utils.cc +++ b/mindspore/lite/src/train/train_utils.cc @@ -20,6 +20,9 @@ #include "include/ms_tensor.h" #include "src/common/utils.h" #include "src/lite_kernel.h" +#ifdef ENABLE_FP16 +#include "src/runtime/kernel/arm/fp16/fp16_op_handler.h" +#endif namespace mindspore { namespace lite { @@ -50,7 +53,7 @@ kernel::LiteKernel *TSFindKernel(const std::vector &where, float CalculateSparseClassification(tensor::MSTensor *input, tensor::MSTensor *output) { if ((input->shape().size() != 1) || (input->data_type() != kNumberTypeInt32) || (output->shape().size() != 2)) { - MS_LOG(WARNING) << "SparceClassification got a " << input->shape() << "-D input tensor, " << output->shape() + MS_LOG(WARNING) << "SparseClassification got a " << input->shape() << "-D input tensor, " << output->shape() << "-D output tensor"; return 0.0; } @@ -106,5 +109,72 @@ float CalculateOneHotClassification(tensor::MSTensor *input, tensor::MSTensor *o return accuracy / (static_cast(batch_size)); } +Tensor *CastTensor(Tensor *tensor, TypeId dst_data_type) { +#ifdef ENABLE_FP16 + MS_ASSERT(tensor != nullptr); + std::vector valid_type = {kNumberTypeFloat32, kNumberTypeFloat16, kNumberTypeFloat}; + std::vector fp32_type = {kNumberTypeFloat32, kNumberTypeFloat}; + if (!IsContain(valid_type, tensor->data_type())) { + MS_LOG(ERROR) << "source data type must be fp32 or fp16"; + return nullptr; + } + + if (!IsContain(valid_type, dst_data_type)) { + MS_LOG(ERROR) << "destination data type must be fp32 or fp16"; + return nullptr; + } + + auto origin_data = tensor->data_c(); + MS_ASSERT(origin_data != nullptr); + auto restore_tensor = Tensor::CopyTensor(*tensor, false); + restore_tensor->set_data(origin_data); + restore_tensor->set_own_data(tensor->own_data()); + restore_tensor->set_allocator(tensor->allocator()); + restore_tensor->set_scale(tensor->get_scale()); + if (IsContain(fp32_type, tensor->data_type()) && dst_data_type == kNumberTypeFloat16) { + tensor->set_data(nullptr); + tensor->set_data_type(kNumberTypeFloat16); + auto ret = tensor->MallocData(); + auto new_tensor_data = tensor->data_c(); + MS_ASSERT(new_tensor_data != nullptr); + if (RET_OK != ret) { + MS_LOG(ERROR) << "malloc data failed"; + delete restore_tensor; + return nullptr; + } + MS_LOG(DEBUG) << "Convert tensor to fp16 " << tensor->tensor_name(); + Float32ToFloat16_fp16_handler(origin_data, new_tensor_data, tensor->ElementsNum()); + } else { + tensor->set_data(nullptr); + tensor->set_data_type(kNumberTypeFloat32); + auto ret = tensor->MallocData(); + if (RET_OK != ret) { + MS_LOG(ERROR) << "malloc data failed"; + delete restore_tensor; + return nullptr; + } + auto new_tensor_data = tensor->data_c(); + MS_ASSERT(new_tensor_data != nullptr); + MS_LOG(DEBUG) << "Convert tensor to fp32 " << tensor->tensor_name(); + Float16ToFloat32_fp16_handler(origin_data, new_tensor_data, tensor->ElementsNum()); + } + return restore_tensor; +#else + return nullptr; +#endif +} + +int ScaleTensor(Tensor *tensor, float scale) { + MS_ASSERT(tensor != nullptr); + std::vector valid_type = {kNumberTypeFloat32, kNumberTypeFloat}; + if (!IsContain(valid_type, tensor->data_type())) { + MS_LOG(DEBUG) << "Tensor: " << tensor->tensor_name() << " type is " << tensor->data_type(); + return RET_OK; + } + + MS_LOG(DEBUG) << "Scale tensor: " << tensor->tensor_name() << " " << scale; + return tensor->Scale(scale); +} + } // namespace lite } // namespace mindspore diff --git a/mindspore/lite/src/train/train_utils.h b/mindspore/lite/src/train/train_utils.h index f14d5306085..e345dd01876 100644 --- a/mindspore/lite/src/train/train_utils.h +++ b/mindspore/lite/src/train/train_utils.h @@ -35,7 +35,8 @@ kernel::LiteKernel *TSFindKernel(const std::vector &where, size_t TSFindTensor(const std::vector &where, const lite::Tensor *searchParameter); float CalculateSparseClassification(tensor::MSTensor *input, tensor::MSTensor *output); float CalculateOneHotClassification(tensor::MSTensor *input, tensor::MSTensor *output); - +Tensor *CastTensor(Tensor *tensor, TypeId dst_data_type); +int ScaleTensor(Tensor *tensor, float scale); } // namespace lite } // namespace mindspore #endif // MINDSPORE_LITE_SRC_TRAIN_TRAIN_UTILS_H_ diff --git a/mindspore/lite/test/config/models_ms_train.cfg b/mindspore/lite/test/config/models_ms_train.cfg index 7627535382c..e6303c9bdcb 100644 --- a/mindspore/lite/test/config/models_ms_train.cfg +++ b/mindspore/lite/test/config/models_ms_train.cfg @@ -10,6 +10,7 @@ effnet_tune googlenet densenet shufflenetv2 +# xception mini_alexnet weight_quant 2 nin weight_quant 9 lenet weight_quant 5 @@ -22,5 +23,15 @@ effnet_tune weight_quant 7 googlenet weight_quant 10 densenet weight_quant 11 shufflenetv2 weight_quant 3 -# xception +#mini_alexnet fp16 6 +#nin fp16 8.0 +#lenet fp16 2 +mobilenetv1 fp16 2 +mobilenetv2 fp16 2 +#mobilenetv3 fp16 7 +effnet fp16 2 +#effnet_tune fp16 5.0 +resnet fp16 2 +#googlenet fp16 10 +#xception fp16 20.0 # LAST diff --git a/mindspore/lite/test/st/run_net_train.sh b/mindspore/lite/test/st/run_net_train.sh index 7eed84fcc97..0400ce0324d 100755 --- a/mindspore/lite/test/st/run_net_train.sh +++ b/mindspore/lite/test/st/run_net_train.sh @@ -53,6 +53,13 @@ function Run_Converter() { if [[ $model_name == \#* ]]; then continue fi + if [[ "${line_array[1]}" == "fp16" ]]; then + ms_file=$ms_models_path'/'$model_name'.ms' + if [ -f "$ms_file" ]; then + echo $model_name'.ms already exist, continue without convert' + continue + fi + fi if [[ "${line_array[1]}" == "weight_quant" ]]; then WEIGHT_QUANT="--quantType=WeightQuant --bitNum=8 --quantWeightSize=0 --quantWeightChannel=0" model_name=${line_array[0]}'_train_quant' @@ -88,6 +95,8 @@ function Run_x86() { if [[ "${line_array[1]}" == "weight_quant" ]]; then model_name=${line_array[0]}'_train_quant' accuracy_limit=${line_array[2]} + elif [[ "${line_array[1]}" == "fp16" ]]; then + continue fi export_file="${ms_models_path}/${model_name}_tod" inference_file="${ms_models_path}/${model_name}_infer" @@ -171,6 +180,8 @@ function Run_arm() { model_prefix=${line_array[0]} model_name=${line_array[0]}'_train' accuracy_limit=0.5 + enable_fp16="false" + suffix_print="" if [[ $model_name == \#* ]]; then continue fi @@ -185,6 +196,17 @@ function Run_arm() { run_result=$1': '${model_name}' irrelevant'; echo ${run_result} >> ${run_benchmark_train_result_file} continue fi + + if [[ "${line_array[1]}" == "fp16" ]]; then + if [[ "$1" == arm64 ]]; then + enable_fp16="true" + suffix_print="_fp16" + accuracy_limit=${line_array[2]} + else + continue + fi + fi + # run benchmark_train test without clib data echo ${model_name} >> "${run_arm_log_file}" adb -s ${device_id} push ${train_io_path}/${model_prefix}_input*.bin ${train_io_path}/${model_prefix}_output*.bin /data/local/tmp/benchmark_train_test >> ${adb_push_log_file} @@ -207,6 +229,7 @@ function Run_arm() { --expectedDataFile=${tmp_dir}/${model_prefix}_output \ --numThreads=${threads} \ --accuracyThreshold=${accuracy_limit} \ + --enableFp16=${enable_fp16} \ --inferenceFile=${inference_file} \ --exportFile=${export_file} ENDM @@ -216,9 +239,9 @@ ENDM adb -s ${device_id} shell < ${adb_cmd_run_file} >> ${run_arm_log_file} # TODO: change to arm_type if [ $? = 0 ]; then - run_result=$1': '${model_name}' pass'; echo ${run_result} >> ${run_benchmark_train_result_file} + run_result=$1': '${model_name}''${suffix_print}' pass'; echo ${run_result} >> ${run_benchmark_train_result_file} else - run_result=$1': '${model_name}' failed'; echo ${run_result} >> ${run_benchmark_train_result_file}; + run_result=$1': '${model_name}''${suffix_print}' failed'; echo ${run_result} >> ${run_benchmark_train_result_file}; fail=1 fi diff --git a/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32/fullconnection_fp32_tests.cc b/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32/fullconnection_fp32_tests.cc index f2cf2c8570f..6c741385daf 100644 --- a/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32/fullconnection_fp32_tests.cc +++ b/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32/fullconnection_fp32_tests.cc @@ -86,8 +86,12 @@ TEST_F(TestFcFp32, FcTest1) { ASSERT_EQ(lite::RET_OK, ctx->Init()); auto *fc = new kernel::FullconnectionCPUKernel(reinterpret_cast(matmul_param), inputs_, outputs_, ctx); fc->Init(); +#ifdef SUPPORT_TRAIN + fc->AllocWorkspace(); +#endif fc->Run(); ASSERT_EQ(0, CompareOutputData(reinterpret_cast(outputs_[0]->MutableData()), correct, total_size, 0.0001)); + delete fc; delete ctx; } @@ -146,8 +150,12 @@ TEST_F(TestFcFp32, FcTest2) { ASSERT_EQ(lite::RET_OK, ctx->Init()); auto *fc = new kernel::FullconnectionCPUKernel(reinterpret_cast(matmul_param), inputs_, outputs_, ctx); fc->Init(); +#ifdef SUPPORT_TRAIN + fc->AllocWorkspace(); +#endif fc->Run(); ASSERT_EQ(0, CompareOutputData(reinterpret_cast(outputs_[0]->MutableData()), correct, total_size, 0.0001)); + delete fc; delete ctx; } @@ -196,11 +204,15 @@ TEST_F(TestFcFp32, FcTest3) { ASSERT_EQ(lite::RET_OK, ctx->Init()); auto *fc = new kernel::FullconnectionCPUKernel(reinterpret_cast(matmul_param), inputs_, outputs_, ctx); fc->Init(); +#ifdef SUPPORT_TRAIN + fc->AllocWorkspace(); +#endif struct timeval start, end; gettimeofday(&start, nullptr); for (int i = 0; i < 100000; ++i) fc->Run(); gettimeofday(&end, nullptr); // printf("## elapsed: %llu\n", 1000000 * (end.tv_sec - start.tv_sec) + end.tv_usec - end.tv_usec); + delete fc; delete ctx; } diff --git a/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/bn_grad_fp32_test.cc b/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/bn_grad_fp32_test.cc index 3be96222300..52cac588702 100644 --- a/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/bn_grad_fp32_test.cc +++ b/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/bn_grad_fp32_test.cc @@ -86,10 +86,9 @@ TEST_F(TestBNGradFp32, BNGradFp32) { ASSERT_NE(creator, nullptr); auto kernel_obj = creator(inputs, outputs, reinterpret_cast(bn_param), &ctx, desc); ASSERT_NE(kernel_obj, nullptr); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel_obj->workspace_size()); - auto ret = kernel_obj->Init(); EXPECT_EQ(0, ret); + kernel_obj->AllocWorkspace(); ret = kernel_obj->Run(); EXPECT_EQ(0, ret); std::cout << "==========dx==========\n"; @@ -114,7 +113,6 @@ TEST_F(TestBNGradFp32, BNGradFp32) { v->set_data(nullptr); delete v; } - mindspore::kernel::InnerKernel::FreeWorkspace(); delete kernel_obj; MS_LOG(INFO) << "BNGradFp32 passed"; } @@ -182,7 +180,7 @@ TEST_F(TestBNGradFp32, BNTtrainFp32) { ASSERT_NE(creator, nullptr); auto kernel_obj = creator(inputs, outputs, reinterpret_cast(bn_param), &context, desc); ASSERT_NE(kernel_obj, nullptr); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel_obj->workspace_size()); + kernel_obj->AllocWorkspace(); float *save_mean = reinterpret_cast(save_mean_tensor.MutableData()); float *save_var = reinterpret_cast(save_var_tensor.MutableData()); for (int i = 0; i < channels; i++) { @@ -213,7 +211,6 @@ TEST_F(TestBNGradFp32, BNTtrainFp32) { x_tensor->set_data(nullptr); delete x_tensor; - mindspore::kernel::InnerKernel::FreeWorkspace(); delete kernel_obj; } } // namespace mindspore diff --git a/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/convolution_grad_fp32_tests.cc b/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/convolution_grad_fp32_tests.cc index ec419dbe3d7..0400be4a80d 100644 --- a/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/convolution_grad_fp32_tests.cc +++ b/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/convolution_grad_fp32_tests.cc @@ -121,8 +121,7 @@ TEST_F(TestConvolutionGradFp32, ConvFp32FilterGrad) { ASSERT_NE(kernel, nullptr); auto ret = kernel->Init(); EXPECT_EQ(0, ret); - - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); + kernel->AllocWorkspace(); // warm up loop for (int i = 0; i < 3; i++) { kernel->Run(); @@ -146,7 +145,6 @@ TEST_F(TestConvolutionGradFp32, ConvFp32FilterGrad) { delete[] input_data; delete[] dy_data; delete[] dw_data; - mindspore::kernel::InnerKernel::FreeWorkspace(); delete kernel; // delete conv_param; dw_tensor.set_data(nullptr); @@ -201,8 +199,7 @@ TEST_F(TestConvolutionGradFp32, ConvFp32InputGrad) { ASSERT_NE(kernel, nullptr); auto ret = kernel->Init(); EXPECT_EQ(0, ret); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); - + kernel->AllocWorkspace(); // warm up loop for (int i = 0; i < 3; i++) { kernel->Run(); @@ -227,7 +224,6 @@ TEST_F(TestConvolutionGradFp32, ConvFp32InputGrad) { w_tensor.set_data(nullptr); dy_tensor.set_data(nullptr); dx_tensor.set_data(nullptr); - mindspore::kernel::InnerKernel::FreeWorkspace(); delete kernel; // delete conv_param; @@ -279,7 +275,7 @@ TEST_F(TestConvolutionGradFp32, ConvFp32GroupFilterGrad) { ASSERT_NE(kernel, nullptr); auto ret = kernel->Init(); EXPECT_EQ(0, ret); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); + kernel->AllocWorkspace(); kernel->Run(); int loop_count = 100; @@ -302,7 +298,6 @@ TEST_F(TestConvolutionGradFp32, ConvFp32GroupFilterGrad) { dw_tensor.set_data(nullptr); x_tensor.set_data(nullptr); dy_tensor.set_data(nullptr); - mindspore::kernel::InnerKernel::FreeWorkspace(); delete kernel; // delete conv_param; MS_LOG(INFO) << "TestConvolutionGradFp32 Filter Grad passed"; @@ -354,7 +349,7 @@ TEST_F(TestConvolutionGradFp32, ConvFp32GroupInputGrad) { ASSERT_NE(kernel, nullptr); auto ret = kernel->Init(); EXPECT_EQ(0, ret); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); + kernel->AllocWorkspace(); // warm up loop for (int i = 0; i < 3; i++) { kernel->Run(); @@ -379,9 +374,8 @@ TEST_F(TestConvolutionGradFp32, ConvFp32GroupInputGrad) { dx_tensor.set_data(nullptr); w_tensor.set_data(nullptr); dy_tensor.set_data(nullptr); - delete kernel; - mindspore::kernel::InnerKernel::FreeWorkspace(); + // delete conv_param; MS_LOG(INFO) << "TestConvolutionGradFp32 Filter Grad passed"; } @@ -431,8 +425,7 @@ TEST_F(TestConvolutionGradFp32, ConvFp32GroupDilationFilterGrad) { ASSERT_NE(kernel, nullptr); auto ret = kernel->Init(); EXPECT_EQ(0, ret); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); - + kernel->AllocWorkspace(); // warm up loop for (int i = 0; i < 3; i++) { kernel->Run(); @@ -457,7 +450,6 @@ TEST_F(TestConvolutionGradFp32, ConvFp32GroupDilationFilterGrad) { dw_tensor.set_data(nullptr); dy_tensor.set_data(nullptr); x_tensor.set_data(nullptr); - mindspore::kernel::InnerKernel::FreeWorkspace(); delete kernel; // delete conv_param; MS_LOG(INFO) << "TestConvolutionGradFp32 Filter Grad passed"; @@ -509,8 +501,7 @@ TEST_F(TestConvolutionGradFp32, ConvFp32GroupDilationInputGrad) { ASSERT_NE(kernel, nullptr); auto ret = kernel->Init(); EXPECT_EQ(0, ret); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); - + kernel->AllocWorkspace(); int loop_count = 100; auto time_start = mindspore::lite::GetTimeUs(); for (int i = 0; i < loop_count; i++) { @@ -530,7 +521,6 @@ TEST_F(TestConvolutionGradFp32, ConvFp32GroupDilationInputGrad) { dx_tensor.set_data(nullptr); dy_tensor.set_data(nullptr); w_tensor.set_data(nullptr); - mindspore::kernel::InnerKernel::FreeWorkspace(); delete kernel; // delete conv_param; MS_LOG(INFO) << "TestConvolutionGradFp32 Filter Grad passed"; @@ -579,7 +569,7 @@ TEST_F(TestConvolutionGradFp32, ConvGroupDilation) { outputs, &context); ASSERT_NE(kernel, nullptr); kernel->Init(); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); + kernel->AllocWorkspace(); kernel->Train(); EXPECT_EQ(kernel->IsTrain(), 1); @@ -609,7 +599,6 @@ TEST_F(TestConvolutionGradFp32, ConvGroupDilation) { x_tensor.set_data(nullptr); y_tensor.set_data(nullptr); w_tensor.set_data(nullptr); - mindspore::kernel::InnerKernel::FreeWorkspace(); delete kernel; MS_LOG(INFO) << "TestConvolutionFp32 Filter Grad passed"; @@ -688,8 +677,7 @@ TEST_F(TestConvolutionGradFp32, ConvFp32Dilation2Group2Stride2FilterGrad) { ASSERT_NE(kernel, nullptr); auto ret = kernel->Init(); EXPECT_EQ(0, ret); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); - + kernel->AllocWorkspace(); // warm up loop for (int i = 0; i < 3; i++) { kernel->Run(); @@ -718,7 +706,6 @@ TEST_F(TestConvolutionGradFp32, ConvFp32Dilation2Group2Stride2FilterGrad) { dw_tensor.set_data(nullptr); x_tensor.set_data(nullptr); dy_tensor.set_data(nullptr); - mindspore::kernel::InnerKernel::FreeWorkspace(); MS_LOG(INFO) << "TestConvolutionGradFp32 Filter Grad passed"; } @@ -797,7 +784,7 @@ TEST_F(TestConvolutionGradFp32, ConvGroup2Dilation2Stride2) { ASSERT_NE(kernel, nullptr); auto ret = kernel->Init(); EXPECT_EQ(0, ret); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); + kernel->AllocWorkspace(); // warm up loop for (int i = 0; i < 3; i++) { @@ -824,7 +811,6 @@ TEST_F(TestConvolutionGradFp32, ConvGroup2Dilation2Stride2) { dy_tensor.set_data(nullptr); w_tensor.set_data(nullptr); delete kernel; - mindspore::kernel::InnerKernel::FreeWorkspace(); MS_LOG(INFO) << "TestConvolutionGradFp32 Filter Grad passed"; } diff --git a/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/deconvolution_grad_fp32_tests.cc b/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/deconvolution_grad_fp32_tests.cc index 15d9f4f8168..f71cb8ea9ed 100644 --- a/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/deconvolution_grad_fp32_tests.cc +++ b/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/deconvolution_grad_fp32_tests.cc @@ -100,7 +100,7 @@ TEST_F(TestDeConvolutionGradFp32, DeConvFp32FilterGrad) { ASSERT_NE(kernel, nullptr); auto ret = kernel->Init(); EXPECT_EQ(0, ret); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); + kernel->AllocWorkspace(); // warm up loop for (int i = 0; i < 3; i++) { @@ -133,7 +133,7 @@ TEST_F(TestDeConvolutionGradFp32, DeConvFp32FilterGrad) { dw_tensor.set_data(nullptr); x_tensor.set_data(nullptr); dy_tensor.set_data(nullptr); - mindspore::kernel::InnerKernel::FreeWorkspace(); + MS_LOG(INFO) << "TestDeConvolutionGradFp32 Filter Grad passed"; } @@ -208,9 +208,7 @@ TEST_F(TestDeConvolutionGradFp32, DeConvFp32Dilation2FilterGrad) { ASSERT_NE(kernel, nullptr); auto ret = kernel->Init(); EXPECT_EQ(0, ret); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); - for (int i = 0; i < 3; i++) { - } + kernel->AllocWorkspace(); // runtime part printf("Calculating runtime cost...\n"); @@ -238,7 +236,6 @@ TEST_F(TestDeConvolutionGradFp32, DeConvFp32Dilation2FilterGrad) { dw_tensor.set_data(nullptr); x_tensor.set_data(nullptr); dy_tensor.set_data(nullptr); - mindspore::kernel::InnerKernel::FreeWorkspace(); MS_LOG(INFO) << "TestDeConvolutionGradFp32 Filter Grad passed"; } @@ -316,7 +313,7 @@ TEST_F(TestDeConvolutionGradFp32, DeConvFp32Dilation2Group3FilterGrad) { ASSERT_NE(kernel, nullptr); auto ret = kernel->Init(); EXPECT_EQ(0, ret); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); + kernel->AllocWorkspace(); // warm up loop for (int i = 0; i < 3; i++) { @@ -346,7 +343,6 @@ TEST_F(TestDeConvolutionGradFp32, DeConvFp32Dilation2Group3FilterGrad) { dw_tensor.set_data(nullptr); x_tensor.set_data(nullptr); dy_tensor.set_data(nullptr); - mindspore::kernel::InnerKernel::FreeWorkspace(); MS_LOG(INFO) << "TestDeConvolutionGradFp32 Filter Grad passed"; } @@ -421,7 +417,7 @@ TEST_F(TestDeConvolutionGradFp32, DeConvFp32Dilation2Group3Stride1FilterGrad) { ASSERT_NE(kernel, nullptr); auto ret = kernel->Init(); EXPECT_EQ(0, ret); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); + kernel->AllocWorkspace(); // warm up loop for (int i = 0; i < 3; i++) { @@ -454,7 +450,6 @@ TEST_F(TestDeConvolutionGradFp32, DeConvFp32Dilation2Group3Stride1FilterGrad) { dw_tensor.set_data(nullptr); x_tensor.set_data(nullptr); dy_tensor.set_data(nullptr); - mindspore::kernel::InnerKernel::FreeWorkspace(); MS_LOG(INFO) << "TestDeConvolutionGradFp32 Filter Grad passed"; } @@ -529,7 +524,7 @@ TEST_F(TestDeConvolutionGradFp32, DeConvFp32Dilation2Group2Stride2FilterGrad) { ASSERT_NE(kernel, nullptr); auto ret = kernel->Init(); EXPECT_EQ(0, ret); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); + kernel->AllocWorkspace(); // warm up loop for (int i = 0; i < 3; i++) { @@ -562,7 +557,6 @@ TEST_F(TestDeConvolutionGradFp32, DeConvFp32Dilation2Group2Stride2FilterGrad) { dw_tensor.set_data(nullptr); x_tensor.set_data(nullptr); dy_tensor.set_data(nullptr); - mindspore::kernel::InnerKernel::FreeWorkspace(); MS_LOG(INFO) << "TestDeConvolutionGradFp32 Filter Grad passed"; } @@ -640,7 +634,7 @@ TEST_F(TestDeConvolutionGradFp32, DeConvFp32Dilation2Group12Stride2FilterGrad) { ASSERT_NE(kernel, nullptr); auto ret = kernel->Init(); EXPECT_EQ(0, ret); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel->workspace_size()); + kernel->AllocWorkspace(); // warm up loop for (int i = 0; i < 3; i++) { @@ -670,7 +664,6 @@ TEST_F(TestDeConvolutionGradFp32, DeConvFp32Dilation2Group12Stride2FilterGrad) { dw_tensor.set_data(nullptr); x_tensor.set_data(nullptr); dy_tensor.set_data(nullptr); - mindspore::kernel::InnerKernel::FreeWorkspace(); MS_LOG(INFO) << "TestDeConvolutionGradFp32 Filter Grad passed"; } diff --git a/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/network_test.cc b/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/network_test.cc index c29bc79f5ab..f2a209a854a 100644 --- a/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/network_test.cc +++ b/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/network_test.cc @@ -148,9 +148,10 @@ TEST_F(NetworkTest, noname) { context.device_list_[0].device_info_.cpu_device_info_.cpu_bind_mode_ = lite::NO_BIND; context.thread_num_ = 1; - auto session = mindspore::session::LiteSession::CreateTrainSession(net, &context); + lite::TrainCfg cfg; + cfg.loss_name_ = "nhwc"; + auto session = mindspore::session::LiteSession::CreateTrainSession(net, &context, true, &cfg); ASSERT_NE(session, nullptr); - auto tensors_map = session->GetOutputs(); auto tensor_names = session->GetOutputTensorNames(); EXPECT_EQ(tensors_map.size(), 1); diff --git a/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/softmax_crossentropy_fp32_tests.cc b/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/softmax_crossentropy_fp32_tests.cc index c0d581c064b..a0f3a090197 100644 --- a/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/softmax_crossentropy_fp32_tests.cc +++ b/mindspore/lite/test/ut/src/runtime/kernel/arm/fp32_grad/softmax_crossentropy_fp32_tests.cc @@ -76,12 +76,11 @@ TEST_F(TestSoftmaxCrossEntropyFp32, SoftmaxCrossEntropyFp32) { ASSERT_NE(creator, nullptr); auto kernel_obj = creator(inputs, outputs, reinterpret_cast(sce_param), &context, desc); ASSERT_NE(kernel_obj, nullptr); - mindspore::kernel::InnerKernel::AllocWorkspace(kernel_obj->workspace_size()); auto ret = kernel_obj->Init(); EXPECT_EQ(0, ret); + kernel_obj->AllocWorkspace(); ret = kernel_obj->Run(); EXPECT_EQ(0, ret); - printf("==================total loss=================\n"); std::cout << loss[0] << " ," << std::endl; @@ -114,7 +113,6 @@ TEST_F(TestSoftmaxCrossEntropyFp32, SoftmaxCrossEntropyFp32) { y_tensor.set_data(nullptr); loss_tensor.set_data(nullptr); grad_tensor.set_data(nullptr); - mindspore::kernel::InnerKernel::FreeWorkspace(); delete kernel_obj; MS_LOG(INFO) << "SoftmaxCrossEntropyFp32 passed"; } diff --git a/mindspore/lite/tools/benchmark_train/net_train.cc b/mindspore/lite/tools/benchmark_train/net_train.cc index b6653c7b78f..40fcc1a7095 100644 --- a/mindspore/lite/tools/benchmark_train/net_train.cc +++ b/mindspore/lite/tools/benchmark_train/net_train.cc @@ -93,7 +93,7 @@ int NetTrain::GenerateInputData(std::vector *ms_i auto status = GenerateRandomData(tensor_byte_size, input_data); if (status != RET_OK) { std::cerr << "GenerateRandomData for inTensor failed: " << status << std::endl; - MS_LOG(ERROR) << "GenerateRandomData for inTensor failed:" << status; + MS_LOG(ERROR) << "GenerateRandomData for inTensor failed: " << status; return status; } } @@ -111,8 +111,8 @@ int NetTrain::LoadInput(std::vector *ms_inputs) { } else { auto status = ReadInputFile(ms_inputs); if (status != RET_OK) { - std::cerr << "ReadInputFile error, " << status << std::endl; - MS_LOG(ERROR) << "ReadInputFile error, " << status; + std::cerr << "Read Input File error, " << status << std::endl; + MS_LOG(ERROR) << "Read Input File error, " << status; return status; } } @@ -172,7 +172,7 @@ int NetTrain::CompareOutput(const session::LiteSession &lite_session) { auto outputs = tensor->data(); size_t size; std::string output_file = flags_->data_file_ + std::to_string(i) + ".bin"; - auto *bin_buf = ReadFileBuf(output_file.c_str(), &size); + auto bin_buf = std::unique_ptr(ReadFileBuf(output_file.c_str(), &size)); if (bin_buf == nullptr) { MS_LOG(ERROR) << "ReadFile return nullptr"; return RET_ERROR; @@ -182,7 +182,7 @@ int NetTrain::CompareOutput(const session::LiteSession &lite_session) { << ", read size: " << size; return RET_ERROR; } - float bias = CompareData(bin_buf, tensor->ElementsNum(), reinterpret_cast(outputs)); + float bias = CompareData(bin_buf.get(), tensor->ElementsNum(), reinterpret_cast(outputs)); if (bias >= 0) { total_bias += bias; total_size++; @@ -191,7 +191,6 @@ int NetTrain::CompareOutput(const session::LiteSession &lite_session) { break; } i++; - delete[] bin_buf; } if (!has_error) { @@ -221,7 +220,7 @@ int NetTrain::CompareOutput(const session::LiteSession &lite_session) { } } -int NetTrain::MarkPerformance(session::LiteSession *session) { +int NetTrain::MarkPerformance(const std::unique_ptr &session) { MS_LOG(INFO) << "Running train loops..."; std::cout << "Running train loops..." << std::endl; uint64_t time_min = 0xFFFFFFFFFFFFFFFF; @@ -266,7 +265,7 @@ int NetTrain::MarkPerformance(session::LiteSession *session) { return RET_OK; } -int NetTrain::MarkAccuracy(session::LiteSession *session, bool enforce_accuracy) { +int NetTrain::MarkAccuracy(const std::unique_ptr &session, bool enforce_accuracy) { MS_LOG(INFO) << "MarkAccuracy"; for (auto &msInput : session->GetInputs()) { switch (msInput->data_type()) { @@ -329,18 +328,17 @@ int NetTrain::CreateAndRunNetwork(const std::string &filename, int train_session if (flags_->loss_name_ != "") { train_cfg.loss_name_ = flags_->loss_name_; } - - session::LiteSession *session = nullptr; + std::unique_ptr session; if (train_session) { MS_LOG(INFO) << "CreateTrainSession from model file" << filename.c_str(); std::cout << "CreateTrainSession from model file " << filename.c_str() << std::endl; - session = session::LiteSession::CreateTrainSession(filename, &context, true, &train_cfg); + session = std::unique_ptr( + session::LiteSession::CreateTrainSession(filename, &context, true, &train_cfg)); if (session == nullptr) { MS_LOG(ERROR) << "RunNetTrain CreateTrainSession failed while running " << model_name.c_str(); std::cout << "RunNetTrain CreateTrainSession failed while running " << model_name.c_str() << std::endl; return RET_ERROR; } - if (epochs > 0) { session->Train(); } @@ -350,14 +348,14 @@ int NetTrain::CreateAndRunNetwork(const std::string &filename, int train_session filenamems = filenamems + ".ms"; } - MS_LOG(INFO) << "start reading model file" << filenamems.c_str(); + MS_LOG(INFO) << "start reading model file " << filenamems.c_str(); std::cout << "start reading model file " << filenamems.c_str() << std::endl; auto *model = mindspore::lite::Model::Import(filenamems.c_str()); if (model == nullptr) { MS_LOG(ERROR) << "create model for train session failed"; return RET_ERROR; } - session = session::LiteSession::CreateSession(&context); + session = std::unique_ptr(session::LiteSession::CreateSession(&context)); if (session == nullptr) { MS_LOG(ERROR) << "ExportedFile CreateSession failed while running " << model_name.c_str(); std::cout << "CreateSession failed while running " << model_name.c_str() << std::endl; @@ -409,8 +407,9 @@ int NetTrain::CreateAndRunNetwork(const std::string &filename, int train_session int NetTrain::RunNetTrain() { auto status = CreateAndRunNetwork(flags_->model_file_, true, flags_->epochs_); if (status != RET_OK) { - MS_LOG(ERROR) << "CreateAndRunNetwork failed for model" << flags_->model_file_ << ". Status is " << status; - std::cout << "CreateAndRunNetwork failed for model" << flags_->model_file_ << ". Status is " << status << std::endl; + MS_LOG(ERROR) << "CreateAndRunNetwork failed for model " << flags_->model_file_ << ". Status is " << status; + std::cout << "CreateAndRunNetwork failed for model " << flags_->model_file_ << ". Status is " << status + << std::endl; return status; } @@ -423,34 +422,34 @@ int NetTrain::RunNetTrain() { return RET_OK; } -int NetTrain::SaveModels(session::LiteSession *session) { +int NetTrain::SaveModels(const std::unique_ptr &session) { if (!flags_->export_file_.empty()) { auto status = session->Export(flags_->export_file_ + "_qt", lite::MT_TRAIN, lite::QT_WEIGHT); if (status != RET_OK) { - MS_LOG(ERROR) << "Export quantized model error" << flags_->export_file_ + "_qt"; - std::cout << "Export quantized model error" << flags_->export_file_ + "_qt" << std::endl; + MS_LOG(ERROR) << "Export quantized model error " << flags_->export_file_ + "_qt"; + std::cout << "Export quantized model error " << flags_->export_file_ + "_qt" << std::endl; return RET_ERROR; } status = session->Export(flags_->export_file_, lite::MT_TRAIN, lite::QT_NONE); if (status != RET_OK) { - MS_LOG(ERROR) << "Export non quantized model error" << flags_->export_file_; - std::cout << "Export non quantized model error" << flags_->export_file_ << std::endl; + MS_LOG(ERROR) << "Export non quantized model error " << flags_->export_file_; + std::cout << "Export non quantized model error " << flags_->export_file_ << std::endl; return RET_ERROR; } } if (!flags_->inference_file_.empty()) { auto status = session->Export(flags_->inference_file_ + "_qt", lite::MT_INFERENCE, lite::QT_WEIGHT); if (status != RET_OK) { - MS_LOG(ERROR) << "Export quantized inference model error" << flags_->inference_file_ + "_qt"; - std::cout << "Export quantized inference model error" << flags_->inference_file_ + "_qt" << std::endl; + MS_LOG(ERROR) << "Export quantized inference model error " << flags_->inference_file_ + "_qt"; + std::cout << "Export quantized inference model error " << flags_->inference_file_ + "_qt" << std::endl; return RET_ERROR; } auto tick = GetTimeUs(); status = session->Export(flags_->inference_file_, lite::MT_INFERENCE, lite::QT_NONE); if (status != RET_OK) { - MS_LOG(ERROR) << "Export non quantized inference model error" << flags_->inference_file_ + "_qt"; - std::cout << "Export non quantized inference model error" << flags_->inference_file_ + "_qt" << std::endl; + MS_LOG(ERROR) << "Export non quantized inference model error " << flags_->inference_file_ + "_qt"; + std::cout << "Export non quantized inference model error " << flags_->inference_file_ + "_qt" << std::endl; return status; } std::cout << "ExportInference() execution time is " << GetTimeUs() - tick << "us\n"; @@ -491,6 +490,29 @@ int NetTrain::CheckExecutionOfSavedModels() { return status; } +void NetTrain::CheckSum(mindspore::tensor::MSTensor *tensor, std::string node_type, int id, std::string in_out) { + int tensor_size = tensor->ElementsNum(); + void *data = tensor->MutableData(); + TypeId type = tensor->data_type(); + std::cout << node_type << " " << in_out << id << " shape=" << tensor->shape() << " sum="; + switch (type) { + case kNumberTypeFloat32: + std::cout << TensorSum(data, tensor_size) << std::endl; + break; + case kNumberTypeInt32: + std::cout << TensorSum(data, tensor_size) << std::endl; + break; +#ifdef ENABLE_FP16 + case kNumberTypeFloat16: + std::cout << TensorSum(data, tensor_size) << std::endl; + break; +#endif + default: + std::cout << "unsupported type:" << type; + break; + } +} + int NetTrain::InitCallbackParameter() { // before callback before_call_back_ = [&](const std::vector &before_inputs, @@ -510,6 +532,12 @@ int NetTrain::InitCallbackParameter() { } op_call_times_total_++; op_begin_ = GetTimeUs(); + if (callParam.node_type == "Adam") { + for (auto tensor : before_outputs) { + std::fill(reinterpret_cast(tensor->MutableData()), + reinterpret_cast(tensor->MutableData()) + tensor->Size(), 0); + } + } return true; }; @@ -518,14 +546,12 @@ int NetTrain::InitCallbackParameter() { const std::vector &after_outputs, const mindspore::CallBackParam &call_param) { uint64_t opEnd = GetTimeUs(); - if (after_inputs.empty()) { MS_LOG(INFO) << "The num of after inputs is empty"; } if (after_outputs.empty()) { MS_LOG(INFO) << "The num of after outputs is empty"; } - float cost = static_cast(opEnd - op_begin_) / 1000.0f; op_cost_total_ += cost; op_times_by_type_[call_param.node_type].first++; @@ -533,26 +559,11 @@ int NetTrain::InitCallbackParameter() { op_times_by_name_[call_param.node_name].first++; op_times_by_name_[call_param.node_name].second += cost; if (flags_->layer_checksum_) { - auto out_tensor = after_outputs.at(0); - void *output = out_tensor->MutableData(); - int tensor_size = out_tensor->ElementsNum(); - TypeId type = out_tensor->data_type(); - std::cout << call_param.node_type << " shape=" << after_outputs.at(0)->shape() << " sum="; - switch (type) { - case kNumberTypeFloat32: - std::cout << TensorSum(output, tensor_size); - break; - case kNumberTypeInt32: - std::cout << TensorSum(output, tensor_size); - break; -#ifdef ENABLE_FP16 - case kNumberTypeFloat16: - std::cout << TensorSum(output, tensor_size); - break; -#endif - default: - std::cout << "unsupported type:" << type; - break; + for (size_t i = 0; i < after_inputs.size(); i++) { + CheckSum(after_inputs.at(i), call_param.node_type, i, "in"); + } + for (size_t i = 0; i < after_outputs.size(); i++) { + CheckSum(after_outputs.at(i), call_param.node_type, i, "out"); } std::cout << std::endl; } diff --git a/mindspore/lite/tools/benchmark_train/net_train.h b/mindspore/lite/tools/benchmark_train/net_train.h index 999aff41e07..699281500a8 100644 --- a/mindspore/lite/tools/benchmark_train/net_train.h +++ b/mindspore/lite/tools/benchmark_train/net_train.h @@ -113,7 +113,7 @@ class MS_API NetTrain { private: // call GenerateInputData or ReadInputFile to init inputTensors int LoadInput(Vector *ms_inputs); - + void CheckSum(mindspore::tensor::MSTensor *tensor, std::string node_type, int id, std::string in_out); // call GenerateRandomData to fill inputTensors int GenerateInputData(std::vector *ms_inputs); @@ -192,11 +192,10 @@ class MS_API NetTrain { return meanError; } - int MarkPerformance(session::LiteSession *session); - - int MarkAccuracy(session::LiteSession *lite_session, bool enforce_accuracy = true); + int MarkPerformance(const std::unique_ptr &session); + int MarkAccuracy(const std::unique_ptr &session, bool enforce_accuracy = true); int CompareOutput(const session::LiteSession &lite_session); - int SaveModels(session::LiteSession *session); + int SaveModels(const std::unique_ptr &session); int CheckExecutionOfSavedModels(); NetTrainFlags *flags_;