From edd151b48f68ebc3a30cbb8b69994a201269cd9d Mon Sep 17 00:00:00 2001 From: zengxianglong Date: Sat, 31 Jul 2021 17:38:39 +0800 Subject: [PATCH] adjust scripts --- .../test/config/models_caffe_posttraining.cfg | 2 -- mindspore/lite/test/config/models_onnx.cfg | 3 +++ ...sttraining.cfg => models_posttraining.cfg} | 2 ++ ...ocess_only.cfg => models_process_only.cfg} | 3 +-- .../test/config/models_process_only_fp16.cfg | 1 + .../lite/test/st/scripts/base_functions.sh | 12 +++++++--- .../test/st/scripts/run_benchmark_arm64.sh | 17 +++++++------- .../lite/test/st/scripts/run_benchmark_x86.sh | 22 +++++++++---------- 8 files changed, 36 insertions(+), 26 deletions(-) delete mode 100644 mindspore/lite/test/config/models_caffe_posttraining.cfg rename mindspore/lite/test/config/{models_tflite_posttraining.cfg => models_posttraining.cfg} (75%) rename mindspore/lite/test/config/{models_for_process_only.cfg => models_process_only.cfg} (94%) create mode 100644 mindspore/lite/test/config/models_process_only_fp16.cfg diff --git a/mindspore/lite/test/config/models_caffe_posttraining.cfg b/mindspore/lite/test/config/models_caffe_posttraining.cfg deleted file mode 100644 index a0cb52ba50b..00000000000 --- a/mindspore/lite/test/config/models_caffe_posttraining.cfg +++ /dev/null @@ -1,2 +0,0 @@ -ml_face_mnet 105 -ml_face_landmark_2 2 diff --git a/mindspore/lite/test/config/models_onnx.cfg b/mindspore/lite/test/config/models_onnx.cfg index e86513bc650..19fda52d682 100644 --- a/mindspore/lite/test/config/models_onnx.cfg +++ b/mindspore/lite/test/config/models_onnx.cfg @@ -100,3 +100,6 @@ gender_lstm_vad.onnx gender_resnet34_lzl.onnx # cur acc for tiny-yolov3-11 is 2.5% because the Unsqueeze_concat_7:0's output of the last op has very small numbers. tiny-yolov3-11.onnx;2;1,224,224,3:1,2 3 +# cur acc for ml_video_edit_art_transfer is 2+% +ml_video_edit_art_transfer.onnx;3 +ssd-10.onnx;;;;calib_only diff --git a/mindspore/lite/test/config/models_tflite_posttraining.cfg b/mindspore/lite/test/config/models_posttraining.cfg similarity index 75% rename from mindspore/lite/test/config/models_tflite_posttraining.cfg rename to mindspore/lite/test/config/models_posttraining.cfg index 59eba326b97..f684576a709 100644 --- a/mindspore/lite/test/config/models_tflite_posttraining.cfg +++ b/mindspore/lite/test/config/models_posttraining.cfg @@ -1,3 +1,5 @@ +ml_face_mnet 105 +ml_face_landmark_2 2 mobilenet.tflite 0.5 transformer_20200831_encoder_fp32.tflite;36 70 transformer_20200831_decoder_fp32.tflite;11 35 diff --git a/mindspore/lite/test/config/models_for_process_only.cfg b/mindspore/lite/test/config/models_process_only.cfg similarity index 94% rename from mindspore/lite/test/config/models_for_process_only.cfg rename to mindspore/lite/test/config/models_process_only.cfg index dde8b698e9a..2786ae0b545 100644 --- a/mindspore/lite/test/config/models_for_process_only.cfg +++ b/mindspore/lite/test/config/models_process_only.cfg @@ -2,6 +2,7 @@ lite-model_arbitrary-image-stylization-inceptionv3_dr_transfer_1.tflite lite-model_arbitrary-image-stylization-inceptionv3_int8_transfer_1.tflite lite-model_arbitrary-image-stylization-inceptionv3-dynamic-shapes_dr_transfer_1.tflite;2;1,1,1,100:1,64,64,3 lite-model_cartoongan_dr_1.tflite +lite-model_deeplabv3-xception65_1_default_1.tflite lite-model_deeplabv3-mobilenetv2_1_default_1.tflite lite-model_deeplabv3-mobilenetv2_dm05_1_default_1.tflite lite-model_deeplabv3-mobilenetv2-int8_1_default_1.tflite @@ -21,8 +22,6 @@ mtk_transformer_decoder_joint.tflite quant_aware_bank_card_detection_inception.onnx quant_aware_bank_card_recognition_fcny.onnx quant_aware_identify_card_detect.onnx -# cur acc for ml_video_edit_art_transfer is 2+% -ml_video_edit_art_transfer.onnx;3 #ml_table_detection.onnx: onnx quantized model ml_table_detection.onnx intent_detect_hi_v2.tflite diff --git a/mindspore/lite/test/config/models_process_only_fp16.cfg b/mindspore/lite/test/config/models_process_only_fp16.cfg new file mode 100644 index 00000000000..6ad9e783445 --- /dev/null +++ b/mindspore/lite/test/config/models_process_only_fp16.cfg @@ -0,0 +1 @@ +lite-model_deeplabv3-xception65_1_default_1.tflite diff --git a/mindspore/lite/test/st/scripts/base_functions.sh b/mindspore/lite/test/st/scripts/base_functions.sh index 1cc37e0e18f..480009512fe 100644 --- a/mindspore/lite/test/st/scripts/base_functions.sh +++ b/mindspore/lite/test/st/scripts/base_functions.sh @@ -146,18 +146,21 @@ function Run_Benchmark() { if [[ $6 == "arm64" && $7 == "CPU" && ! ${cfg_file_name} =~ "fp16" ]]; then benchmark_mode="calib+loop" fi - # adjust file name - infix="" + # adjust precision mode mode="fp32" if [[ ${cfg_file_name} =~ "fp16" ]]; then mode="fp16" - elif [[ ${cfg_file_name} =~ "bit" ]]; then + fi + # adjust file name + infix="" + if [[ ${cfg_file_name} =~ "bit" ]]; then infix="_${cfg_file##*_}" infix=${infix%.*} elif [[ ${cfg_file_name} =~ "_train" ]]; then infix="_train" elif [[ ${cfg_file_name} =~ "_weightquant" ]]; then infix="_weightquant" + benchmark_mode="calib" elif [[ ${cfg_file_name} =~ "_posttraining" ]]; then model_name=${model_name}"_posttraining" elif [[ ${cfg_file_name} =~ "_process_only" ]]; then @@ -198,6 +201,9 @@ function Run_Benchmark() { if [[ ${mode} == "fp16" ]]; then enableFp16="true" fi + if [[ ${extra_info} =~ "calib_only" ]]; then + benchmark_mode="calib" + fi # start running benchmark echo "---------------------------------------------------------" >> "$4" if [[ ${benchmark_mode} = "calib" || ${benchmark_mode} = "calib+loop" ]]; then diff --git a/mindspore/lite/test/st/scripts/run_benchmark_arm64.sh b/mindspore/lite/test/st/scripts/run_benchmark_arm64.sh index e020b15f03c..e1b807887c0 100644 --- a/mindspore/lite/test/st/scripts/run_benchmark_arm64.sh +++ b/mindspore/lite/test/st/scripts/run_benchmark_arm64.sh @@ -15,9 +15,9 @@ function Run_Converter() { mkdir -p ${ms_models_path} # Prepare the config file list local fp32_cfg_file_list=("$models_tf_config" "$models_tflite_config" "$models_caffe_config" "$models_onnx_config" "$models_mindspore_config" \ - "$models_mindspore_train_config" "$models_tflite_posttraining_config" "$models_caffe_posttraining_config" \ + "$models_mindspore_train_config" "$models_posttraining_config" "$models_process_only_fp16_config" \ "$models_tflite_awaretraining_config" "$models_weightquant_config" "$models_weightquant_7bit_config" \ - "$models_weightquant_9bit_config" "$models_for_process_only_config") + "$models_weightquant_9bit_config" "$models_process_only_config") local fp16_cfg_file_list=("$models_onnx_fp16_config" "$models_caffe_fp16_config" "$models_tflite_fp16_config" "$models_tf_fp16_config") # Convert models: @@ -34,9 +34,9 @@ function Run_Converter() { function Run_arm64() { # Prepare the config file list local arm64_cfg_file_list=("$models_tf_config" "$models_tflite_config" "$models_caffe_config" "$models_onnx_config" "$models_mindspore_config" \ - "$models_mindspore_train_config" "$models_tflite_posttraining_config" "$models_caffe_posttraining_config" \ + "$models_mindspore_train_config" "$models_posttraining_config" "$models_compatibility_config" \ "$models_tflite_awaretraining_config" "$models_weightquant_config" "$models_weightquant_7bit_config" \ - "$models_weightquant_9bit_config" "$models_for_process_only_config" "$models_compatibility_config") + "$models_weightquant_9bit_config" "$models_process_only_config" "$models_process_only_fp16_config") # Run converted models: # $1:cfgFileList; $2:modelPath; $3:dataPath; $4:logFile; $5:resultFile; $6:platform; $7:processor; $8:phoneId; Run_Benchmark "${arm64_cfg_file_list[*]}" . '/data/local/tmp' $run_arm64_fp32_log_file $run_benchmark_result_file 'arm64' 'CPU' $device_id @@ -44,7 +44,8 @@ function Run_arm64() { # Run on arm64-fp16 platform: function Run_arm64_fp16() { - local arm64_cfg_file_list=("$models_onnx_fp16_config" "$models_caffe_fp16_config" "$models_tflite_fp16_config" "$models_tf_fp16_config") + local arm64_cfg_file_list=("$models_onnx_fp16_config" "$models_caffe_fp16_config" "$models_tflite_fp16_config" "$models_tf_fp16_config" \ + "$models_process_only_fp16_config") # $1:cfgFileList; $2:modelPath; $3:dataPath; $4:logFile; $5:resultFile; $6:platform; $7:processor; $8:phoneId; Run_Benchmark "${arm64_cfg_file_list[*]}" . '/data/local/tmp' $run_arm64_fp16_log_file $run_benchmark_result_file 'arm64' 'CPU' $device_id } @@ -90,8 +91,7 @@ models_tflite_config=${basepath}/../config/models_tflite.cfg models_tf_config=${basepath}/../config/models_tf.cfg models_caffe_config=${basepath}/../config/models_caffe.cfg models_tflite_awaretraining_config=${basepath}/../config/models_tflite_awaretraining.cfg -models_tflite_posttraining_config=${basepath}/../config/models_tflite_posttraining.cfg -models_caffe_posttraining_config=${basepath}/../config/models_caffe_posttraining.cfg +models_posttraining_config=${basepath}/../config/models_posttraining.cfg models_onnx_config=${basepath}/../config/models_onnx.cfg models_onnx_fp16_config=${basepath}/../config/models_onnx_fp16.cfg models_caffe_fp16_config=${basepath}/../config/models_caffe_fp16.cfg @@ -103,7 +103,8 @@ models_weightquant_7bit_config=${basepath}/../config/models_weightquant_7bit.cfg models_weightquant_9bit_config=${basepath}/../config/models_weightquant_9bit.cfg models_weightquant_config=${basepath}/../config/models_weightquant.cfg models_compatibility_config=${basepath}/../config/models_compatibility.cfg -models_for_process_only_config=${basepath}/../config/models_for_process_only.cfg +models_process_only_config=${basepath}/../config/models_process_only.cfg +models_process_only_fp16_config=${basepath}/../config/models_process_only_fp16.cfg ms_models_path=${basepath}/ms_models diff --git a/mindspore/lite/test/st/scripts/run_benchmark_x86.sh b/mindspore/lite/test/st/scripts/run_benchmark_x86.sh index e1e137e8b1e..dd61c255e70 100644 --- a/mindspore/lite/test/st/scripts/run_benchmark_x86.sh +++ b/mindspore/lite/test/st/scripts/run_benchmark_x86.sh @@ -51,9 +51,9 @@ function Run_Converter() { # Prepare the config file list local x86_cfg_file_list=("$models_tf_config" "$models_tflite_config" "$models_caffe_config" "$models_onnx_config" "$models_mindspore_config" \ - "$models_mindspore_train_config" "$models_tflite_posttraining_config" "$models_caffe_posttraining_config" \ + "$models_mindspore_train_config" "$models_posttraining_config" "$models_process_only_config" \ "$models_tflite_awaretraining_config" "$models_weightquant_config" "$models_weightquant_7bit_config" \ - "$models_weightquant_9bit_config" "$models_for_process_only_config") + "$models_weightquant_9bit_config") # Convert models: # $1:cfgFileList; $2:inModelPath; $3:outModelPath; $4:logFile; $5:resultFile; Convert "${x86_cfg_file_list[*]}" $models_path $ms_models_path $run_converter_log_file $run_converter_result_file @@ -102,9 +102,9 @@ function Run_x86() { # Prepare the config file list local x86_cfg_file_list=("$models_tf_config" "$models_tflite_config" "$models_caffe_config" "$models_onnx_config" "$models_mindspore_config" \ - "$models_mindspore_train_config" "$models_tflite_posttraining_config" "$models_caffe_posttraining_config" \ + "$models_mindspore_train_config" "$models_posttraining_config" "$models_process_only_fp16_config" \ "$models_tflite_awaretraining_config" "$models_weightquant_config" "$models_weightquant_7bit_config" \ - "$models_weightquant_9bit_config" "$models_for_process_only_config") + "$models_weightquant_9bit_config" "$models_process_only_config") # Run converted models: # $1:cfgFileList; $2:modelPath; $3:dataPath; $4:logFile; $5:resultFile; $6:platform; $7:processor; $8:phoneId; Run_Benchmark "${x86_cfg_file_list[*]}" $ms_models_path $models_path $run_x86_log_file $run_benchmark_result_file 'x86' 'CPU' '' @@ -120,9 +120,9 @@ function Run_x86_sse() { # Prepare the config file list local sse_cfg_file_list=("$models_tf_config" "$models_tflite_config" "$models_caffe_config" "$models_onnx_config" "$models_mindspore_config" \ - "$models_mindspore_train_config" "$models_tflite_posttraining_config" "$models_caffe_posttraining_config" \ + "$models_mindspore_train_config" "$models_posttraining_config" "$models_process_only_fp16_config" \ "$models_tflite_awaretraining_config" "$models_weightquant_config" "$models_weightquant_7bit_config" \ - "$models_weightquant_9bit_config" "$models_for_process_only_config") + "$models_weightquant_9bit_config" "$models_process_only_config") # Run converted models: # $1:cfgFileList; $2:modelPath; $3:dataPath; $4:logFile; $5:resultFile; $6:platform; $7:processor; $8:phoneId; Run_Benchmark "${sse_cfg_file_list[*]}" $ms_models_path $models_path $run_x86_sse_log_file $run_benchmark_result_file 'x86' 'CPU' '' @@ -138,9 +138,9 @@ function Run_x86_avx() { # Prepare the config file list local avx_cfg_file_list=("$models_tf_config" "$models_tflite_config" "$models_caffe_config" "$models_onnx_config" "$models_mindspore_config" \ - "$models_mindspore_train_config" "$models_tflite_posttraining_config" "$models_caffe_posttraining_config" \ + "$models_mindspore_train_config" "$models_posttraining_config" "$models_process_only_fp16_config" \ "$models_tflite_awaretraining_config" "$models_weightquant_config" "$models_weightquant_7bit_config" \ - "$models_weightquant_9bit_config" "$models_for_process_only_config") + "$models_weightquant_9bit_config" "$models_process_only_config") # Run converted models: # $1:cfgFileList; $2:modelPath; $3:dataPath; $4:logFile; $5:resultFile; $6:platform; $7:processor; $8:phoneId; $9:benchmark_mode Run_Benchmark "${avx_cfg_file_list[*]}" $ms_models_path $models_path $run_x86_avx_log_file $run_benchmark_result_file 'x86' 'CPU' '' @@ -219,15 +219,15 @@ models_tflite_config=${basepath}/../config/models_tflite.cfg models_tf_config=${basepath}/../config/models_tf.cfg models_caffe_config=${basepath}/../config/models_caffe.cfg models_tflite_awaretraining_config=${basepath}/../config/models_tflite_awaretraining.cfg -models_tflite_posttraining_config=${basepath}/../config/models_tflite_posttraining.cfg -models_caffe_posttraining_config=${basepath}/../config/models_caffe_posttraining.cfg +models_posttraining_config=${basepath}/../config/models_tflite_posttraining.cfg models_onnx_config=${basepath}/../config/models_onnx.cfg models_mindspore_config=${basepath}/../config/models_mindspore.cfg models_mindspore_train_config=${basepath}/../config/models_mindspore_train.cfg models_weightquant_7bit_config=${basepath}/../config/models_weightquant_7bit.cfg models_weightquant_9bit_config=${basepath}/../config/models_weightquant_9bit.cfg models_weightquant_config=${basepath}/../config/models_weightquant.cfg -models_for_process_only_config=${basepath}/../config/models_for_process_only.cfg +models_process_only_config=${basepath}/../config/models_process_only.cfg +models_process_only_fp16_config=${basepath}/../config/models_process_only_fp16.cfg ms_models_path=${basepath}/ms_models