Col_lin
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix centerface scripts

This commit is contained in:
panfengfeng 2021-08-07 05:36:18 +08:00
parent 9f42da3d7c
commit 75da2b0f2a
13 changed files with 502 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
model_zoo/official/cv/centerface/README.md
View File

@ -151,10 +151,7 @@ ls ./dataset/centerface/images/train/images # img_dir
```python
# enter script dir, train CenterFace
bash train_distribute_gpu.sh
# after training
mkdir ./model
cp train_distribute_gpu/output/*/*.ckpt ./model # cp model to [MODEL_PATH]
bash train_distribute_gpu.sh [DEVICE_NUM] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [PRETRAINED_BACKBONE] [ANNOTATIONS] [DATASET]
```
step5: test
@ -186,7 +183,7 @@ ls ./dataset/centerface/ground_truth/val.mat # annot_path
```bash
# test CenterFace
bash test_distribute GPU
bash test_distribute_gpu.sh [DEVICE_NUM] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [CKPT_PATH] [DATASET] [GROUND_TRUTH_MAT]
```
step6: eval
@ -321,10 +318,14 @@ bash eval_all.sh [ground_truth_path]
├── scripts
│ ├──run_infer_310.sh // shell script for infer on ascend310
│ ├──eval.sh // evaluate a single testing result
│ ├──eval.sh // evaluate a single testing result
│ ├──eval_all.sh // choose a range of testing results to evaluate
│ ├──test.sh // testing a single model
│ ├──test_gpu.sh // testing a single model on GPU
│ ├──test_distribute.sh // testing a range of models
│ ├──test_distribute_gpu.sh // testing a range of models on GPU
│ ├──test_and_eval.sh // test then evaluate a single model
│ ├──test_and_eval_gpu.sh // test then evaluate a single model on GPU
│ ├──train_standalone.sh // train in ascend with single npu
│ ├──train_standalone_gpu.sh // train on GPU with single npu
│ ├──train_distribute.sh // train in ascend with multi npu
@ -519,12 +520,9 @@ Major parameters eval.py as follows:
# or use the command as follow:
# USE_DEVICE_ID: your device
# PRETRAINED_BACKBONE: your pretrained model path
# DATASET: dataset path
# ANNOTATIONS: annotation path
# images: img_dir in dataset path
bash train_standalone_gpu.sh [USE_DEVICE_ID] [PRETRAINED_BACKBONE] [DATASET] [ANNOTATIONS] [IMAGES]
# after training
cp train_standalone_gpu/output/*/*.ckpt [MODEL_PATH]
# DATASET: image dataset path
bash train_standalone_gpu.sh [USE_DEVICE_ID] [PRETRAINED_BACKBONE] [ANNOTATIONS] [DATASET]
```
- Multi-device (recommended)
@ -536,9 +534,7 @@ Major parameters eval.py as follows:
# or use symbolic link as quick start
# or use the command as follow, most are the same as train_standalone_gpu.sh, the different is DEVICE_NUM
# DEVICE_NUM: for multi-device only, number of devices
bash train_distribute_gpu.sh [DEVICE_NUM] [PRETRAINED_BACKBONE] [DATASET] [ANNOTATIONS] [IMAGES]
# after training
cp train_distribute_gpu/output/*/*.ckpt [MODEL_PATH]
bash train_distribute_gpu.sh [DEVICE_NUM] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [PRETRAINED_BACKBONE] [ANNOTATIONS] [DATASET]
```
After training with 8 device, the loss value will be achieved as follows:
@ -581,15 +577,21 @@ mkdir [SAVE_PATH]
```python
# you need to change the parameter in test.sh
# or use symbolic link as quick start
- On Ascend
# or use the command as follow:
# DEVICE_TARGET: device where the code will be implemented. Either Ascend or GPU (default: Ascend)
# MODEL_PATH: ckpt path saved during training
# DATASET: img dir
# GROUND_TRUTH_MAT: ground_truth file, mat type
# SAVE_PATH: save_path for evaluate
# DEVICE_ID: use device id
# CKPT: test model name
bash test.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_ID] [CKPT]
bash test.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_ID] [CKPT]
- On GPU
# or use the command as follow:
# CKPT: test model name
# DATASET: img dir
# GROUND_TRUTH_MAT: ground_truth file, mat type
bash test_gpu.sh [DEVICE_ID] [CKPT] [DATASET] [GROUND_TRUTH_MAT]
```
2. test many out ckpt for user to choose the best one
@ -597,13 +599,20 @@ mkdir [SAVE_PATH]
```python
# you need to change the parameter in test.sh
# or use symbolic link as quick start
- On Ascend
# or use the command as follow, most are the same as test.sh, the different are:
# DEVICE_TARGET: device where the code will be implemented. Either Ascend or GPU (default: Ascend)
# DEVICE_NUM: training device number
# STEPS_PER_EPOCH: steps for each epoch
# START: start loop number, used to calculate first epoch number
# END: end loop number, used to calculate last epoch number
bash test_distribute.sh [DEVICE_TARGET][MODEL_PATH] [DATASET][GROUND_TRUTH_MAT] [SAVE_PATH][DEVICE_NUM] [STEPS_PER_EPOCH][START] [END]
bash test_distribute.sh [MODEL_PATH] [DATASET][GROUND_TRUTH_MAT] [SAVE_PATH][DEVICE_NUM] [STEPS_PER_EPOCH][START] [END]
- On GPU
# or use the command as follow, most are the same as test.sh, the different are:
# DEVICE_NUM: training device number
# CKPT_PATH: test model path
# DATASET: img dir
# GROUND_TRUTH_MAT: ground_truth file, mat type
bash test_distribute_gpu.sh [DEVICE_NUM] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [CKPT_PATH] [DATASET] [GROUND_TRUTH_MAT]
```
=======
@ -648,11 +657,14 @@ cd ../../../scripts;
3. test+eval
```python
- On Ascend
# you need to change the parameter in test_and_eval.sh
# or use symbolic link as quick start, default eval the ckpt saved in ./scripts/output/centerface/999
# or use the command as follow, most are the same as test.sh, the different are:
# GROUND_TRUTH_PATH: ground truth path
bash test_and_eval.sh [DEVICE_TARGET][MODEL_PATH] [DATASET][GROUND_TRUTH_MAT] [SAVE_PATH][CKPT] [GROUND_TRUTH_PATH]
bash test_and_eval.sh [MODEL_PATH] [DATASET][GROUND_TRUTH_MAT] [SAVE_PATH][CKPT] [GROUND_TRUTH_PATH]
- On GPU
bash test_and_eval_gpu.sh [DEVICE_ID] [CKPT] [DATASET] [GROUND_TRUTH_MAT]
```
- Running on Ascend

2
model_zoo/official/cv/centerface/dependency/evaluate/eval.py
View File

@ -39,7 +39,7 @@ from bbox import bbox_overlaps
def get_gt_boxes(gt_dir):
""" gt dir: (wider_face_val.mat, wider_easy_val.mat, wider_medium_val.mat, wider_hard_val.mat)"""
gt_mat = loadmat(os.path.join(gt_dir, 'val.mat')) # you own ground_truth name
gt_mat = loadmat(os.path.join(gt_dir, 'wider_face_val.mat')) # you own ground_truth name
hard_mat = loadmat(os.path.join(gt_dir, 'wider_hard_val.mat'))
medium_mat = loadmat(os.path.join(gt_dir, 'wider_medium_val.mat'))
easy_mat = loadmat(os.path.join(gt_dir, 'wider_easy_val.mat'))

12
model_zoo/official/cv/centerface/scripts/eval.sh
View File

@ -16,7 +16,19 @@
root=$PWD
save_path=$root/output/centerface/
if [ ! -d $save_path ]
then
echo "error: save_path=$save_path is not a dir"
exit 1
fi
ground_truth_path=$1
if [ ! -d $ground_truth_path ]
then
echo "error: ground_truth_path=$ground_truth_path is not a dir"
exit 1
fi
echo "start eval"
python ../dependency/evaluate/eval.py --pred=$save_path --gt=$ground_truth_path
echo "end eval"

12
model_zoo/official/cv/centerface/scripts/eval_all.sh
View File

@ -16,7 +16,19 @@
root=$PWD
save_path=$root/output/centerface/
if [ ! -d $save_path ]
then
echo "error: save_path=$save_path is not a dir"
exit 1
fi
ground_truth_path=$1
if [ ! -d $ground_truth_path ]
then
echo "error: ground_truth_path=$ground_truth_path is not a dir"
exit 1
fi
#for i in $(seq start_epoch end_epoch+1)
for i in $(seq 89 200)
do

61
model_zoo/official/cv/centerface/scripts/test.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash
# Copyright 2020-21 Huawei Technologies Co., Ltd
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -14,15 +14,14 @@
# limitations under the License.
# ============================================================================
if [ $# -gt 7 ]
if [ $# -gt 6 ]
then
echo "Usage: sh test.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_ID] [CKPT]"
echo " or: sh test.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_ID]"
echo " or: sh test.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH]"
echo " or: sh test.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT]"
echo " or: sh test.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET]"
echo " or: sh test.sh [DEVICE_TARGET] [MODEL_PATH]"
echo " or: sh test.sh [DEVICE_TARGET]"
echo "Usage: sh test.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_ID] [CKPT]"
echo " or: sh test.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_ID]"
echo " or: sh test.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH]"
echo " or: sh test.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT]"
echo " or: sh test.sh [MODEL_PATH] [DATASET]"
echo " or: sh test.sh [MODEL_PATH]"
echo " or: sh test.sh "
exit 1
fi
@ -51,43 +50,32 @@ dataset_root=$root/dataset
dataset_path=$dataset_root/centerface/images/val/images/
ground_truth_mat=$dataset_root/centerface/ground_truth/val.mat
save_path=$root/output/centerface/
device_target="Ascend"
device_id=0
ckpt="0-140_221620.ckpt" # the model saved for epoch=140
ckpt="0-125_24750.ckpt" # the model saved for epoch=125
if [ $# -ge 1 ]
if [ $# == 1 ]
then
device_target="$1"
if [ "$device_target" != "Ascend" ] && [ "$device_target" != "GPU" ]
then
echo "error: device_target=$device_target is not a valid option (Ascend or GPU)"
exit 1
fi
fi
if [ $# -ge 2 ]
then
model_path=$(get_real_path $2)
if [ ! -d $model_path ]
model_path=$(get_real_path $1)
if [ ! -f $model_path ]
then
echo "error: model_path=$model_path is not a file"
exit 1
fi
fi
if [ $# -ge 3 ]
if [ $# == 2 ]
then
dataset_path=$(get_real_path $3)
if [ ! -d $dataset_path ]
dataset_path=$(get_real_path $2)
if [ ! -f $dataset_path ]
then
echo "error: dataset_path=$dataset_path is not a file"
exit 1
fi
fi
if [ $# -ge 4 ]
if [ $# == 3 ]
then
ground_truth_mat=$(get_real_path $4)
ground_truth_mat=$(get_real_path $3)
if [ ! -f $ground_truth_mat ]
then
echo "error: ground_truth_mat=$ground_truth_mat is not a file"
@ -95,24 +83,24 @@ then
fi
fi
if [ $# -ge 5 ]
if [ $# == 4 ]
then
save_path=$(get_real_path $5)
if [ ! -d $save_path ]
save_path=$(get_real_path $4)
if [ ! -f $save_path ]
then
echo "error: save_path=$save_path is not a file"
exit 1
fi
fi
if [ $# -ge 6 ]
if [ $# == 5 ]
then
device_id=$6
device_id=$5
fi
if [ $# == 7 ]
if [ $# == 6 ]
then
ckpt=$7
ckpt=$6
fi
echo $model_path
@ -138,7 +126,6 @@ python ${dirname_path}/${SCRIPT_NAME} \
--ground_truth_mat=$ground_truth_mat \
--save_dir=$save_path \
--rank=$device_id \
--device_target=$device_target \
--ckpt_name=$ckpt > test.log 2>&1 &
echo 'running'

62
model_zoo/official/cv/centerface/scripts/test_and_eval.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash
# Copyright 2020-21 Huawei Technologies Co., Ltd
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -14,16 +14,15 @@
# limitations under the License.
# ============================================================================
if [ $# -gt 8 ]
if [ $# -gt 6 ]
then
echo "Usage: sh test_and_eval.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_ID] [CKPT] [GROUND_TRUTH_PATH]"
echo " or: sh test_and_eval.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_ID] [CKPT]"
echo " or: sh test_and_eval.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_ID]"
echo " or: sh test_and_eval.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH]"
echo " or: sh test_and_eval.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT]"
echo " or: sh test_and_eval.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET]"
echo " or: sh test_and_eval.sh [DEVICE_TARGET] [MODEL_PATH]"
echo " or: sh test_and_eval.sh [DEVICE_TARGET]"
echo "Usage: sh test_and_eval.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_ID] [CKPT] [GROUND_TRUTH_PATH]"
echo " or: sh test_and_eval.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_ID] [CKPT]"
echo " or: sh test_and_eval.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_ID]"
echo " or: sh test_and_eval.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH]"
echo " or: sh test_and_eval.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT]"
echo " or: sh test_and_eval.sh [MODEL_PATH] [DATASET]"
echo " or: sh test_and_eval.sh [MODEL_PATH]"
echo " or: sh test_and_eval.sh "
exit 1
fi
@ -52,24 +51,14 @@ dataset_root=$root/dataset
dataset_path=$dataset_root/centerface/images/val/images/
ground_truth_mat=$dataset_root/centerface/ground_truth/val.mat
save_path=$root/output/centerface/999
device_target="Ascend"
device_id=0
ckpt="0-140_221620.ckpt" # the model saved for epoch=125
ckpt="0-125_24750.ckpt" # the model saved for epoch=125
ground_truth_path=$root/dataset/centerface/ground_truth
if [ $# -ge 1 ]
then
device_target="$1"
if [ "$device_target" != "Ascend" ] && [ "$device_target" != "GPU" ]
then
echo "error: device_target=$device_target is not a valid option (Ascend or GPU)"
exit 1
fi
fi
if [ $# -ge 2 ]
then
model_path=$(get_real_path $2)
model_path=$(get_real_path $1)
# if [ ! -f $model_path ]
if [ ! -d $model_path ]
then
echo "error: model_path=$model_path is not a dir"
@ -77,9 +66,9 @@ then
fi
fi
if [ $# -ge 3 ]
if [ $# -ge 2 ]
then
dataset_path=$(get_real_path $3)
dataset_path=$(get_real_path $2)
if [ ! -d $dataset_path ]
then
echo "error: dataset_path=$dataset_path is not a dir"
@ -87,9 +76,9 @@ then
fi
fi
if [ $# -ge 4 ]
if [ $# -ge 3 ]
then
ground_truth_mat=$(get_real_path $4)
ground_truth_mat=$(get_real_path $3)
if [ ! -f $ground_truth_mat ]
then
echo "error: ground_truth_mat=$ground_truth_mat is not a file"
@ -97,9 +86,9 @@ then
fi
fi
if [ $# -ge 5 ]
if [ $# -ge 4 ]
then
save_path=$(get_real_path $5)
save_path=$(get_real_path $4)
if [ ! -d $save_path ]
then
echo "error: save_path=$save_path is not a dir"
@ -107,19 +96,19 @@ then
fi
fi
if [ $# -ge 5 ]
then
device_id=$5
fi
if [ $# -ge 6 ]
then
device_id=$6
ckpt=$6
fi
if [ $# -ge 7 ]
then
ckpt=$7
fi
if [ $# == 8 ]
then
ground_truth_path=$(get_real_path $8)
ground_truth_path=$(get_real_path $7)
if [ ! -f $ground_truth_path ]
then
echo "error: ground_truth_path=$ground_truth_path is not a file"
@ -153,7 +142,6 @@ python ${dirname_path}/${SCRIPT_NAME} \
--rank=$device_id \
--ckpt_name=$ckpt \
--eval=1 \
--device_target=$device_target \
--ground_truth_path=$ground_truth_path > test.log 2>&1 &
echo 'running'

113
model_zoo/official/cv/centerface/scripts/test_and_eval_gpu.sh Normal file
View File

@ -0,0 +1,113 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
if [ $# != 4 ]
then
echo "Usage: bash test_and_eval_gpu.sh [DEVICE_ID] [CKPT] [DATASET] [GROUND_TRUTH_MAT]"
exit 1
fi
get_real_path(){
if [ "${1:0:1}" == "/" ]; then
echo "$1"
else
echo "$(realpath -m $PWD/$1)"
fi
}
current_exec_path=$(pwd)
echo ${current_exec_path}
dirname_path=$(dirname "$(pwd)")
echo ${dirname_path}
SCRIPT_NAME='test.py'
ulimit -c unlimited
if [ $1 -lt 0 ] && [ $1 -gt 7 ]
then
echo "error: DEVICE_ID=$1 is not in (0-7)"
exit 1
fi
device_id=$1
export CUDA_VISIBLE_DEVICES="$1"
root=${current_exec_path} # your script path
save_path=$root/output/centerface/999
ckpt=$(get_real_path $2)
if [ ! -f $ckpt ]
then
echo "error: ckpt=$ckpt is not a file"
exit 1
fi
ckpt_name=$(basename $ckpt)
ckpt_dir=$(dirname $ckpt)
echo $ckpt
echo $ckpt_name
echo $ckpt_dir
dataset_path=$(get_real_path $3)
if [ ! -d $dataset_path ]
then
echo "error: dataset_path=$dataset_path is not a dir"
exit 1
fi
ground_truth_mat=$(get_real_path $4)
if [ ! -f $ground_truth_mat ]
then
echo "error: ground_truth_mat=$ground_truth_mat is not a file"
exit 1
fi
ground_truth_path=$(dirname $ground_truth_mat)
echo $dataset_path
echo $ground_truth_mat
echo $save_path
echo $ground_truth_path
export PYTHONPATH=${dirname_path}:$PYTHONPATH
export RANK_SIZE=1
echo 'start testing'
rm -rf ${current_exec_path}/device_test$device_id
rm -rf $save_path
echo 'start rank '$device_id
mkdir ${current_exec_path}/device_test$device_id
mkdir -p $save_path
cd ${current_exec_path}/device_test$device_id || exit
export RANK_ID=0
dev=`expr $device_id + 0`
export DEVICE_ID=$dev
python ${dirname_path}/${SCRIPT_NAME} \
--is_distributed=0 \
--data_dir=$dataset_path \
--test_model=$ckpt_dir \
--ground_truth_mat=$ground_truth_mat \
--save_dir=$save_path \
--rank=$device_id \
--ckpt_name=$ckpt_name \
--eval=1 \
--ground_truth_path=$ground_truth_path > test.log 2>&1 &
echo 'running'

70
model_zoo/official/cv/centerface/scripts/test_distribute.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash
# Copyright 2020-21 Huawei Technologies Co., Ltd
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -14,19 +14,18 @@
# limitations under the License.
# ============================================================================
if [ $# -gt 9 ]
if [ $# -gt 8 ]
then
echo "Usage: sh test_distribute.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_NUM] [STEPS_PER_EPOCH] [START] [END]"
echo " or: sh test_distribute.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_NUM] [STEPS_PER_EPOCH] [START]"
echo " or: sh test_distribute.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_NUM] [STEPS_PER_EPOCH]"
echo " or: sh test_distribute.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_NUM]"
echo " or: sh test_distribute.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_NUM]"
echo " or: sh test_distribute.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH]"
echo " or: sh test_distribute.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT]"
echo " or: sh test_distribute.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET]"
echo " or: sh test_distribute.sh [DEVICE_TARGET] [MODEL_PATH] [DATASET]"
echo " or: sh test_distribute.sh [DEVICE_TARGET] [MODEL_PATH]"
echo " or: sh test_distribute.sh [DEVICE_TARGET]"
echo "Usage: sh test_distribute.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_NUM] [STEPS_PER_EPOCH] [START] [END]"
echo " or: sh test_distribute.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_NUM] [STEPS_PER_EPOCH] [START]"
echo " or: sh test_distribute.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_NUM] [STEPS_PER_EPOCH]"
echo " or: sh test_distribute.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_NUM]"
echo " or: sh test_distribute.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH] [DEVICE_NUM]"
echo " or: sh test_distribute.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT] [SAVE_PATH]"
echo " or: sh test_distribute.sh [MODEL_PATH] [DATASET] [GROUND_TRUTH_MAT]"
echo " or: sh test_distribute.sh [MODEL_PATH] [DATASET]"
echo " or: sh test_distribute.sh [MODEL_PATH] [DATASET]"
echo " or: sh test_distribute.sh [MODEL_PATH]"
echo " or: sh test_distribute.sh "
exit 1
fi
@ -59,7 +58,6 @@ save_path=$root/output/centerface/
# model/ckpt name is "0-" + str(ckpt_num) + "_" + str(198*ckpt_num) + ".ckpt";
# ckpt_num is epoch number, can be calculated by device_num
# detail can be found in "test.py"
device_target="Ascend"
device_num=8
steps_per_epoch=198 #198 for 8P; 1583 for 1p
start=11 # start epoch number = start * device_num + min(device_phy_id) + 1
@ -67,17 +65,8 @@ end=18 # end epoch number = end * device_num + max(device_phy_id) + 1
if [ $# -ge 1 ]
then
device_target="$1"
if [ "$device_target" != "Ascend" ] && [ "$device_target" != "GPU" ]
then
echo "error: device_target=$device_target is not a valid option (Ascend or GPU)"
exit 1
fi
fi
if [ $# -ge 2 ]
then
model_path=$(get_real_path $2)
model_path=$(get_real_path $1)
# if [ ! -f $model_path ]
if [ ! -d $model_path ]
then
echo "error: model_path=$model_path is not a dir"
@ -85,9 +74,9 @@ then
fi
fi
if [ $# -ge 3 ]
if [ $# -ge 2 ]
then
dataset_path=$(get_real_path $3)
dataset_path=$(get_real_path $2)
if [ ! -d $dataset_path ]
then
echo "error: dataset_path=$dataset_path is not a dir"
@ -95,9 +84,9 @@ then
fi
fi
if [ $# -ge 4 ]
if [ $# -ge 3 ]
then
ground_truth_mat=$(get_real_path $4)
ground_truth_mat=$(get_real_path $3)
if [ ! -f $ground_truth_mat ]
then
echo "error: ground_truth_mat=$ground_truth_mat is not a file"
@ -105,9 +94,9 @@ then
fi
fi
if [ $# -ge 5 ]
if [ $# -ge 4 ]
then
save_path=$(get_real_path $5)
save_path=$(get_real_path $4)
if [ ! -d $save_path ]
then
echo "error: save_path=$save_path is not a dir"
@ -115,24 +104,24 @@ then
fi
fi
if [ $# -ge 5 ]
then
device_num=$5
fi
if [ $# -ge 6 ]
then
device_num=$6
steps_per_epoch=$6
fi
if [ $# -ge 7 ]
then
steps_per_epoch=$7
start=$7
fi
if [ $# -ge 8 ]
if [ $# == 8 ]
then
start=$8
fi
if [ $# == 9 ]
then
end=$9
end=$8
fi
echo $model_path
@ -161,7 +150,6 @@ do
--save_dir=$save_path \
--rank=$i \
--device_num=$device_num \
--device_target=$device_target \
--steps_per_epoch=$steps_per_epoch \
--start=$start \
--end=$end > test.log 2>&1 &

103
model_zoo/official/cv/centerface/scripts/test_distribute_gpu.sh Normal file
View File

@ -0,0 +1,103 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
if [ $# != 5 ]
then
echo "Usage: bash test_distribute_gpu.sh [DEVICE_NUM] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [CKPT_PATH] [DATASET] [GROUND_TRUTH_MAT]"
exit 1
fi
get_real_path(){
if [ "${1:0:1}" == "/" ]; then
echo "$1"
else
echo "$(realpath -m $PWD/$1)"
fi
}
current_exec_path=$(pwd)
echo ${current_exec_path}
dirname_path=$(dirname "$(pwd)")
echo ${dirname_path}
SCRIPT_NAME='test.py'
ulimit -c unlimited
# blow are used for calculate model name
# model/ckpt name is "0-" + str(ckpt_num) + "_" + str(198*ckpt_num) + ".ckpt";
# ckpt_num is epoch number, can be calculated by device_num
# detail can be found in "test.py"
start=11 # start epoch number = start * device_num + min(device_phy_id) + 1
end=18 # end epoch number = end * device_num + max(device_phy_id) + 1
model_path=$(get_real_path $3)
if [ ! -d $model_path ]
then
echo "error: model_path=$model_path is not a dir"
exit 1
fi
dataset_path=$(get_real_path $4)
if [ ! -d $dataset_path ]
then
echo "error: dataset_path=$dataset_path is not a dir"
exit 1
fi
ground_truth_mat=$(get_real_path $5)
if [ ! -f $ground_truth_mat ]
then
echo "error: ground_truth_mat=$ground_truth_mat is not a file"
exit 1
fi
save_path=${current_exec_path}/output/centerface/
echo $model_path
echo $dataset_path
echo $ground_truth_mat
echo $save_path
export PYTHONPATH=${dirname_path}:$PYTHONPATH
export RANK_SIZE=1
echo 'start testing'
rm -rf ${current_exec_path}/device_test*
rm -rf $save_path
mkdir -p $save_path
for((i=0;i<=$1-1;i++));
do
echo 'start rank '$i
mkdir ${current_exec_path}/device_test$i
cd ${current_exec_path}/device_test$i || exit
export RANK_ID=0
dev=`expr $i + 0`
export DEVICE_ID=$dev
python ${dirname_path}/${SCRIPT_NAME} \
--is_distributed=0 \
--data_dir=$dataset_path \
--test_model=$model_path \
--ground_truth_mat=$ground_truth_mat \
--save_dir=$save_path \
--rank=$i \
--device_num=$1 \
--device_target="GPU" \
--start=$start \
--end=$end > test.log 2>&1 &
done
echo 'running'

106
model_zoo/official/cv/centerface/scripts/test_gpu.sh Normal file
View File

@ -0,0 +1,106 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
if [ $# != 4 ]
then
echo "Usage: bash test_gpu.sh [DEVICE_ID] [CKPT] [DATASET] [GROUND_TRUTH_MAT]"
exit 1
fi
get_real_path(){
if [ "${1:0:1}" == "/" ]; then
echo "$1"
else
echo "$(realpath -m $PWD/$1)"
fi
}
current_exec_path=$(pwd)
echo ${current_exec_path}
dirname_path=$(dirname "$(pwd)")
echo ${dirname_path}
SCRIPT_NAME='test.py'
ulimit -c unlimited
if [ $1 -lt 0 ] && [ $1 -gt 7 ]
then
echo "error: DEVICE_ID=$1 is not in (0-7)"
exit 1
fi
device_id=$1
export CUDA_VISIBLE_DEVICES="$1"
root=${current_exec_path} # your script path
save_path=$root/output/centerface/
ckpt=$(get_real_path $2)
if [ ! -f $ckpt ]
then
echo "error: ckpt=$ckpt is not a file"
exit 1
fi
ckpt_name=$(basename $ckpt)
ckpt_dir=$(dirname $ckpt)
echo $ckpt
echo $ckpt_name
echo $ckpt_dir
dataset_path=$(get_real_path $3)
if [ ! -d $dataset_path ]
then
echo "error: dataset_path=$dataset_path is not a dir"
exit 1
fi
ground_truth_mat=$(get_real_path $4)
if [ ! -f $ground_truth_mat ]
then
echo "error: ground_truth_mat=$ground_truth_mat is not a file"
exit 1
fi
echo $dataset_path
echo $ground_truth_mat
echo $save_path
export PYTHONPATH=${dirname_path}:$PYTHONPATH
export RANK_SIZE=1
echo 'start testing'
rm -rf ${current_exec_path}/device_test$device_id
rm -rf $save_path
echo 'start rank '$device_id
mkdir ${current_exec_path}/device_test$device_id
mkdir -p $save_path
cd ${current_exec_path}/device_test$device_id || exit
export RANK_ID=0
python ${dirname_path}/${SCRIPT_NAME} \
--is_distributed=0 \
--data_dir=$dataset_path \
--test_model=$ckpt_dir \
--ground_truth_mat=$ground_truth_mat \
--save_dir=$save_path \
--rank=$device_id \
--ckpt_name=$ckpt_name > test.log 2>&1 &
echo 'running'

74
model_zoo/official/cv/centerface/scripts/train_distribute_gpu.sh
View File

@ -14,15 +14,10 @@
# limitations under the License.
# ============================================================================
if [ $# != 0 ] && [ $# != 1 ] && [ $# != 2 ] && [ $# != 3 ] && [ $# != 4 ] && [ $# != 5 ]
if [ $# != 5 ]
then
echo "Usage: sh train_distribute_gpu.sh [DEVICE_NUM] [PRETRAINED_BACKBONE] [DATASET] [ANNOTATIONS] [IMAGES]"
echo " or: sh train_distribute_gpu.sh [DEVICE_NUM] [PRETRAINED_BACKBONE] [DATASET] [ANNOTATIONS]"
echo " or: sh train_distribute_gpu.sh [DEVICE_NUM] [PRETRAINED_BACKBONE] [DATASET]"
echo " or: sh train_distribute_gpu.sh [DEVICE_NUM] [PRETRAINED_BACKBONE]"
echo " or: sh train_distribute_gpu.sh [DEVICE_NUM]"
echo " or: sh train_distribute_gpu.sh "
exit 1
echo "Usage: bash train_distribute_gpu.sh [DEVICE_NUM] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [PRETRAINED_BACKBONE] [ANNOTATIONS] [DATASET]"
exit 1
fi
get_real_path(){
@ -44,73 +39,48 @@ SCRIPT_NAME='train.py'
ulimit -c unlimited
root=${current_exec_path} # your script path
pretrained_backbone=${dirname_path}/mobilenet_v2.ckpt # or mobilenet_v2-b0353104.ckpt
dataset_path=$root/dataset/centerface
annot_path=$dataset_path/annotations/train.json
img_dir=$dataset_path/images/train/images
num_devices=8
if [ $# == 1 ]
if [ $1 -lt 1 ] && [ $1 -gt 8 ]
then
num_devices=$1
echo "error: DEVICE_NUM=$1 is not in (1-8)"
exit 1
fi
if [ $# == 2 ]
export CUDA_VISIBLE_DEVICES="$2"
pretrained_backbone=$(get_real_path $3)
if [ ! -f $pretrained_backbone ]
then
pretrained_backbone=$(get_real_path $2)
if [ ! -f $pretrained_backbone ]
then
echo "error: pretrained_backbone=$pretrained_backbone is not a file"
echo "error: pretrained_backbone=$pretrained_backbone is not a file"
exit 1
fi
fi
if [ $# == 3 ]
annot_path=$(get_real_path $4)
if [ ! -f $annot_path ]
then
dataset_path=$(get_real_path $3)
if [ ! -f $dataset_path ]
then
echo "error: dataset_path=$dataset_path is not a file"
echo "error: annot_path=$annot_path is not a file"
exit 1
fi
fi
if [ $# == 4 ]
dataset_path=$(get_real_path $5)
if [ ! -d $dataset_path ]
then
annot_path=$(get_real_path $4)
if [ ! -f $annot_path ]
then
echo "error: annot_path=$annot_path is not a file"
echo "error: dataset_path=$dataset_path is not a dir"
exit 1
fi
fi
if [ $# == 5 ]
then
img_dir=$(get_real_path $5)
if [ ! -f $img_dir ]
then
echo "error: img_dir=$img_dir is not a file"
exit 1
fi
fi
echo $pretrained_backbone
echo $dataset_path
echo $annot_path
echo $img_dir
echo $dataset_path
export PYTHONPATH=${dirname_path}:$PYTHONPATH
export RANK_SIZE=$num_devices
export DEVICE_ID=0
export RANK_SIZE=$1
echo "start training on $RANK_SIZE devices"
mkdir ${current_exec_path}/train_distribute_gpu
cd ${current_exec_path}/train_distribute_gpu || exit
mpirun -n $RANK_SIZE \
mpirun -n $1 \
python ${dirname_path}/${SCRIPT_NAME} \
--lr=4e-3 \
--per_batch_size=8 \
@ -123,10 +93,8 @@ mpirun -n $RANK_SIZE \
--weight_decay=0.0000 \
--loss_scale=1024 \
--pretrained_backbone=$pretrained_backbone \
--data_dir=$dataset_path \
--annot_path=$annot_path \
--img_dir=$img_dir \
--img_dir=$dataset_path \
--device_target="GPU" > train.log 2>&1 &
echo 'running'

89
model_zoo/official/cv/centerface/scripts/train_standalone_gpu.sh
View File

@ -14,15 +14,10 @@
# limitations under the License.
# ============================================================================
if [ $# != 0 ] && [ $# != 1 ] && [ $# != 2 ] && [ $# != 3 ] && [ $# != 4 ] && [ $# != 5 ]
if [ $# != 1 ]
then
echo "Usage: sh train_standalone_gpu.sh [USE_DEVICE_ID] [PRETRAINED_BACKBONE] [DATASET] [ANNOTATIONS] [IMAGES]"
echo " or: sh train_standalone_gpu.sh [USE_DEVICE_ID] [PRETRAINED_BACKBONE] [DATASET] [ANNOTATIONS]"
echo " or: sh train_standalone_gpu.sh [USE_DEVICE_ID] [PRETRAINED_BACKBONE] [DATASET]"
echo " or: sh train_standalone_gpu.sh [USE_DEVICE_ID] [PRETRAINED_BACKBONE]"
echo " or: sh train_standalone_gpu.sh [USE_DEVICE_ID]"
echo " or: sh train_standalone_gpu.sh "
exit 1
echo "Usage: bash train_standalone_gpu.sh [USE_DEVICE_ID] [PRETRAINED_BACKBONE] [ANNOTATIONS] [DATASET]"
exit 1
fi
get_real_path(){
@ -43,89 +38,48 @@ SCRIPT_NAME='train.py'
ulimit -c unlimited
root=${current_exec_path} # your script path
pretrained_backbone=${dirname_path}/mobilenet_v2.ckpt # or mobilenet_v2-b0353104.ckpt
dataset_path=$root/dataset/centerface
annot_path=$dataset_path/annotations/train.json
img_dir=$dataset_path/images/train/images
use_device_id=0
if [ $# == 1 ]
if [ $1 -lt 0 ] && [ $1 -gt 7 ]
then
use_device_id=$1
echo "error: DEVICE_ID=$1 is not in (0-7)"
exit 1
fi
if [ $# == 2 ]
then
use_device_id=$1
pretrained_backbone=$(get_real_path $2)
fi
if [ $# == 3 ]
then
use_device_id=$1
pretrained_backbone=$(get_real_path $2)
dataset_path=$(get_real_path $3)
fi
if [ $# == 4 ]
then
use_device_id=$1
pretrained_backbone=$(get_real_path $2)
dataset_path=$(get_real_path $3)
annot_path=$(get_real_path $4)
fi
if [ $# == 5 ]
then
use_device_id=$1
pretrained_backbone=$(get_real_path $2)
dataset_path=$(get_real_path $3)
annot_path=$(get_real_path $4)
img_dir=$(get_real_path $5)
fi
echo "use_device_id: " $use_device_id
echo "pretrained_backbone: " $pretrained_backbone
echo "dataset_path: " $dataset_path
echo "annot_path: " $annot_path
echo "img_dir: " $img_dir
export CUDA_VISIBLE_DEVICES="$1"
pretrained_backbone=$(get_real_path $2)
if [ ! -f $pretrained_backbone ]
then
echo "error: pretrained_backbone=$pretrained_backbone is not a file"
exit 1
fi
if [ ! -d $dataset_path ]
then
echo "error: dataset_path=$dataset_path is not a directory"
exit 1
exit 1
fi
annot_path=$(get_real_path $3)
if [ ! -f $annot_path ]
then
echo "error: annot_path=$annot_path is not a file"
exit 1
exit 1
fi
if [ ! -d $img_dir ]
dataset_path=$(get_real_path $4)
if [ ! -d $dataset_path ]
then
echo "error: img_dir=$img_dir is not a directory"
exit 1
echo "error: dataset_path=$dataset_path is not a dir"
exit 1
fi
echo $pretrained_backbone
echo $annot_path
echo $dataset_path
export PYTHONPATH=${dirname_path}:$PYTHONPATH
export RANK_SIZE=1
echo 'start training'
echo 'start rank '$use_device_id
rm -rf ${current_exec_path}/train_standalone_gpu
mkdir ${current_exec_path}/train_standalone_gpu
cd ${current_exec_path}/train_standalone_gpu || exit
export RANK_ID=0
dev=`expr $use_device_id + 0`
export DEVICE_ID=$dev
python ${dirname_path}/${SCRIPT_NAME} \
--lr=5e-4 \
--per_batch_size=8 \
@ -138,9 +92,8 @@ python ${dirname_path}/${SCRIPT_NAME} \
--weight_decay=0.0000 \
--loss_scale=1024 \
--pretrained_backbone=$pretrained_backbone \
--data_dir=$dataset_path \
--annot_path=$annot_path \
--img_dir=$img_dir \
--img_dir=$dataset_path \
--device_target="GPU" > train.log 2>&1 &
echo 'running'

9
model_zoo/official/cv/centerface/test.py
View File

@ -35,9 +35,10 @@ from dependency.evaluate.eval import evaluation
dev_id = get_device_id()
context.set_context(mode=context.GRAPH_MODE,
device_target=config.device_target, save_graphs=False, device_id=dev_id)
device_target=config.device_target, save_graphs=False)
if config.device_target == "Ascend":
context.set_context(device_id=dev_id)
context.set_context(enable_auto_mixed_precision=False)
def modelarts_process():
@ -65,7 +66,7 @@ def test_centerface():
else:
ckpt_name = config.ckpt_name
test_model = config.test_model + ckpt_name
test_model = config.test_model + "/" + ckpt_name
if not test_model:
print('load_model {} none'.format(test_model))
continue
@ -112,8 +113,8 @@ def test_centerface():
if not os.path.exists(save_path + im_dir):
os.makedirs(save_path + im_dir)
print('save_path + im_dir={}'.format(save_path + im_dir))
for num, file in enumerate(file_list_item):
im_name = file[0][0]
for num, file_obj in enumerate(file_list_item):
im_name = file_obj[0][0]
zip_name = '%s/%s.jpg' % (im_dir, im_name)
img_path = os.path.join(config.data_dir, zip_name)
print('img_path={}'.format(img_path))