Col_lin
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

clould

This commit is contained in:
huchunmei 2021-05-19 15:01:59 +08:00
parent 8721b1db1f
commit a554ace6ca
22 changed files with 737 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
model_zoo/official/cv/inceptionv4/README.md
View File

@ -78,14 +78,21 @@ For FP16 operators, if the input data type is FP32, the backend of MindSpore wil
├─run_infer_310.sh # shell script for 310 inference
└─run_eval_ascend.sh # launch evaluating with ascend platform
├─src
├─config.py # parameter configuration
├─dataset.py # data preprocessing
├─inceptionv4.py # network definition
└─callback.py # eval callback function
├─eval.py # eval net
├─export.py # export checkpoint, surpport .onnx, .air, .mindir convert
├─postprogress.py # post process for 310 inference
└─train.py # train net
├─callback.py # eval callback function
└─model_utils
├─config.py # Processing configuration parameters
├─device_adapter.py # Get cloud ID
├─local_adapter.py # Get local ID
└─moxing_adapter.py # Parameter processing
├─default_config.yaml # Training parameter profile(ascend)
├─default_config_cpu.yaml # Training parameter profile(cpu)
├─default_config_gpu.yaml # Training parameter profile(gpu)
├─eval.py # eval net
├─export.py # export checkpoint, surpport .onnx, .air, .mindir convert
├─postprogress.py # post process for 310 inference
└─train.py # train net
```
## [Script Parameters](#contents)

74
model_zoo/official/cv/inceptionv4/default_config.yaml Normal file
View File

@ -0,0 +1,74 @@
# Builtin Configurations(DO NOT CHANGE THESE CONFIGURATIONS unless you know exactly what you are doing)
enable_modelarts: False
data_url: ""
train_url: ""
checkpoint_url: ""
data_path: "/cache/data"
output_path: "/cache/train"
load_path: "/cache/checkpoint_path"
device_target: Ascend
enable_profiling: False
# ==============================================================================
dataset_path: "/cache/data"
ckpt_path: '/cache/data/'
checkpoint_path: '/cache/data/inceptionv3/inceptionv3-rank3_1-247_1251.ckpt'
ckpt_file: '/cache/data/inceptionv3/inceptionv3-rank3_1-247_1251.ckpt'
resume: ''
is_distributed: False
device_id: 0
platform: 'Ascend'
file_name: 'inceptionv4'
file_format: 'MINDIR'
width: 299
height: 299
# fasterrcnn_export
result_path: '' # "result file path"
label_file: '' # "label file"
# Training options
is_save_on_master: False
batch_size: 128
epoch_size: 250
num_classes: 1000
work_nums: 8
ds_type: 'imagenet'
ds_sink_mode: True
loss_scale: 1024
smooth_factor: 0.1
weight_decay: 0.00004
momentum: 0.9
amp_level: 'O3'
decay: 0.9
epsilon: 1.0
keep_checkpoint_max: 10
save_checkpoint_epochs: 10
lr_init: 0.00004
lr_end: 0.000004
lr_max: 0.4
warmup_epochs: 1
start_epoch: 1
---
# Config description for each option
enable_modelarts: 'Whether training on modelarts, default: False'
data_url: 'Dataset url for obs'
train_url: 'Training output url for obs'
data_path: 'Dataset path for local'
output_path: 'Training output path for local'
device_target: 'Target device type'
enable_profiling: 'Whether enable profiling while training, default: False'
file_name: 'output file name.'
file_format: 'file format'
resume: 'resume training with existed checkpoint'
---
device_target: ['Ascend', 'GPU', 'CPU']
file_format: ['AIR', 'ONNX', 'MINDIR']

67
model_zoo/official/cv/inceptionv4/default_config_cpu.yaml Normal file
View File

@ -0,0 +1,67 @@
# Builtin Configurations(DO NOT CHANGE THESE CONFIGURATIONS unless you know exactly what you are doing)
enable_modelarts: False
data_url: ""
train_url: ""
checkpoint_url: ""
data_path: "/cache/data"
output_path: "/cache/train"
load_path: "/cache/checkpoint_path"
device_target: Ascend
enable_profiling: False
# ==============================================================================
dataset_path: "/cache/data"
ckpt_path: '/cache/data/'
checkpoint: '/cache/data/inceptionv3/inceptionv3-rank3_1-247_1251.ckpt'
ckpt_file: '/cache/data/inceptionv3/inceptionv3-rank3_1-247_1251.ckpt'
resume: ''
is_distributed: False
device_id: 0
platform: 'GPU'
file_name: 'inceptionv3'
file_format: 'AIR'
width: 299
height: 299
# Training options
batch_size: 128
epoch_size: 250
num_classes: 10
work_nums: 8
ds_type: 'cifar10'
ds_sink_mode: False
loss_scale: 1024
smooth_factor: 0.1
weight_decay: 0.00004
momentum: 0.9
amp_level: 'O0'
decay: 0.9
epsilon: 1.0
keep_checkpoint_max: 10
save_checkpoint_epochs: 10
lr_init: 0.00004
lr_end: 0.000004
lr_max: 0.4
warmup_epochs: 1
start_epoch: 1
---
# Config description for each option
enable_modelarts: 'Whether training on modelarts, default: False'
data_url: 'Dataset url for obs'
train_url: 'Training output url for obs'
data_path: 'Dataset path for local'
output_path: 'Training output path for local'
device_target: 'Target device type'
enable_profiling: 'Whether enable profiling while training, default: False'
file_name: 'inceptionv3 output air name.'
file_format: 'file format'
---
device_target: ['Ascend', 'GPU', 'CPU']
file_format: ['AIR', 'ONNX', 'MINDIR']

69
model_zoo/official/cv/inceptionv4/default_config_gpu.yaml Normal file
View File

@ -0,0 +1,69 @@
# Builtin Configurations(DO NOT CHANGE THESE CONFIGURATIONS unless you know exactly what you are doing)
enable_modelarts: False
data_url: ""
train_url: ""
checkpoint_url: ""
data_path: "/cache/data"
output_path: "/cache/train"
load_path: "/cache/checkpoint_path"
device_target: Ascend
enable_profiling: False
# ==============================================================================
dataset_path: "/cache/data"
ckpt_path: '/cache/data/'
checkpoint: '/cache/data/inceptionv3/inceptionv3-rank3_1-247_1251.ckpt'
ckpt_file: '/cache/data/inceptionv3/inceptionv3-rank3_1-247_1251.ckpt'
resume: ''
is_distributed: False
device_id: 0
platform: 'GPU'
file_name: 'inceptionv3'
file_format: 'AIR'
width: 299
height: 299
# Training options
is_save_on_master: False
batch_size: 128
epoch_size: 250
num_classes: 1000
work_nums: 8
ds_type: 'imagenet'
ds_sink_mode: True
loss_scale: 1024
smooth_factor: 0.1
weight_decay: 0.00004
momentum: 0.9
amp_level: 'O0'
decay: 0.9
epsilon: 1.0
keep_checkpoint_max: 10
save_checkpoint_epochs: 10
lr_init: 0.00004
lr_end: 0.000004
lr_max: 0.4
warmup_epochs: 1
start_epoch: 1
---
# Config description for each option
enable_modelarts: 'Whether training on modelarts, default: False'
data_url: 'Dataset url for obs'
train_url: 'Training output url for obs'
data_path: 'Dataset path for local'
output_path: 'Training output path for local'
device_target: 'Target device type'
enable_profiling: 'Whether enable profiling while training, default: False'
file_name: 'output file name.'
file_format: 'file format'
---
device_target: ['Ascend', 'GPU', 'CPU']
file_format: ['AIR', 'ONNX', 'MINDIR']

101
model_zoo/official/cv/inceptionv4/eval.py
View File

@ -13,58 +13,100 @@
# limitations under the License.
# ============================================================================
"""evaluate_imagenet"""
import argparse
import time
import os
from src.model_utils.config import config
from src.model_utils.moxing_adapter import moxing_wrapper
from src.model_utils.device_adapter import get_device_id, get_device_num
from src.dataset import create_dataset_imagenet, create_dataset_cifar10
from src.inceptionv4 import Inceptionv4
import mindspore.nn as nn
from mindspore import context
from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits
from mindspore.train.model import Model
from mindspore.train.serialization import load_checkpoint, load_param_into_net
from src.config import config_ascend, config_gpu, config_cpu
from src.dataset import create_dataset_imagenet, create_dataset_cifar10
from src.inceptionv4 import Inceptionv4
CFG_DICT = {
"Ascend": config_ascend,
"GPU": config_gpu,
"CPU": config_cpu,
}
def modelarts_process():
""" modelarts process """
def unzip(zip_file, save_dir):
import zipfile
s_time = time.time()
if not os.path.exists(os.path.join(save_dir, config.modelarts_dataset_unzip_name)):
zip_isexist = zipfile.is_zipfile(zip_file)
if zip_isexist:
fz = zipfile.ZipFile(zip_file, 'r')
data_num = len(fz.namelist())
print('Extract Start...')
print('unzip file num: {}'.format(data_num))
data_print = int(data_num / 100) if data_num > 100 else 1
i = 0
for file in fz.namelist():
if i % data_print == 0:
print('unzip percent: {}%'.format(int(i * 100 / data_num)), flush=True)
i += 1
fz.extract(file, save_dir)
print('cost time: {}min:{}s.'.format(int((time.time() - s_time) / 60),\
int(int(time.time() - s_time) % 60)))
print('Extract Done')
else:
print('This is not zip.')
else:
print('Zip has been extracted.')
if config.need_modelarts_dataset_unzip:
zip_file_1 = os.path.join(config.data_path, config.modelarts_dataset_unzip_name + '.zip')
save_dir_1 = os.path.join(config.data_path)
sync_lock = '/tmp/unzip_sync.lock'
# Each server contains 8 devices as most
if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock):
print('Zip file path: ', zip_file_1)
print('Unzip file save dir: ', save_dir_1)
unzip(zip_file_1, save_dir_1)
print('===Finish extract data synchronization===')
try:
os.mknod(sync_lock)
except IOError:
pass
while True:
if os.path.exists(sync_lock):
break
time.sleep(1)
print('Device: {}, Finish sync unzip data from {} to {}.'.format(get_device_id(), zip_file_1, save_dir_1))
print('#' * 200, os.listdir(save_dir_1))
print('#' * 200, os.listdir(os.path.join(config.data_path, config.modelarts_dataset_unzip_name)))
config.dataset_path = os.path.join(config.data_path, config.modelarts_dataset_unzip_name)
DS_DICT = {
"imagenet": create_dataset_imagenet,
"cifar10": create_dataset_cifar10,
}
@moxing_wrapper(pre_process=modelarts_process)
def inception_v4_eval():
def parse_args():
'''parse_args'''
parser = argparse.ArgumentParser(description='image classification evaluation')
parser.add_argument('--platform', type=str, default='Ascend', choices=('Ascend', 'GPU', 'CPU'), help='run platform')
parser.add_argument('--dataset_path', type=str, default='', help='Dataset path')
parser.add_argument('--checkpoint_path', type=str, default='', help='checkpoint of inceptionV4')
args_opt = parser.parse_args()
return args_opt
if __name__ == '__main__':
args = parse_args()
if args.platform == 'Ascend':
if config.platform == 'Ascend':
device_id = int(os.getenv('DEVICE_ID', '0'))
context.set_context(device_id=device_id)
config = CFG_DICT[args.platform]
create_dataset = DS_DICT[config.ds_type]
context.set_context(mode=context.GRAPH_MODE, device_target=args.platform)
context.set_context(mode=context.GRAPH_MODE, device_target=config.platform)
net = Inceptionv4(classes=config.num_classes)
ckpt = load_checkpoint(args.checkpoint_path)
ckpt = load_checkpoint(config.checkpoint_path)
load_param_into_net(net, ckpt)
net.set_train(False)
config.rank = 0
config.group_size = 1
dataset = create_dataset(dataset_path=args.dataset_path, do_train=False, cfg=config)
dataset = create_dataset(dataset_path=config.dataset_path, do_train=False, cfg=config)
loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
eval_metrics = {'Loss': nn.Loss(),
'Top1-Acc': nn.Top1CategoricalAccuracy(),
@ -73,3 +115,8 @@ if __name__ == '__main__':
print('=' * 20, 'Evalute start', '=' * 20)
metrics = model.eval(dataset, dataset_sink_mode=config.ds_sink_mode)
print("metric: ", metrics)
if __name__ == '__main__':
config.dataset_path = os.path.join(config.dataset_path, 'validation_preprocess')
inception_v4_eval()

38
model_zoo/official/cv/inceptionv4/export.py
View File

@ -13,43 +13,27 @@
# limitations under the License.
# ============================================================================
"""export checkpoint file into air, onnx, mindir models"""
import argparse
import numpy as np
from src.model_utils.config import config
from src.model_utils.device_adapter import get_device_id
from src.inceptionv4 import Inceptionv4
import mindspore as ms
from mindspore import Tensor
from mindspore.train.serialization import load_checkpoint, load_param_into_net, export, context
from src.config import config_ascend, config_gpu, config_cpu
from src.inceptionv4 import Inceptionv4
parser = argparse.ArgumentParser(description='inceptionv4 export')
parser.add_argument("--device_id", type=int, default=0, help="Device id")
parser.add_argument("--batch_size", type=int, default=1, help="batch size")
parser.add_argument('--ckpt_file', type=str, required=True, help='inceptionv4 ckpt file.')
parser.add_argument('--file_name', type=str, default='inceptionv4', help='inceptionv4 output air name.')
parser.add_argument('--file_format', type=str, choices=["AIR", "MINDIR"], default='AIR', help='file format')
parser.add_argument('--width', type=int, default=299, help='input width')
parser.add_argument('--height', type=int, default=299, help='input height')
parser.add_argument("--device_target", type=str, choices=["Ascend", "GPU", "CPU"], default="Ascend",
help="device target")
args = parser.parse_args()
config.batch_size = 1
context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target)
if args.device_target == "Ascend":
context.set_context(device_id=args.device_id)
CFG_DICT = {
"Ascend": config_ascend,
"GPU": config_gpu,
"CPU": config_cpu,
}
config = CFG_DICT[args.device_target]
context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target)
if config.device_target == "Ascend":
context.set_context(device_id=get_device_id())
if __name__ == '__main__':
net = Inceptionv4(classes=config.num_classes)
param_dict = load_checkpoint(args.ckpt_file)
param_dict = load_checkpoint(config.ckpt_file)
load_param_into_net(net, param_dict)
input_arr = Tensor(np.ones([args.batch_size, 3, args.width, args.height]), ms.float32)
export(net, input_arr, file_name=args.file_name, file_format=args.file_format)
input_arr = Tensor(np.ones([config.batch_size, 3, config.width, config.height]), ms.float32)
export(net, input_arr, file_name=config.file_name, file_format=config.file_format)

9
model_zoo/official/cv/inceptionv4/postprocess.py
View File

@ -14,13 +14,10 @@
# ============================================================================
'''post process for 310 inference'''
import os
import argparse
import numpy as np
parser = argparse.ArgumentParser(description='fasterrcnn_export')
parser.add_argument("--result_path", type=str, required=True, help="result file path")
parser.add_argument("--label_file", type=str, required=True, help="label file")
args = parser.parse_args()
from src.model_utils.config import config
def read_label(label_file):
f = open(label_file, "r")
@ -55,4 +52,4 @@ def cal_acc(result_path, label_file):
print("========accuraty:{}========".format(accuracy))
if __name__ == "__main__":
cal_acc(args.result_path, args.label_file)
cal_acc(config.result_path, config.label_file)

6
model_zoo/official/cv/inceptionv4/scripts/run_distribute_train_ascend.sh
View File

@ -19,6 +19,8 @@ export RANK_TABLE_FILE=$1
DATA_DIR=$2
export RANK_SIZE=8
BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd)
CONFIG_FILE="${BASE_PATH}/../default_config.yaml"
cores=`cat /proc/cpuinfo|grep "processor" |wc -l`
echo "the number of logical core" $cores
@ -39,11 +41,13 @@ do
rm -rf train_parallel$i
mkdir ./train_parallel$i
cp *.py ./train_parallel$i
cp *.yaml ./train_parallel$i
cp -r ./src ./train_parallel$i
cd ./train_parallel$i || exit
echo "start training for rank $i, device $DEVICE_ID rank_id $RANK_ID"
env > env.log
taskset -c $cmdopt python -u ../train.py \
taskset -c $cmdopt python -u ../train.py --config_path=$CONFIG_FILE \
--device_id $i \
--dataset_path=$DATA_DIR > log.txt 2>&1 &
cd ../

7
model_zoo/official/cv/inceptionv4/scripts/run_distribute_train_gpu.sh
View File

@ -17,14 +17,17 @@
rm -rf device
mkdir device
cp ./*.py ./device
cp ./*.yaml ./device
cp -r ./src ./device
cd ./device || exit
DATA_DIR=$1
export DEVICE_ID=0
export RANK_SIZE=8
BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd)
CONFIG_FILE="${BASE_PATH}/../default_config_gpu.yaml"
echo "start training"
mpirun -n $RANK_SIZE --allow-run-as-root python train.py --dataset_path=$DATA_DIR --platform='GPU' > train.log 2>&1 &
mpirun -n $RANK_SIZE --allow-run-as-root python train.py --config_path=$CONFIG_FILE --dataset_path=$DATA_DIR --platform='GPU' > train.log 2>&1 &

5
model_zoo/official/cv/inceptionv4/scripts/run_eval_ascend.sh
View File

@ -19,10 +19,13 @@ DATA_DIR=$2
CHECKPOINT_PATH=$3
export RANK_SIZE=1
BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd)
CONFIG_FILE="${BASE_PATH}/../default_config.yaml"
rm -rf evaluation_ascend
mkdir ./evaluation_ascend
cd ./evaluation_ascend || exit
echo "start training for device id $DEVICE_ID"
env > env.log
python ../eval.py --platform=Ascend --dataset_path=$DATA_DIR --checkpoint_path=$CHECKPOINT_PATH > eval.log 2>&1 &
python ../eval.py --config_path=$CONFIG_FILE --platform=Ascend --dataset_path=$DATA_DIR --checkpoint_path=$CHECKPOINT_PATH > eval.log 2>&1 &
cd ../

5
model_zoo/official/cv/inceptionv4/scripts/run_eval_cpu.sh
View File

@ -17,12 +17,15 @@
rm -rf evaluation
mkdir evaluation
cp ./*.py ./evaluation
cp ./*.yaml ./evaluation
cp -r ./src ./evaluation
cd ./evaluation || exit
DATA_DIR=$1
CKPT_DIR=$2
BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd)
CONFIG_FILE="${BASE_PATH}/../default_config_cpu.yaml"
echo "start evaluation"
python eval.py --dataset_path=$DATA_DIR --checkpoint_path=$CKPT_DIR --platform='CPU' > eval.log 2>&1 &
python eval.py --config_path=$CONFIG_FILE --dataset_path=$DATA_DIR --checkpoint_path=$CKPT_DIR --platform='CPU' > eval.log 2>&1 &

6
model_zoo/official/cv/inceptionv4/scripts/run_eval_gpu.sh
View File

@ -17,6 +17,7 @@
rm -rf evaluation
mkdir evaluation
cp ./*.py ./evaluation
cp ./*.yaml ./evaluation
cp -r ./src ./evaluation
cd ./evaluation || exit
@ -26,6 +27,9 @@ export RANK_SIZE=1
DATA_DIR=$1
CKPT_DIR=$2
BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd)
CONFIG_FILE="${BASE_PATH}/../default_config_gpu.yaml"
echo "start evaluation"
python eval.py --dataset_path=$DATA_DIR --checkpoint_path=$CKPT_DIR --platform='GPU' > eval.log 2>&1 &
python eval.py --config_path=$CONFIG_FILE --dataset_path=$DATA_DIR --checkpoint_path=$CKPT_DIR --platform='GPU' > eval.log 2>&1 &

7
model_zoo/official/cv/inceptionv4/scripts/run_infer_310.sh
View File

@ -43,6 +43,9 @@ echo $data_path
echo $label_file
echo $device_id
BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd)
CONFIG_FILE="${BASE_PATH}/../default_config.yaml"
export ASCEND_HOME=/usr/local/Ascend/
if [ -d ${ASCEND_HOME}/ascend-toolkit ]; then
export PATH=$ASCEND_HOME/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin:$ASCEND_HOME/ascend-toolkit/latest/atc/bin:$PATH
@ -82,7 +85,7 @@ function infer()
fi
mkdir result_Files
mkdir time_Result
../ascend310_infer/out/main --model_path=$model --dataset_path=$data_path --device_id=$device_id &> infer.log
../ascend310_infer/out/main --config_path=$CONFIG_FILE --model_path=$model --dataset_path=$data_path --device_id=$device_id &> infer.log
if [ $? -ne 0 ]; then
echo "execute inference failed"
@ -92,7 +95,7 @@ function infer()
function cal_acc()
{
python ../postprocess.py --label_file=$label_file --result_path=result_Files &> acc.log
python ../postprocess.py --config_path=$CONFIG_FILE --label_file=$label_file --result_path=result_Files &> acc.log
if [ $? -ne 0 ]; then
echo "calculate accuracy failed"
exit 1

5
model_zoo/official/cv/inceptionv4/scripts/run_standalone_train_ascend.sh
View File

@ -18,12 +18,15 @@ export RANK_SIZE=1
export DEVICE_ID=$1
DATA_DIR=$2
BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd)
CONFIG_FILE="${BASE_PATH}/../default_config.yaml"
rm -rf train_standalone
mkdir ./train_standalone
cd ./train_standalone || exit
echo "start training for device id $DEVICE_ID"
env > env.log
python -u ../train.py \
python -u ../train.py --config_path=$CONFIG_FILE \
--device_id=$1 \
--dataset_path=$DATA_DIR > log.txt 2>&1 &
cd ../

5
model_zoo/official/cv/inceptionv4/scripts/run_standalone_train_cpu.sh
View File

@ -16,10 +16,13 @@
DATA_DIR=$1
BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd)
CONFIG_FILE="${BASE_PATH}/../default_config_cpu.yaml"
rm -rf train_standalone
mkdir ./train_standalone
cd ./train_standalone || exit
env > env.log
python -u ../train.py \
python -u ../train.py --config_path=$CONFIG_FILE \
--dataset_path=$DATA_DIR --platform=CPU> log.txt 2>&1 &
cd ../

100
model_zoo/official/cv/inceptionv4/src/config.py
View File

@ -1,100 +0,0 @@
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""
network config setting, will be used in main.py
"""
from easydict import EasyDict as edict
config_ascend = edict({
'is_save_on_master': False,
'batch_size': 128,
'epoch_size': 250,
'num_classes': 1000,
'work_nums': 8,
'ds_type': 'imagenet',
'ds_sink_mode': True,
'loss_scale': 1024,
'smooth_factor': 0.1,
'weight_decay': 0.00004,
'momentum': 0.9,
'amp_level': 'O3',
'decay': 0.9,
'epsilon': 1.0,
'keep_checkpoint_max': 10,
'save_checkpoint_epochs': 10,
'lr_init': 0.00004,
'lr_end': 0.000004,
'lr_max': 0.4,
'warmup_epochs': 1,
'start_epoch': 1,
})
config_gpu = edict({
'is_save_on_master': False,
'batch_size': 128,
'epoch_size': 250,
'num_classes': 1000,
'work_nums': 8,
'ds_type': 'imagenet',
'ds_sink_mode': True,
'loss_scale': 1024,
'smooth_factor': 0.1,
'weight_decay': 0.00004,
'momentum': 0.9,
'amp_level': 'O0',
'decay': 0.9,
'epsilon': 1.0,
'keep_checkpoint_max': 10,
'save_checkpoint_epochs': 10,
'lr_init': 0.00004,
'lr_end': 0.000004,
'lr_max': 0.4,
'warmup_epochs': 1,
'start_epoch': 1,
})
config_cpu = edict({
'batch_size': 128,
'epoch_size': 250,
'num_classes': 10,
'work_nums': 8,
'ds_type': 'cifar10',
'ds_sink_mode': False,
'loss_scale': 1024,
'smooth_factor': 0.1,
'weight_decay': 0.00004,
'momentum': 0.9,
'amp_level': 'O0',
'decay': 0.9,
'epsilon': 1.0,
'keep_checkpoint_max': 10,
'save_checkpoint_epochs': 10,
'lr_init': 0.00004,
'lr_end': 0.000004,
'lr_max': 0.4,
'warmup_epochs': 1,
'start_epoch': 1,
})

0
model_zoo/official/cv/inceptionv4/src/model_utils/__init__.py Normal file
View File

127
model_zoo/official/cv/inceptionv4/src/model_utils/config.py Normal file
View File

@ -0,0 +1,127 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Parse arguments"""
import os
import ast
import argparse
from pprint import pprint, pformat
import yaml
class Config:
"""
Configuration namespace. Convert dictionary to members.
"""
def __init__(self, cfg_dict):
for k, v in cfg_dict.items():
if isinstance(v, (list, tuple)):
setattr(self, k, [Config(x) if isinstance(x, dict) else x for x in v])
else:
setattr(self, k, Config(v) if isinstance(v, dict) else v)
def __str__(self):
return pformat(self.__dict__)
def __repr__(self):
return self.__str__()
def parse_cli_to_yaml(parser, cfg, helper=None, choices=None, cfg_path="default_config.yaml"):
"""
Parse command line arguments to the configuration according to the default yaml.
Args:
parser: Parent parser.
cfg: Base configuration.
helper: Helper description.
cfg_path: Path to the default yaml config.
"""
parser = argparse.ArgumentParser(description="[REPLACE THIS at config.py]",
parents=[parser])
helper = {} if helper is None else helper
choices = {} if choices is None else choices
for item in cfg:
if not isinstance(cfg[item], list) and not isinstance(cfg[item], dict):
help_description = helper[item] if item in helper else "Please reference to {}".format(cfg_path)
choice = choices[item] if item in choices else None
if isinstance(cfg[item], bool):
parser.add_argument("--" + item, type=ast.literal_eval, default=cfg[item], choices=choice,
help=help_description)
else:
parser.add_argument("--" + item, type=type(cfg[item]), default=cfg[item], choices=choice,
help=help_description)
args = parser.parse_args()
return args
def parse_yaml(yaml_path):
"""
Parse the yaml config file.
Args:
yaml_path: Path to the yaml config.
"""
with open(yaml_path, 'r') as fin:
try:
cfgs = yaml.load_all(fin.read(), Loader=yaml.FullLoader)
cfgs = [x for x in cfgs]
if len(cfgs) == 1:
cfg_helper = {}
cfg = cfgs[0]
cfg_choices = {}
elif len(cfgs) == 2:
cfg, cfg_helper = cfgs
cfg_choices = {}
elif len(cfgs) == 3:
cfg, cfg_helper, cfg_choices = cfgs
else:
raise ValueError("At most 3 docs (config, description for help, choices) are supported in config yaml")
print(cfg_helper)
except:
raise ValueError("Failed to parse yaml")
return cfg, cfg_helper, cfg_choices
def merge(args, cfg):
"""
Merge the base config from yaml file and command line arguments.
Args:
args: Command line arguments.
cfg: Base configuration.
"""
args_var = vars(args)
for item in args_var:
cfg[item] = args_var[item]
return cfg
def get_config():
"""
Get Config according to the yaml file and cli arguments.
"""
parser = argparse.ArgumentParser(description="default name", add_help=False)
current_dir = os.path.dirname(os.path.abspath(__file__))
parser.add_argument("--config_path", type=str, default=os.path.join(current_dir, "../../default_config.yaml"),
help="Config file path")
path_args, _ = parser.parse_known_args()
default, helper, choices = parse_yaml(path_args.config_path)
pprint(default)
args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path)
final_config = merge(args, default)
return Config(final_config)
config = get_config()

27
model_zoo/official/cv/inceptionv4/src/model_utils/device_adapter.py Normal file
View File

@ -0,0 +1,27 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Device adapter for ModelArts"""
from .config import config
if config.enable_modelarts:
from .moxing_adapter import get_device_id, get_device_num, get_rank_id, get_job_id
else:
from .local_adapter import get_device_id, get_device_num, get_rank_id, get_job_id
__all__ = [
"get_device_id", "get_device_num", "get_rank_id", "get_job_id"
]

36
model_zoo/official/cv/inceptionv4/src/model_utils/local_adapter.py Normal file
View File

@ -0,0 +1,36 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Local adapter"""
import os
def get_device_id():
device_id = os.getenv('DEVICE_ID', '0')
return int(device_id)
def get_device_num():
device_num = os.getenv('RANK_SIZE', '1')
return int(device_num)
def get_rank_id():
global_rank_id = os.getenv('RANK_ID', '0')
return int(global_rank_id)
def get_job_id():
return "Local Job"

122
model_zoo/official/cv/inceptionv4/src/model_utils/moxing_adapter.py Normal file
View File

@ -0,0 +1,122 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Moxing adapter for ModelArts"""
import os
import functools
from mindspore import context
from mindspore.profiler import Profiler
from .config import config
_global_sync_count = 0
def get_device_id():
device_id = os.getenv('DEVICE_ID', '0')
return int(device_id)
def get_device_num():
device_num = os.getenv('RANK_SIZE', '1')
return int(device_num)
def get_rank_id():
global_rank_id = os.getenv('RANK_ID', '0')
return int(global_rank_id)
def get_job_id():
job_id = os.getenv('JOB_ID')
job_id = job_id if job_id != "" else "default"
return job_id
def sync_data(from_path, to_path):
"""
Download data from remote obs to local directory if the first url is remote url and the second one is local path
Upload data from local directory to remote obs in contrast.
"""
import moxing as mox
import time
global _global_sync_count
sync_lock = "/tmp/copy_sync.lock" + str(_global_sync_count)
_global_sync_count += 1
# Each server contains 8 devices as most.
if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock):
print("from path: ", from_path)
print("to path: ", to_path)
mox.file.copy_parallel(from_path, to_path)
print("===finish data synchronization===")
try:
os.mknod(sync_lock)
except IOError:
pass
print("===save flag===")
while True:
if os.path.exists(sync_lock):
break
time.sleep(1)
print("Finish sync data from {} to {}.".format(from_path, to_path))
def moxing_wrapper(pre_process=None, post_process=None):
"""
Moxing wrapper to download dataset and upload outputs.
"""
def wrapper(run_func):
@functools.wraps(run_func)
def wrapped_func(*args, **kwargs):
# Download data from data_url
if config.enable_modelarts:
if config.data_url:
sync_data(config.data_url, config.data_path)
print("Dataset downloaded: ", os.listdir(config.data_path))
if config.checkpoint_url:
sync_data(config.checkpoint_url, config.load_path)
print("Preload downloaded: ", os.listdir(config.load_path))
if config.train_url:
sync_data(config.train_url, config.output_path)
print("Workspace downloaded: ", os.listdir(config.output_path))
context.set_context(save_graphs_path=os.path.join(config.output_path, str(get_rank_id())))
config.device_num = get_device_num()
config.device_id = get_device_id()
if not os.path.exists(config.output_path):
os.makedirs(config.output_path)
if pre_process:
pre_process()
if config.enable_profiling:
profiler = Profiler()
run_func(*args, **kwargs)
if config.enable_profiling:
profiler.analyse()
# Upload data to train_url
if config.enable_modelarts:
if post_process:
post_process()
if config.train_url:
print("Start to copy output directory")
sync_data(config.output_path, config.train_url)
return wrapped_func
return wrapper

117
model_zoo/official/cv/inceptionv4/train.py
View File

@ -13,12 +13,17 @@
# limitations under the License.
# ============================================================================
"""train imagenet"""
import argparse
import time
import math
import os
import numpy as np
from src.model_utils.config import config
from src.model_utils.moxing_adapter import moxing_wrapper
from src.model_utils.device_adapter import get_device_id, get_device_num
from src.dataset import create_dataset_imagenet, create_dataset_cifar10
from src.inceptionv4 import Inceptionv4
from mindspore import Model
from mindspore import Tensor
from mindspore import context
@ -31,42 +36,19 @@ from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, TimeMoni
from mindspore.train.loss_scale_manager import FixedLossScaleManager
from mindspore.train.model import ParallelMode
from mindspore.train.serialization import load_checkpoint, load_param_into_net
from src.config import config_ascend, config_gpu, config_cpu
from src.dataset import create_dataset_imagenet, create_dataset_cifar10
from src.inceptionv4 import Inceptionv4
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python'
set_seed(1)
CFG_DICT = {
"Ascend": config_ascend,
"GPU": config_gpu,
"CPU": config_cpu,
}
DS_DICT = {
"imagenet": create_dataset_imagenet,
"cifar10": create_dataset_cifar10,
}
device_num = int(os.getenv('RANK_SIZE', '1'))
def parse_args():
'''parse_args'''
arg_parser = argparse.ArgumentParser(description='InceptionV4 image classification training')
arg_parser.add_argument('--dataset_path', type=str, default='', help='Dataset path')
arg_parser.add_argument('--device_id', type=int, default=0, help='device id')
arg_parser.add_argument('--platform', type=str, default='Ascend', choices=("Ascend", "GPU", "CPU"),
help='Platform, support Ascend, GPU, CPU.')
arg_parser.add_argument('--resume', type=str, default='', help='resume training with existed checkpoint')
args_opt = arg_parser.parse_args()
return args_opt
args = parse_args()
config = CFG_DICT[args.platform]
config.device_id = get_device_id()
config.device_num = get_device_num()
device_num = config.device_num
create_dataset = DS_DICT[config.ds_type]
@ -107,21 +89,77 @@ def generate_cosine_lr(steps_per_epoch, total_epochs,
return learning_rate
def modelarts_pre_process():
def unzip(zip_file, save_dir):
import zipfile
s_time = time.time()
if not os.path.exists(os.path.join(save_dir, config.modelarts_dataset_unzip_name)):
zip_isexist = zipfile.is_zipfile(zip_file)
if zip_isexist:
fz = zipfile.ZipFile(zip_file, 'r')
data_num = len(fz.namelist())
print('Extract Start...')
print('unzip file num: {}'.format(data_num))
data_print = int(data_num / 100) if data_num > 100 else 1
i = 0
for file in fz.namelist():
if i % data_print == 0:
print('unzip percent: {}%'.format(int(i * 100 / data_num)), flush=True)
i += 1
fz.extract(file, save_dir)
print('cost time: {}min:{}s.'.format(int((time.time() - s_time) / 60),\
int(int(time.time() - s_time) % 60)))
print('Extract Done')
else:
print('This is not zip.')
else:
print('Zip has been extracted.')
if config.need_modelarts_dataset_unzip:
zip_file_1 = os.path.join(config.data_path, config.modelarts_dataset_unzip_name + '.zip')
save_dir_1 = os.path.join(config.data_path)
sync_lock = '/tmp/unzip_sync.lock'
# Each server contains 8 devices as most
if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock):
print('Zip file path: ', zip_file_1)
print('Unzip file save dir: ', save_dir_1)
unzip(zip_file_1, save_dir_1)
print('===Finish extract data synchronization===')
try:
os.mknod(sync_lock)
except IOError:
pass
while True:
if os.path.exists(sync_lock):
break
time.sleep(1)
print('Device: {}, Finish sync unzip data from {} to {}.'.format(get_device_id(), zip_file_1, save_dir_1))
print('#' * 200, os.listdir(save_dir_1))
print('#' * 200, os.listdir(os.path.join(config.data_path, config.modelarts_dataset_unzip_name)))
config.dataset_path = os.path.join(config.data_path, config.modelarts_dataset_unzip_name)
@moxing_wrapper(pre_process=modelarts_pre_process)
def inception_v4_train():
"""
Train Inceptionv4 in data parallelism
"""
print('epoch_size: {} batch_size: {} class_num {}'.format(config.epoch_size, config.batch_size, config.num_classes))
context.set_context(mode=context.GRAPH_MODE, device_target=args.platform)
if args.platform == "Ascend":
context.set_context(device_id=args.device_id)
context.set_context(mode=context.GRAPH_MODE, device_target=config.platform)
if config.platform == "Ascend":
context.set_context(device_id=get_device_id())
context.set_context(enable_graph_kernel=False)
if device_num > 1:
if args.platform == "Ascend":
if config.platform == "Ascend":
init(backend_name='hccl')
elif args.platform == "GPU":
elif config.platform == "GPU":
init()
else:
raise ValueError("Unsupported device target.")
@ -137,7 +175,7 @@ def inception_v4_train():
config.group_size = 1
# create dataset
train_dataset = create_dataset(dataset_path=args.dataset_path, do_train=True, cfg=config)
train_dataset = create_dataset(dataset_path=config.dataset_path, do_train=True, cfg=config)
train_step_size = train_dataset.get_dataset_size()
# create model
@ -164,11 +202,11 @@ def inception_v4_train():
opt = RMSProp(group_params, lr, decay=config.decay, epsilon=config.epsilon, weight_decay=config.weight_decay,
momentum=config.momentum, loss_scale=config.loss_scale)
if args.device_id == 0:
if get_device_id() == 0:
print(lr)
print(train_step_size)
if args.resume:
ckpt = load_checkpoint(args.resume)
if config.resume:
ckpt = load_checkpoint(config.resume)
load_param_into_net(net, ckpt)
loss_scale_manager = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False)
@ -185,7 +223,7 @@ def inception_v4_train():
directory='ckpts_rank_' + str(config.rank), config=config_ck)
callbacks = [performance_cb, loss_cb]
if device_num > 1 and config.is_save_on_master:
if args.device_id == 0:
if get_device_id() == 0:
callbacks.append(ckpoint_cb)
else:
callbacks.append(ckpoint_cb)
@ -195,5 +233,6 @@ def inception_v4_train():
if __name__ == '__main__':
config.dataset_path = os.path.join(config.dataset_path, 'train')
inception_v4_train()
print('Inceptionv4 training success!')