diff --git a/model_zoo/official/cv/inceptionv4/README.md b/model_zoo/official/cv/inceptionv4/README.md index 23f80335723..306c0f1cec4 100644 --- a/model_zoo/official/cv/inceptionv4/README.md +++ b/model_zoo/official/cv/inceptionv4/README.md @@ -78,14 +78,21 @@ For FP16 operators, if the input data type is FP32, the backend of MindSpore wil ├─run_infer_310.sh # shell script for 310 inference └─run_eval_ascend.sh # launch evaluating with ascend platform ├─src - ├─config.py # parameter configuration ├─dataset.py # data preprocessing ├─inceptionv4.py # network definition - └─callback.py # eval callback function - ├─eval.py # eval net - ├─export.py # export checkpoint, surpport .onnx, .air, .mindir convert - ├─postprogress.py # post process for 310 inference - └─train.py # train net + ├─callback.py # eval callback function + └─model_utils + ├─config.py # Processing configuration parameters + ├─device_adapter.py # Get cloud ID + ├─local_adapter.py # Get local ID + └─moxing_adapter.py # Parameter processing + ├─default_config.yaml # Training parameter profile(ascend) + ├─default_config_cpu.yaml # Training parameter profile(cpu) + ├─default_config_gpu.yaml # Training parameter profile(gpu) + ├─eval.py # eval net + ├─export.py # export checkpoint, surpport .onnx, .air, .mindir convert + ├─postprogress.py # post process for 310 inference + └─train.py # train net ``` ## [Script Parameters](#contents) diff --git a/model_zoo/official/cv/inceptionv4/default_config.yaml b/model_zoo/official/cv/inceptionv4/default_config.yaml new file mode 100644 index 00000000000..eee0fde951d --- /dev/null +++ b/model_zoo/official/cv/inceptionv4/default_config.yaml @@ -0,0 +1,74 @@ +# Builtin Configurations(DO NOT CHANGE THESE CONFIGURATIONS unless you know exactly what you are doing) +enable_modelarts: False +data_url: "" +train_url: "" +checkpoint_url: "" +data_path: "/cache/data" +output_path: "/cache/train" +load_path: "/cache/checkpoint_path" +device_target: Ascend +enable_profiling: False + +# ============================================================================== +dataset_path: "/cache/data" +ckpt_path: '/cache/data/' +checkpoint_path: '/cache/data/inceptionv3/inceptionv3-rank3_1-247_1251.ckpt' +ckpt_file: '/cache/data/inceptionv3/inceptionv3-rank3_1-247_1251.ckpt' +resume: '' +is_distributed: False +device_id: 0 +platform: 'Ascend' +file_name: 'inceptionv4' +file_format: 'MINDIR' +width: 299 +height: 299 + +# fasterrcnn_export +result_path: '' # "result file path" +label_file: '' # "label file" + +# Training options +is_save_on_master: False + +batch_size: 128 +epoch_size: 250 +num_classes: 1000 +work_nums: 8 +ds_type: 'imagenet' +ds_sink_mode: True + +loss_scale: 1024 +smooth_factor: 0.1 +weight_decay: 0.00004 +momentum: 0.9 +amp_level: 'O3' +decay: 0.9 +epsilon: 1.0 + +keep_checkpoint_max: 10 +save_checkpoint_epochs: 10 + +lr_init: 0.00004 +lr_end: 0.000004 +lr_max: 0.4 +warmup_epochs: 1 +start_epoch: 1 + + +--- +# Config description for each option +enable_modelarts: 'Whether training on modelarts, default: False' +data_url: 'Dataset url for obs' +train_url: 'Training output url for obs' +data_path: 'Dataset path for local' +output_path: 'Training output path for local' + +device_target: 'Target device type' +enable_profiling: 'Whether enable profiling while training, default: False' +file_name: 'output file name.' +file_format: 'file format' +resume: 'resume training with existed checkpoint' + +--- +device_target: ['Ascend', 'GPU', 'CPU'] +file_format: ['AIR', 'ONNX', 'MINDIR'] diff --git a/model_zoo/official/cv/inceptionv4/default_config_cpu.yaml b/model_zoo/official/cv/inceptionv4/default_config_cpu.yaml new file mode 100644 index 00000000000..e6f403b2230 --- /dev/null +++ b/model_zoo/official/cv/inceptionv4/default_config_cpu.yaml @@ -0,0 +1,67 @@ +# Builtin Configurations(DO NOT CHANGE THESE CONFIGURATIONS unless you know exactly what you are doing) +enable_modelarts: False +data_url: "" +train_url: "" +checkpoint_url: "" +data_path: "/cache/data" +output_path: "/cache/train" +load_path: "/cache/checkpoint_path" +device_target: Ascend +enable_profiling: False + +# ============================================================================== +dataset_path: "/cache/data" +ckpt_path: '/cache/data/' +checkpoint: '/cache/data/inceptionv3/inceptionv3-rank3_1-247_1251.ckpt' +ckpt_file: '/cache/data/inceptionv3/inceptionv3-rank3_1-247_1251.ckpt' +resume: '' +is_distributed: False +device_id: 0 +platform: 'GPU' +file_name: 'inceptionv3' +file_format: 'AIR' +width: 299 +height: 299 + +# Training options +batch_size: 128 +epoch_size: 250 +num_classes: 10 +work_nums: 8 +ds_type: 'cifar10' +ds_sink_mode: False + +loss_scale: 1024 +smooth_factor: 0.1 +weight_decay: 0.00004 +momentum: 0.9 +amp_level: 'O0' +decay: 0.9 +epsilon: 1.0 + +keep_checkpoint_max: 10 +save_checkpoint_epochs: 10 + +lr_init: 0.00004 +lr_end: 0.000004 +lr_max: 0.4 +warmup_epochs: 1 +start_epoch: 1 + + +--- +# Config description for each option +enable_modelarts: 'Whether training on modelarts, default: False' +data_url: 'Dataset url for obs' +train_url: 'Training output url for obs' +data_path: 'Dataset path for local' +output_path: 'Training output path for local' + +device_target: 'Target device type' +enable_profiling: 'Whether enable profiling while training, default: False' +file_name: 'inceptionv3 output air name.' +file_format: 'file format' + +--- +device_target: ['Ascend', 'GPU', 'CPU'] +file_format: ['AIR', 'ONNX', 'MINDIR'] diff --git a/model_zoo/official/cv/inceptionv4/default_config_gpu.yaml b/model_zoo/official/cv/inceptionv4/default_config_gpu.yaml new file mode 100644 index 00000000000..e53162651d1 --- /dev/null +++ b/model_zoo/official/cv/inceptionv4/default_config_gpu.yaml @@ -0,0 +1,69 @@ +# Builtin Configurations(DO NOT CHANGE THESE CONFIGURATIONS unless you know exactly what you are doing) +enable_modelarts: False +data_url: "" +train_url: "" +checkpoint_url: "" +data_path: "/cache/data" +output_path: "/cache/train" +load_path: "/cache/checkpoint_path" +device_target: Ascend +enable_profiling: False + +# ============================================================================== +dataset_path: "/cache/data" +ckpt_path: '/cache/data/' +checkpoint: '/cache/data/inceptionv3/inceptionv3-rank3_1-247_1251.ckpt' +ckpt_file: '/cache/data/inceptionv3/inceptionv3-rank3_1-247_1251.ckpt' +resume: '' +is_distributed: False +device_id: 0 +platform: 'GPU' +file_name: 'inceptionv3' +file_format: 'AIR' +width: 299 +height: 299 + +# Training options +is_save_on_master: False + +batch_size: 128 +epoch_size: 250 +num_classes: 1000 +work_nums: 8 +ds_type: 'imagenet' +ds_sink_mode: True + +loss_scale: 1024 +smooth_factor: 0.1 +weight_decay: 0.00004 +momentum: 0.9 +amp_level: 'O0' +decay: 0.9 +epsilon: 1.0 + +keep_checkpoint_max: 10 +save_checkpoint_epochs: 10 + +lr_init: 0.00004 +lr_end: 0.000004 +lr_max: 0.4 +warmup_epochs: 1 +start_epoch: 1 + + +--- +# Config description for each option +enable_modelarts: 'Whether training on modelarts, default: False' +data_url: 'Dataset url for obs' +train_url: 'Training output url for obs' +data_path: 'Dataset path for local' +output_path: 'Training output path for local' + +device_target: 'Target device type' +enable_profiling: 'Whether enable profiling while training, default: False' +file_name: 'output file name.' +file_format: 'file format' + +--- +device_target: ['Ascend', 'GPU', 'CPU'] +file_format: ['AIR', 'ONNX', 'MINDIR'] diff --git a/model_zoo/official/cv/inceptionv4/eval.py b/model_zoo/official/cv/inceptionv4/eval.py index 64269bb0345..d83836adfe0 100644 --- a/model_zoo/official/cv/inceptionv4/eval.py +++ b/model_zoo/official/cv/inceptionv4/eval.py @@ -13,58 +13,100 @@ # limitations under the License. # ============================================================================ """evaluate_imagenet""" -import argparse +import time import os +from src.model_utils.config import config +from src.model_utils.moxing_adapter import moxing_wrapper +from src.model_utils.device_adapter import get_device_id, get_device_num +from src.dataset import create_dataset_imagenet, create_dataset_cifar10 +from src.inceptionv4 import Inceptionv4 + import mindspore.nn as nn from mindspore import context from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits from mindspore.train.model import Model from mindspore.train.serialization import load_checkpoint, load_param_into_net -from src.config import config_ascend, config_gpu, config_cpu -from src.dataset import create_dataset_imagenet, create_dataset_cifar10 -from src.inceptionv4 import Inceptionv4 -CFG_DICT = { - "Ascend": config_ascend, - "GPU": config_gpu, - "CPU": config_cpu, -} + +def modelarts_process(): + """ modelarts process """ + def unzip(zip_file, save_dir): + import zipfile + s_time = time.time() + if not os.path.exists(os.path.join(save_dir, config.modelarts_dataset_unzip_name)): + zip_isexist = zipfile.is_zipfile(zip_file) + if zip_isexist: + fz = zipfile.ZipFile(zip_file, 'r') + data_num = len(fz.namelist()) + print('Extract Start...') + print('unzip file num: {}'.format(data_num)) + data_print = int(data_num / 100) if data_num > 100 else 1 + i = 0 + for file in fz.namelist(): + if i % data_print == 0: + print('unzip percent: {}%'.format(int(i * 100 / data_num)), flush=True) + i += 1 + fz.extract(file, save_dir) + print('cost time: {}min:{}s.'.format(int((time.time() - s_time) / 60),\ + int(int(time.time() - s_time) % 60))) + print('Extract Done') + else: + print('This is not zip.') + else: + print('Zip has been extracted.') + + if config.need_modelarts_dataset_unzip: + zip_file_1 = os.path.join(config.data_path, config.modelarts_dataset_unzip_name + '.zip') + save_dir_1 = os.path.join(config.data_path) + + sync_lock = '/tmp/unzip_sync.lock' + + # Each server contains 8 devices as most + if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock): + print('Zip file path: ', zip_file_1) + print('Unzip file save dir: ', save_dir_1) + unzip(zip_file_1, save_dir_1) + print('===Finish extract data synchronization===') + try: + os.mknod(sync_lock) + except IOError: + pass + + while True: + if os.path.exists(sync_lock): + break + time.sleep(1) + + print('Device: {}, Finish sync unzip data from {} to {}.'.format(get_device_id(), zip_file_1, save_dir_1)) + print('#' * 200, os.listdir(save_dir_1)) + print('#' * 200, os.listdir(os.path.join(config.data_path, config.modelarts_dataset_unzip_name))) + + config.dataset_path = os.path.join(config.data_path, config.modelarts_dataset_unzip_name) + DS_DICT = { "imagenet": create_dataset_imagenet, "cifar10": create_dataset_cifar10, } +@moxing_wrapper(pre_process=modelarts_process) +def inception_v4_eval(): -def parse_args(): - '''parse_args''' - parser = argparse.ArgumentParser(description='image classification evaluation') - parser.add_argument('--platform', type=str, default='Ascend', choices=('Ascend', 'GPU', 'CPU'), help='run platform') - parser.add_argument('--dataset_path', type=str, default='', help='Dataset path') - parser.add_argument('--checkpoint_path', type=str, default='', help='checkpoint of inceptionV4') - args_opt = parser.parse_args() - return args_opt - - -if __name__ == '__main__': - args = parse_args() - - if args.platform == 'Ascend': + if config.platform == 'Ascend': device_id = int(os.getenv('DEVICE_ID', '0')) context.set_context(device_id=device_id) - config = CFG_DICT[args.platform] create_dataset = DS_DICT[config.ds_type] - context.set_context(mode=context.GRAPH_MODE, device_target=args.platform) + context.set_context(mode=context.GRAPH_MODE, device_target=config.platform) net = Inceptionv4(classes=config.num_classes) - ckpt = load_checkpoint(args.checkpoint_path) + ckpt = load_checkpoint(config.checkpoint_path) load_param_into_net(net, ckpt) net.set_train(False) config.rank = 0 config.group_size = 1 - dataset = create_dataset(dataset_path=args.dataset_path, do_train=False, cfg=config) + dataset = create_dataset(dataset_path=config.dataset_path, do_train=False, cfg=config) loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") eval_metrics = {'Loss': nn.Loss(), 'Top1-Acc': nn.Top1CategoricalAccuracy(), @@ -73,3 +115,8 @@ if __name__ == '__main__': print('=' * 20, 'Evalute start', '=' * 20) metrics = model.eval(dataset, dataset_sink_mode=config.ds_sink_mode) print("metric: ", metrics) + + +if __name__ == '__main__': + config.dataset_path = os.path.join(config.dataset_path, 'validation_preprocess') + inception_v4_eval() diff --git a/model_zoo/official/cv/inceptionv4/export.py b/model_zoo/official/cv/inceptionv4/export.py index eab136ed51a..ea1144d742a 100644 --- a/model_zoo/official/cv/inceptionv4/export.py +++ b/model_zoo/official/cv/inceptionv4/export.py @@ -13,43 +13,27 @@ # limitations under the License. # ============================================================================ """export checkpoint file into air, onnx, mindir models""" -import argparse import numpy as np +from src.model_utils.config import config +from src.model_utils.device_adapter import get_device_id +from src.inceptionv4 import Inceptionv4 + import mindspore as ms from mindspore import Tensor from mindspore.train.serialization import load_checkpoint, load_param_into_net, export, context -from src.config import config_ascend, config_gpu, config_cpu -from src.inceptionv4 import Inceptionv4 -parser = argparse.ArgumentParser(description='inceptionv4 export') -parser.add_argument("--device_id", type=int, default=0, help="Device id") -parser.add_argument("--batch_size", type=int, default=1, help="batch size") -parser.add_argument('--ckpt_file', type=str, required=True, help='inceptionv4 ckpt file.') -parser.add_argument('--file_name', type=str, default='inceptionv4', help='inceptionv4 output air name.') -parser.add_argument('--file_format', type=str, choices=["AIR", "MINDIR"], default='AIR', help='file format') -parser.add_argument('--width', type=int, default=299, help='input width') -parser.add_argument('--height', type=int, default=299, help='input height') -parser.add_argument("--device_target", type=str, choices=["Ascend", "GPU", "CPU"], default="Ascend", - help="device target") -args = parser.parse_args() +config.batch_size = 1 -context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target) -if args.device_target == "Ascend": - context.set_context(device_id=args.device_id) - -CFG_DICT = { - "Ascend": config_ascend, - "GPU": config_gpu, - "CPU": config_cpu, -} -config = CFG_DICT[args.device_target] +context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target) +if config.device_target == "Ascend": + context.set_context(device_id=get_device_id()) if __name__ == '__main__': net = Inceptionv4(classes=config.num_classes) - param_dict = load_checkpoint(args.ckpt_file) + param_dict = load_checkpoint(config.ckpt_file) load_param_into_net(net, param_dict) - input_arr = Tensor(np.ones([args.batch_size, 3, args.width, args.height]), ms.float32) - export(net, input_arr, file_name=args.file_name, file_format=args.file_format) + input_arr = Tensor(np.ones([config.batch_size, 3, config.width, config.height]), ms.float32) + export(net, input_arr, file_name=config.file_name, file_format=config.file_format) diff --git a/model_zoo/official/cv/inceptionv4/postprocess.py b/model_zoo/official/cv/inceptionv4/postprocess.py index 0b032de2b3f..1ba9a16a633 100644 --- a/model_zoo/official/cv/inceptionv4/postprocess.py +++ b/model_zoo/official/cv/inceptionv4/postprocess.py @@ -14,13 +14,10 @@ # ============================================================================ '''post process for 310 inference''' import os -import argparse import numpy as np -parser = argparse.ArgumentParser(description='fasterrcnn_export') -parser.add_argument("--result_path", type=str, required=True, help="result file path") -parser.add_argument("--label_file", type=str, required=True, help="label file") -args = parser.parse_args() +from src.model_utils.config import config + def read_label(label_file): f = open(label_file, "r") @@ -55,4 +52,4 @@ def cal_acc(result_path, label_file): print("========accuraty:{}========".format(accuracy)) if __name__ == "__main__": - cal_acc(args.result_path, args.label_file) + cal_acc(config.result_path, config.label_file) diff --git a/model_zoo/official/cv/inceptionv4/scripts/run_distribute_train_ascend.sh b/model_zoo/official/cv/inceptionv4/scripts/run_distribute_train_ascend.sh index 49768b95a64..bf91ed469df 100644 --- a/model_zoo/official/cv/inceptionv4/scripts/run_distribute_train_ascend.sh +++ b/model_zoo/official/cv/inceptionv4/scripts/run_distribute_train_ascend.sh @@ -19,6 +19,8 @@ export RANK_TABLE_FILE=$1 DATA_DIR=$2 export RANK_SIZE=8 +BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd) +CONFIG_FILE="${BASE_PATH}/../default_config.yaml" cores=`cat /proc/cpuinfo|grep "processor" |wc -l` echo "the number of logical core" $cores @@ -39,11 +41,13 @@ do rm -rf train_parallel$i mkdir ./train_parallel$i cp *.py ./train_parallel$i + cp *.yaml ./train_parallel$i + cp -r ./src ./train_parallel$i cd ./train_parallel$i || exit echo "start training for rank $i, device $DEVICE_ID rank_id $RANK_ID" env > env.log - taskset -c $cmdopt python -u ../train.py \ + taskset -c $cmdopt python -u ../train.py --config_path=$CONFIG_FILE \ --device_id $i \ --dataset_path=$DATA_DIR > log.txt 2>&1 & cd ../ diff --git a/model_zoo/official/cv/inceptionv4/scripts/run_distribute_train_gpu.sh b/model_zoo/official/cv/inceptionv4/scripts/run_distribute_train_gpu.sh index ff0a892a1a9..ca1ee884bb5 100644 --- a/model_zoo/official/cv/inceptionv4/scripts/run_distribute_train_gpu.sh +++ b/model_zoo/official/cv/inceptionv4/scripts/run_distribute_train_gpu.sh @@ -17,14 +17,17 @@ rm -rf device mkdir device cp ./*.py ./device +cp ./*.yaml ./device cp -r ./src ./device cd ./device || exit DATA_DIR=$1 - export DEVICE_ID=0 export RANK_SIZE=8 +BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd) +CONFIG_FILE="${BASE_PATH}/../default_config_gpu.yaml" + echo "start training" -mpirun -n $RANK_SIZE --allow-run-as-root python train.py --dataset_path=$DATA_DIR --platform='GPU' > train.log 2>&1 & +mpirun -n $RANK_SIZE --allow-run-as-root python train.py --config_path=$CONFIG_FILE --dataset_path=$DATA_DIR --platform='GPU' > train.log 2>&1 & diff --git a/model_zoo/official/cv/inceptionv4/scripts/run_eval_ascend.sh b/model_zoo/official/cv/inceptionv4/scripts/run_eval_ascend.sh index a33555ae249..02f42e11a63 100644 --- a/model_zoo/official/cv/inceptionv4/scripts/run_eval_ascend.sh +++ b/model_zoo/official/cv/inceptionv4/scripts/run_eval_ascend.sh @@ -19,10 +19,13 @@ DATA_DIR=$2 CHECKPOINT_PATH=$3 export RANK_SIZE=1 +BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd) +CONFIG_FILE="${BASE_PATH}/../default_config.yaml" + rm -rf evaluation_ascend mkdir ./evaluation_ascend cd ./evaluation_ascend || exit echo "start training for device id $DEVICE_ID" env > env.log -python ../eval.py --platform=Ascend --dataset_path=$DATA_DIR --checkpoint_path=$CHECKPOINT_PATH > eval.log 2>&1 & +python ../eval.py --config_path=$CONFIG_FILE --platform=Ascend --dataset_path=$DATA_DIR --checkpoint_path=$CHECKPOINT_PATH > eval.log 2>&1 & cd ../ diff --git a/model_zoo/official/cv/inceptionv4/scripts/run_eval_cpu.sh b/model_zoo/official/cv/inceptionv4/scripts/run_eval_cpu.sh index 0dd105b77d6..018c0abd39a 100644 --- a/model_zoo/official/cv/inceptionv4/scripts/run_eval_cpu.sh +++ b/model_zoo/official/cv/inceptionv4/scripts/run_eval_cpu.sh @@ -17,12 +17,15 @@ rm -rf evaluation mkdir evaluation cp ./*.py ./evaluation +cp ./*.yaml ./evaluation cp -r ./src ./evaluation cd ./evaluation || exit DATA_DIR=$1 CKPT_DIR=$2 +BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd) +CONFIG_FILE="${BASE_PATH}/../default_config_cpu.yaml" echo "start evaluation" -python eval.py --dataset_path=$DATA_DIR --checkpoint_path=$CKPT_DIR --platform='CPU' > eval.log 2>&1 & +python eval.py --config_path=$CONFIG_FILE --dataset_path=$DATA_DIR --checkpoint_path=$CKPT_DIR --platform='CPU' > eval.log 2>&1 & diff --git a/model_zoo/official/cv/inceptionv4/scripts/run_eval_gpu.sh b/model_zoo/official/cv/inceptionv4/scripts/run_eval_gpu.sh index cbe12e5b2e4..818b24efe0f 100644 --- a/model_zoo/official/cv/inceptionv4/scripts/run_eval_gpu.sh +++ b/model_zoo/official/cv/inceptionv4/scripts/run_eval_gpu.sh @@ -17,6 +17,7 @@ rm -rf evaluation mkdir evaluation cp ./*.py ./evaluation +cp ./*.yaml ./evaluation cp -r ./src ./evaluation cd ./evaluation || exit @@ -26,6 +27,9 @@ export RANK_SIZE=1 DATA_DIR=$1 CKPT_DIR=$2 +BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd) +CONFIG_FILE="${BASE_PATH}/../default_config_gpu.yaml" + echo "start evaluation" -python eval.py --dataset_path=$DATA_DIR --checkpoint_path=$CKPT_DIR --platform='GPU' > eval.log 2>&1 & +python eval.py --config_path=$CONFIG_FILE --dataset_path=$DATA_DIR --checkpoint_path=$CKPT_DIR --platform='GPU' > eval.log 2>&1 & diff --git a/model_zoo/official/cv/inceptionv4/scripts/run_infer_310.sh b/model_zoo/official/cv/inceptionv4/scripts/run_infer_310.sh index faa08fba0c1..9e6f29790ff 100755 --- a/model_zoo/official/cv/inceptionv4/scripts/run_infer_310.sh +++ b/model_zoo/official/cv/inceptionv4/scripts/run_infer_310.sh @@ -43,6 +43,9 @@ echo $data_path echo $label_file echo $device_id +BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd) +CONFIG_FILE="${BASE_PATH}/../default_config.yaml" + export ASCEND_HOME=/usr/local/Ascend/ if [ -d ${ASCEND_HOME}/ascend-toolkit ]; then export PATH=$ASCEND_HOME/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin:$ASCEND_HOME/ascend-toolkit/latest/atc/bin:$PATH @@ -82,7 +85,7 @@ function infer() fi mkdir result_Files mkdir time_Result - ../ascend310_infer/out/main --model_path=$model --dataset_path=$data_path --device_id=$device_id &> infer.log + ../ascend310_infer/out/main --config_path=$CONFIG_FILE --model_path=$model --dataset_path=$data_path --device_id=$device_id &> infer.log if [ $? -ne 0 ]; then echo "execute inference failed" @@ -92,7 +95,7 @@ function infer() function cal_acc() { - python ../postprocess.py --label_file=$label_file --result_path=result_Files &> acc.log + python ../postprocess.py --config_path=$CONFIG_FILE --label_file=$label_file --result_path=result_Files &> acc.log if [ $? -ne 0 ]; then echo "calculate accuracy failed" exit 1 diff --git a/model_zoo/official/cv/inceptionv4/scripts/run_standalone_train_ascend.sh b/model_zoo/official/cv/inceptionv4/scripts/run_standalone_train_ascend.sh index 326f2a7d5fa..0956920983b 100644 --- a/model_zoo/official/cv/inceptionv4/scripts/run_standalone_train_ascend.sh +++ b/model_zoo/official/cv/inceptionv4/scripts/run_standalone_train_ascend.sh @@ -18,12 +18,15 @@ export RANK_SIZE=1 export DEVICE_ID=$1 DATA_DIR=$2 +BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd) +CONFIG_FILE="${BASE_PATH}/../default_config.yaml" + rm -rf train_standalone mkdir ./train_standalone cd ./train_standalone || exit echo "start training for device id $DEVICE_ID" env > env.log -python -u ../train.py \ +python -u ../train.py --config_path=$CONFIG_FILE \ --device_id=$1 \ --dataset_path=$DATA_DIR > log.txt 2>&1 & cd ../ diff --git a/model_zoo/official/cv/inceptionv4/scripts/run_standalone_train_cpu.sh b/model_zoo/official/cv/inceptionv4/scripts/run_standalone_train_cpu.sh index 2279d933442..9c7a5b8830c 100644 --- a/model_zoo/official/cv/inceptionv4/scripts/run_standalone_train_cpu.sh +++ b/model_zoo/official/cv/inceptionv4/scripts/run_standalone_train_cpu.sh @@ -16,10 +16,13 @@ DATA_DIR=$1 +BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd) +CONFIG_FILE="${BASE_PATH}/../default_config_cpu.yaml" + rm -rf train_standalone mkdir ./train_standalone cd ./train_standalone || exit env > env.log -python -u ../train.py \ +python -u ../train.py --config_path=$CONFIG_FILE \ --dataset_path=$DATA_DIR --platform=CPU> log.txt 2>&1 & cd ../ diff --git a/model_zoo/official/cv/inceptionv4/src/config.py b/model_zoo/official/cv/inceptionv4/src/config.py deleted file mode 100644 index a2be6987082..00000000000 --- a/model_zoo/official/cv/inceptionv4/src/config.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -network config setting, will be used in main.py -""" -from easydict import EasyDict as edict - -config_ascend = edict({ - 'is_save_on_master': False, - - 'batch_size': 128, - 'epoch_size': 250, - 'num_classes': 1000, - 'work_nums': 8, - 'ds_type': 'imagenet', - 'ds_sink_mode': True, - - 'loss_scale': 1024, - 'smooth_factor': 0.1, - 'weight_decay': 0.00004, - 'momentum': 0.9, - 'amp_level': 'O3', - 'decay': 0.9, - 'epsilon': 1.0, - - 'keep_checkpoint_max': 10, - 'save_checkpoint_epochs': 10, - - 'lr_init': 0.00004, - 'lr_end': 0.000004, - 'lr_max': 0.4, - 'warmup_epochs': 1, - 'start_epoch': 1, -}) - -config_gpu = edict({ - 'is_save_on_master': False, - - 'batch_size': 128, - 'epoch_size': 250, - 'num_classes': 1000, - 'work_nums': 8, - 'ds_type': 'imagenet', - 'ds_sink_mode': True, - - 'loss_scale': 1024, - 'smooth_factor': 0.1, - 'weight_decay': 0.00004, - 'momentum': 0.9, - 'amp_level': 'O0', - 'decay': 0.9, - 'epsilon': 1.0, - - 'keep_checkpoint_max': 10, - 'save_checkpoint_epochs': 10, - - 'lr_init': 0.00004, - 'lr_end': 0.000004, - 'lr_max': 0.4, - 'warmup_epochs': 1, - 'start_epoch': 1, -}) - -config_cpu = edict({ - 'batch_size': 128, - 'epoch_size': 250, - 'num_classes': 10, - 'work_nums': 8, - 'ds_type': 'cifar10', - 'ds_sink_mode': False, - - 'loss_scale': 1024, - 'smooth_factor': 0.1, - 'weight_decay': 0.00004, - 'momentum': 0.9, - 'amp_level': 'O0', - 'decay': 0.9, - 'epsilon': 1.0, - - 'keep_checkpoint_max': 10, - 'save_checkpoint_epochs': 10, - - 'lr_init': 0.00004, - 'lr_end': 0.000004, - 'lr_max': 0.4, - 'warmup_epochs': 1, - 'start_epoch': 1, -}) diff --git a/model_zoo/official/cv/inceptionv4/src/model_utils/__init__.py b/model_zoo/official/cv/inceptionv4/src/model_utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/model_zoo/official/cv/inceptionv4/src/model_utils/config.py b/model_zoo/official/cv/inceptionv4/src/model_utils/config.py new file mode 100644 index 00000000000..7f1ff6e2b8d --- /dev/null +++ b/model_zoo/official/cv/inceptionv4/src/model_utils/config.py @@ -0,0 +1,127 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Parse arguments""" + +import os +import ast +import argparse +from pprint import pprint, pformat +import yaml + +class Config: + """ + Configuration namespace. Convert dictionary to members. + """ + def __init__(self, cfg_dict): + for k, v in cfg_dict.items(): + if isinstance(v, (list, tuple)): + setattr(self, k, [Config(x) if isinstance(x, dict) else x for x in v]) + else: + setattr(self, k, Config(v) if isinstance(v, dict) else v) + + def __str__(self): + return pformat(self.__dict__) + + def __repr__(self): + return self.__str__() + + +def parse_cli_to_yaml(parser, cfg, helper=None, choices=None, cfg_path="default_config.yaml"): + """ + Parse command line arguments to the configuration according to the default yaml. + + Args: + parser: Parent parser. + cfg: Base configuration. + helper: Helper description. + cfg_path: Path to the default yaml config. + """ + parser = argparse.ArgumentParser(description="[REPLACE THIS at config.py]", + parents=[parser]) + helper = {} if helper is None else helper + choices = {} if choices is None else choices + for item in cfg: + if not isinstance(cfg[item], list) and not isinstance(cfg[item], dict): + help_description = helper[item] if item in helper else "Please reference to {}".format(cfg_path) + choice = choices[item] if item in choices else None + if isinstance(cfg[item], bool): + parser.add_argument("--" + item, type=ast.literal_eval, default=cfg[item], choices=choice, + help=help_description) + else: + parser.add_argument("--" + item, type=type(cfg[item]), default=cfg[item], choices=choice, + help=help_description) + args = parser.parse_args() + return args + + +def parse_yaml(yaml_path): + """ + Parse the yaml config file. + + Args: + yaml_path: Path to the yaml config. + """ + with open(yaml_path, 'r') as fin: + try: + cfgs = yaml.load_all(fin.read(), Loader=yaml.FullLoader) + cfgs = [x for x in cfgs] + if len(cfgs) == 1: + cfg_helper = {} + cfg = cfgs[0] + cfg_choices = {} + elif len(cfgs) == 2: + cfg, cfg_helper = cfgs + cfg_choices = {} + elif len(cfgs) == 3: + cfg, cfg_helper, cfg_choices = cfgs + else: + raise ValueError("At most 3 docs (config, description for help, choices) are supported in config yaml") + print(cfg_helper) + except: + raise ValueError("Failed to parse yaml") + return cfg, cfg_helper, cfg_choices + + +def merge(args, cfg): + """ + Merge the base config from yaml file and command line arguments. + + Args: + args: Command line arguments. + cfg: Base configuration. + """ + args_var = vars(args) + for item in args_var: + cfg[item] = args_var[item] + return cfg + + +def get_config(): + """ + Get Config according to the yaml file and cli arguments. + """ + parser = argparse.ArgumentParser(description="default name", add_help=False) + current_dir = os.path.dirname(os.path.abspath(__file__)) + parser.add_argument("--config_path", type=str, default=os.path.join(current_dir, "../../default_config.yaml"), + help="Config file path") + path_args, _ = parser.parse_known_args() + default, helper, choices = parse_yaml(path_args.config_path) + pprint(default) + args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path) + final_config = merge(args, default) + return Config(final_config) + +config = get_config() diff --git a/model_zoo/official/cv/inceptionv4/src/model_utils/device_adapter.py b/model_zoo/official/cv/inceptionv4/src/model_utils/device_adapter.py new file mode 100644 index 00000000000..7c5d7f837dd --- /dev/null +++ b/model_zoo/official/cv/inceptionv4/src/model_utils/device_adapter.py @@ -0,0 +1,27 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Device adapter for ModelArts""" + +from .config import config + +if config.enable_modelarts: + from .moxing_adapter import get_device_id, get_device_num, get_rank_id, get_job_id +else: + from .local_adapter import get_device_id, get_device_num, get_rank_id, get_job_id + +__all__ = [ + "get_device_id", "get_device_num", "get_rank_id", "get_job_id" +] diff --git a/model_zoo/official/cv/inceptionv4/src/model_utils/local_adapter.py b/model_zoo/official/cv/inceptionv4/src/model_utils/local_adapter.py new file mode 100644 index 00000000000..769fa6dc78e --- /dev/null +++ b/model_zoo/official/cv/inceptionv4/src/model_utils/local_adapter.py @@ -0,0 +1,36 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Local adapter""" + +import os + +def get_device_id(): + device_id = os.getenv('DEVICE_ID', '0') + return int(device_id) + + +def get_device_num(): + device_num = os.getenv('RANK_SIZE', '1') + return int(device_num) + + +def get_rank_id(): + global_rank_id = os.getenv('RANK_ID', '0') + return int(global_rank_id) + + +def get_job_id(): + return "Local Job" diff --git a/model_zoo/official/cv/inceptionv4/src/model_utils/moxing_adapter.py b/model_zoo/official/cv/inceptionv4/src/model_utils/moxing_adapter.py new file mode 100644 index 00000000000..830d19a6fc9 --- /dev/null +++ b/model_zoo/official/cv/inceptionv4/src/model_utils/moxing_adapter.py @@ -0,0 +1,122 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Moxing adapter for ModelArts""" + +import os +import functools +from mindspore import context +from mindspore.profiler import Profiler +from .config import config + +_global_sync_count = 0 + +def get_device_id(): + device_id = os.getenv('DEVICE_ID', '0') + return int(device_id) + + +def get_device_num(): + device_num = os.getenv('RANK_SIZE', '1') + return int(device_num) + + +def get_rank_id(): + global_rank_id = os.getenv('RANK_ID', '0') + return int(global_rank_id) + + +def get_job_id(): + job_id = os.getenv('JOB_ID') + job_id = job_id if job_id != "" else "default" + return job_id + +def sync_data(from_path, to_path): + """ + Download data from remote obs to local directory if the first url is remote url and the second one is local path + Upload data from local directory to remote obs in contrast. + """ + import moxing as mox + import time + global _global_sync_count + sync_lock = "/tmp/copy_sync.lock" + str(_global_sync_count) + _global_sync_count += 1 + + # Each server contains 8 devices as most. + if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock): + print("from path: ", from_path) + print("to path: ", to_path) + mox.file.copy_parallel(from_path, to_path) + print("===finish data synchronization===") + try: + os.mknod(sync_lock) + except IOError: + pass + print("===save flag===") + + while True: + if os.path.exists(sync_lock): + break + time.sleep(1) + + print("Finish sync data from {} to {}.".format(from_path, to_path)) + + +def moxing_wrapper(pre_process=None, post_process=None): + """ + Moxing wrapper to download dataset and upload outputs. + """ + def wrapper(run_func): + @functools.wraps(run_func) + def wrapped_func(*args, **kwargs): + # Download data from data_url + if config.enable_modelarts: + if config.data_url: + sync_data(config.data_url, config.data_path) + print("Dataset downloaded: ", os.listdir(config.data_path)) + if config.checkpoint_url: + sync_data(config.checkpoint_url, config.load_path) + print("Preload downloaded: ", os.listdir(config.load_path)) + if config.train_url: + sync_data(config.train_url, config.output_path) + print("Workspace downloaded: ", os.listdir(config.output_path)) + + context.set_context(save_graphs_path=os.path.join(config.output_path, str(get_rank_id()))) + config.device_num = get_device_num() + config.device_id = get_device_id() + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) + + if pre_process: + pre_process() + + if config.enable_profiling: + profiler = Profiler() + + run_func(*args, **kwargs) + + if config.enable_profiling: + profiler.analyse() + + # Upload data to train_url + if config.enable_modelarts: + if post_process: + post_process() + + if config.train_url: + print("Start to copy output directory") + sync_data(config.output_path, config.train_url) + return wrapped_func + return wrapper diff --git a/model_zoo/official/cv/inceptionv4/train.py b/model_zoo/official/cv/inceptionv4/train.py index 4f8dc54614f..8ceaed7dc85 100644 --- a/model_zoo/official/cv/inceptionv4/train.py +++ b/model_zoo/official/cv/inceptionv4/train.py @@ -13,12 +13,17 @@ # limitations under the License. # ============================================================================ """train imagenet""" -import argparse +import time import math import os - import numpy as np +from src.model_utils.config import config +from src.model_utils.moxing_adapter import moxing_wrapper +from src.model_utils.device_adapter import get_device_id, get_device_num +from src.dataset import create_dataset_imagenet, create_dataset_cifar10 +from src.inceptionv4 import Inceptionv4 + from mindspore import Model from mindspore import Tensor from mindspore import context @@ -31,42 +36,19 @@ from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, TimeMoni from mindspore.train.loss_scale_manager import FixedLossScaleManager from mindspore.train.model import ParallelMode from mindspore.train.serialization import load_checkpoint, load_param_into_net -from src.config import config_ascend, config_gpu, config_cpu -from src.dataset import create_dataset_imagenet, create_dataset_cifar10 -from src.inceptionv4 import Inceptionv4 + os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python' set_seed(1) -CFG_DICT = { - "Ascend": config_ascend, - "GPU": config_gpu, - "CPU": config_cpu, -} - DS_DICT = { "imagenet": create_dataset_imagenet, "cifar10": create_dataset_cifar10, } -device_num = int(os.getenv('RANK_SIZE', '1')) - - -def parse_args(): - '''parse_args''' - arg_parser = argparse.ArgumentParser(description='InceptionV4 image classification training') - arg_parser.add_argument('--dataset_path', type=str, default='', help='Dataset path') - arg_parser.add_argument('--device_id', type=int, default=0, help='device id') - arg_parser.add_argument('--platform', type=str, default='Ascend', choices=("Ascend", "GPU", "CPU"), - help='Platform, support Ascend, GPU, CPU.') - arg_parser.add_argument('--resume', type=str, default='', help='resume training with existed checkpoint') - args_opt = arg_parser.parse_args() - return args_opt - - -args = parse_args() - -config = CFG_DICT[args.platform] +config.device_id = get_device_id() +config.device_num = get_device_num() +device_num = config.device_num create_dataset = DS_DICT[config.ds_type] @@ -107,21 +89,77 @@ def generate_cosine_lr(steps_per_epoch, total_epochs, return learning_rate +def modelarts_pre_process(): + def unzip(zip_file, save_dir): + import zipfile + s_time = time.time() + if not os.path.exists(os.path.join(save_dir, config.modelarts_dataset_unzip_name)): + zip_isexist = zipfile.is_zipfile(zip_file) + if zip_isexist: + fz = zipfile.ZipFile(zip_file, 'r') + data_num = len(fz.namelist()) + print('Extract Start...') + print('unzip file num: {}'.format(data_num)) + data_print = int(data_num / 100) if data_num > 100 else 1 + i = 0 + for file in fz.namelist(): + if i % data_print == 0: + print('unzip percent: {}%'.format(int(i * 100 / data_num)), flush=True) + i += 1 + fz.extract(file, save_dir) + print('cost time: {}min:{}s.'.format(int((time.time() - s_time) / 60),\ + int(int(time.time() - s_time) % 60))) + print('Extract Done') + else: + print('This is not zip.') + else: + print('Zip has been extracted.') + + if config.need_modelarts_dataset_unzip: + zip_file_1 = os.path.join(config.data_path, config.modelarts_dataset_unzip_name + '.zip') + save_dir_1 = os.path.join(config.data_path) + + sync_lock = '/tmp/unzip_sync.lock' + + # Each server contains 8 devices as most + if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock): + print('Zip file path: ', zip_file_1) + print('Unzip file save dir: ', save_dir_1) + unzip(zip_file_1, save_dir_1) + print('===Finish extract data synchronization===') + try: + os.mknod(sync_lock) + except IOError: + pass + + while True: + if os.path.exists(sync_lock): + break + time.sleep(1) + + print('Device: {}, Finish sync unzip data from {} to {}.'.format(get_device_id(), zip_file_1, save_dir_1)) + print('#' * 200, os.listdir(save_dir_1)) + print('#' * 200, os.listdir(os.path.join(config.data_path, config.modelarts_dataset_unzip_name))) + + config.dataset_path = os.path.join(config.data_path, config.modelarts_dataset_unzip_name) + + +@moxing_wrapper(pre_process=modelarts_pre_process) def inception_v4_train(): """ Train Inceptionv4 in data parallelism """ print('epoch_size: {} batch_size: {} class_num {}'.format(config.epoch_size, config.batch_size, config.num_classes)) - context.set_context(mode=context.GRAPH_MODE, device_target=args.platform) - if args.platform == "Ascend": - context.set_context(device_id=args.device_id) + context.set_context(mode=context.GRAPH_MODE, device_target=config.platform) + if config.platform == "Ascend": + context.set_context(device_id=get_device_id()) context.set_context(enable_graph_kernel=False) if device_num > 1: - if args.platform == "Ascend": + if config.platform == "Ascend": init(backend_name='hccl') - elif args.platform == "GPU": + elif config.platform == "GPU": init() else: raise ValueError("Unsupported device target.") @@ -137,7 +175,7 @@ def inception_v4_train(): config.group_size = 1 # create dataset - train_dataset = create_dataset(dataset_path=args.dataset_path, do_train=True, cfg=config) + train_dataset = create_dataset(dataset_path=config.dataset_path, do_train=True, cfg=config) train_step_size = train_dataset.get_dataset_size() # create model @@ -164,11 +202,11 @@ def inception_v4_train(): opt = RMSProp(group_params, lr, decay=config.decay, epsilon=config.epsilon, weight_decay=config.weight_decay, momentum=config.momentum, loss_scale=config.loss_scale) - if args.device_id == 0: + if get_device_id() == 0: print(lr) print(train_step_size) - if args.resume: - ckpt = load_checkpoint(args.resume) + if config.resume: + ckpt = load_checkpoint(config.resume) load_param_into_net(net, ckpt) loss_scale_manager = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) @@ -185,7 +223,7 @@ def inception_v4_train(): directory='ckpts_rank_' + str(config.rank), config=config_ck) callbacks = [performance_cb, loss_cb] if device_num > 1 and config.is_save_on_master: - if args.device_id == 0: + if get_device_id() == 0: callbacks.append(ckpoint_cb) else: callbacks.append(ckpoint_cb) @@ -195,5 +233,6 @@ def inception_v4_train(): if __name__ == '__main__': + config.dataset_path = os.path.join(config.dataset_path, 'train') inception_v4_train() print('Inceptionv4 training success!')