diff --git a/model_zoo/official/cv/yolov4/README.md b/model_zoo/official/cv/yolov4/README.md index 8dc8af0d1db..5412ac32e36 100644 --- a/model_zoo/official/cv/yolov4/README.md +++ b/model_zoo/official/cv/yolov4/README.md @@ -93,58 +93,148 @@ other datasets need to use the same format as MS COCO. python hccl_tools.py --device_num "[0,8)" ``` -```text -# The parameter of training_shape define image shape for network, default is - [416, 416], - [448, 448], - [480, 480], - [512, 512], - [544, 544], - [576, 576], - [608, 608], - [640, 640], - [672, 672], - [704, 704], - [736, 736]. -# It means use 11 kinds of shape as input shape, or it can be set some kind of shape. -``` +- Run on local -```bash -#run training example(1p) by python command (Training with a single scale) -python train.py \ - --data_dir=./dataset/xxx \ - --pretrained_backbone=cspdarknet53_backbone.ckpt \ - --is_distributed=0 \ - --lr=0.1 \ - --t_max=320 \ - --max_epoch=320 \ - --warmup_epochs=4 \ - --training_shape=416 \ - --lr_scheduler=cosine_annealing > log.txt 2>&1 & -``` + ```text + # The parameter of training_shape define image shape for network, default is + [416, 416], + [448, 448], + [480, 480], + [512, 512], + [544, 544], + [576, 576], + [608, 608], + [640, 640], + [672, 672], + [704, 704], + [736, 736]. + # It means use 11 kinds of shape as input shape, or it can be set some kind of shape. -```bash -# standalone training example(1p) by shell script (Training with a single scale) -sh run_standalone_train.sh dataset/xxx cspdarknet53_backbone.ckpt -``` + #run training example(1p) by python command (Training with a single scale) + python train.py \ + --data_dir=./dataset/xxx \ + --pretrained_backbone=cspdarknet53_backbone.ckpt \ + --is_distributed=0 \ + --lr=0.1 \ + --t_max=320 \ + --max_epoch=320 \ + --warmup_epochs=4 \ + --training_shape=416 \ + --lr_scheduler=cosine_annealing > log.txt 2>&1 & -```bash -# For Ascend device, distributed training example(8p) by shell script (Training with multi scale) -sh run_distribute_train.sh dataset/xxx cspdarknet53_backbone.ckpt rank_table_8p.json -``` + # standalone training example(1p) by shell script (Training with a single scale) + sh run_standalone_train.sh dataset/xxx cspdarknet53_backbone.ckpt -```bash -# run evaluation by python command -python eval.py \ - --data_dir=./dataset/xxx \ - --pretrained=yolov4.ckpt \ - --testing_shape=608 > log.txt 2>&1 & -``` + # For Ascend device, distributed training example(8p) by shell script (Training with multi scale) + sh run_distribute_train.sh dataset/xxx cspdarknet53_backbone.ckpt rank_table_8p.json -```bash -# run evaluation by shell script -sh run_eval.sh dataset/xxx checkpoint/xxx.ckpt -``` + # run evaluation by python command + python eval.py \ + --data_dir=./dataset/xxx \ + --pretrained=yolov4.ckpt \ + --testing_shape=608 > log.txt 2>&1 & + + # run evaluation by shell script + sh run_eval.sh dataset/xxx checkpoint/xxx.ckpt + ``` + +- Train on [ModelArts](https://support.huaweicloud.com/modelarts/) + + ```python + # Train 8p with Ascend + # (1) Perform a or b. + # a. Set "enable_modelarts=True" on base_config.yaml file. + # Set "data_dir='/cache/data/coco/'" on base_config.yaml file. + # Set "checkpoint_url='s3://dir_to_your_pretrain/'" on base_config.yaml file. + # Set "pretrained_backbone='/cache/checkpoint_path/cspdarknet53_backbone.ckpt'" on base_config.yaml file. + # Set other parameters on base_config.yaml file you need. + # b. Add "enable_modelarts=True" on the website UI interface. + # Add "data_dir=/cache/data/coco/" on the website UI interface. + # Add "checkpoint_url=s3://dir_to_your_pretrain/" on the website UI interface. + # Add "pretrained_backbone=/cache/checkpoint_path/cspdarknet53_backbone.ckpt" on the website UI interface. + # Add other parameters on the website UI interface. + # (3) Upload or copy your pretrained model to S3 bucket. + # (4) Upload a zip dataset to S3 bucket. (you could also upload the origin dataset, but it can be so slow.) + # (5) Set the code directory to "/path/yolov4" on the website UI interface. + # (6) Set the startup file to "train.py" on the website UI interface. + # (7) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface. + # (8) Create your job. + # + # Train 1p with Ascend + # (1) Perform a or b. + # a. Set "enable_modelarts=True" on base_config.yaml file. + # Set "data_dir='/cache/data/coco/'" on base_config.yaml file. + # Set "checkpoint_url='s3://dir_to_your_pretrain/'" on base_config.yaml file. + # Set "pretrained_backbone='/cache/checkpoint_path/cspdarknet53_backbone.ckpt'" on base_config.yaml file. + # Set "is_distributed=0" on base_config.yaml file. + # Set "warmup_epochs=4" on base_config.yaml file. + # Set "training_shape=416" on base_config.yaml file. + # Set other parameters on base_config.yaml file you need. + # b. Add "enable_modelarts=True" on the website UI interface. + # Add "data_dir=/cache/data/coco/" on the website UI interface. + # Add "checkpoint_url=s3://dir_to_your_pretrain/" on the website UI interface. + # Add "pretrained_backbone=/cache/checkpoint_path/cspdarknet53_backbone.ckpt" on the website UI interface. + # Add "is_distributed=0" on the website UI interface. + # Add "warmup_epochs=4" on the website UI interface. + # Add "training_shape=416" on the website UI interface. + # Add other parameters on the website UI interface. + # (3) Upload or copy your pretrained model to S3 bucket. + # (4) Upload a zip dataset to S3 bucket. (you could also upload the origin dataset, but it can be so slow.) + # (5) Set the code directory to "/path/yolov4" on the website UI interface. + # (6) Set the startup file to "train.py" on the website UI interface. + # (7) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface. + # (8) Create your job. + # + # Eval 1p with Ascend + # (1) Perform a or b. + # a. Set "enable_modelarts=True" on base_config.yaml file. + # Set "data_dir='/cache/data/coco/'" on base_config.yaml file. + # Set "checkpoint_url='s3://dir_to_your_trained_ckpt/'" on base_config.yaml file. + # Set "pretrained='/cache/checkpoint_path/model.ckpt'" on base_config.yaml file. + # Set "is_distributed=0" on base_config.yaml file. + # Set "per_batch_size=1" on base_config.yaml file. + # Set other parameters on base_config.yaml file you need. + # b. Add "enable_modelarts=True" on the website UI interface. + # Add "data_dir=/cache/data/coco/" on the website UI interface. + # Add "checkpoint_url=s3://dir_to_your_trained_ckpt/" on the website UI interface. + # Add "pretrained=/cache/checkpoint_path/model.ckpt" on the website UI interface. + # Add "is_distributed=0" on the website UI interface. + # Add "per_batch_size=1" on the website UI interface. + # Add other parameters on the website UI interface. + # (3) Upload or copy your trained model to S3 bucket. + # (4) Upload a zip dataset to S3 bucket. (you could also upload the origin dataset, but it can be so slow.) + # (5) Set the code directory to "/path/yolov4" on the website UI interface. + # (6) Set the startup file to "eval.py" on the website UI interface. + # (7) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface. + # (8) Create your job. + # + # Test 1p with Ascend + # (1) Perform a or b. + # a. Set "enable_modelarts=True" on base_config.yaml file. + # Set "data_dir='/cache/data/coco/'" on base_config.yaml file. + # Set "checkpoint_url='s3://dir_to_your_trained_ckpt/'" on base_config.yaml file. + # Set "pretrained='/cache/checkpoint_path/model.ckpt'" on base_config.yaml file. + # Set "is_distributed=0" on base_config.yaml file. + # Set "per_batch_size=1" on base_config.yaml file. + # Set "test_nms_thresh=0.45" on base_config.yaml file. + # Set "test_ignore_threshold=0.001" on base_config.yaml file. + # Set other parameters on base_config.yaml file you need. + # b. Add "enable_modelarts=True" on the website UI interface. + # Add "data_dir=/cache/data/coco/" on the website UI interface. + # Add "checkpoint_url=s3://dir_to_your_trained_ckpt/" on the website UI interface. + # Add "pretrained=/cache/checkpoint_path/model.ckpt" on the website UI interface. + # Add "is_distributed=0" on the website UI interface. + # Add "per_batch_size=1" on the website UI interface. + # Add "test_nms_thresh=0.45" on the website UI interface. + # Add "test_ignore_threshold=0.001" on the website UI interface. + # Add other parameters on the website UI interface. + # (3) Upload or copy your trained model to S3 bucket. + # (4) Upload a zip dataset to S3 bucket. (you could also upload the origin dataset, but it can be so slow.) + # (5) Set the code directory to "/path/yolov4" on the website UI interface. + # (6) Set the startup file to "test.py" on the website UI interface. + # (7) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface. + # (8) Create your job. + ``` # [Script Description](#contents) @@ -448,7 +538,7 @@ YOLOv4 on 118K images(The annotation and data format must be the same as coco201 | Parameters | YOLOv4 | | -------------------------- | ----------------------------------------------------------- | -| Resource | Ascend 910; CPU 2.60GHz, 192cores; Memory 755G; OS Euler2.8; System, Euleros 2.8;| +| Resource | Ascend 910; CPU 2.60GHz, 192cores; Memory, 755G; System, Euleros 2.8;| | uploaded Date | 10/16/2020 (month/day/year) | | MindSpore Version | 1.0.0-alpha | | Dataset | 118K images | @@ -468,7 +558,7 @@ YOLOv4 on 20K images(The annotation and data format must be the same as coco tes | Parameters | YOLOv4 | | -------------------------- | ----------------------------------------------------------- | -| Resource | Ascend 910; CPU 2.60GHz, 192cores; Memory 755G; OS Euler2.8 | +| Resource | Ascend 910; CPU 2.60GHz, 192cores; Memory, 755G | | uploaded Date | 10/16/2020 (month/day/year) | | MindSpore Version | 1.0.0-alpha | | Dataset | 20K images | diff --git a/model_zoo/official/cv/yolov4/default_config.yaml b/model_zoo/official/cv/yolov4/default_config.yaml new file mode 100644 index 00000000000..454fcf90d0a --- /dev/null +++ b/model_zoo/official/cv/yolov4/default_config.yaml @@ -0,0 +1,165 @@ +# Builtin Configurations(DO NOT CHANGE THESE CONFIGURATIONS unless you know exactly what you are doing) +enable_modelarts: False +# Url for modelarts +data_url: "" +train_url: "" +checkpoint_url: "" +# Path for local +data_path: "/cache/data" +output_path: "/cache/train" +load_path: "/cache/checkpoint_path" +device_target: "Ascend" +need_modelarts_dataset_unzip: True +modelarts_dataset_unzip_name: "coco" + +# ============================================================================== +# Train options +data_dir: "" +per_batch_size: 8 +pretrained_backbone: "" +resume_yolov4: "" +pretrained_checkpoint: "" +filter_weight: False +lr_scheduler: "cosine_annealing" +lr: 0.012 +lr_epochs: "220,250" +lr_gamma: 0.1 +eta_min: 0.0 +t_max: 320 +max_epoch: 320 +warmup_epochs: 20 +weight_decay: 0.0005 +momentum: 0.9 +loss_scale: 64 +label_smooth: 0 +label_smooth_factor: 0.1 +log_interval: 100 +ckpt_path: "outputs/" +ckpt_interval: -1 +is_save_on_master: 1 +is_distributed: 1 +rank: 0 +group_size: 1 +need_profiler: 0 +training_shape: "" +run_eval: False +save_best_ckpt: True +eval_start_epoch: 200 +eval_interval: 1 +ann_file: "" + +# Eval options +pretrained: "" +log_path: "outputs/" +ann_val_file: "" + +# Test option +test_nms_thresh: 0.45 +test_ignore_threshold: 0.001 + +# Export options +device_id: 0 +batch_size: 1 +testing_shape: 608 +ckpt_file: "" +file_name: "yolov4" +file_format: "AIR" + + +# Other default config +hue: 0.1 +saturation: 1.5 +value: 1.5 +jitter: 0.3 +resize_rate: 10 + +multi_scale: [[416, 416], + [448, 448], + [480, 480], + [512, 512], + [544, 544], + [576, 576], + [608, 608], + [640, 640], + [672, 672], + [704, 704], + [736, 736] + ] + +max_box: 90 +backbone_input_shape: [32, 64, 128, 256, 512] +backbone_shape: [64, 128, 256, 512, 1024] +backbone_layers: [1, 2, 8, 8, 4] + +ignore_threshold: 0.7 +eval_ignore_threshold: 0.001 +nms_thresh: 0.5 +anchor_scales: [[12, 16], + [19, 36], + [40, 28], + [36, 75], + [76, 55], + [72, 146], + [142, 110], + [192, 243], + [459, 401]] + +num_classes: 80 +out_channel: 255 # 3 * (num_classes + 5) +test_img_shape: [608, 608] +checkpoint_filter_list: ['feature_map.backblock0.conv6.weight', 'feature_map.backblock0.conv6.bias', + 'feature_map.backblock1.conv6.weight', 'feature_map.backblock1.conv6.bias', + 'feature_map.backblock2.conv6.weight', 'feature_map.backblock2.conv6.bias', + 'feature_map.backblock3.conv6.weight', 'feature_map.backblock3.conv6.bias'] + +--- + +# Help description for each configuration +# Train options +data_dir: "Train dataset directory." +per_batch_size: "Batch size for Training." +pretrained_backbone: "The ckpt file of CspDarkNet53." +resume_yolov4: "The ckpt file of YOLOv4, which used to fine tune." +pretrained_checkpoint: "The ckpt file of YoloV4CspDarkNet53." +filter_weight: "Filter the last weight parameters" +lr_scheduler: "Learning rate scheduler, options: exponential, cosine_annealing." +lr: "Learning rate." +lr_epochs: "Epoch of changing of lr changing, split with ','." +lr_gamma: "Decrease lr by a factor of exponential lr_scheduler." +eta_min: "Eta_min in cosine_annealing scheduler." +t_max: "T-max in cosine_annealing scheduler." +max_epoch: "Max epoch num to train the model." +warmup_epochs: "Warmup epochs." +weight_decay: "Weight decay factor." +momentum: "Momentum." +loss_scale: "Static loss scale." +label_smooth: "Whether to use label smooth in CE." +label_smooth_factor: "Smooth strength of original one-hot." +log_interval: "Logging interval steps." +ckpt_path: "Checkpoint save location." +ckpt_interval: "Save checkpoint interval." +is_save_on_master: "Save ckpt on master or all rank, 1 for master, 0 for all ranks." +is_distributed: "Distribute train or not, 1 for yes, 0 for no." +rank: "Local rank of distributed." +group_size: "World size of device." +need_profiler: "Whether use profiler. 0 for no, 1 for yes." +training_shape: "Fix training shape." +resize_rate: "Resize rate for multi-scale training." +run_eval: "Run evaluation when training." +save_best_ckpt: "Save best checkpoint when run_eval is True." +eval_start_epoch: "Evaluation start epoch when run_eval is True." +eval_interval: "Evaluation interval when run_eval is True" +ann_file: "path to annotation" + +# Eval options +pretrained: "model_path, local pretrained model to load" +log_path: "checkpoint save location" +ann_val_file: "path to annotation" + +# Export options +device_id: "Device id for export" +batch_size: "batch size for export" +testing_shape: "shape for test" +ckpt_file: "Checkpoint file path for export" +file_name: "output file name for export" +file_format: "file format for export" \ No newline at end of file diff --git a/model_zoo/official/cv/yolov4/eval.py b/model_zoo/official/cv/yolov4/eval.py index 75b9ad89bc6..414b0d73f1a 100644 --- a/model_zoo/official/cv/yolov4/eval.py +++ b/model_zoo/official/cv/yolov4/eval.py @@ -14,78 +14,100 @@ # ============================================================================ """YoloV4 eval.""" import os -import argparse import datetime import time -from mindspore import Tensor from mindspore.context import ParallelMode from mindspore import context from mindspore.train.serialization import load_checkpoint, load_param_into_net -import mindspore as ms from src.yolo import YOLOV4CspDarkNet53 from src.logger import get_logger from src.yolo_dataset import create_yolo_dataset -from src.config import ConfigYOLOV4CspDarkNet53 from src.eval_utils import apply_eval -parser = argparse.ArgumentParser('mindspore coco testing') +from model_utils.config import config +from model_utils.moxing_adapter import moxing_wrapper +from model_utils.device_adapter import get_device_id, get_device_num -# device related -parser.add_argument('--device_target', type=str, default='Ascend', - help='device where the code will be implemented. (Default: Ascend)') +config.data_root = os.path.join(config.data_dir, 'val2017') +config.ann_val_file = os.path.join(config.data_dir, 'annotations/instances_val2017.json') -# dataset related -parser.add_argument('--data_dir', type=str, default='', help='train data dir') -parser.add_argument('--per_batch_size', default=1, type=int, help='batch size for per gpu') +def modelarts_pre_process(): + '''modelarts pre process function.''' + def unzip(zip_file, save_dir): + import zipfile + s_time = time.time() + if not os.path.exists(os.path.join(save_dir, config.modelarts_dataset_unzip_name)): + zip_isexist = zipfile.is_zipfile(zip_file) + if zip_isexist: + fz = zipfile.ZipFile(zip_file, 'r') + data_num = len(fz.namelist()) + print("Extract Start...") + print("unzip file num: {}".format(data_num)) + data_print = int(data_num / 100) if data_num > 100 else 1 + i = 0 + for file in fz.namelist(): + if i % data_print == 0: + print("unzip percent: {}%".format(int(i * 100 / data_num)), flush=True) + i += 1 + fz.extract(file, save_dir) + print("cost time: {}min:{}s.".format(int((time.time() - s_time) / 60), + int(int(time.time() - s_time) % 60))) + print("Extract Done.") + else: + print("This is not zip.") + else: + print("Zip has been extracted.") -# network related -parser.add_argument('--pretrained', default='', type=str, help='model_path, local pretrained model to load') + if config.need_modelarts_dataset_unzip: + zip_file_1 = os.path.join(config.data_path, config.modelarts_dataset_unzip_name + ".zip") + save_dir_1 = os.path.join(config.data_path) -# logging related -parser.add_argument('--log_path', type=str, default='outputs/', help='checkpoint save location') + sync_lock = "/tmp/unzip_sync.lock" -# detect_related -parser.add_argument('--ann_val_file', type=str, default='', help='path to annotation') -parser.add_argument('--testing_shape', type=str, default='', help='shape for test ') + # Each server contains 8 devices as most. + if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock): + print("Zip file path: ", zip_file_1) + print("Unzip file save dir: ", save_dir_1) + unzip(zip_file_1, save_dir_1) + print("===Finish extract data synchronization===") + try: + os.mknod(sync_lock) + except IOError: + pass -args, _ = parser.parse_known_args() + while True: + if os.path.exists(sync_lock): + break + time.sleep(1) -config = ConfigYOLOV4CspDarkNet53() -args.nms_thresh = config.nms_thresh -args.ignore_threshold = config.eval_ignore_threshold -args.data_root = os.path.join(args.data_dir, 'val2017') -args.ann_val_file = os.path.join(args.data_dir, 'annotations/instances_val2017.json') + print("Device: {}, Finish sync unzip data from {} to {}.".format(get_device_id(), zip_file_1, save_dir_1)) + config.log_path = os.path.join(config.output_path, config.log_path) -def convert_testing_shape(args_testing_shape): - """Convert testing shape to list.""" - testing_shape = [int(args_testing_shape), int(args_testing_shape)] - return testing_shape - - -if __name__ == "__main__": +@moxing_wrapper(pre_process=modelarts_pre_process) +def run_eval(): start_time = time.time() device_id = int(os.getenv('DEVICE_ID')) if os.getenv('DEVICE_ID') else 0 - context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target, device_id=device_id) + context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target, device_id=device_id) # logger - args.outputs_dir = os.path.join(args.log_path, - datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S')) + config.outputs_dir = os.path.join(config.log_path, + datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S')) rank_id = int(os.environ.get('RANK_ID')) if os.environ.get('RANK_ID') else 0 - args.logger = get_logger(args.outputs_dir, rank_id) + config.logger = get_logger(config.outputs_dir, rank_id) context.reset_auto_parallel_context() parallel_mode = ParallelMode.STAND_ALONE context.set_auto_parallel_context(parallel_mode=parallel_mode, gradients_mean=True, device_num=1) - args.logger.info('Creating Network....') + config.logger.info('Creating Network....') network = YOLOV4CspDarkNet53() - args.logger.info(args.pretrained) - if os.path.isfile(args.pretrained): - param_dict = load_checkpoint(args.pretrained) + config.logger.info(config.pretrained) + if os.path.isfile(config.pretrained): + param_dict = load_checkpoint(config.pretrained) param_dict_new = {} for key, values in param_dict.items(): if key.startswith('moments.'): @@ -95,33 +117,34 @@ if __name__ == "__main__": else: param_dict_new[key] = values load_param_into_net(network, param_dict_new) - args.logger.info('load_model {} success'.format(args.pretrained)) + config.logger.info('load_model %s success', config.pretrained) else: - args.logger.info('{} not exists or not a pre-trained file'.format(args.pretrained)) - assert FileNotFoundError('{} not exists or not a pre-trained file'.format(args.pretrained)) + config.logger.info('%s not exists or not a pre-trained file', config.pretrained) + assert FileNotFoundError('{} not exists or not a pre-trained file'.format(config.pretrained)) exit(1) - data_root = args.data_root - ann_val_file = args.ann_val_file + data_root = config.data_root + ann_val_file = config.ann_val_file - if args.testing_shape: - config.test_img_shape = convert_testing_shape(args.testing_shape) - - ds, data_size = create_yolo_dataset(data_root, ann_val_file, is_training=False, batch_size=args.per_batch_size, + ds, data_size = create_yolo_dataset(data_root, ann_val_file, is_training=False, batch_size=config.per_batch_size, max_epoch=1, device_num=1, rank=rank_id, shuffle=False, config=config) - args.logger.info('testing shape : {}'.format(config.test_img_shape)) - args.logger.info('totol {} images to eval'.format(data_size)) + config.logger.info('testing shape : %s', config.test_img_shape) + config.logger.info('totol %d images to eval', data_size) network.set_train(False) # init detection engine - input_shape = Tensor(tuple(config.test_img_shape), ms.float32) - args.logger.info('Start inference....') + config.logger.info('Start inference....') eval_param_dict = {"net": network, "dataset": ds, "data_size": data_size, - "anno_json": args.ann_val_file, "input_shape": input_shape, "args": args} + "anno_json": config.ann_val_file, "args": config} eval_result, _ = apply_eval(eval_param_dict) cost_time = time.time() - start_time - args.logger.info('\n=============coco eval reulst=========\n' + eval_result) - args.logger.info('testing cost time {:.2f}h'.format(cost_time / 3600.)) + eval_log_string = '\n=============coco eval reulst=========\n' + eval_result + config.logger.info(eval_log_string) + config.logger.info('testing cost time %.2f h', cost_time / 3600.) + + +if __name__ == "__main__": + run_eval() diff --git a/model_zoo/official/cv/yolov4/export.py b/model_zoo/official/cv/yolov4/export.py index ba046864106..9bf0fe4e863 100644 --- a/model_zoo/official/cv/yolov4/export.py +++ b/model_zoo/official/cv/yolov4/export.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -import argparse import numpy as np import mindspore @@ -21,30 +20,21 @@ from mindspore.train.serialization import export, load_checkpoint, load_param_in from src.yolo import YOLOV4CspDarkNet53 -parser = argparse.ArgumentParser(description='yolov4 export') -parser.add_argument("--device_id", type=int, default=0, help="Device id") -parser.add_argument("--batch_size", type=int, default=1, help="batch size") -parser.add_argument("--testing_shape", type=int, default=608, help="test shape") -parser.add_argument("--ckpt_file", type=str, required=True, help="Checkpoint file path.") -parser.add_argument("--file_name", type=str, default="yolov4", help="output file name.") -parser.add_argument('--file_format', type=str, choices=["AIR", "ONNX", "MINDIR"], default='AIR', help='file format') -parser.add_argument("--device_target", type=str, choices=["Ascend", "GPU", "CPU"], default="Ascend", - help="device target") -args = parser.parse_args() +from model_utils.config import config -context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target) -if args.device_target == "Ascend": - context.set_context(device_id=args.device_id) +context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target) +if config.device_target == "Ascend": + context.set_context(device_id=config.device_id) if __name__ == "__main__": - ts_shape = args.testing_shape + ts_shape = config.testing_shape network = YOLOV4CspDarkNet53() network.set_train(False) - param_dict = load_checkpoint(args.ckpt_file) + param_dict = load_checkpoint(config.ckpt_file) load_param_into_net(network, param_dict) - input_data = Tensor(np.zeros([args.batch_size, 3, ts_shape, ts_shape]), mindspore.float32) + input_data = Tensor(np.zeros([config.batch_size, 3, ts_shape, ts_shape]), mindspore.float32) - export(network, input_data, file_name=args.file_name, file_format=args.file_format) + export(network, input_data, file_name=config.file_name, file_format=config.file_format) diff --git a/model_zoo/official/cv/yolov4/model_utils/__init__.py b/model_zoo/official/cv/yolov4/model_utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/model_zoo/official/cv/yolov4/model_utils/config.py b/model_zoo/official/cv/yolov4/model_utils/config.py new file mode 100644 index 00000000000..ad0d7497a8e --- /dev/null +++ b/model_zoo/official/cv/yolov4/model_utils/config.py @@ -0,0 +1,126 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Parse arguments""" + +import os +import ast +import argparse +from pprint import pformat +import yaml + +class Config: + """ + Configuration namespace. Convert dictionary to members. + """ + def __init__(self, cfg_dict): + for k, v in cfg_dict.items(): + if isinstance(v, (list, tuple)): + setattr(self, k, [Config(x) if isinstance(x, dict) else x for x in v]) + else: + setattr(self, k, Config(v) if isinstance(v, dict) else v) + + def __str__(self): + return pformat(self.__dict__) + + def __repr__(self): + return self.__str__() + + +def parse_cli_to_yaml(parser, cfg, helper=None, choices=None, cfg_path="default_config.yaml"): + """ + Parse command line arguments to the configuration according to the default yaml. + + Args: + parser: Parent parser. + cfg: Base configuration. + helper: Helper description. + cfg_path: Path to the default yaml config. + """ + parser = argparse.ArgumentParser(description="[REPLACE THIS at config.py]", + parents=[parser]) + helper = {} if helper is None else helper + choices = {} if choices is None else choices + for item in cfg: + if not isinstance(cfg[item], list) and not isinstance(cfg[item], dict): + help_description = helper[item] if item in helper else "Please reference to {}".format(cfg_path) + choice = choices[item] if item in choices else None + if isinstance(cfg[item], bool): + parser.add_argument("--" + item, type=ast.literal_eval, default=cfg[item], choices=choice, + help=help_description) + else: + parser.add_argument("--" + item, type=type(cfg[item]), default=cfg[item], choices=choice, + help=help_description) + args = parser.parse_args() + return args + + +def parse_yaml(yaml_path): + """ + Parse the yaml config file. + + Args: + yaml_path: Path to the yaml config. + """ + with open(yaml_path, 'r') as fin: + try: + cfgs = yaml.load_all(fin.read(), Loader=yaml.FullLoader) + cfgs = [x for x in cfgs] + if len(cfgs) == 1: + cfg_helper = {} + cfg = cfgs[0] + cfg_choices = {} + elif len(cfgs) == 2: + cfg, cfg_helper = cfgs + cfg_choices = {} + elif len(cfgs) == 3: + cfg, cfg_helper, cfg_choices = cfgs + else: + raise ValueError("At most 3 docs (config, description for help, choices) are supported in config yaml") + print(cfg_helper) + except: + raise ValueError("Failed to parse yaml") + return cfg, cfg_helper, cfg_choices + + +def merge(args, cfg): + """ + Merge the base config from yaml file and command line arguments. + + Args: + args: Command line arguments. + cfg: Base configuration. + """ + args_var = vars(args) + for item in args_var: + cfg[item] = args_var[item] + return cfg + + +def get_config(): + """ + Get Config according to the yaml file and cli arguments. + """ + parser = argparse.ArgumentParser(description="default name", add_help=False) + current_dir = os.path.dirname(os.path.abspath(__file__)) + parser.add_argument("--config_path", type=str, default=os.path.join(current_dir, "../default_config.yaml"), + help="Config file path") + path_args, _ = parser.parse_known_args() + default, helper, choices = parse_yaml(path_args.config_path) + args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path) + final_config = merge(args, default) + return Config(final_config) + +config = get_config() diff --git a/model_zoo/official/cv/yolov4/model_utils/device_adapter.py b/model_zoo/official/cv/yolov4/model_utils/device_adapter.py new file mode 100644 index 00000000000..7c5d7f837dd --- /dev/null +++ b/model_zoo/official/cv/yolov4/model_utils/device_adapter.py @@ -0,0 +1,27 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Device adapter for ModelArts""" + +from .config import config + +if config.enable_modelarts: + from .moxing_adapter import get_device_id, get_device_num, get_rank_id, get_job_id +else: + from .local_adapter import get_device_id, get_device_num, get_rank_id, get_job_id + +__all__ = [ + "get_device_id", "get_device_num", "get_rank_id", "get_job_id" +] diff --git a/model_zoo/official/cv/yolov4/model_utils/local_adapter.py b/model_zoo/official/cv/yolov4/model_utils/local_adapter.py new file mode 100644 index 00000000000..769fa6dc78e --- /dev/null +++ b/model_zoo/official/cv/yolov4/model_utils/local_adapter.py @@ -0,0 +1,36 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Local adapter""" + +import os + +def get_device_id(): + device_id = os.getenv('DEVICE_ID', '0') + return int(device_id) + + +def get_device_num(): + device_num = os.getenv('RANK_SIZE', '1') + return int(device_num) + + +def get_rank_id(): + global_rank_id = os.getenv('RANK_ID', '0') + return int(global_rank_id) + + +def get_job_id(): + return "Local Job" diff --git a/model_zoo/official/cv/yolov4/model_utils/moxing_adapter.py b/model_zoo/official/cv/yolov4/model_utils/moxing_adapter.py new file mode 100644 index 00000000000..25838a7da99 --- /dev/null +++ b/model_zoo/official/cv/yolov4/model_utils/moxing_adapter.py @@ -0,0 +1,116 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Moxing adapter for ModelArts""" + +import os +import functools +from mindspore import context +from .config import config + +_global_sync_count = 0 + +def get_device_id(): + device_id = os.getenv('DEVICE_ID', '0') + return int(device_id) + + +def get_device_num(): + device_num = os.getenv('RANK_SIZE', '1') + return int(device_num) + + +def get_rank_id(): + global_rank_id = os.getenv('RANK_ID', '0') + return int(global_rank_id) + + +def get_job_id(): + job_id = os.getenv('JOB_ID') + job_id = job_id if job_id != "" else "default" + return job_id + +def sync_data(from_path, to_path): + """ + Download data from remote obs to local directory if the first url is remote url and the second one is local path + Upload data from local directory to remote obs in contrast. + """ + import moxing as mox + import time + global _global_sync_count + sync_lock = "/tmp/copy_sync.lock" + str(_global_sync_count) + _global_sync_count += 1 + + # Each server contains 8 devices as most. + if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock): + print("from path: ", from_path) + print("to path: ", to_path) + mox.file.copy_parallel(from_path, to_path) + print("===finish data synchronization===") + try: + os.mknod(sync_lock) + except IOError: + pass + print("===save flag===") + + while True: + if os.path.exists(sync_lock): + break + time.sleep(1) + + print("Finish sync data from {} to {}.".format(from_path, to_path)) + + +def moxing_wrapper(pre_process=None, post_process=None): + """ + Moxing wrapper to download dataset and upload outputs. + """ + def wrapper(run_func): + @functools.wraps(run_func) + def wrapped_func(*args, **kwargs): + # Download data from data_url + if config.enable_modelarts: + if config.data_url: + sync_data(config.data_url, config.data_path) + print("Dataset downloaded: ", os.listdir(config.data_path)) + if config.checkpoint_url: + sync_data(config.checkpoint_url, config.load_path) + print("Preload downloaded: ", os.listdir(config.load_path)) + if config.train_url: + sync_data(config.train_url, config.output_path) + print("Workspace downloaded: ", os.listdir(config.output_path)) + + context.set_context(save_graphs_path=os.path.join(config.output_path, str(get_rank_id()))) + config.device_num = get_device_num() + config.device_id = get_device_id() + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) + + if pre_process: + pre_process() + + # Run the main function + run_func(*args, **kwargs) + + # Upload data to train_url + if config.enable_modelarts: + if post_process: + post_process() + + if config.train_url: + print("Start to copy output directory") + sync_data(config.output_path, config.train_url) + return wrapped_func + return wrapper diff --git a/model_zoo/official/cv/yolov4/postprocess.py b/model_zoo/official/cv/yolov4/postprocess.py index 64432f87e3d..59fdf612744 100644 --- a/model_zoo/official/cv/yolov4/postprocess.py +++ b/model_zoo/official/cv/yolov4/postprocess.py @@ -35,7 +35,7 @@ parser.add_argument('--log_path', type=str, default='outputs/', help='checkpoint # detect_related parser.add_argument('--nms_thresh', type=float, default=0.5, help='threshold for NMS') parser.add_argument('--ann_file', type=str, default='', help='path to annotation') -parser.add_argument('--ignore_threshold', type=float, default=0.001, help='threshold to throw low quality boxes') +parser.add_argument('--eval_ignore_threshold', type=float, default=0.001, help='threshold to throw low quality boxes') parser.add_argument('--img_id_file_path', type=str, default='', help='path of image dataset') parser.add_argument('--result_files', type=str, default='./result_Files', help='path to 310 infer result floder') diff --git a/model_zoo/official/cv/yolov4/scripts/run_distribute_train.sh b/model_zoo/official/cv/yolov4/scripts/run_distribute_train.sh index 3f4334997a5..793fee9498b 100644 --- a/model_zoo/official/cv/yolov4/scripts/run_distribute_train.sh +++ b/model_zoo/official/cv/yolov4/scripts/run_distribute_train.sh @@ -65,7 +65,9 @@ do rm -rf ./train_parallel$i mkdir ./train_parallel$i cp ../*.py ./train_parallel$i + cp ../*.yaml ./train_parallel$i cp -r ../src ./train_parallel$i + cp -r ../model_utils ./train_parallel$i cd ./train_parallel$i || exit echo "start training for rank $RANK_ID, device $DEVICE_ID" env > env.log diff --git a/model_zoo/official/cv/yolov4/scripts/run_eval.sh b/model_zoo/official/cv/yolov4/scripts/run_eval.sh index 6046eb58af2..f4f1f7bf00d 100644 --- a/model_zoo/official/cv/yolov4/scripts/run_eval.sh +++ b/model_zoo/official/cv/yolov4/scripts/run_eval.sh @@ -55,12 +55,15 @@ then fi mkdir ./eval cp ../*.py ./eval +cp ../*.yaml ./eval cp -r ../src ./eval +cp -r ../model_utils ./eval cd ./eval || exit env > env.log echo "start inferring for device $DEVICE_ID" python eval.py \ --data_dir=$DATASET_PATH \ - --pretrained=$CHECKPOINT_PATH \ - --testing_shape=608 > log.txt 2>&1 & + --is_distributed=0 \ + --per_batch_size=1 \ + --pretrained=$CHECKPOINT_PATH > log.txt 2>&1 & cd .. diff --git a/model_zoo/official/cv/yolov4/scripts/run_standalone_train.sh b/model_zoo/official/cv/yolov4/scripts/run_standalone_train.sh index 3333ff63ea4..6f402f3c69f 100644 --- a/model_zoo/official/cv/yolov4/scripts/run_standalone_train.sh +++ b/model_zoo/official/cv/yolov4/scripts/run_standalone_train.sh @@ -56,7 +56,9 @@ then fi mkdir ./train cp ../*.py ./train +cp ../*.yaml ./train cp -r ../src ./train +cp -r ../model_utils ./train cd ./train || exit echo "start training for device $DEVICE_ID" env > env.log diff --git a/model_zoo/official/cv/yolov4/scripts/run_test.sh b/model_zoo/official/cv/yolov4/scripts/run_test.sh index 420d4b01ece..6afabc0f4d1 100644 --- a/model_zoo/official/cv/yolov4/scripts/run_test.sh +++ b/model_zoo/official/cv/yolov4/scripts/run_test.sh @@ -55,12 +55,17 @@ then fi mkdir ./test cp ../*.py ./test +cp ../*.yaml ./test cp -r ../src ./test +cp -r ../model_utils ./test cd ./test || exit env > env.log echo "start inferring for device $DEVICE_ID" python test.py \ --data_dir=$DATASET_PATH \ - --pretrained=$CHECKPOINT_PATH \ - --testing_shape=608 > log.txt 2>&1 & + --is_distributed=0 \ + --per_batch_size=1 \ + --test_nms_thresh=0.45 \ + --test_ignore_threshold=0.001 \ + --pretrained=$CHECKPOINT_PATH > log.txt 2>&1 & cd .. diff --git a/model_zoo/official/cv/yolov4/src/config.py b/model_zoo/official/cv/yolov4/src/config.py deleted file mode 100644 index cc18774c2a1..00000000000 --- a/model_zoo/official/cv/yolov4/src/config.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""Config parameters for Darknet based yolov4_cspdarknet53 models.""" - - -class ConfigYOLOV4CspDarkNet53: - """ - Config parameters for the yolov4_cspdarknet53. - - Examples: - ConfigYOLOV4CspDarkNet53() - """ - # train_param - # data augmentation related - hue = 0.1 - saturation = 1.5 - value = 1.5 - jitter = 0.3 - - resize_rate = 10 - multi_scale = [[416, 416], - [448, 448], - [480, 480], - [512, 512], - [544, 544], - [576, 576], - [608, 608], - [640, 640], - [672, 672], - [704, 704], - [736, 736] - ] - - num_classes = 80 - max_box = 90 - - backbone_input_shape = [32, 64, 128, 256, 512] - backbone_shape = [64, 128, 256, 512, 1024] - backbone_layers = [1, 2, 8, 8, 4] - - # confidence under ignore_threshold means no object when training - ignore_threshold = 0.7 - # threshold to throw low quality boxes when eval - eval_ignore_threshold = 0.001 - nms_thresh = 0.5 - - # h->w - anchor_scales = [(12, 16), - (19, 36), - (40, 28), - (36, 75), - (76, 55), - (72, 146), - (142, 110), - (192, 243), - (459, 401)] - out_channel = 3 * (num_classes + 5) - - # test_param - test_img_shape = [608, 608] - - # transfer training - checkpoint_filter_list = ['feature_map.backblock0.conv6.weight', 'feature_map.backblock0.conv6.bias', - 'feature_map.backblock1.conv6.weight', 'feature_map.backblock1.conv6.bias', - 'feature_map.backblock2.conv6.weight', 'feature_map.backblock2.conv6.bias', - 'feature_map.backblock3.conv6.weight', 'feature_map.backblock3.conv6.bias'] diff --git a/model_zoo/official/cv/yolov4/src/eval_utils.py b/model_zoo/official/cv/yolov4/src/eval_utils.py index ed742b456e2..5d40aeecd9a 100644 --- a/model_zoo/official/cv/yolov4/src/eval_utils.py +++ b/model_zoo/official/cv/yolov4/src/eval_utils.py @@ -23,7 +23,7 @@ class Redirct: class DetectionEngine: """Detection engine.""" def __init__(self, args_detection): - self.ignore_threshold = args_detection.ignore_threshold + self.eval_ignore_threshold = args_detection.eval_ignore_threshold self.labels = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', @@ -147,10 +147,8 @@ class DetectionEngine: def get_eval_result(self): """Get eval result.""" - up_path = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) - self.file_path = os.path.join(up_path, self.file_path) if not self.results: - args.logger.info("[WARNING] result is {}") + logger.warning("[WARNING] result is None.") return 0.0, 0.0 coco_gt = COCO(self.ann_file) coco_dt = coco_gt.loadRes(self.file_path) @@ -208,7 +206,7 @@ class DetectionEngine: flag[i, c] = True confidence = cls_emb[flag] * conf for x_lefti, y_lefti, wi, hi, confi, clsi in zip(x_top_left, y_top_left, w, h, confidence, cls_argmax): - if confi < self.ignore_threshold: + if confi < self.eval_ignore_threshold: continue if img_id not in self.results: self.results[img_id] = defaultdict(list) @@ -291,20 +289,18 @@ class EvalCallBack(Callback): self.args.logger.info("End training, the best {0} is: {1}, " "the best {0} epoch is {2}".format(self.metrics_name, self.best_res, self.best_epoch)) - def apply_eval(eval_param_dict): network = eval_param_dict["net"] network.set_train(False) ds = eval_param_dict["dataset"] data_size = eval_param_dict["data_size"] - input_shape = eval_param_dict["input_shape"] args = eval_param_dict["args"] detection = DetectionEngine(args) for index, data in enumerate(ds.create_dict_iterator(num_epochs=1)): image = data["image"] image_shape_ = data["image_shape"] image_id_ = data["img_id"] - prediction = network(image, input_shape) + prediction = network(image) output_big, output_me, output_small = prediction output_big = output_big.asnumpy() output_me = output_me.asnumpy() diff --git a/model_zoo/official/cv/yolov4/src/logger.py b/model_zoo/official/cv/yolov4/src/logger.py index a8816bc22ca..4c7eb824342 100644 --- a/model_zoo/official/cv/yolov4/src/logger.py +++ b/model_zoo/official/cv/yolov4/src/logger.py @@ -30,11 +30,12 @@ class LOGGER(logging.Logger): def __init__(self, logger_name, rank=0): super(LOGGER, self).__init__(logger_name) self.rank = rank - console = logging.StreamHandler(sys.stdout) - console.setLevel(logging.INFO) - formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s') - console.setFormatter(formatter) - self.addHandler(console) + if rank % 8 == 0: + console = logging.StreamHandler(sys.stdout) + console.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s') + console.setFormatter(formatter) + self.addHandler(console) def setup_logging_file(self, log_dir, rank=0): """Setup logging file.""" @@ -61,7 +62,7 @@ class LOGGER(logging.Logger): self.info('') def important_info(self, msg, *args, **kwargs): - if self.isEnabledFor(logging.INFO): + if self.isEnabledFor(logging.INFO) and self.rank == 0: line_width = 2 important_msg = '\n' important_msg += ('*'*70 + '\n')*line_width diff --git a/model_zoo/official/cv/yolov4/src/yolo.py b/model_zoo/official/cv/yolov4/src/yolo.py index 7d6881d0a18..9701226816c 100644 --- a/model_zoo/official/cv/yolov4/src/yolo.py +++ b/model_zoo/official/cv/yolov4/src/yolo.py @@ -25,9 +25,9 @@ from mindspore.ops import functional as F from mindspore.ops import composite as C from src.cspdarknet53 import CspDarkNet53, ResidualBlock -from src.config import ConfigYOLOV4CspDarkNet53 from src.loss import XYLoss, WHLoss, ConfidenceLoss, ClassLoss +from model_utils.config import config as default_config def _conv_bn_leakyrelu(in_channel, out_channel, @@ -210,7 +210,7 @@ class DetectionBlock(nn.Cell): Args: scale: Character. - config: ConfigYOLOV4CspDarkNet53, Configuration instance. + config: Configuration. is_training: Bool, Whether train or not, default True. Returns: @@ -220,7 +220,7 @@ class DetectionBlock(nn.Cell): DetectionBlock(scale='l',stride=32) """ - def __init__(self, scale, config=ConfigYOLOV4CspDarkNet53()): + def __init__(self, scale, config=default_config): super(DetectionBlock, self).__init__() self.config = config if scale == 's': @@ -330,7 +330,7 @@ class YoloLossBlock(nn.Cell): """ Loss block cell of YOLOV4 network. """ - def __init__(self, scale, config=ConfigYOLOV4CspDarkNet53()): + def __init__(self, scale, config=default_config): super(YoloLossBlock, self).__init__() self.config = config if scale == 's': @@ -431,7 +431,8 @@ class YOLOV4CspDarkNet53(nn.Cell): def __init__(self): super(YOLOV4CspDarkNet53, self).__init__() - self.config = ConfigYOLOV4CspDarkNet53() + self.config = default_config + self.test_img_shape = Tensor(tuple(self.config.test_img_shape), ms.float32) # YOLOv4 network self.feature_map = YOLOv4(backbone=CspDarkNet53(ResidualBlock, detect=True), @@ -443,7 +444,9 @@ class YOLOV4CspDarkNet53(nn.Cell): self.detect_2 = DetectionBlock('m') self.detect_3 = DetectionBlock('s') - def construct(self, x, input_shape): + def construct(self, x, input_shape=None): + if input_shape is None: + input_shape = self.test_img_shape big_object_output, medium_object_output, small_object_output = self.feature_map(x) output_big = self.detect_1(big_object_output, input_shape) output_me = self.detect_2(medium_object_output, input_shape) @@ -457,7 +460,7 @@ class YoloWithLossCell(nn.Cell): def __init__(self, network): super(YoloWithLossCell, self).__init__() self.yolo_network = network - self.config = ConfigYOLOV4CspDarkNet53() + self.config = default_config self.loss_big = YoloLossBlock('l', self.config) self.loss_me = YoloLossBlock('m', self.config) self.loss_small = YoloLossBlock('s', self.config) diff --git a/model_zoo/official/cv/yolov4/test.py b/model_zoo/official/cv/yolov4/test.py index eb863ca0b6c..7c76961d01c 100644 --- a/model_zoo/official/cv/yolov4/test.py +++ b/model_zoo/official/cv/yolov4/test.py @@ -16,7 +16,7 @@ import os import sys -import argparse +import time import datetime from collections import defaultdict import json @@ -27,48 +27,25 @@ from mindspore import Tensor from mindspore.context import ParallelMode from mindspore.communication.management import init, get_rank, get_group_size from mindspore.train.serialization import load_checkpoint, load_param_into_net -import mindspore as ms from src.yolo import YOLOV4CspDarkNet53 from src.logger import get_logger from src.yolo_dataset import create_yolo_datasetv2 -from src.config import ConfigYOLOV4CspDarkNet53 + +from model_utils.config import config +from model_utils.moxing_adapter import moxing_wrapper +from model_utils.device_adapter import get_device_id, get_device_num devid = int(os.getenv('DEVICE_ID')) context.set_context(mode=context.GRAPH_MODE, device_target="Davinci", save_graphs=False, device_id=devid) -parser = argparse.ArgumentParser('mindspore coco testing') - -# dataset related -parser.add_argument('--data_dir', type=str, default='', help='train data dir') -parser.add_argument('--per_batch_size', default=1, type=int, help='batch size for per gpu') - -# network related -parser.add_argument('--pretrained', default='', type=str, help='model_path, local pretrained model to load') - -# logging related -parser.add_argument('--log_path', type=str, default='outputs/', help='checkpoint save location') - -# distributed related -parser.add_argument('--is_distributed', type=int, default=0, help='if multi device') -parser.add_argument('--rank', type=int, default=0, help='local rank of distributed') -parser.add_argument('--group_size', type=int, default=1, help='world size of distributed') - -# detect_related -parser.add_argument('--nms_thresh', type=float, default=0.45, help='threshold for NMS') -parser.add_argument('--annFile', type=str, default='', help='path to annotation') -parser.add_argument('--testing_shape', type=str, default='', help='shape for test ') -parser.add_argument('--ignore_threshold', type=float, default=0.001, help='threshold to throw low quality boxes') - -args, _ = parser.parse_known_args() - -args.data_root = os.path.join(args.data_dir, 'test2017') - +config.data_root = os.path.join(config.data_dir, 'test2017') +config.nms_thresh = config.test_nms_thresh class DetectionEngine(): """Detection engine""" def __init__(self, args_engine): - self.ignore_threshold = args_engine.ignore_threshold + self.test_ignore_threshold = args_engine.test_ignore_threshold self.labels = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', @@ -230,7 +207,7 @@ class DetectionEngine(): flag[i, c] = True confidence = cls_emb[flag] * conf for x_lefti, y_lefti, wi, hi, confi, clsi in zip(x_top_left, y_top_left, w, h, confidence, cls_argmax): - if confi < self.ignore_threshold: + if confi < self.test_ignore_threshold: continue if img_id not in self.results: self.results[img_id] = defaultdict(list) @@ -243,39 +220,87 @@ class DetectionEngine(): self.results[img_id][coco_clsi].append([x_lefti, y_lefti, wi, hi, confi]) -def convert_testing_shape(args_test): - testing_shape = [int(args_test.testing_shape), int(args_test.testing_shape)] - return testing_shape +def modelarts_pre_process(): + '''modelarts pre process function.''' + def unzip(zip_file, save_dir): + import zipfile + s_time = time.time() + if not os.path.exists(os.path.join(save_dir, config.modelarts_dataset_unzip_name)): + zip_isexist = zipfile.is_zipfile(zip_file) + if zip_isexist: + fz = zipfile.ZipFile(zip_file, 'r') + data_num = len(fz.namelist()) + print("Extract Start...") + print("unzip file num: {}".format(data_num)) + data_print = int(data_num / 100) if data_num > 100 else 1 + i = 0 + for file in fz.namelist(): + if i % data_print == 0: + print("unzip percent: {}%".format(int(i * 100 / data_num)), flush=True) + i += 1 + fz.extract(file, save_dir) + print("cost time: {}min:{}s.".format(int((time.time() - s_time) / 60), + int(int(time.time() - s_time) % 60))) + print("Extract Done.") + else: + print("This is not zip.") + else: + print("Zip has been extracted.") + + if config.need_modelarts_dataset_unzip: + zip_file_1 = os.path.join(config.data_path, config.modelarts_dataset_unzip_name + ".zip") + save_dir_1 = os.path.join(config.data_path) + + sync_lock = "/tmp/unzip_sync.lock" + + # Each server contains 8 devices as most. + if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock): + print("Zip file path: ", zip_file_1) + print("Unzip file save dir: ", save_dir_1) + unzip(zip_file_1, save_dir_1) + print("===Finish extract data synchronization===") + try: + os.mknod(sync_lock) + except IOError: + pass + + while True: + if os.path.exists(sync_lock): + break + time.sleep(1) + + print("Device: {}, Finish sync unzip data from {} to {}.".format(get_device_id(), zip_file_1, save_dir_1)) -def test(): +@moxing_wrapper(pre_process=modelarts_pre_process) +def run_test(): """test method""" # init distributed - if args.is_distributed: + if config.is_distributed: init() - args.rank = get_rank() - args.group_size = get_group_size() + config.rank = get_rank() + config.group_size = get_group_size() # logger - args.outputs_dir = os.path.join(args.log_path, - datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S')) + config.outputs_dir = os.path.join(config.log_path, + datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S')) - args.logger = get_logger(args.outputs_dir, args.rank) + config.logger = get_logger(config.outputs_dir, config.rank) context.reset_auto_parallel_context() - if args.is_distributed: + if config.is_distributed: parallel_mode = ParallelMode.DATA_PARALLEL else: parallel_mode = ParallelMode.STAND_ALONE context.set_auto_parallel_context(parallel_mode=parallel_mode, gradients_mean=True, device_num=1) - args.logger.info('Creating Network....') + config.logger.info('Creating Network....') network = YOLOV4CspDarkNet53() - args.logger.info(args.pretrained) - if os.path.isfile(args.pretrained): - param_dict = load_checkpoint(args.pretrained) + config.logger.info(config.pretrained) + if os.path.isfile(config.pretrained): + param_dict = load_checkpoint(config.pretrained) param_dict_new = {} for key, values in param_dict.items(): if key.startswith('moments.'): @@ -285,40 +310,35 @@ def test(): else: param_dict_new[key] = values load_param_into_net(network, param_dict_new) - args.logger.info('load_model {} success'.format(args.pretrained)) + config.logger.info('load_model %s success', config.pretrained) else: - args.logger.info('{} not exists or not a pre-trained file'.format(args.pretrained)) - assert FileNotFoundError('{} not exists or not a pre-trained file'.format(args.pretrained)) + config.logger.info('%s not exists or not a pre-trained file', config.pretrained) + assert FileNotFoundError('{} not exists or not a pre-trained file'.format(config.pretrained)) exit(1) - data_root = args.data_root + data_root = config.data_root - config = ConfigYOLOV4CspDarkNet53() - if args.testing_shape: - config.test_img_shape = convert_testing_shape(args) - - data_txt = os.path.join(args.data_dir, 'testdev2017.txt') - ds, data_size = create_yolo_datasetv2(data_root, data_txt=data_txt, batch_size=args.per_batch_size, - max_epoch=1, device_num=args.group_size, rank=args.rank, shuffle=False, + data_txt = os.path.join(config.data_dir, 'testdev2017.txt') + ds, data_size = create_yolo_datasetv2(data_root, data_txt=data_txt, batch_size=config.per_batch_size, + max_epoch=1, device_num=config.group_size, rank=config.rank, shuffle=False, config=config) - args.logger.info('testing shape : {}'.format(config.test_img_shape)) - args.logger.info('totol {} images to eval'.format(data_size)) + config.logger.info('testing shape : %s', config.test_img_shape) + config.logger.info('totol %d images to eval', data_size) network.set_train(False) # init detection engine - detection = DetectionEngine(args) + detection = DetectionEngine(config) - input_shape = Tensor(tuple(config.test_img_shape), ms.float32) - args.logger.info('Start inference....') + config.logger.info('Start inference....') for i, data in enumerate(ds.create_dict_iterator()): image = Tensor(data["image"]) image_shape = Tensor(data["image_shape"]) image_id = Tensor(data["img_id"]) - prediction = network(image, input_shape) + prediction = network(image) output_big, output_me, output_small = prediction output_big = output_big.asnumpy() output_me = output_me.asnumpy() @@ -326,14 +346,14 @@ def test(): image_id = image_id.asnumpy() image_shape = image_shape.asnumpy() - detection.detect([output_small, output_me, output_big], args.per_batch_size, image_shape, image_id) + detection.detect([output_small, output_me, output_big], config.per_batch_size, image_shape, image_id) if i % 1000 == 0: - args.logger.info('Processing... {:.2f}% '.format(i * args.per_batch_size / data_size * 100)) + config.logger.info('Processing... {:.2f}% '.format(i * config.per_batch_size / data_size * 100)) - args.logger.info('Calculating mAP...') + config.logger.info('Calculating mAP...') detection.do_nms_for_results() result_file_path = detection.write_result() - args.logger.info('result file path: {}'.format(result_file_path)) + config.logger.info('result file path: %s', result_file_path) if __name__ == "__main__": - test() + run_test() diff --git a/model_zoo/official/cv/yolov4/train.py b/model_zoo/official/cv/yolov4/train.py index c4f1a053a15..ed4aa3e25e1 100644 --- a/model_zoo/official/cv/yolov4/train.py +++ b/model_zoo/official/cv/yolov4/train.py @@ -15,9 +15,7 @@ """YoloV4 train.""" import os import time -import argparse import datetime -import ast from mindspore.context import ParallelMode from mindspore.nn.optim.momentum import Momentum @@ -39,131 +37,59 @@ from src.util import AverageMeter, get_param_groups from src.lr_scheduler import get_lr from src.yolo_dataset import create_yolo_dataset from src.initializer import default_recurisive_init, load_yolov4_params -from src.config import ConfigYOLOV4CspDarkNet53 from src.util import keep_loss_fp32 from src.eval_utils import apply_eval, EvalCallBack +from model_utils.config import config +from model_utils.moxing_adapter import moxing_wrapper +from model_utils.device_adapter import get_device_id, get_device_num + set_seed(1) -parser = argparse.ArgumentParser('mindspore coco training') +def set_default(): + if config.lr_scheduler == 'cosine_annealing' and config.max_epoch > config.t_max: + config.t_max = config.max_epoch -# device related -parser.add_argument('--device_target', type=str, default='Ascend', - help='device where the code will be implemented. (Default: Ascend)') + config.lr_epochs = list(map(int, config.lr_epochs.split(','))) + config.data_root = os.path.join(config.data_dir, 'train2017') + config.annFile = os.path.join(config.data_dir, 'annotations/instances_train2017.json') -# dataset related -parser.add_argument('--data_dir', type=str, help='Train dataset directory.') -parser.add_argument('--per_batch_size', default=8, type=int, help='Batch size for Training. Default: 8.') + config.data_val_root = os.path.join(config.data_dir, 'val2017') + config.ann_val_file = os.path.join(config.data_dir, 'annotations/instances_val2017.json') -# network related -parser.add_argument('--pretrained_backbone', default='', type=str, - help='The ckpt file of CspDarkNet53. Default: "".') -parser.add_argument('--resume_yolov4', default='', type=str, - help='The ckpt file of YOLOv4, which used to fine tune. Default: ""') -parser.add_argument('--pretrained_checkpoint', default='', type=str, - help='The ckpt file of YoloV4CspDarkNet53. Default: "".') -parser.add_argument("--filter_weight", type=ast.literal_eval, default=False, - help="Filter the last weight parameters, default is False.") + device_id = int(os.getenv('DEVICE_ID', '0')) + context.set_context(mode=context.GRAPH_MODE, enable_auto_mixed_precision=True, + device_target=config.device_target, save_graphs=False, device_id=device_id) -# optimizer and lr related -parser.add_argument('--lr_scheduler', default='cosine_annealing', type=str, - help='Learning rate scheduler, options: exponential, cosine_annealing. Default: exponential') -parser.add_argument('--lr', default=0.012, type=float, help='Learning rate. Default: 0.001') -parser.add_argument('--lr_epochs', type=str, default='220,250', - help='Epoch of changing of lr changing, split with ",". Default: 220,250') -parser.add_argument('--lr_gamma', type=float, default=0.1, - help='Decrease lr by a factor of exponential lr_scheduler. Default: 0.1') -parser.add_argument('--eta_min', type=float, default=0., help='Eta_min in cosine_annealing scheduler. Default: 0') -parser.add_argument('--t_max', type=int, default=320, help='T-max in cosine_annealing scheduler. Default: 320') -parser.add_argument('--max_epoch', type=int, default=320, help='Max epoch num to train the model. Default: 320') -parser.add_argument('--warmup_epochs', default=20, type=float, help='Warmup epochs. Default: 0') -parser.add_argument('--weight_decay', type=float, default=0.0005, help='Weight decay factor. Default: 0.0005') -parser.add_argument('--momentum', type=float, default=0.9, help='Momentum. Default: 0.9') - -# loss related -parser.add_argument('--loss_scale', type=int, default=64, help='Static loss scale. Default: 1024') -parser.add_argument('--label_smooth', type=int, default=0, help='Whether to use label smooth in CE. Default:0') -parser.add_argument('--label_smooth_factor', type=float, default=0.1, - help='Smooth strength of original one-hot. Default: 0.1') - -# logging related -parser.add_argument('--log_interval', type=int, default=100, help='Logging interval steps. Default: 100') -parser.add_argument('--ckpt_path', type=str, default='outputs/', help='Checkpoint save location. Default: outputs/') -parser.add_argument('--ckpt_interval', type=int, default=None, help='Save checkpoint interval. Default: None') - -parser.add_argument('--is_save_on_master', type=int, default=1, - help='Save ckpt on master or all rank, 1 for master, 0 for all ranks. Default: 1') - -# distributed related -parser.add_argument('--is_distributed', type=int, default=1, - help='Distribute train or not, 1 for yes, 0 for no. Default: 1') -parser.add_argument('--rank', type=int, default=0, help='Local rank of distributed. Default: 0') -parser.add_argument('--group_size', type=int, default=1, help='World size of device. Default: 1') - -# profiler init -parser.add_argument('--need_profiler', type=int, default=0, - help='Whether use profiler. 0 for no, 1 for yes. Default: 0') - -# reset default config -parser.add_argument('--training_shape', type=str, default="", help='Fix training shape. Default: ""') -parser.add_argument('--resize_rate', type=int, default=10, - help='Resize rate for multi-scale training. Default: None') - -parser.add_argument("--run_eval", type=ast.literal_eval, default=False, - help="Run evaluation when training, default is False.") -parser.add_argument("--save_best_ckpt", type=ast.literal_eval, default=True, - help="Save best checkpoint when run_eval is True, default is True.") -parser.add_argument("--eval_start_epoch", type=int, default=200, - help="Evaluation start epoch when run_eval is True, default is 200.") -parser.add_argument("--eval_interval", type=int, default=1, - help="Evaluation interval when run_eval is True, default is 1.") -parser.add_argument('--ann_file', type=str, default='', help='path to annotation') - -args, _ = parser.parse_known_args() - -if args.lr_scheduler == 'cosine_annealing' and args.max_epoch > args.t_max: - args.t_max = args.max_epoch - -args.lr_epochs = list(map(int, args.lr_epochs.split(','))) -args.data_root = os.path.join(args.data_dir, 'train2017') -args.annFile = os.path.join(args.data_dir, 'annotations/instances_train2017.json') - -args.data_val_root = os.path.join(args.data_dir, 'val2017') -args.ann_val_file = os.path.join(args.data_dir, 'annotations/instances_val2017.json') - -config = ConfigYOLOV4CspDarkNet53() -args.nms_thresh = config.nms_thresh -args.ignore_threshold = config.eval_ignore_threshold - -device_id = int(os.getenv('DEVICE_ID', '0')) -context.set_context(mode=context.GRAPH_MODE, enable_auto_mixed_precision=True, - device_target=args.device_target, save_graphs=False, device_id=device_id) - -if args.need_profiler: - profiler = Profiler(output_path=args.outputs_dir, is_detail=True, is_show_op_path=True) - -# init distributed -if args.is_distributed: - if args.device_target == "Ascend": - init() + if config.need_profiler: + profiler = Profiler(output_path=config.outputs_dir, is_detail=True, is_show_op_path=True) else: - init("nccl") - args.rank = get_rank() - args.group_size = get_group_size() + profiler = None -# select for master rank save ckpt or all rank save, compatible for model parallel -args.rank_save_ckpt_flag = 0 -if args.is_save_on_master: - if args.rank == 0: - args.rank_save_ckpt_flag = 1 -else: - args.rank_save_ckpt_flag = 1 + # init distributed + if config.is_distributed: + if config.device_target == "Ascend": + init() + else: + init("nccl") + config.rank = get_rank() + config.group_size = get_group_size() -# logger -args.outputs_dir = os.path.join(args.ckpt_path, - datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S')) -args.logger = get_logger(args.outputs_dir, args.rank) -args.logger.save_args(args) + # select for master rank save ckpt or all rank save, compatible for model parallel + config.rank_save_ckpt_flag = 0 + if config.is_save_on_master: + if config.rank == 0: + config.rank_save_ckpt_flag = 1 + else: + config.rank_save_ckpt_flag = 1 + + # logger + config.outputs_dir = os.path.join(config.ckpt_path, + datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S')) + config.logger = get_logger(config.outputs_dir, config.rank) + config.logger.save_args(config) + + return profiler def convert_training_shape(args_training_shape): @@ -183,92 +109,151 @@ class BuildTrainNetwork(nn.Cell): return loss_ -if __name__ == "__main__": - loss_meter = AverageMeter('loss') +def modelarts_pre_process(): + '''modelarts pre process function.''' + def unzip(zip_file, save_dir): + import zipfile + s_time = time.time() + if not os.path.exists(os.path.join(save_dir, config.modelarts_dataset_unzip_name)): + zip_isexist = zipfile.is_zipfile(zip_file) + if zip_isexist: + fz = zipfile.ZipFile(zip_file, 'r') + data_num = len(fz.namelist()) + print("Extract Start...") + print("unzip file num: {}".format(data_num)) + data_print = int(data_num / 100) if data_num > 100 else 1 + i = 0 + for file in fz.namelist(): + if i % data_print == 0: + print("unzip percent: {}%".format(int(i * 100 / data_num)), flush=True) + i += 1 + fz.extract(file, save_dir) + print("cost time: {}min:{}s.".format(int((time.time() - s_time) / 60), + int(int(time.time() - s_time) % 60))) + print("Extract Done.") + else: + print("This is not zip.") + else: + print("Zip has been extracted.") + if config.need_modelarts_dataset_unzip: + zip_file_1 = os.path.join(config.data_path, config.modelarts_dataset_unzip_name + ".zip") + save_dir_1 = os.path.join(config.data_path) + + sync_lock = "/tmp/unzip_sync.lock" + + # Each server contains 8 devices as most. + if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock): + print("Zip file path: ", zip_file_1) + print("Unzip file save dir: ", save_dir_1) + unzip(zip_file_1, save_dir_1) + print("===Finish extract data synchronization===") + try: + os.mknod(sync_lock) + except IOError: + pass + + while True: + if os.path.exists(sync_lock): + break + time.sleep(1) + + print("Device: {}, Finish sync unzip data from {} to {}.".format(get_device_id(), zip_file_1, save_dir_1)) + + config.ckpt_path = os.path.join(config.output_path, config.ckpt_path) + + +def get_network(net, cfg, learning_rate): + opt = Momentum(params=get_param_groups(net), + learning_rate=Tensor(learning_rate), + momentum=cfg.momentum, + weight_decay=cfg.weight_decay, + loss_scale=cfg.loss_scale) + is_gpu = context.get_context("device_target") == "GPU" + if is_gpu: + loss_scale_value = 1.0 + loss_scale = FixedLossScaleManager(loss_scale_value, drop_overflow_update=False) + net = amp.build_train_network(net, optimizer=opt, loss_scale_manager=loss_scale, + level="O2", keep_batchnorm_fp32=False) + keep_loss_fp32(net) + else: + net = TrainingWrapper(net, opt) + net.set_train() + + return net + + +@moxing_wrapper(pre_process=modelarts_pre_process) +def run_train(): + profiler = set_default() + + loss_meter = AverageMeter('loss') context.reset_auto_parallel_context() parallel_mode = ParallelMode.STAND_ALONE degree = 1 - if args.is_distributed: + if config.is_distributed: parallel_mode = ParallelMode.DATA_PARALLEL degree = get_group_size() context.set_auto_parallel_context(parallel_mode=parallel_mode, gradients_mean=True, device_num=degree) network = YOLOV4CspDarkNet53() - network_eval = network + if config.run_eval: + network_eval = network # default is kaiming-normal - args.checkpoint_filter_list = config.checkpoint_filter_list default_recurisive_init(network) - load_yolov4_params(args, network) + load_yolov4_params(config, network) network = YoloWithLossCell(network) - args.logger.info('finish get network') + config.logger.info('finish get network') - config.label_smooth = args.label_smooth - config.label_smooth_factor = args.label_smooth_factor + if config.training_shape: + config.multi_scale = [convert_training_shape(config.training_shape)] - if args.training_shape: - config.multi_scale = [convert_training_shape(args.training_shape)] - if args.resize_rate: - config.resize_rate = args.resize_rate + ds, data_size = create_yolo_dataset(image_dir=config.data_root, anno_path=config.annFile, is_training=True, + batch_size=config.per_batch_size, max_epoch=config.max_epoch, + device_num=config.group_size, rank=config.rank, config=config) + config.logger.info('Finish loading dataset') - ds, data_size = create_yolo_dataset(image_dir=args.data_root, anno_path=args.annFile, is_training=True, - batch_size=args.per_batch_size, max_epoch=args.max_epoch, - device_num=args.group_size, rank=args.rank, config=config) - args.logger.info('Finish loading dataset') + config.steps_per_epoch = int(data_size / config.per_batch_size / config.group_size) - args.steps_per_epoch = int(data_size / args.per_batch_size / args.group_size) + if config.ckpt_interval <= 0: + config.ckpt_interval = config.steps_per_epoch - if not args.ckpt_interval: - args.ckpt_interval = args.steps_per_epoch + lr = get_lr(config) + network = get_network(network, config, lr) + network.set_train(True) - lr = get_lr(args) + if config.rank_save_ckpt_flag or config.run_eval: + cb_params = _InternalCallbackParam() + cb_params.train_network = network + cb_params.epoch_num = config.max_epoch * config.steps_per_epoch // config.ckpt_interval + cb_params.cur_epoch_num = 1 + run_context = RunContext(cb_params) - opt = Momentum(params=get_param_groups(network), - learning_rate=Tensor(lr), - momentum=args.momentum, - weight_decay=args.weight_decay, - loss_scale=args.loss_scale) - is_gpu = context.get_context("device_target") == "GPU" - if is_gpu: - loss_scale_value = 1.0 - loss_scale = FixedLossScaleManager(loss_scale_value, drop_overflow_update=False) - network = amp.build_train_network(network, optimizer=opt, loss_scale_manager=loss_scale, - level="O2", keep_batchnorm_fp32=False) - keep_loss_fp32(network) - else: - network = TrainingWrapper(network, opt) - network.set_train() + if config.rank_save_ckpt_flag: + # checkpoint save + ckpt_max_num = 10 + ckpt_config = CheckpointConfig(save_checkpoint_steps=config.ckpt_interval, + keep_checkpoint_max=ckpt_max_num) + save_ckpt_path = os.path.join(config.outputs_dir, 'ckpt_' + str(config.rank) + '/') + ckpt_cb = ModelCheckpoint(config=ckpt_config, + directory=save_ckpt_path, + prefix='{}'.format(config.rank)) + ckpt_cb.begin(run_context) - # checkpoint save - ckpt_max_num = 10 - ckpt_config = CheckpointConfig(save_checkpoint_steps=args.ckpt_interval, - keep_checkpoint_max=ckpt_max_num) - save_ckpt_path = os.path.join(args.outputs_dir, 'ckpt_' + str(args.rank) + '/') - ckpt_cb = ModelCheckpoint(config=ckpt_config, - directory=save_ckpt_path, - prefix='{}'.format(args.rank)) - cb_params = _InternalCallbackParam() - cb_params.train_network = network - cb_params.epoch_num = args.max_epoch * args.steps_per_epoch // args.ckpt_interval - cb_params.cur_epoch_num = 1 - run_context = RunContext(cb_params) - ckpt_cb.begin(run_context) - - if args.run_eval: - rank_id = int(os.environ.get('RANK_ID')) if os.environ.get('RANK_ID') else 0 - data_val_root = args.data_val_root - ann_val_file = args.ann_val_file - save_ckpt_path = os.path.join(args.outputs_dir, 'ckpt_' + str(args.rank) + '/') + if config.run_eval: + data_val_root = config.data_val_root + ann_val_file = config.ann_val_file + save_ckpt_path = os.path.join(config.outputs_dir, 'ckpt_' + str(config.rank) + '/') input_val_shape = Tensor(tuple(config.test_img_shape), ms.float32) # init detection engine eval_dataset, eval_data_size = create_yolo_dataset(data_val_root, ann_val_file, is_training=False, - batch_size=args.per_batch_size, max_epoch=1, device_num=1, + batch_size=config.per_batch_size, max_epoch=1, device_num=1, rank=0, shuffle=False, config=config) eval_param_dict = {"net": network_eval, "dataset": eval_dataset, "data_size": eval_data_size, - "anno_json": ann_val_file, "input_shape": input_val_shape, "args": args} - eval_cb = EvalCallBack(apply_eval, eval_param_dict, interval=args.eval_interval, - eval_start_epoch=args.eval_start_epoch, save_best_ckpt=True, + "anno_json": ann_val_file, "input_shape": input_val_shape, "args": config} + eval_cb = EvalCallBack(apply_eval, eval_param_dict, interval=config.eval_interval, + eval_start_epoch=config.eval_start_epoch, save_best_ckpt=True, ckpt_directory=save_ckpt_path, besk_ckpt_name="best_map.ckpt", metrics_name="mAP") @@ -277,13 +262,11 @@ if __name__ == "__main__": data_loader = ds.create_dict_iterator(output_numpy=True, num_epochs=1) for i, data in enumerate(data_loader): - network.set_train() images = data["image"] input_shape = images.shape[2:4] - args.logger.info('iter[{}], shape{}'.format(i, input_shape[0])) + config.logger.info('iter[%d], shape%d', i, input_shape[0]) images = Tensor.from_numpy(images) - batch_y_true_0 = Tensor.from_numpy(data['bbox1']) batch_y_true_1 = Tensor.from_numpy(data['bbox2']) batch_y_true_2 = Tensor.from_numpy(data['bbox3']) @@ -297,29 +280,35 @@ if __name__ == "__main__": loss_meter.update(loss.asnumpy()) # ckpt progress - if args.rank_save_ckpt_flag: + if config.rank_save_ckpt_flag: cb_params.cur_step_num = i + 1 # current step number cb_params.batch_num = i + 2 ckpt_cb.step_end(run_context) - if i % args.log_interval == 0: + if i % config.log_interval == 0: time_used = time.time() - t_end - epoch = int(i / args.steps_per_epoch) - fps = args.per_batch_size * (i - old_progress) * args.group_size / time_used - args.logger.info('epoch[{}], iter[{}], {}, {:.2f} imgs/sec, lr:{}'.format(epoch, i, loss_meter, fps, lr[i])) + epoch = int(i / config.steps_per_epoch) + fps = config.per_batch_size * (i - old_progress) * config.group_size / time_used + if config.rank == 0: + config.logger.info( + 'epoch[{}], iter[{}], {}, {:.2f} imgs/sec, lr:{}'.format(epoch, i, loss_meter, fps, lr[i])) t_end = time.time() loss_meter.reset() old_progress = i - if args.run_eval and (i + 1) % args.steps_per_epoch == 0: - eval_cb.epoch_end(run_context) - - if (i + 1) % args.steps_per_epoch == 0: + if (i + 1) % config.steps_per_epoch == 0 and (config.run_eval or config.rank_save_ckpt_flag): + if config.run_eval: + eval_cb.epoch_end(run_context) + network.set_train() cb_params.cur_epoch_num += 1 - if args.need_profiler: + if config.need_profiler and profiler is not None: if i == 10: profiler.analyse() break - args.logger.info('==========end training===============') + config.logger.info('==========end training===============') + + +if __name__ == "__main__": + run_train()