Col_lin
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

modify yolov4 fo clould

This commit is contained in:
zhanghuiyao 2021-05-12 18:08:33 +08:00
parent d32f041248
commit d53bda663a
20 changed files with 1000 additions and 484 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
model_zoo/official/cv/yolov4/README.md
View File

@ -93,58 +93,148 @@ other datasets need to use the same format as MS COCO.
python hccl_tools.py --device_num "[0,8)"
```
```text
# The parameter of training_shape define image shape for network, default is
[416, 416],
[448, 448],
[480, 480],
[512, 512],
[544, 544],
[576, 576],
[608, 608],
[640, 640],
[672, 672],
[704, 704],
[736, 736].
# It means use 11 kinds of shape as input shape, or it can be set some kind of shape.
```
- Run on local
```bash
#run training example(1p) by python command (Training with a single scale)
python train.py \
--data_dir=./dataset/xxx \
--pretrained_backbone=cspdarknet53_backbone.ckpt \
--is_distributed=0 \
--lr=0.1 \
--t_max=320 \
--max_epoch=320 \
--warmup_epochs=4 \
--training_shape=416 \
--lr_scheduler=cosine_annealing > log.txt 2>&1 &
```
```text
# The parameter of training_shape define image shape for network, default is
[416, 416],
[448, 448],
[480, 480],
[512, 512],
[544, 544],
[576, 576],
[608, 608],
[640, 640],
[672, 672],
[704, 704],
[736, 736].
# It means use 11 kinds of shape as input shape, or it can be set some kind of shape.
```bash
# standalone training example(1p) by shell script (Training with a single scale)
sh run_standalone_train.sh dataset/xxx cspdarknet53_backbone.ckpt
```
#run training example(1p) by python command (Training with a single scale)
python train.py \
--data_dir=./dataset/xxx \
--pretrained_backbone=cspdarknet53_backbone.ckpt \
--is_distributed=0 \
--lr=0.1 \
--t_max=320 \
--max_epoch=320 \
--warmup_epochs=4 \
--training_shape=416 \
--lr_scheduler=cosine_annealing > log.txt 2>&1 &
```bash
# For Ascend device, distributed training example(8p) by shell script (Training with multi scale)
sh run_distribute_train.sh dataset/xxx cspdarknet53_backbone.ckpt rank_table_8p.json
```
# standalone training example(1p) by shell script (Training with a single scale)
sh run_standalone_train.sh dataset/xxx cspdarknet53_backbone.ckpt
```bash
# run evaluation by python command
python eval.py \
--data_dir=./dataset/xxx \
--pretrained=yolov4.ckpt \
--testing_shape=608 > log.txt 2>&1 &
```
# For Ascend device, distributed training example(8p) by shell script (Training with multi scale)
sh run_distribute_train.sh dataset/xxx cspdarknet53_backbone.ckpt rank_table_8p.json
```bash
# run evaluation by shell script
sh run_eval.sh dataset/xxx checkpoint/xxx.ckpt
```
# run evaluation by python command
python eval.py \
--data_dir=./dataset/xxx \
--pretrained=yolov4.ckpt \
--testing_shape=608 > log.txt 2>&1 &
# run evaluation by shell script
sh run_eval.sh dataset/xxx checkpoint/xxx.ckpt
```
- Train on [ModelArts](https://support.huaweicloud.com/modelarts/)
```python
# Train 8p with Ascend
# (1) Perform a or b.
# a. Set "enable_modelarts=True" on base_config.yaml file.
# Set "data_dir='/cache/data/coco/'" on base_config.yaml file.
# Set "checkpoint_url='s3://dir_to_your_pretrain/'" on base_config.yaml file.
# Set "pretrained_backbone='/cache/checkpoint_path/cspdarknet53_backbone.ckpt'" on base_config.yaml file.
# Set other parameters on base_config.yaml file you need.
# b. Add "enable_modelarts=True" on the website UI interface.
# Add "data_dir=/cache/data/coco/" on the website UI interface.
# Add "checkpoint_url=s3://dir_to_your_pretrain/" on the website UI interface.
# Add "pretrained_backbone=/cache/checkpoint_path/cspdarknet53_backbone.ckpt" on the website UI interface.
# Add other parameters on the website UI interface.
# (3) Upload or copy your pretrained model to S3 bucket.
# (4) Upload a zip dataset to S3 bucket. (you could also upload the origin dataset, but it can be so slow.)
# (5) Set the code directory to "/path/yolov4" on the website UI interface.
# (6) Set the startup file to "train.py" on the website UI interface.
# (7) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
# (8) Create your job.
#
# Train 1p with Ascend
# (1) Perform a or b.
# a. Set "enable_modelarts=True" on base_config.yaml file.
# Set "data_dir='/cache/data/coco/'" on base_config.yaml file.
# Set "checkpoint_url='s3://dir_to_your_pretrain/'" on base_config.yaml file.
# Set "pretrained_backbone='/cache/checkpoint_path/cspdarknet53_backbone.ckpt'" on base_config.yaml file.
# Set "is_distributed=0" on base_config.yaml file.
# Set "warmup_epochs=4" on base_config.yaml file.
# Set "training_shape=416" on base_config.yaml file.
# Set other parameters on base_config.yaml file you need.
# b. Add "enable_modelarts=True" on the website UI interface.
# Add "data_dir=/cache/data/coco/" on the website UI interface.
# Add "checkpoint_url=s3://dir_to_your_pretrain/" on the website UI interface.
# Add "pretrained_backbone=/cache/checkpoint_path/cspdarknet53_backbone.ckpt" on the website UI interface.
# Add "is_distributed=0" on the website UI interface.
# Add "warmup_epochs=4" on the website UI interface.
# Add "training_shape=416" on the website UI interface.
# Add other parameters on the website UI interface.
# (3) Upload or copy your pretrained model to S3 bucket.
# (4) Upload a zip dataset to S3 bucket. (you could also upload the origin dataset, but it can be so slow.)
# (5) Set the code directory to "/path/yolov4" on the website UI interface.
# (6) Set the startup file to "train.py" on the website UI interface.
# (7) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
# (8) Create your job.
#
# Eval 1p with Ascend
# (1) Perform a or b.
# a. Set "enable_modelarts=True" on base_config.yaml file.
# Set "data_dir='/cache/data/coco/'" on base_config.yaml file.
# Set "checkpoint_url='s3://dir_to_your_trained_ckpt/'" on base_config.yaml file.
# Set "pretrained='/cache/checkpoint_path/model.ckpt'" on base_config.yaml file.
# Set "is_distributed=0" on base_config.yaml file.
# Set "per_batch_size=1" on base_config.yaml file.
# Set other parameters on base_config.yaml file you need.
# b. Add "enable_modelarts=True" on the website UI interface.
# Add "data_dir=/cache/data/coco/" on the website UI interface.
# Add "checkpoint_url=s3://dir_to_your_trained_ckpt/" on the website UI interface.
# Add "pretrained=/cache/checkpoint_path/model.ckpt" on the website UI interface.
# Add "is_distributed=0" on the website UI interface.
# Add "per_batch_size=1" on the website UI interface.
# Add other parameters on the website UI interface.
# (3) Upload or copy your trained model to S3 bucket.
# (4) Upload a zip dataset to S3 bucket. (you could also upload the origin dataset, but it can be so slow.)
# (5) Set the code directory to "/path/yolov4" on the website UI interface.
# (6) Set the startup file to "eval.py" on the website UI interface.
# (7) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
# (8) Create your job.
#
# Test 1p with Ascend
# (1) Perform a or b.
# a. Set "enable_modelarts=True" on base_config.yaml file.
# Set "data_dir='/cache/data/coco/'" on base_config.yaml file.
# Set "checkpoint_url='s3://dir_to_your_trained_ckpt/'" on base_config.yaml file.
# Set "pretrained='/cache/checkpoint_path/model.ckpt'" on base_config.yaml file.
# Set "is_distributed=0" on base_config.yaml file.
# Set "per_batch_size=1" on base_config.yaml file.
# Set "test_nms_thresh=0.45" on base_config.yaml file.
# Set "test_ignore_threshold=0.001" on base_config.yaml file.
# Set other parameters on base_config.yaml file you need.
# b. Add "enable_modelarts=True" on the website UI interface.
# Add "data_dir=/cache/data/coco/" on the website UI interface.
# Add "checkpoint_url=s3://dir_to_your_trained_ckpt/" on the website UI interface.
# Add "pretrained=/cache/checkpoint_path/model.ckpt" on the website UI interface.
# Add "is_distributed=0" on the website UI interface.
# Add "per_batch_size=1" on the website UI interface.
# Add "test_nms_thresh=0.45" on the website UI interface.
# Add "test_ignore_threshold=0.001" on the website UI interface.
# Add other parameters on the website UI interface.
# (3) Upload or copy your trained model to S3 bucket.
# (4) Upload a zip dataset to S3 bucket. (you could also upload the origin dataset, but it can be so slow.)
# (5) Set the code directory to "/path/yolov4" on the website UI interface.
# (6) Set the startup file to "test.py" on the website UI interface.
# (7) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
# (8) Create your job.
```
# [Script Description](#contents)
@ -448,7 +538,7 @@ YOLOv4 on 118K images(The annotation and data format must be the same as coco201
| Parameters | YOLOv4 |
| -------------------------- | ----------------------------------------------------------- |
| Resource | Ascend 910; CPU 2.60GHz, 192cores; Memory 755G; OS Euler2.8; System, Euleros 2.8;|
| Resource | Ascend 910; CPU 2.60GHz, 192cores; Memory, 755G; System, Euleros 2.8;|
| uploaded Date | 10/16/2020 (month/day/year) |
| MindSpore Version | 1.0.0-alpha |
| Dataset | 118K images |
@ -468,7 +558,7 @@ YOLOv4 on 20K images(The annotation and data format must be the same as coco tes
| Parameters | YOLOv4 |
| -------------------------- | ----------------------------------------------------------- |
| Resource | Ascend 910; CPU 2.60GHz, 192cores; Memory 755G; OS Euler2.8 |
| Resource | Ascend 910; CPU 2.60GHz, 192cores; Memory, 755G |
| uploaded Date | 10/16/2020 (month/day/year) |
| MindSpore Version | 1.0.0-alpha |
| Dataset | 20K images |

165
model_zoo/official/cv/yolov4/default_config.yaml Normal file
View File

@ -0,0 +1,165 @@
# Builtin Configurations(DO NOT CHANGE THESE CONFIGURATIONS unless you know exactly what you are doing)
enable_modelarts: False
# Url for modelarts
data_url: ""
train_url: ""
checkpoint_url: ""
# Path for local
data_path: "/cache/data"
output_path: "/cache/train"
load_path: "/cache/checkpoint_path"
device_target: "Ascend"
need_modelarts_dataset_unzip: True
modelarts_dataset_unzip_name: "coco"
# ==============================================================================
# Train options
data_dir: ""
per_batch_size: 8
pretrained_backbone: ""
resume_yolov4: ""
pretrained_checkpoint: ""
filter_weight: False
lr_scheduler: "cosine_annealing"
lr: 0.012
lr_epochs: "220,250"
lr_gamma: 0.1
eta_min: 0.0
t_max: 320
max_epoch: 320
warmup_epochs: 20
weight_decay: 0.0005
momentum: 0.9
loss_scale: 64
label_smooth: 0
label_smooth_factor: 0.1
log_interval: 100
ckpt_path: "outputs/"
ckpt_interval: -1
is_save_on_master: 1
is_distributed: 1
rank: 0
group_size: 1
need_profiler: 0
training_shape: ""
run_eval: False
save_best_ckpt: True
eval_start_epoch: 200
eval_interval: 1
ann_file: ""
# Eval options
pretrained: ""
log_path: "outputs/"
ann_val_file: ""
# Test option
test_nms_thresh: 0.45
test_ignore_threshold: 0.001
# Export options
device_id: 0
batch_size: 1
testing_shape: 608
ckpt_file: ""
file_name: "yolov4"
file_format: "AIR"
# Other default config
hue: 0.1
saturation: 1.5
value: 1.5
jitter: 0.3
resize_rate: 10
multi_scale: [[416, 416],
[448, 448],
[480, 480],
[512, 512],
[544, 544],
[576, 576],
[608, 608],
[640, 640],
[672, 672],
[704, 704],
[736, 736]
]
max_box: 90
backbone_input_shape: [32, 64, 128, 256, 512]
backbone_shape: [64, 128, 256, 512, 1024]
backbone_layers: [1, 2, 8, 8, 4]
ignore_threshold: 0.7
eval_ignore_threshold: 0.001
nms_thresh: 0.5
anchor_scales: [[12, 16],
[19, 36],
[40, 28],
[36, 75],
[76, 55],
[72, 146],
[142, 110],
[192, 243],
[459, 401]]
num_classes: 80
out_channel: 255 # 3 * (num_classes + 5)
test_img_shape: [608, 608]
checkpoint_filter_list: ['feature_map.backblock0.conv6.weight', 'feature_map.backblock0.conv6.bias',
'feature_map.backblock1.conv6.weight', 'feature_map.backblock1.conv6.bias',
'feature_map.backblock2.conv6.weight', 'feature_map.backblock2.conv6.bias',
'feature_map.backblock3.conv6.weight', 'feature_map.backblock3.conv6.bias']
---
# Help description for each configuration
# Train options
data_dir: "Train dataset directory."
per_batch_size: "Batch size for Training."
pretrained_backbone: "The ckpt file of CspDarkNet53."
resume_yolov4: "The ckpt file of YOLOv4, which used to fine tune."
pretrained_checkpoint: "The ckpt file of YoloV4CspDarkNet53."
filter_weight: "Filter the last weight parameters"
lr_scheduler: "Learning rate scheduler, options: exponential, cosine_annealing."
lr: "Learning rate."
lr_epochs: "Epoch of changing of lr changing, split with ','."
lr_gamma: "Decrease lr by a factor of exponential lr_scheduler."
eta_min: "Eta_min in cosine_annealing scheduler."
t_max: "T-max in cosine_annealing scheduler."
max_epoch: "Max epoch num to train the model."
warmup_epochs: "Warmup epochs."
weight_decay: "Weight decay factor."
momentum: "Momentum."
loss_scale: "Static loss scale."
label_smooth: "Whether to use label smooth in CE."
label_smooth_factor: "Smooth strength of original one-hot."
log_interval: "Logging interval steps."
ckpt_path: "Checkpoint save location."
ckpt_interval: "Save checkpoint interval."
is_save_on_master: "Save ckpt on master or all rank, 1 for master, 0 for all ranks."
is_distributed: "Distribute train or not, 1 for yes, 0 for no."
rank: "Local rank of distributed."
group_size: "World size of device."
need_profiler: "Whether use profiler. 0 for no, 1 for yes."
training_shape: "Fix training shape."
resize_rate: "Resize rate for multi-scale training."
run_eval: "Run evaluation when training."
save_best_ckpt: "Save best checkpoint when run_eval is True."
eval_start_epoch: "Evaluation start epoch when run_eval is True."
eval_interval: "Evaluation interval when run_eval is True"
ann_file: "path to annotation"
# Eval options
pretrained: "model_path, local pretrained model to load"
log_path: "checkpoint save location"
ann_val_file: "path to annotation"
# Export options
device_id: "Device id for export"
batch_size: "batch size for export"
testing_shape: "shape for test"
ckpt_file: "Checkpoint file path for export"
file_name: "output file name for export"
file_format: "file format for export"

133
model_zoo/official/cv/yolov4/eval.py
View File

@ -14,78 +14,100 @@
# ============================================================================
"""YoloV4 eval."""
import os
import argparse
import datetime
import time
from mindspore import Tensor
from mindspore.context import ParallelMode
from mindspore import context
from mindspore.train.serialization import load_checkpoint, load_param_into_net
import mindspore as ms
from src.yolo import YOLOV4CspDarkNet53
from src.logger import get_logger
from src.yolo_dataset import create_yolo_dataset
from src.config import ConfigYOLOV4CspDarkNet53
from src.eval_utils import apply_eval
parser = argparse.ArgumentParser('mindspore coco testing')
from model_utils.config import config
from model_utils.moxing_adapter import moxing_wrapper
from model_utils.device_adapter import get_device_id, get_device_num
# device related
parser.add_argument('--device_target', type=str, default='Ascend',
help='device where the code will be implemented. (Default: Ascend)')
config.data_root = os.path.join(config.data_dir, 'val2017')
config.ann_val_file = os.path.join(config.data_dir, 'annotations/instances_val2017.json')
# dataset related
parser.add_argument('--data_dir', type=str, default='', help='train data dir')
parser.add_argument('--per_batch_size', default=1, type=int, help='batch size for per gpu')
def modelarts_pre_process():
'''modelarts pre process function.'''
def unzip(zip_file, save_dir):
import zipfile
s_time = time.time()
if not os.path.exists(os.path.join(save_dir, config.modelarts_dataset_unzip_name)):
zip_isexist = zipfile.is_zipfile(zip_file)
if zip_isexist:
fz = zipfile.ZipFile(zip_file, 'r')
data_num = len(fz.namelist())
print("Extract Start...")
print("unzip file num: {}".format(data_num))
data_print = int(data_num / 100) if data_num > 100 else 1
i = 0
for file in fz.namelist():
if i % data_print == 0:
print("unzip percent: {}%".format(int(i * 100 / data_num)), flush=True)
i += 1
fz.extract(file, save_dir)
print("cost time: {}min:{}s.".format(int((time.time() - s_time) / 60),
int(int(time.time() - s_time) % 60)))
print("Extract Done.")
else:
print("This is not zip.")
else:
print("Zip has been extracted.")
# network related
parser.add_argument('--pretrained', default='', type=str, help='model_path, local pretrained model to load')
if config.need_modelarts_dataset_unzip:
zip_file_1 = os.path.join(config.data_path, config.modelarts_dataset_unzip_name + ".zip")
save_dir_1 = os.path.join(config.data_path)
# logging related
parser.add_argument('--log_path', type=str, default='outputs/', help='checkpoint save location')
sync_lock = "/tmp/unzip_sync.lock"
# detect_related
parser.add_argument('--ann_val_file', type=str, default='', help='path to annotation')
parser.add_argument('--testing_shape', type=str, default='', help='shape for test ')
# Each server contains 8 devices as most.
if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock):
print("Zip file path: ", zip_file_1)
print("Unzip file save dir: ", save_dir_1)
unzip(zip_file_1, save_dir_1)
print("===Finish extract data synchronization===")
try:
os.mknod(sync_lock)
except IOError:
pass
args, _ = parser.parse_known_args()
while True:
if os.path.exists(sync_lock):
break
time.sleep(1)
config = ConfigYOLOV4CspDarkNet53()
args.nms_thresh = config.nms_thresh
args.ignore_threshold = config.eval_ignore_threshold
args.data_root = os.path.join(args.data_dir, 'val2017')
args.ann_val_file = os.path.join(args.data_dir, 'annotations/instances_val2017.json')
print("Device: {}, Finish sync unzip data from {} to {}.".format(get_device_id(), zip_file_1, save_dir_1))
config.log_path = os.path.join(config.output_path, config.log_path)
def convert_testing_shape(args_testing_shape):
"""Convert testing shape to list."""
testing_shape = [int(args_testing_shape), int(args_testing_shape)]
return testing_shape
if __name__ == "__main__":
@moxing_wrapper(pre_process=modelarts_pre_process)
def run_eval():
start_time = time.time()
device_id = int(os.getenv('DEVICE_ID')) if os.getenv('DEVICE_ID') else 0
context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target, device_id=device_id)
context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target, device_id=device_id)
# logger
args.outputs_dir = os.path.join(args.log_path,
datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S'))
config.outputs_dir = os.path.join(config.log_path,
datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S'))
rank_id = int(os.environ.get('RANK_ID')) if os.environ.get('RANK_ID') else 0
args.logger = get_logger(args.outputs_dir, rank_id)
config.logger = get_logger(config.outputs_dir, rank_id)
context.reset_auto_parallel_context()
parallel_mode = ParallelMode.STAND_ALONE
context.set_auto_parallel_context(parallel_mode=parallel_mode, gradients_mean=True, device_num=1)
args.logger.info('Creating Network....')
config.logger.info('Creating Network....')
network = YOLOV4CspDarkNet53()
args.logger.info(args.pretrained)
if os.path.isfile(args.pretrained):
param_dict = load_checkpoint(args.pretrained)
config.logger.info(config.pretrained)
if os.path.isfile(config.pretrained):
param_dict = load_checkpoint(config.pretrained)
param_dict_new = {}
for key, values in param_dict.items():
if key.startswith('moments.'):
@ -95,33 +117,34 @@ if __name__ == "__main__":
else:
param_dict_new[key] = values
load_param_into_net(network, param_dict_new)
args.logger.info('load_model {} success'.format(args.pretrained))
config.logger.info('load_model %s success', config.pretrained)
else:
args.logger.info('{} not exists or not a pre-trained file'.format(args.pretrained))
assert FileNotFoundError('{} not exists or not a pre-trained file'.format(args.pretrained))
config.logger.info('%s not exists or not a pre-trained file', config.pretrained)
assert FileNotFoundError('{} not exists or not a pre-trained file'.format(config.pretrained))
exit(1)
data_root = args.data_root
ann_val_file = args.ann_val_file
data_root = config.data_root
ann_val_file = config.ann_val_file
if args.testing_shape:
config.test_img_shape = convert_testing_shape(args.testing_shape)
ds, data_size = create_yolo_dataset(data_root, ann_val_file, is_training=False, batch_size=args.per_batch_size,
ds, data_size = create_yolo_dataset(data_root, ann_val_file, is_training=False, batch_size=config.per_batch_size,
max_epoch=1, device_num=1, rank=rank_id, shuffle=False,
config=config)
args.logger.info('testing shape : {}'.format(config.test_img_shape))
args.logger.info('totol {} images to eval'.format(data_size))
config.logger.info('testing shape : %s', config.test_img_shape)
config.logger.info('totol %d images to eval', data_size)
network.set_train(False)
# init detection engine
input_shape = Tensor(tuple(config.test_img_shape), ms.float32)
args.logger.info('Start inference....')
config.logger.info('Start inference....')
eval_param_dict = {"net": network, "dataset": ds, "data_size": data_size,
"anno_json": args.ann_val_file, "input_shape": input_shape, "args": args}
"anno_json": config.ann_val_file, "args": config}
eval_result, _ = apply_eval(eval_param_dict)
cost_time = time.time() - start_time
args.logger.info('\n=============coco eval reulst=========\n' + eval_result)
args.logger.info('testing cost time {:.2f}h'.format(cost_time / 3600.))
eval_log_string = '\n=============coco eval reulst=========\n' + eval_result
config.logger.info(eval_log_string)
config.logger.info('testing cost time %.2f h', cost_time / 3600.)
if __name__ == "__main__":
run_eval()

26
model_zoo/official/cv/yolov4/export.py
View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import argparse
import numpy as np
import mindspore
@ -21,30 +20,21 @@ from mindspore.train.serialization import export, load_checkpoint, load_param_in
from src.yolo import YOLOV4CspDarkNet53
parser = argparse.ArgumentParser(description='yolov4 export')
parser.add_argument("--device_id", type=int, default=0, help="Device id")
parser.add_argument("--batch_size", type=int, default=1, help="batch size")
parser.add_argument("--testing_shape", type=int, default=608, help="test shape")
parser.add_argument("--ckpt_file", type=str, required=True, help="Checkpoint file path.")
parser.add_argument("--file_name", type=str, default="yolov4", help="output file name.")
parser.add_argument('--file_format', type=str, choices=["AIR", "ONNX", "MINDIR"], default='AIR', help='file format')
parser.add_argument("--device_target", type=str, choices=["Ascend", "GPU", "CPU"], default="Ascend",
help="device target")
args = parser.parse_args()
from model_utils.config import config
context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target)
if args.device_target == "Ascend":
context.set_context(device_id=args.device_id)
context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target)
if config.device_target == "Ascend":
context.set_context(device_id=config.device_id)
if __name__ == "__main__":
ts_shape = args.testing_shape
ts_shape = config.testing_shape
network = YOLOV4CspDarkNet53()
network.set_train(False)
param_dict = load_checkpoint(args.ckpt_file)
param_dict = load_checkpoint(config.ckpt_file)
load_param_into_net(network, param_dict)
input_data = Tensor(np.zeros([args.batch_size, 3, ts_shape, ts_shape]), mindspore.float32)
input_data = Tensor(np.zeros([config.batch_size, 3, ts_shape, ts_shape]), mindspore.float32)
export(network, input_data, file_name=args.file_name, file_format=args.file_format)
export(network, input_data, file_name=config.file_name, file_format=config.file_format)

0
model_zoo/official/cv/yolov4/model_utils/__init__.py Normal file
View File

126
model_zoo/official/cv/yolov4/model_utils/config.py Normal file
View File

@ -0,0 +1,126 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Parse arguments"""
import os
import ast
import argparse
from pprint import pformat
import yaml
class Config:
"""
Configuration namespace. Convert dictionary to members.
"""
def __init__(self, cfg_dict):
for k, v in cfg_dict.items():
if isinstance(v, (list, tuple)):
setattr(self, k, [Config(x) if isinstance(x, dict) else x for x in v])
else:
setattr(self, k, Config(v) if isinstance(v, dict) else v)
def __str__(self):
return pformat(self.__dict__)
def __repr__(self):
return self.__str__()
def parse_cli_to_yaml(parser, cfg, helper=None, choices=None, cfg_path="default_config.yaml"):
"""
Parse command line arguments to the configuration according to the default yaml.
Args:
parser: Parent parser.
cfg: Base configuration.
helper: Helper description.
cfg_path: Path to the default yaml config.
"""
parser = argparse.ArgumentParser(description="[REPLACE THIS at config.py]",
parents=[parser])
helper = {} if helper is None else helper
choices = {} if choices is None else choices
for item in cfg:
if not isinstance(cfg[item], list) and not isinstance(cfg[item], dict):
help_description = helper[item] if item in helper else "Please reference to {}".format(cfg_path)
choice = choices[item] if item in choices else None
if isinstance(cfg[item], bool):
parser.add_argument("--" + item, type=ast.literal_eval, default=cfg[item], choices=choice,
help=help_description)
else:
parser.add_argument("--" + item, type=type(cfg[item]), default=cfg[item], choices=choice,
help=help_description)
args = parser.parse_args()
return args
def parse_yaml(yaml_path):
"""
Parse the yaml config file.
Args:
yaml_path: Path to the yaml config.
"""
with open(yaml_path, 'r') as fin:
try:
cfgs = yaml.load_all(fin.read(), Loader=yaml.FullLoader)
cfgs = [x for x in cfgs]
if len(cfgs) == 1:
cfg_helper = {}
cfg = cfgs[0]
cfg_choices = {}
elif len(cfgs) == 2:
cfg, cfg_helper = cfgs
cfg_choices = {}
elif len(cfgs) == 3:
cfg, cfg_helper, cfg_choices = cfgs
else:
raise ValueError("At most 3 docs (config, description for help, choices) are supported in config yaml")
print(cfg_helper)
except:
raise ValueError("Failed to parse yaml")
return cfg, cfg_helper, cfg_choices
def merge(args, cfg):
"""
Merge the base config from yaml file and command line arguments.
Args:
args: Command line arguments.
cfg: Base configuration.
"""
args_var = vars(args)
for item in args_var:
cfg[item] = args_var[item]
return cfg
def get_config():
"""
Get Config according to the yaml file and cli arguments.
"""
parser = argparse.ArgumentParser(description="default name", add_help=False)
current_dir = os.path.dirname(os.path.abspath(__file__))
parser.add_argument("--config_path", type=str, default=os.path.join(current_dir, "../default_config.yaml"),
help="Config file path")
path_args, _ = parser.parse_known_args()
default, helper, choices = parse_yaml(path_args.config_path)
args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path)
final_config = merge(args, default)
return Config(final_config)
config = get_config()

27
model_zoo/official/cv/yolov4/model_utils/device_adapter.py Normal file
View File

@ -0,0 +1,27 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Device adapter for ModelArts"""
from .config import config
if config.enable_modelarts:
from .moxing_adapter import get_device_id, get_device_num, get_rank_id, get_job_id
else:
from .local_adapter import get_device_id, get_device_num, get_rank_id, get_job_id
__all__ = [
"get_device_id", "get_device_num", "get_rank_id", "get_job_id"
]

36
model_zoo/official/cv/yolov4/model_utils/local_adapter.py Normal file
View File

@ -0,0 +1,36 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Local adapter"""
import os
def get_device_id():
device_id = os.getenv('DEVICE_ID', '0')
return int(device_id)
def get_device_num():
device_num = os.getenv('RANK_SIZE', '1')
return int(device_num)
def get_rank_id():
global_rank_id = os.getenv('RANK_ID', '0')
return int(global_rank_id)
def get_job_id():
return "Local Job"

116
model_zoo/official/cv/yolov4/model_utils/moxing_adapter.py Normal file
View File

@ -0,0 +1,116 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Moxing adapter for ModelArts"""
import os
import functools
from mindspore import context
from .config import config
_global_sync_count = 0
def get_device_id():
device_id = os.getenv('DEVICE_ID', '0')
return int(device_id)
def get_device_num():
device_num = os.getenv('RANK_SIZE', '1')
return int(device_num)
def get_rank_id():
global_rank_id = os.getenv('RANK_ID', '0')
return int(global_rank_id)
def get_job_id():
job_id = os.getenv('JOB_ID')
job_id = job_id if job_id != "" else "default"
return job_id
def sync_data(from_path, to_path):
"""
Download data from remote obs to local directory if the first url is remote url and the second one is local path
Upload data from local directory to remote obs in contrast.
"""
import moxing as mox
import time
global _global_sync_count
sync_lock = "/tmp/copy_sync.lock" + str(_global_sync_count)
_global_sync_count += 1
# Each server contains 8 devices as most.
if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock):
print("from path: ", from_path)
print("to path: ", to_path)
mox.file.copy_parallel(from_path, to_path)
print("===finish data synchronization===")
try:
os.mknod(sync_lock)
except IOError:
pass
print("===save flag===")
while True:
if os.path.exists(sync_lock):
break
time.sleep(1)
print("Finish sync data from {} to {}.".format(from_path, to_path))
def moxing_wrapper(pre_process=None, post_process=None):
"""
Moxing wrapper to download dataset and upload outputs.
"""
def wrapper(run_func):
@functools.wraps(run_func)
def wrapped_func(*args, **kwargs):
# Download data from data_url
if config.enable_modelarts:
if config.data_url:
sync_data(config.data_url, config.data_path)
print("Dataset downloaded: ", os.listdir(config.data_path))
if config.checkpoint_url:
sync_data(config.checkpoint_url, config.load_path)
print("Preload downloaded: ", os.listdir(config.load_path))
if config.train_url:
sync_data(config.train_url, config.output_path)
print("Workspace downloaded: ", os.listdir(config.output_path))
context.set_context(save_graphs_path=os.path.join(config.output_path, str(get_rank_id())))
config.device_num = get_device_num()
config.device_id = get_device_id()
if not os.path.exists(config.output_path):
os.makedirs(config.output_path)
if pre_process:
pre_process()
# Run the main function
run_func(*args, **kwargs)
# Upload data to train_url
if config.enable_modelarts:
if post_process:
post_process()
if config.train_url:
print("Start to copy output directory")
sync_data(config.output_path, config.train_url)
return wrapped_func
return wrapper

2
model_zoo/official/cv/yolov4/postprocess.py
View File

@ -35,7 +35,7 @@ parser.add_argument('--log_path', type=str, default='outputs/', help='checkpoint
# detect_related
parser.add_argument('--nms_thresh', type=float, default=0.5, help='threshold for NMS')
parser.add_argument('--ann_file', type=str, default='', help='path to annotation')
parser.add_argument('--ignore_threshold', type=float, default=0.001, help='threshold to throw low quality boxes')
parser.add_argument('--eval_ignore_threshold', type=float, default=0.001, help='threshold to throw low quality boxes')
parser.add_argument('--img_id_file_path', type=str, default='', help='path of image dataset')
parser.add_argument('--result_files', type=str, default='./result_Files', help='path to 310 infer result floder')

2
model_zoo/official/cv/yolov4/scripts/run_distribute_train.sh
View File

@ -65,7 +65,9 @@ do
rm -rf ./train_parallel$i
mkdir ./train_parallel$i
cp ../*.py ./train_parallel$i
cp ../*.yaml ./train_parallel$i
cp -r ../src ./train_parallel$i
cp -r ../model_utils ./train_parallel$i
cd ./train_parallel$i || exit
echo "start training for rank $RANK_ID, device $DEVICE_ID"
env > env.log

7
model_zoo/official/cv/yolov4/scripts/run_eval.sh
View File

@ -55,12 +55,15 @@ then
fi
mkdir ./eval
cp ../*.py ./eval
cp ../*.yaml ./eval
cp -r ../src ./eval
cp -r ../model_utils ./eval
cd ./eval || exit
env > env.log
echo "start inferring for device $DEVICE_ID"
python eval.py \
--data_dir=$DATASET_PATH \
--pretrained=$CHECKPOINT_PATH \
--testing_shape=608 > log.txt 2>&1 &
--is_distributed=0 \
--per_batch_size=1 \
--pretrained=$CHECKPOINT_PATH > log.txt 2>&1 &
cd ..

2
model_zoo/official/cv/yolov4/scripts/run_standalone_train.sh
View File

@ -56,7 +56,9 @@ then
fi
mkdir ./train
cp ../*.py ./train
cp ../*.yaml ./train
cp -r ../src ./train
cp -r ../model_utils ./train
cd ./train || exit
echo "start training for device $DEVICE_ID"
env > env.log

9
model_zoo/official/cv/yolov4/scripts/run_test.sh
View File

@ -55,12 +55,17 @@ then
fi
mkdir ./test
cp ../*.py ./test
cp ../*.yaml ./test
cp -r ../src ./test
cp -r ../model_utils ./test
cd ./test || exit
env > env.log
echo "start inferring for device $DEVICE_ID"
python test.py \
--data_dir=$DATASET_PATH \
--pretrained=$CHECKPOINT_PATH \
--testing_shape=608 > log.txt 2>&1 &
--is_distributed=0 \
--per_batch_size=1 \
--test_nms_thresh=0.45 \
--test_ignore_threshold=0.001 \
--pretrained=$CHECKPOINT_PATH > log.txt 2>&1 &
cd ..

78
model_zoo/official/cv/yolov4/src/config.py
View File

@ -1,78 +0,0 @@
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Config parameters for Darknet based yolov4_cspdarknet53 models."""
class ConfigYOLOV4CspDarkNet53:
"""
Config parameters for the yolov4_cspdarknet53.
Examples:
ConfigYOLOV4CspDarkNet53()
"""
# train_param
# data augmentation related
hue = 0.1
saturation = 1.5
value = 1.5
jitter = 0.3
resize_rate = 10
multi_scale = [[416, 416],
[448, 448],
[480, 480],
[512, 512],
[544, 544],
[576, 576],
[608, 608],
[640, 640],
[672, 672],
[704, 704],
[736, 736]
]
num_classes = 80
max_box = 90
backbone_input_shape = [32, 64, 128, 256, 512]
backbone_shape = [64, 128, 256, 512, 1024]
backbone_layers = [1, 2, 8, 8, 4]
# confidence under ignore_threshold means no object when training
ignore_threshold = 0.7
# threshold to throw low quality boxes when eval
eval_ignore_threshold = 0.001
nms_thresh = 0.5
# h->w
anchor_scales = [(12, 16),
(19, 36),
(40, 28),
(36, 75),
(76, 55),
(72, 146),
(142, 110),
(192, 243),
(459, 401)]
out_channel = 3 * (num_classes + 5)
# test_param
test_img_shape = [608, 608]
# transfer training
checkpoint_filter_list = ['feature_map.backblock0.conv6.weight', 'feature_map.backblock0.conv6.bias',
'feature_map.backblock1.conv6.weight', 'feature_map.backblock1.conv6.bias',
'feature_map.backblock2.conv6.weight', 'feature_map.backblock2.conv6.bias',
'feature_map.backblock3.conv6.weight', 'feature_map.backblock3.conv6.bias']

12
model_zoo/official/cv/yolov4/src/eval_utils.py
View File

@ -23,7 +23,7 @@ class Redirct:
class DetectionEngine:
"""Detection engine."""
def __init__(self, args_detection):
self.ignore_threshold = args_detection.ignore_threshold
self.eval_ignore_threshold = args_detection.eval_ignore_threshold
self.labels = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat',
'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat',
'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack',
@ -147,10 +147,8 @@ class DetectionEngine:
def get_eval_result(self):
"""Get eval result."""
up_path = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
self.file_path = os.path.join(up_path, self.file_path)
if not self.results:
args.logger.info("[WARNING] result is {}")
logger.warning("[WARNING] result is None.")
return 0.0, 0.0
coco_gt = COCO(self.ann_file)
coco_dt = coco_gt.loadRes(self.file_path)
@ -208,7 +206,7 @@ class DetectionEngine:
flag[i, c] = True
confidence = cls_emb[flag] * conf
for x_lefti, y_lefti, wi, hi, confi, clsi in zip(x_top_left, y_top_left, w, h, confidence, cls_argmax):
if confi < self.ignore_threshold:
if confi < self.eval_ignore_threshold:
continue
if img_id not in self.results:
self.results[img_id] = defaultdict(list)
@ -291,20 +289,18 @@ class EvalCallBack(Callback):
self.args.logger.info("End training, the best {0} is: {1}, "
"the best {0} epoch is {2}".format(self.metrics_name, self.best_res, self.best_epoch))
def apply_eval(eval_param_dict):
network = eval_param_dict["net"]
network.set_train(False)
ds = eval_param_dict["dataset"]
data_size = eval_param_dict["data_size"]
input_shape = eval_param_dict["input_shape"]
args = eval_param_dict["args"]
detection = DetectionEngine(args)
for index, data in enumerate(ds.create_dict_iterator(num_epochs=1)):
image = data["image"]
image_shape_ = data["image_shape"]
image_id_ = data["img_id"]
prediction = network(image, input_shape)
prediction = network(image)
output_big, output_me, output_small = prediction
output_big = output_big.asnumpy()
output_me = output_me.asnumpy()

13
model_zoo/official/cv/yolov4/src/logger.py
View File

@ -30,11 +30,12 @@ class LOGGER(logging.Logger):
def __init__(self, logger_name, rank=0):
super(LOGGER, self).__init__(logger_name)
self.rank = rank
console = logging.StreamHandler(sys.stdout)
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s')
console.setFormatter(formatter)
self.addHandler(console)
if rank % 8 == 0:
console = logging.StreamHandler(sys.stdout)
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s')
console.setFormatter(formatter)
self.addHandler(console)
def setup_logging_file(self, log_dir, rank=0):
"""Setup logging file."""
@ -61,7 +62,7 @@ class LOGGER(logging.Logger):
self.info('')
def important_info(self, msg, *args, **kwargs):
if self.isEnabledFor(logging.INFO):
if self.isEnabledFor(logging.INFO) and self.rank == 0:
line_width = 2
important_msg = '\n'
important_msg += ('*'*70 + '\n')*line_width

17
model_zoo/official/cv/yolov4/src/yolo.py
View File

@ -25,9 +25,9 @@ from mindspore.ops import functional as F
from mindspore.ops import composite as C
from src.cspdarknet53 import CspDarkNet53, ResidualBlock
from src.config import ConfigYOLOV4CspDarkNet53
from src.loss import XYLoss, WHLoss, ConfidenceLoss, ClassLoss
from model_utils.config import config as default_config
def _conv_bn_leakyrelu(in_channel,
out_channel,
@ -210,7 +210,7 @@ class DetectionBlock(nn.Cell):
Args:
scale: Character.
config: ConfigYOLOV4CspDarkNet53, Configuration instance.
config: Configuration.
is_training: Bool, Whether train or not, default True.
Returns:
@ -220,7 +220,7 @@ class DetectionBlock(nn.Cell):
DetectionBlock(scale='l',stride=32)
"""
def __init__(self, scale, config=ConfigYOLOV4CspDarkNet53()):
def __init__(self, scale, config=default_config):
super(DetectionBlock, self).__init__()
self.config = config
if scale == 's':
@ -330,7 +330,7 @@ class YoloLossBlock(nn.Cell):
"""
Loss block cell of YOLOV4 network.
"""
def __init__(self, scale, config=ConfigYOLOV4CspDarkNet53()):
def __init__(self, scale, config=default_config):
super(YoloLossBlock, self).__init__()
self.config = config
if scale == 's':
@ -431,7 +431,8 @@ class YOLOV4CspDarkNet53(nn.Cell):
def __init__(self):
super(YOLOV4CspDarkNet53, self).__init__()
self.config = ConfigYOLOV4CspDarkNet53()
self.config = default_config
self.test_img_shape = Tensor(tuple(self.config.test_img_shape), ms.float32)
# YOLOv4 network
self.feature_map = YOLOv4(backbone=CspDarkNet53(ResidualBlock, detect=True),
@ -443,7 +444,9 @@ class YOLOV4CspDarkNet53(nn.Cell):
self.detect_2 = DetectionBlock('m')
self.detect_3 = DetectionBlock('s')
def construct(self, x, input_shape):
def construct(self, x, input_shape=None):
if input_shape is None:
input_shape = self.test_img_shape
big_object_output, medium_object_output, small_object_output = self.feature_map(x)
output_big = self.detect_1(big_object_output, input_shape)
output_me = self.detect_2(medium_object_output, input_shape)
@ -457,7 +460,7 @@ class YoloWithLossCell(nn.Cell):
def __init__(self, network):
super(YoloWithLossCell, self).__init__()
self.yolo_network = network
self.config = ConfigYOLOV4CspDarkNet53()
self.config = default_config
self.loss_big = YoloLossBlock('l', self.config)
self.loss_me = YoloLossBlock('m', self.config)
self.loss_small = YoloLossBlock('s', self.config)

158
model_zoo/official/cv/yolov4/test.py
View File

@ -16,7 +16,7 @@
import os
import sys
import argparse
import time
import datetime
from collections import defaultdict
import json
@ -27,48 +27,25 @@ from mindspore import Tensor
from mindspore.context import ParallelMode
from mindspore.communication.management import init, get_rank, get_group_size
from mindspore.train.serialization import load_checkpoint, load_param_into_net
import mindspore as ms
from src.yolo import YOLOV4CspDarkNet53
from src.logger import get_logger
from src.yolo_dataset import create_yolo_datasetv2
from src.config import ConfigYOLOV4CspDarkNet53
from model_utils.config import config
from model_utils.moxing_adapter import moxing_wrapper
from model_utils.device_adapter import get_device_id, get_device_num
devid = int(os.getenv('DEVICE_ID'))
context.set_context(mode=context.GRAPH_MODE, device_target="Davinci", save_graphs=False, device_id=devid)
parser = argparse.ArgumentParser('mindspore coco testing')
# dataset related
parser.add_argument('--data_dir', type=str, default='', help='train data dir')
parser.add_argument('--per_batch_size', default=1, type=int, help='batch size for per gpu')
# network related
parser.add_argument('--pretrained', default='', type=str, help='model_path, local pretrained model to load')
# logging related
parser.add_argument('--log_path', type=str, default='outputs/', help='checkpoint save location')
# distributed related
parser.add_argument('--is_distributed', type=int, default=0, help='if multi device')
parser.add_argument('--rank', type=int, default=0, help='local rank of distributed')
parser.add_argument('--group_size', type=int, default=1, help='world size of distributed')
# detect_related
parser.add_argument('--nms_thresh', type=float, default=0.45, help='threshold for NMS')
parser.add_argument('--annFile', type=str, default='', help='path to annotation')
parser.add_argument('--testing_shape', type=str, default='', help='shape for test ')
parser.add_argument('--ignore_threshold', type=float, default=0.001, help='threshold to throw low quality boxes')
args, _ = parser.parse_known_args()
args.data_root = os.path.join(args.data_dir, 'test2017')
config.data_root = os.path.join(config.data_dir, 'test2017')
config.nms_thresh = config.test_nms_thresh
class DetectionEngine():
"""Detection engine"""
def __init__(self, args_engine):
self.ignore_threshold = args_engine.ignore_threshold
self.test_ignore_threshold = args_engine.test_ignore_threshold
self.labels = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat',
'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat',
'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack',
@ -230,7 +207,7 @@ class DetectionEngine():
flag[i, c] = True
confidence = cls_emb[flag] * conf
for x_lefti, y_lefti, wi, hi, confi, clsi in zip(x_top_left, y_top_left, w, h, confidence, cls_argmax):
if confi < self.ignore_threshold:
if confi < self.test_ignore_threshold:
continue
if img_id not in self.results:
self.results[img_id] = defaultdict(list)
@ -243,39 +220,87 @@ class DetectionEngine():
self.results[img_id][coco_clsi].append([x_lefti, y_lefti, wi, hi, confi])
def convert_testing_shape(args_test):
testing_shape = [int(args_test.testing_shape), int(args_test.testing_shape)]
return testing_shape
def modelarts_pre_process():
'''modelarts pre process function.'''
def unzip(zip_file, save_dir):
import zipfile
s_time = time.time()
if not os.path.exists(os.path.join(save_dir, config.modelarts_dataset_unzip_name)):
zip_isexist = zipfile.is_zipfile(zip_file)
if zip_isexist:
fz = zipfile.ZipFile(zip_file, 'r')
data_num = len(fz.namelist())
print("Extract Start...")
print("unzip file num: {}".format(data_num))
data_print = int(data_num / 100) if data_num > 100 else 1
i = 0
for file in fz.namelist():
if i % data_print == 0:
print("unzip percent: {}%".format(int(i * 100 / data_num)), flush=True)
i += 1
fz.extract(file, save_dir)
print("cost time: {}min:{}s.".format(int((time.time() - s_time) / 60),
int(int(time.time() - s_time) % 60)))
print("Extract Done.")
else:
print("This is not zip.")
else:
print("Zip has been extracted.")
if config.need_modelarts_dataset_unzip:
zip_file_1 = os.path.join(config.data_path, config.modelarts_dataset_unzip_name + ".zip")
save_dir_1 = os.path.join(config.data_path)
sync_lock = "/tmp/unzip_sync.lock"
# Each server contains 8 devices as most.
if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock):
print("Zip file path: ", zip_file_1)
print("Unzip file save dir: ", save_dir_1)
unzip(zip_file_1, save_dir_1)
print("===Finish extract data synchronization===")
try:
os.mknod(sync_lock)
except IOError:
pass
while True:
if os.path.exists(sync_lock):
break
time.sleep(1)
print("Device: {}, Finish sync unzip data from {} to {}.".format(get_device_id(), zip_file_1, save_dir_1))
def test():
@moxing_wrapper(pre_process=modelarts_pre_process)
def run_test():
"""test method"""
# init distributed
if args.is_distributed:
if config.is_distributed:
init()
args.rank = get_rank()
args.group_size = get_group_size()
config.rank = get_rank()
config.group_size = get_group_size()
# logger
args.outputs_dir = os.path.join(args.log_path,
datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S'))
config.outputs_dir = os.path.join(config.log_path,
datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S'))
args.logger = get_logger(args.outputs_dir, args.rank)
config.logger = get_logger(config.outputs_dir, config.rank)
context.reset_auto_parallel_context()
if args.is_distributed:
if config.is_distributed:
parallel_mode = ParallelMode.DATA_PARALLEL
else:
parallel_mode = ParallelMode.STAND_ALONE
context.set_auto_parallel_context(parallel_mode=parallel_mode, gradients_mean=True, device_num=1)
args.logger.info('Creating Network....')
config.logger.info('Creating Network....')
network = YOLOV4CspDarkNet53()
args.logger.info(args.pretrained)
if os.path.isfile(args.pretrained):
param_dict = load_checkpoint(args.pretrained)
config.logger.info(config.pretrained)
if os.path.isfile(config.pretrained):
param_dict = load_checkpoint(config.pretrained)
param_dict_new = {}
for key, values in param_dict.items():
if key.startswith('moments.'):
@ -285,40 +310,35 @@ def test():
else:
param_dict_new[key] = values
load_param_into_net(network, param_dict_new)
args.logger.info('load_model {} success'.format(args.pretrained))
config.logger.info('load_model %s success', config.pretrained)
else:
args.logger.info('{} not exists or not a pre-trained file'.format(args.pretrained))
assert FileNotFoundError('{} not exists or not a pre-trained file'.format(args.pretrained))
config.logger.info('%s not exists or not a pre-trained file', config.pretrained)
assert FileNotFoundError('{} not exists or not a pre-trained file'.format(config.pretrained))
exit(1)
data_root = args.data_root
data_root = config.data_root
config = ConfigYOLOV4CspDarkNet53()
if args.testing_shape:
config.test_img_shape = convert_testing_shape(args)
data_txt = os.path.join(args.data_dir, 'testdev2017.txt')
ds, data_size = create_yolo_datasetv2(data_root, data_txt=data_txt, batch_size=args.per_batch_size,
max_epoch=1, device_num=args.group_size, rank=args.rank, shuffle=False,
data_txt = os.path.join(config.data_dir, 'testdev2017.txt')
ds, data_size = create_yolo_datasetv2(data_root, data_txt=data_txt, batch_size=config.per_batch_size,
max_epoch=1, device_num=config.group_size, rank=config.rank, shuffle=False,
config=config)
args.logger.info('testing shape : {}'.format(config.test_img_shape))
args.logger.info('totol {} images to eval'.format(data_size))
config.logger.info('testing shape : %s', config.test_img_shape)
config.logger.info('totol %d images to eval', data_size)
network.set_train(False)
# init detection engine
detection = DetectionEngine(args)
detection = DetectionEngine(config)
input_shape = Tensor(tuple(config.test_img_shape), ms.float32)
args.logger.info('Start inference....')
config.logger.info('Start inference....')
for i, data in enumerate(ds.create_dict_iterator()):
image = Tensor(data["image"])
image_shape = Tensor(data["image_shape"])
image_id = Tensor(data["img_id"])
prediction = network(image, input_shape)
prediction = network(image)
output_big, output_me, output_small = prediction
output_big = output_big.asnumpy()
output_me = output_me.asnumpy()
@ -326,14 +346,14 @@ def test():
image_id = image_id.asnumpy()
image_shape = image_shape.asnumpy()
detection.detect([output_small, output_me, output_big], args.per_batch_size, image_shape, image_id)
detection.detect([output_small, output_me, output_big], config.per_batch_size, image_shape, image_id)
if i % 1000 == 0:
args.logger.info('Processing... {:.2f}% '.format(i * args.per_batch_size / data_size * 100))
config.logger.info('Processing... {:.2f}% '.format(i * config.per_batch_size / data_size * 100))
args.logger.info('Calculating mAP...')
config.logger.info('Calculating mAP...')
detection.do_nms_for_results()
result_file_path = detection.write_result()
args.logger.info('result file path: {}'.format(result_file_path))
config.logger.info('result file path: %s', result_file_path)
if __name__ == "__main__":
test()
run_test()

367
model_zoo/official/cv/yolov4/train.py
View File

@ -15,9 +15,7 @@
"""YoloV4 train."""
import os
import time
import argparse
import datetime
import ast
from mindspore.context import ParallelMode
from mindspore.nn.optim.momentum import Momentum
@ -39,131 +37,59 @@ from src.util import AverageMeter, get_param_groups
from src.lr_scheduler import get_lr
from src.yolo_dataset import create_yolo_dataset
from src.initializer import default_recurisive_init, load_yolov4_params
from src.config import ConfigYOLOV4CspDarkNet53
from src.util import keep_loss_fp32
from src.eval_utils import apply_eval, EvalCallBack
from model_utils.config import config
from model_utils.moxing_adapter import moxing_wrapper
from model_utils.device_adapter import get_device_id, get_device_num
set_seed(1)
parser = argparse.ArgumentParser('mindspore coco training')
def set_default():
if config.lr_scheduler == 'cosine_annealing' and config.max_epoch > config.t_max:
config.t_max = config.max_epoch
# device related
parser.add_argument('--device_target', type=str, default='Ascend',
help='device where the code will be implemented. (Default: Ascend)')
config.lr_epochs = list(map(int, config.lr_epochs.split(',')))
config.data_root = os.path.join(config.data_dir, 'train2017')
config.annFile = os.path.join(config.data_dir, 'annotations/instances_train2017.json')
# dataset related
parser.add_argument('--data_dir', type=str, help='Train dataset directory.')
parser.add_argument('--per_batch_size', default=8, type=int, help='Batch size for Training. Default: 8.')
config.data_val_root = os.path.join(config.data_dir, 'val2017')
config.ann_val_file = os.path.join(config.data_dir, 'annotations/instances_val2017.json')
# network related
parser.add_argument('--pretrained_backbone', default='', type=str,
help='The ckpt file of CspDarkNet53. Default: "".')
parser.add_argument('--resume_yolov4', default='', type=str,
help='The ckpt file of YOLOv4, which used to fine tune. Default: ""')
parser.add_argument('--pretrained_checkpoint', default='', type=str,
help='The ckpt file of YoloV4CspDarkNet53. Default: "".')
parser.add_argument("--filter_weight", type=ast.literal_eval, default=False,
help="Filter the last weight parameters, default is False.")
device_id = int(os.getenv('DEVICE_ID', '0'))
context.set_context(mode=context.GRAPH_MODE, enable_auto_mixed_precision=True,
device_target=config.device_target, save_graphs=False, device_id=device_id)
# optimizer and lr related
parser.add_argument('--lr_scheduler', default='cosine_annealing', type=str,
help='Learning rate scheduler, options: exponential, cosine_annealing. Default: exponential')
parser.add_argument('--lr', default=0.012, type=float, help='Learning rate. Default: 0.001')
parser.add_argument('--lr_epochs', type=str, default='220,250',
help='Epoch of changing of lr changing, split with ",". Default: 220,250')
parser.add_argument('--lr_gamma', type=float, default=0.1,
help='Decrease lr by a factor of exponential lr_scheduler. Default: 0.1')
parser.add_argument('--eta_min', type=float, default=0., help='Eta_min in cosine_annealing scheduler. Default: 0')
parser.add_argument('--t_max', type=int, default=320, help='T-max in cosine_annealing scheduler. Default: 320')
parser.add_argument('--max_epoch', type=int, default=320, help='Max epoch num to train the model. Default: 320')
parser.add_argument('--warmup_epochs', default=20, type=float, help='Warmup epochs. Default: 0')
parser.add_argument('--weight_decay', type=float, default=0.0005, help='Weight decay factor. Default: 0.0005')
parser.add_argument('--momentum', type=float, default=0.9, help='Momentum. Default: 0.9')
# loss related
parser.add_argument('--loss_scale', type=int, default=64, help='Static loss scale. Default: 1024')
parser.add_argument('--label_smooth', type=int, default=0, help='Whether to use label smooth in CE. Default:0')
parser.add_argument('--label_smooth_factor', type=float, default=0.1,
help='Smooth strength of original one-hot. Default: 0.1')
# logging related
parser.add_argument('--log_interval', type=int, default=100, help='Logging interval steps. Default: 100')
parser.add_argument('--ckpt_path', type=str, default='outputs/', help='Checkpoint save location. Default: outputs/')
parser.add_argument('--ckpt_interval', type=int, default=None, help='Save checkpoint interval. Default: None')
parser.add_argument('--is_save_on_master', type=int, default=1,
help='Save ckpt on master or all rank, 1 for master, 0 for all ranks. Default: 1')
# distributed related
parser.add_argument('--is_distributed', type=int, default=1,
help='Distribute train or not, 1 for yes, 0 for no. Default: 1')
parser.add_argument('--rank', type=int, default=0, help='Local rank of distributed. Default: 0')
parser.add_argument('--group_size', type=int, default=1, help='World size of device. Default: 1')
# profiler init
parser.add_argument('--need_profiler', type=int, default=0,
help='Whether use profiler. 0 for no, 1 for yes. Default: 0')
# reset default config
parser.add_argument('--training_shape', type=str, default="", help='Fix training shape. Default: ""')
parser.add_argument('--resize_rate', type=int, default=10,
help='Resize rate for multi-scale training. Default: None')
parser.add_argument("--run_eval", type=ast.literal_eval, default=False,
help="Run evaluation when training, default is False.")
parser.add_argument("--save_best_ckpt", type=ast.literal_eval, default=True,
help="Save best checkpoint when run_eval is True, default is True.")
parser.add_argument("--eval_start_epoch", type=int, default=200,
help="Evaluation start epoch when run_eval is True, default is 200.")
parser.add_argument("--eval_interval", type=int, default=1,
help="Evaluation interval when run_eval is True, default is 1.")
parser.add_argument('--ann_file', type=str, default='', help='path to annotation')
args, _ = parser.parse_known_args()
if args.lr_scheduler == 'cosine_annealing' and args.max_epoch > args.t_max:
args.t_max = args.max_epoch
args.lr_epochs = list(map(int, args.lr_epochs.split(',')))
args.data_root = os.path.join(args.data_dir, 'train2017')
args.annFile = os.path.join(args.data_dir, 'annotations/instances_train2017.json')
args.data_val_root = os.path.join(args.data_dir, 'val2017')
args.ann_val_file = os.path.join(args.data_dir, 'annotations/instances_val2017.json')
config = ConfigYOLOV4CspDarkNet53()
args.nms_thresh = config.nms_thresh
args.ignore_threshold = config.eval_ignore_threshold
device_id = int(os.getenv('DEVICE_ID', '0'))
context.set_context(mode=context.GRAPH_MODE, enable_auto_mixed_precision=True,
device_target=args.device_target, save_graphs=False, device_id=device_id)
if args.need_profiler:
profiler = Profiler(output_path=args.outputs_dir, is_detail=True, is_show_op_path=True)
# init distributed
if args.is_distributed:
if args.device_target == "Ascend":
init()
if config.need_profiler:
profiler = Profiler(output_path=config.outputs_dir, is_detail=True, is_show_op_path=True)
else:
init("nccl")
args.rank = get_rank()
args.group_size = get_group_size()
profiler = None
# select for master rank save ckpt or all rank save, compatible for model parallel
args.rank_save_ckpt_flag = 0
if args.is_save_on_master:
if args.rank == 0:
args.rank_save_ckpt_flag = 1
else:
args.rank_save_ckpt_flag = 1
# init distributed
if config.is_distributed:
if config.device_target == "Ascend":
init()
else:
init("nccl")
config.rank = get_rank()
config.group_size = get_group_size()
# logger
args.outputs_dir = os.path.join(args.ckpt_path,
datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S'))
args.logger = get_logger(args.outputs_dir, args.rank)
args.logger.save_args(args)
# select for master rank save ckpt or all rank save, compatible for model parallel
config.rank_save_ckpt_flag = 0
if config.is_save_on_master:
if config.rank == 0:
config.rank_save_ckpt_flag = 1
else:
config.rank_save_ckpt_flag = 1
# logger
config.outputs_dir = os.path.join(config.ckpt_path,
datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S'))
config.logger = get_logger(config.outputs_dir, config.rank)
config.logger.save_args(config)
return profiler
def convert_training_shape(args_training_shape):
@ -183,92 +109,151 @@ class BuildTrainNetwork(nn.Cell):
return loss_
if __name__ == "__main__":
loss_meter = AverageMeter('loss')
def modelarts_pre_process():
'''modelarts pre process function.'''
def unzip(zip_file, save_dir):
import zipfile
s_time = time.time()
if not os.path.exists(os.path.join(save_dir, config.modelarts_dataset_unzip_name)):
zip_isexist = zipfile.is_zipfile(zip_file)
if zip_isexist:
fz = zipfile.ZipFile(zip_file, 'r')
data_num = len(fz.namelist())
print("Extract Start...")
print("unzip file num: {}".format(data_num))
data_print = int(data_num / 100) if data_num > 100 else 1
i = 0
for file in fz.namelist():
if i % data_print == 0:
print("unzip percent: {}%".format(int(i * 100 / data_num)), flush=True)
i += 1
fz.extract(file, save_dir)
print("cost time: {}min:{}s.".format(int((time.time() - s_time) / 60),
int(int(time.time() - s_time) % 60)))
print("Extract Done.")
else:
print("This is not zip.")
else:
print("Zip has been extracted.")
if config.need_modelarts_dataset_unzip:
zip_file_1 = os.path.join(config.data_path, config.modelarts_dataset_unzip_name + ".zip")
save_dir_1 = os.path.join(config.data_path)
sync_lock = "/tmp/unzip_sync.lock"
# Each server contains 8 devices as most.
if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock):
print("Zip file path: ", zip_file_1)
print("Unzip file save dir: ", save_dir_1)
unzip(zip_file_1, save_dir_1)
print("===Finish extract data synchronization===")
try:
os.mknod(sync_lock)
except IOError:
pass
while True:
if os.path.exists(sync_lock):
break
time.sleep(1)
print("Device: {}, Finish sync unzip data from {} to {}.".format(get_device_id(), zip_file_1, save_dir_1))
config.ckpt_path = os.path.join(config.output_path, config.ckpt_path)
def get_network(net, cfg, learning_rate):
opt = Momentum(params=get_param_groups(net),
learning_rate=Tensor(learning_rate),
momentum=cfg.momentum,
weight_decay=cfg.weight_decay,
loss_scale=cfg.loss_scale)
is_gpu = context.get_context("device_target") == "GPU"
if is_gpu:
loss_scale_value = 1.0
loss_scale = FixedLossScaleManager(loss_scale_value, drop_overflow_update=False)
net = amp.build_train_network(net, optimizer=opt, loss_scale_manager=loss_scale,
level="O2", keep_batchnorm_fp32=False)
keep_loss_fp32(net)
else:
net = TrainingWrapper(net, opt)
net.set_train()
return net
@moxing_wrapper(pre_process=modelarts_pre_process)
def run_train():
profiler = set_default()
loss_meter = AverageMeter('loss')
context.reset_auto_parallel_context()
parallel_mode = ParallelMode.STAND_ALONE
degree = 1
if args.is_distributed:
if config.is_distributed:
parallel_mode = ParallelMode.DATA_PARALLEL
degree = get_group_size()
context.set_auto_parallel_context(parallel_mode=parallel_mode, gradients_mean=True, device_num=degree)
network = YOLOV4CspDarkNet53()
network_eval = network
if config.run_eval:
network_eval = network
# default is kaiming-normal
args.checkpoint_filter_list = config.checkpoint_filter_list
default_recurisive_init(network)
load_yolov4_params(args, network)
load_yolov4_params(config, network)
network = YoloWithLossCell(network)
args.logger.info('finish get network')
config.logger.info('finish get network')
config.label_smooth = args.label_smooth
config.label_smooth_factor = args.label_smooth_factor
if config.training_shape:
config.multi_scale = [convert_training_shape(config.training_shape)]
if args.training_shape:
config.multi_scale = [convert_training_shape(args.training_shape)]
if args.resize_rate:
config.resize_rate = args.resize_rate
ds, data_size = create_yolo_dataset(image_dir=config.data_root, anno_path=config.annFile, is_training=True,
batch_size=config.per_batch_size, max_epoch=config.max_epoch,
device_num=config.group_size, rank=config.rank, config=config)
config.logger.info('Finish loading dataset')
ds, data_size = create_yolo_dataset(image_dir=args.data_root, anno_path=args.annFile, is_training=True,
batch_size=args.per_batch_size, max_epoch=args.max_epoch,
device_num=args.group_size, rank=args.rank, config=config)
args.logger.info('Finish loading dataset')
config.steps_per_epoch = int(data_size / config.per_batch_size / config.group_size)
args.steps_per_epoch = int(data_size / args.per_batch_size / args.group_size)
if config.ckpt_interval <= 0:
config.ckpt_interval = config.steps_per_epoch
if not args.ckpt_interval:
args.ckpt_interval = args.steps_per_epoch
lr = get_lr(config)
network = get_network(network, config, lr)
network.set_train(True)
lr = get_lr(args)
if config.rank_save_ckpt_flag or config.run_eval:
cb_params = _InternalCallbackParam()
cb_params.train_network = network
cb_params.epoch_num = config.max_epoch * config.steps_per_epoch // config.ckpt_interval
cb_params.cur_epoch_num = 1
run_context = RunContext(cb_params)
opt = Momentum(params=get_param_groups(network),
learning_rate=Tensor(lr),
momentum=args.momentum,
weight_decay=args.weight_decay,
loss_scale=args.loss_scale)
is_gpu = context.get_context("device_target") == "GPU"
if is_gpu:
loss_scale_value = 1.0
loss_scale = FixedLossScaleManager(loss_scale_value, drop_overflow_update=False)
network = amp.build_train_network(network, optimizer=opt, loss_scale_manager=loss_scale,
level="O2", keep_batchnorm_fp32=False)
keep_loss_fp32(network)
else:
network = TrainingWrapper(network, opt)
network.set_train()
if config.rank_save_ckpt_flag:
# checkpoint save
ckpt_max_num = 10
ckpt_config = CheckpointConfig(save_checkpoint_steps=config.ckpt_interval,
keep_checkpoint_max=ckpt_max_num)
save_ckpt_path = os.path.join(config.outputs_dir, 'ckpt_' + str(config.rank) + '/')
ckpt_cb = ModelCheckpoint(config=ckpt_config,
directory=save_ckpt_path,
prefix='{}'.format(config.rank))
ckpt_cb.begin(run_context)
# checkpoint save
ckpt_max_num = 10
ckpt_config = CheckpointConfig(save_checkpoint_steps=args.ckpt_interval,
keep_checkpoint_max=ckpt_max_num)
save_ckpt_path = os.path.join(args.outputs_dir, 'ckpt_' + str(args.rank) + '/')
ckpt_cb = ModelCheckpoint(config=ckpt_config,
directory=save_ckpt_path,
prefix='{}'.format(args.rank))
cb_params = _InternalCallbackParam()
cb_params.train_network = network
cb_params.epoch_num = args.max_epoch * args.steps_per_epoch // args.ckpt_interval
cb_params.cur_epoch_num = 1
run_context = RunContext(cb_params)
ckpt_cb.begin(run_context)
if args.run_eval:
rank_id = int(os.environ.get('RANK_ID')) if os.environ.get('RANK_ID') else 0
data_val_root = args.data_val_root
ann_val_file = args.ann_val_file
save_ckpt_path = os.path.join(args.outputs_dir, 'ckpt_' + str(args.rank) + '/')
if config.run_eval:
data_val_root = config.data_val_root
ann_val_file = config.ann_val_file
save_ckpt_path = os.path.join(config.outputs_dir, 'ckpt_' + str(config.rank) + '/')
input_val_shape = Tensor(tuple(config.test_img_shape), ms.float32)
# init detection engine
eval_dataset, eval_data_size = create_yolo_dataset(data_val_root, ann_val_file, is_training=False,
batch_size=args.per_batch_size, max_epoch=1, device_num=1,
batch_size=config.per_batch_size, max_epoch=1, device_num=1,
rank=0, shuffle=False, config=config)
eval_param_dict = {"net": network_eval, "dataset": eval_dataset, "data_size": eval_data_size,
"anno_json": ann_val_file, "input_shape": input_val_shape, "args": args}
eval_cb = EvalCallBack(apply_eval, eval_param_dict, interval=args.eval_interval,
eval_start_epoch=args.eval_start_epoch, save_best_ckpt=True,
"anno_json": ann_val_file, "input_shape": input_val_shape, "args": config}
eval_cb = EvalCallBack(apply_eval, eval_param_dict, interval=config.eval_interval,
eval_start_epoch=config.eval_start_epoch, save_best_ckpt=True,
ckpt_directory=save_ckpt_path, besk_ckpt_name="best_map.ckpt",
metrics_name="mAP")
@ -277,13 +262,11 @@ if __name__ == "__main__":
data_loader = ds.create_dict_iterator(output_numpy=True, num_epochs=1)
for i, data in enumerate(data_loader):
network.set_train()
images = data["image"]
input_shape = images.shape[2:4]
args.logger.info('iter[{}], shape{}'.format(i, input_shape[0]))
config.logger.info('iter[%d], shape%d', i, input_shape[0])
images = Tensor.from_numpy(images)
batch_y_true_0 = Tensor.from_numpy(data['bbox1'])
batch_y_true_1 = Tensor.from_numpy(data['bbox2'])
batch_y_true_2 = Tensor.from_numpy(data['bbox3'])
@ -297,29 +280,35 @@ if __name__ == "__main__":
loss_meter.update(loss.asnumpy())
# ckpt progress
if args.rank_save_ckpt_flag:
if config.rank_save_ckpt_flag:
cb_params.cur_step_num = i + 1 # current step number
cb_params.batch_num = i + 2
ckpt_cb.step_end(run_context)
if i % args.log_interval == 0:
if i % config.log_interval == 0:
time_used = time.time() - t_end
epoch = int(i / args.steps_per_epoch)
fps = args.per_batch_size * (i - old_progress) * args.group_size / time_used
args.logger.info('epoch[{}], iter[{}], {}, {:.2f} imgs/sec, lr:{}'.format(epoch, i, loss_meter, fps, lr[i]))
epoch = int(i / config.steps_per_epoch)
fps = config.per_batch_size * (i - old_progress) * config.group_size / time_used
if config.rank == 0:
config.logger.info(
'epoch[{}], iter[{}], {}, {:.2f} imgs/sec, lr:{}'.format(epoch, i, loss_meter, fps, lr[i]))
t_end = time.time()
loss_meter.reset()
old_progress = i
if args.run_eval and (i + 1) % args.steps_per_epoch == 0:
eval_cb.epoch_end(run_context)
if (i + 1) % args.steps_per_epoch == 0:
if (i + 1) % config.steps_per_epoch == 0 and (config.run_eval or config.rank_save_ckpt_flag):
if config.run_eval:
eval_cb.epoch_end(run_context)
network.set_train()
cb_params.cur_epoch_num += 1
if args.need_profiler:
if config.need_profiler and profiler is not None:
if i == 10:
profiler.analyse()
break
args.logger.info('==========end training===============')
config.logger.info('==========end training===============')
if __name__ == "__main__":
run_train()