diff --git a/model_zoo/official/cv/FCN8s/README.md b/model_zoo/official/cv/FCN8s/README.md index 597bc3865c3..47febab7b90 100644 --- a/model_zoo/official/cv/FCN8s/README.md +++ b/model_zoo/official/cv/FCN8s/README.md @@ -85,21 +85,27 @@ Dataset used: │ ├──loss │ ├──loss.py // loss function │ ├──utils - │ ├──lr_scheduler.py // getting learning_rateFCN-8s + │ ├──lr_scheduler.py // getting learning_rateFCN-8s + │ ├──model_utils + │ ├──config.py // getting config parameters + │ ├──device_adapter.py // getting device info + │ ├──local_adapter.py // getting device info + │ ├──moxing_adapter.py // Decorator + ├── default_config.yaml // Parameters config ├── train.py // training script ├── eval.py // evaluation script ``` ## [脚本参数](#contents) -训练以及评估的参数可以在config.py中设置 +训练以及评估的参数可以在default_config.yaml中设置 - config for FCN8s ```python # dataset 'data_file': '/data/workspace/mindspore_dataset/FCN/FCN/dataset/MINDRECORED_NAME.mindrecord', # path and name of one mindrecord file - 'batch_size': 32, + 'train_batch_size': 32, 'crop_size': 512, 'image_mean': [103.53, 116.28, 123.675], 'image_std': [57.375, 57.120, 58.395], @@ -124,7 +130,7 @@ Dataset used: 'ckpt_dir': './ckpt', ``` -如需获取更多信息，请查看`config.py`. +如需获取更多信息，请查看`default_config.yaml`. ## [生成数据步骤](#contents) @@ -151,11 +157,15 @@ Dataset used: - running on Ascend with default parameters - ```python - python train.py --device_id device_id + ```python 单卡训练 + sh scripts/run_standalone_train.sh DEVICE_ID ``` - 训练时，训练过程中的epch和step以及此时的loss和精确度会呈现在终端上： + ```python 分布式训练 + sh scripts/run_train.sh DEVICE_NUM RANK_TABLE_FILES + ``` + + 训练时，训练过程中的epch和step以及此时的loss和精确度会呈现log.txt中: ```python epoch: * step: **, loss is **** @@ -176,6 +186,10 @@ Dataset used: python eval.py ``` + ```python shell脚本验证 + sh scripts/run_eval.sh DATA_ROOT DATA_LST CKPT_PATH + ``` + 以上的python命令会在终端上运行，你可以在终端上查看此次评估的结果。测试集的精确度会以如下方式呈现： ```python diff --git a/model_zoo/official/cv/FCN8s/default_config.yaml b/model_zoo/official/cv/FCN8s/default_config.yaml index eaffd2b9647..b0468554e2c 100644 --- a/model_zoo/official/cv/FCN8s/default_config.yaml +++ b/model_zoo/official/cv/FCN8s/default_config.yaml @@ -10,7 +10,6 @@ output_path: "/cache/train" load_path: "/cache/checkpoint_path" device_target: "Ascend" enable_profiling: False - checkpoint_path: "./checkpoint/" checkpoint_file: "./checkpoint/.ckpt" # ====================================================================================== @@ -28,7 +27,7 @@ model: "FCN8s" train_batch_size: 32 min_scale: 0.5 max_scale: 2.0 -data_file: "voctrain.mindrecord0" +data_file: "/data/mjq/dataset/vocaug_local_mindrecords/vocaug_local_mindrecords.mindrecords" # optimizer train_epochs: 500 @@ -36,19 +35,22 @@ base_lr: 0.015 loss_scale: 1024 # model -ckpt_vgg16: "" -ckpt_pre_trained: "FCN8s-500_5.ckpt" +ckpt_vgg16: "/data/mjq/ckpt/vgg16_predtrain.ckpt" +ckpt_pre_trained: "" save_steps: 330 keep_checkpoint_max: 5 +ckpt_dir: "./ckpt" + # ====================================================================================== # Eval options eval_batch_size: 16 -data_lst: "" +data_root: "/data/mjq/dataset/VOCdevkit/VOC2012" +data_lst: "/data/mjq/dataset/VOCdevkit/VOC2012/ImageSets/Segmentation/val.txt" scales: [1.0] flip: False freeze_bn: False -ckpt_file: "" +ckpt_file: "/data/mjq/ckpt/FCN8s_1-133_300.ckpt" --- diff --git a/model_zoo/official/cv/FCN8s/eval.py b/model_zoo/official/cv/FCN8s/eval.py index 98cbfca000e..dad18cc553d 100644 --- a/model_zoo/official/cv/FCN8s/eval.py +++ b/model_zoo/official/cv/FCN8s/eval.py @@ -14,7 +14,7 @@ # ============================================================================ """eval FCN8s.""" -import os + import numpy as np import cv2 from PIL import Image @@ -102,7 +102,7 @@ def eval_batch(configs, eval_net, img_lst, crop_size=512, flip=True): for bs in range(batch_size): probs_ = net_out[bs][:, :resize_hw[bs][0], :resize_hw[bs][1]].transpose((1, 2, 0)) ori_h, ori_w = img_lst[bs].shape[0], img_lst[bs].shape[1] - probs_ = cv2.resize(probs_, (ori_w, ori_h)) + probs_ = cv2.resize(probs_.astype(np.float32), (ori_w, ori_h)) result_lst.append(probs_) return result_lst @@ -130,14 +130,12 @@ def net_eval(): save_graphs=False) # data list - data_lst = os.path.join(config.data_path, config.data_lst) - with open(data_lst) as f: + with open(config.data_lst) as f: img_lst = f.readlines() net = FCN8s(n_class=config.num_classes) # load model - config.ckpt_file = os.path.join(config.data_path, config.ckpt_file) param_dict = load_checkpoint(config.ckpt_file) load_param_into_net(net, param_dict) @@ -150,7 +148,7 @@ def net_eval(): for i, line in enumerate(img_lst): img_name = line.strip('\n') - data_root = config.data_path + data_root = config.data_root img_path = data_root + '/JPEGImages/' + str(img_name) + '.jpg' msk_path = data_root + '/SegmentationClass/' + str(img_name) + '.png' diff --git a/model_zoo/official/cv/FCN8s/scripts/build_data.sh b/model_zoo/official/cv/FCN8s/scripts/build_data.sh index f24469ecc67..1e12bf162a6 100644 --- a/model_zoo/official/cv/FCN8s/scripts/build_data.sh +++ b/model_zoo/official/cv/FCN8s/scripts/build_data.sh @@ -15,8 +15,8 @@ # ============================================================================ export DEVICE_ID=0 -python src/data/build_seg_data.py --data_root=/home/sun/data/Mindspore/benchmark_RELEASE/dataset \ - --data_lst=/home/sun/data/Mindspore/benchmark_RELEASE/dataset/trainaug.txt \ - --dst_path=dataset/MINDRECORED_NAME.mindrecord \ +python src/data/build_seg_data.py --data_root=/data/mjq/dataset \ + --data_lst=/data/mjq/dataset/vocaug_train_lst.txt \ + --dst_path=./mindrecords/vocaug_train.mindrecords \ --num_shards=1 \ --shuffle=True \ No newline at end of file diff --git a/model_zoo/official/cv/FCN8s/scripts/run_eval.sh b/model_zoo/official/cv/FCN8s/scripts/run_eval.sh index 6e9ec6b4ccb..aab0e934db8 100644 --- a/model_zoo/official/cv/FCN8s/scripts/run_eval.sh +++ b/model_zoo/official/cv/FCN8s/scripts/run_eval.sh @@ -18,26 +18,27 @@ echo "==============================================================================================================" echo "Please run the script as: " echo "sh run_distribute_eval.sh DEVICE_NUM RANK_TABLE_FILE DATASET CKPT_PATH" -echo "for example: sh run_eval.sh [RANK_TABLE_FILE] /path/to/dataset /path/to/ckpt device_id" +echo "for example: sh scripts/run_eval.sh path/to/data_root /path/to/dataset /path/to/ckpt" echo "It is better to use absolute path." echo "=================================================================================================================" -export DATA_PATH=$1 -CKPT_PATH=$2 -DEVICE_ID=$3 +export DATA_ROOT=$1 +DATA_PATH=$2 +CKPT_PATH=$3 rm -rf eval mkdir ./eval cp ./*.py ./eval +cp ./*.yaml ./eval cp -r ./src ./eval cd ./eval || exit echo "start testing" env > env.log python eval.py \ ---device_id=$DEVICE_ID \ ---data_path=$DATA_PATH \ ---ckpt_path=$CKPT_PATH #> log.txt 2>&1 & +--data_root=$DATA_ROOT \ +--data_lst=$DATA_PATH \ +--ckpt_file=$CKPT_PATH #> log.txt 2>&1 & cd ../ diff --git a/model_zoo/official/cv/FCN8s/scripts/run_standalone_train.sh b/model_zoo/official/cv/FCN8s/scripts/run_standalone_train.sh index 2c15f97a9f2..c1319aeed9c 100644 --- a/model_zoo/official/cv/FCN8s/scripts/run_standalone_train.sh +++ b/model_zoo/official/cv/FCN8s/scripts/run_standalone_train.sh @@ -16,7 +16,7 @@ if [ $# != 1 ] then - echo "Usage: sh run_standalone_train.sh [device_num]" + echo "Usage: sh scripts/run_standalone_train.sh DEVICE_ID" exit 1 fi @@ -30,9 +30,10 @@ fi mkdir -p ${train_path} cp -r ./src ${train_path} cp ./train.py ${train_path} +cp ./*.yaml ${train_path} echo "start training for device $DEVICE_ID" cd ${train_path}|| exit -python train.py --device_id=${DEVICE_ID} > log 2>&1 & +python train.py > log 2>&1 & cd .. diff --git a/model_zoo/official/cv/FCN8s/scripts/run_train.sh b/model_zoo/official/cv/FCN8s/scripts/run_train.sh index e398efd1050..7f0181f2a51 100644 --- a/model_zoo/official/cv/FCN8s/scripts/run_train.sh +++ b/model_zoo/official/cv/FCN8s/scripts/run_train.sh @@ -16,7 +16,7 @@ if [ $# != 2 ] then - echo "Usage: sh run_train.sh [device_num][RANK_TABLE_FILE]" + echo "Usage: sh scripts/run_train.sh [device_num][RANK_TABLE_FILE]" exit 1 fi @@ -44,9 +44,10 @@ do mkdir ./train_parallel$i cp -r ./src ./train_parallel$i cp ./train.py ./train_parallel$i + cp ./*.yaml ./train_parallel$i echo "start training for rank $RANK_ID, device $DEVICE_ID" cd ./train_parallel$i ||exit env > env.log - python train.py --device_id=$i > log 2>&1 & + python train.py > log 2>&1 & cd .. done diff --git a/model_zoo/official/cv/FCN8s/src/config.py b/model_zoo/official/cv/FCN8s/src/config.py deleted file mode 100644 index 0dcc60cd359..00000000000 --- a/model_zoo/official/cv/FCN8s/src/config.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2021 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -network config setting, will be used in train.py -""" - -from easydict import EasyDict as edict - - -FCN8s_VOC2012_cfg = edict({ - # dataset - 'data_file': '/data/workspace/mindspore_dataset/FCN/FCN/dataset/MINDRECORED_NAME.mindrecord', - 'batch_size': 32, - 'crop_size': 512, - 'image_mean': [103.53, 116.28, 123.675], - 'image_std': [57.375, 57.120, 58.395], - 'min_scale': 0.5, - 'max_scale': 2.0, - 'ignore_label': 255, - 'num_classes': 21, - - # optimizer - 'train_epochs': 500, - 'base_lr': 0.015, - 'loss_scale': 1024.0, - - # model - 'model': 'FCN8s', - 'ckpt_vgg16': '', - 'ckpt_pre_trained': '', - - # train - 'save_steps': 330, - 'keep_checkpoint_max': 5, - 'ckpt_dir': './ckpt', -}) diff --git a/model_zoo/official/cv/FCN8s/src/data/build_seg_data.py b/model_zoo/official/cv/FCN8s/src/data/build_seg_data.py index 78cee800869..b75f4de0b4f 100644 --- a/model_zoo/official/cv/FCN8s/src/data/build_seg_data.py +++ b/model_zoo/official/cv/FCN8s/src/data/build_seg_data.py @@ -54,10 +54,8 @@ if __name__ == '__main__': cnt = 0 for l in lines: - img_name = l.strip('\n') - - img_path = 'img/' + str(img_name) + '.jpg' - label_path = 'cls_png/' + str(img_name) + '.png' + img_path = l.split(' ')[0].strip('\n') + label_path = l.split(' ')[1].strip('\n') sample_ = {"file_name": img_path.split('/')[-1]} diff --git a/model_zoo/official/cv/FCN8s/src/model_utils/config.py b/model_zoo/official/cv/FCN8s/src/model_utils/config.py index a9879fb0089..3badc574716 100644 --- a/model_zoo/official/cv/FCN8s/src/model_utils/config.py +++ b/model_zoo/official/cv/FCN8s/src/model_utils/config.py @@ -21,6 +21,9 @@ from pprint import pprint, pformat import yaml +global_yaml = '../../default_config.yaml' + + class Config: """ Configuration namespace. Convert dictionary to members @@ -115,7 +118,7 @@ def get_config(): """ parser = argparse.ArgumentParser(description='default name', add_help=False) current_dir = os.path.dirname(os.path.abspath(__file__)) - parser.add_argument('--config_path', type=str, default=os.path.join(current_dir, '../../default_config.yaml'), + parser.add_argument('--config_path', type=str, default=os.path.join(current_dir, global_yaml), help='Config file path') path_args, _ = parser.parse_known_args() default, helper, choices = parse_yaml(path_args.config_path) diff --git a/model_zoo/official/cv/FCN8s/train.py b/model_zoo/official/cv/FCN8s/train.py index 22e7cc494cd..bab42bdcd85 100644 --- a/model_zoo/official/cv/FCN8s/train.py +++ b/model_zoo/official/cv/FCN8s/train.py @@ -14,7 +14,7 @@ # ============================================================================ """train FCN8s.""" -import os + from mindspore import context, Tensor from mindspore.train.model import Model from mindspore.context import ParallelMode @@ -38,7 +38,7 @@ set_seed(1) def modelarts_pre_process(): - config.checkpoint_path = os.path.join(config.output_path, str(get_rank_id()), config.checkpoint_path) + pass @moxing_wrapper(pre_process=modelarts_pre_process) @@ -59,7 +59,7 @@ def train(): # dataset dataset = data_generator.SegDataset(image_mean=config.image_mean, image_std=config.image_std, - data_file=os.path.join(config.data_path, config.data_file), + data_file=config.data_file, batch_size=config.train_batch_size, crop_size=config.crop_size, max_scale=config.max_scale, @@ -77,7 +77,6 @@ def train(): # load pretrained vgg16 parameters to init FCN8s if config.ckpt_vgg16: - config.ckpt_vgg16 = os.path.join(config.data_path, config.ckpt_vgg16) param_vgg = load_checkpoint(config.ckpt_vgg16) param_dict = {} for layer_id in range(1, 6): @@ -97,7 +96,6 @@ def train(): load_param_into_net(net, param_dict) # load pretrained FCN8s elif config.ckpt_pre_trained: - config.ckpt_pre_trained = os.path.join(config.data_path, config.ckpt_pre_trained) param_dict = load_checkpoint(config.ckpt_pre_trained) load_param_into_net(net, param_dict) @@ -117,7 +115,6 @@ def train(): optimizer = nn.Momentum(params=net.trainable_params(), learning_rate=lr, momentum=0.9, weight_decay=0.0001, loss_scale=config.loss_scale) - print(optimizer.get_lr()) model = Model(net, loss_fn=loss_, loss_scale_manager=manager_loss_scale, optimizer=optimizer, amp_level="O3") # callback for saving ckpts @@ -128,7 +125,7 @@ def train(): if config.rank == 0: config_ck = CheckpointConfig(save_checkpoint_steps=config.save_steps, keep_checkpoint_max=config.keep_checkpoint_max) - ckpoint_cb = ModelCheckpoint(prefix=config.model, directory=config.checkpoint_path, config=config_ck) + ckpoint_cb = ModelCheckpoint(prefix=config.model, directory=config.ckpt_dir, config=config_ck) cbs.append(ckpoint_cb) model.train(config.train_epochs, dataset, callbacks=cbs)