change shell scripts and modify eval.py and train.py

model_zoo/official/cv/FCN8s/README.md

@@ -85,21 +85,27 @@ Dataset used:
         │   ├──loss
         │       ├──loss.py            // loss function
         │   ├──utils
-        │       ├──lr_scheduler.py            // getting learning_rateFCN-8s  
+        │       ├──lr_scheduler.py            // getting learning_rateFCN-8s
+        │   ├──model_utils
+        │       ├──config.py                     // getting config parameters
+        │       ├──device_adapter.py            // getting device info
+        │       ├──local_adapter.py            // getting device info
+        │       ├──moxing_adapter.py          // Decorator
+        ├── default_config.yaml               // Parameters config
         ├── train.py                 // training script
         ├── eval.py                  //  evaluation script
 ```
 
 ## [脚本参数](#contents)
 
-训练以及评估的参数可以在config.py中设置
+训练以及评估的参数可以在default_config.yaml中设置
 
 - config for FCN8s
 
   ```python
      # dataset
     'data_file': '/data/workspace/mindspore_dataset/FCN/FCN/dataset/MINDRECORED_NAME.mindrecord', # path and name of one mindrecord file
-    'batch_size': 32,
+    'train_batch_size': 32,
     'crop_size': 512,
     'image_mean': [103.53, 116.28, 123.675],
     'image_std': [57.375, 57.120, 58.395],
@@ -124,7 +130,7 @@ Dataset used:
     'ckpt_dir': './ckpt',
   ```
 
-如需获取更多信息，请查看`config.py`.
+如需获取更多信息，请查看`default_config.yaml`.
 
 ## [生成数据步骤](#contents)
 
@@ -151,11 +157,15 @@ Dataset used:
 
 - running on Ascend with default parameters
 
-  ```python
-  python train.py --device_id device_id
+  ```python 单卡训练
+  sh scripts/run_standalone_train.sh DEVICE_ID
   ```
 
-  训练时，训练过程中的epch和step以及此时的loss和精确度会呈现在终端上：
+  ```python 分布式训练
+  sh scripts/run_train.sh DEVICE_NUM RANK_TABLE_FILES
+  ```
+
+  训练时，训练过程中的epch和step以及此时的loss和精确度会呈现log.txt中:
 
   ```python
   epoch: * step: **, loss is ****
@@ -176,6 +186,10 @@ Dataset used:
   python eval.py
   ```
 
+  ```python shell脚本验证
+  sh scripts/run_eval.sh DATA_ROOT DATA_LST CKPT_PATH
+  ```
+
   以上的python命令会在终端上运行，你可以在终端上查看此次评估的结果。测试集的精确度会以如下方式呈现：
 
   ```python

model_zoo/official/cv/FCN8s/default_config.yaml

@@ -10,7 +10,6 @@ output_path: "/cache/train"
 load_path: "/cache/checkpoint_path"
 device_target: "Ascend"
 enable_profiling: False
-
 checkpoint_path: "./checkpoint/"
 checkpoint_file: "./checkpoint/.ckpt"
 # ======================================================================================
@@ -28,7 +27,7 @@ model: "FCN8s"
 train_batch_size: 32
 min_scale: 0.5
 max_scale: 2.0
-data_file: "voctrain.mindrecord0"
+data_file: "/data/mjq/dataset/vocaug_local_mindrecords/vocaug_local_mindrecords.mindrecords"
 
 # optimizer
 train_epochs: 500
@@ -36,19 +35,22 @@ base_lr: 0.015
 loss_scale: 1024
 
 # model
-ckpt_vgg16: ""
-ckpt_pre_trained: "FCN8s-500_5.ckpt"
+ckpt_vgg16: "/data/mjq/ckpt/vgg16_predtrain.ckpt"
+ckpt_pre_trained: ""
 save_steps: 330
 keep_checkpoint_max: 5
+ckpt_dir: "./ckpt"
+
 
 # ======================================================================================
 # Eval options
 eval_batch_size: 16
-data_lst: ""
+data_root: "/data/mjq/dataset/VOCdevkit/VOC2012"
+data_lst: "/data/mjq/dataset/VOCdevkit/VOC2012/ImageSets/Segmentation/val.txt"
 scales: [1.0]
 flip: False
 freeze_bn: False
-ckpt_file: ""
+ckpt_file: "/data/mjq/ckpt/FCN8s_1-133_300.ckpt"
 
 
 ---

model_zoo/official/cv/FCN8s/eval.py

@@ -14,7 +14,7 @@
 # ============================================================================
 """eval FCN8s."""
 
-import os
+
 import numpy as np
 import cv2
 from PIL import Image
@@ -102,7 +102,7 @@ def eval_batch(configs, eval_net, img_lst, crop_size=512, flip=True):
     for bs in range(batch_size):
         probs_ = net_out[bs][:, :resize_hw[bs][0], :resize_hw[bs][1]].transpose((1, 2, 0))
         ori_h, ori_w = img_lst[bs].shape[0], img_lst[bs].shape[1]
-        probs_ = cv2.resize(probs_, (ori_w, ori_h))
+        probs_ = cv2.resize(probs_.astype(np.float32), (ori_w, ori_h))
         result_lst.append(probs_)
 
     return result_lst
@@ -130,14 +130,12 @@ def net_eval():
                         save_graphs=False)
 
     # data list
-    data_lst = os.path.join(config.data_path, config.data_lst)
-    with open(data_lst) as f:
+    with open(config.data_lst) as f:
         img_lst = f.readlines()
 
     net = FCN8s(n_class=config.num_classes)
 
     # load model
-    config.ckpt_file = os.path.join(config.data_path, config.ckpt_file)
     param_dict = load_checkpoint(config.ckpt_file)
     load_param_into_net(net, param_dict)
 
@@ -150,7 +148,7 @@ def net_eval():
     for i, line in enumerate(img_lst):
 
         img_name = line.strip('\n')
-        data_root = config.data_path
+        data_root = config.data_root
         img_path = data_root + '/JPEGImages/' + str(img_name) + '.jpg'
         msk_path = data_root + '/SegmentationClass/' + str(img_name) + '.png'
 

model_zoo/official/cv/FCN8s/scripts/build_data.sh

@@ -15,8 +15,8 @@
 # ============================================================================
 
 export DEVICE_ID=0
-python src/data/build_seg_data.py  --data_root=/home/sun/data/Mindspore/benchmark_RELEASE/dataset  \
-                    --data_lst=/home/sun/data/Mindspore/benchmark_RELEASE/dataset/trainaug.txt  \
-                    --dst_path=dataset/MINDRECORED_NAME.mindrecord  \
+python src/data/build_seg_data.py  --data_root=/data/mjq/dataset  \
+                    --data_lst=/data/mjq/dataset/vocaug_train_lst.txt  \
+                    --dst_path=./mindrecords/vocaug_train.mindrecords  \
                     --num_shards=1  \
                     --shuffle=True

model_zoo/official/cv/FCN8s/scripts/run_eval.sh

@@ -18,26 +18,27 @@
 echo "=============================================================================================================="
 echo "Please run the script as: "
 echo "sh run_distribute_eval.sh DEVICE_NUM RANK_TABLE_FILE DATASET CKPT_PATH"
-echo "for example: sh run_eval.sh [RANK_TABLE_FILE] /path/to/dataset /path/to/ckpt device_id"
+echo "for example: sh scripts/run_eval.sh path/to/data_root /path/to/dataset /path/to/ckpt"
 echo "It is better to use absolute path."
 echo "================================================================================================================="
 
 
-export DATA_PATH=$1
-CKPT_PATH=$2
-DEVICE_ID=$3
+export DATA_ROOT=$1
+DATA_PATH=$2
+CKPT_PATH=$3
 
 rm -rf eval
 mkdir ./eval
 cp ./*.py ./eval
+cp ./*.yaml ./eval
 cp -r ./src ./eval
 cd ./eval || exit
 echo "start testing"
 env > env.log
 python eval.py  \
---device_id=$DEVICE_ID  \
---data_path=$DATA_PATH   \
---ckpt_path=$CKPT_PATH #> log.txt 2>&1 &
+--data_root=$DATA_ROOT  \
+--data_lst=$DATA_PATH   \
+--ckpt_file=$CKPT_PATH #> log.txt 2>&1 &
 
 cd ../
 

model_zoo/official/cv/FCN8s/scripts/run_standalone_train.sh

@@ -16,7 +16,7 @@
 
 if [ $# != 1 ]
 then
-    echo "Usage: sh run_standalone_train.sh [device_num]"
+    echo "Usage: sh scripts/run_standalone_train.sh DEVICE_ID"
 exit 1
 fi
 
@@ -30,9 +30,10 @@ fi
 mkdir -p ${train_path}
 cp -r ./src ${train_path}
 cp ./train.py ${train_path}
+cp ./*.yaml ${train_path}
 
 echo "start training for device $DEVICE_ID"
 
 cd ${train_path}|| exit
-python train.py --device_id=${DEVICE_ID} > log 2>&1 &
+python train.py > log 2>&1 &
 cd ..

model_zoo/official/cv/FCN8s/scripts/run_train.sh

@@ -16,7 +16,7 @@
 
 if [ $# != 2 ]
 then
-    echo "Usage: sh run_train.sh [device_num][RANK_TABLE_FILE]"
+    echo "Usage: sh scripts/run_train.sh [device_num][RANK_TABLE_FILE]"
 exit 1
 fi
 
@@ -44,9 +44,10 @@ do
     mkdir ./train_parallel$i
     cp -r ./src ./train_parallel$i
     cp ./train.py ./train_parallel$i
+    cp ./*.yaml ./train_parallel$i
     echo "start training for rank $RANK_ID, device $DEVICE_ID"
     cd ./train_parallel$i ||exit
     env > env.log
-    python train.py --device_id=$i  > log 2>&1 &
+    python train.py > log 2>&1 &
     cd ..
 done

model_zoo/official/cv/FCN8s/src/config.py

@@ -1,48 +0,0 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""
-network config setting, will be used in train.py
-"""
-
-from easydict import EasyDict as edict
-
-
-FCN8s_VOC2012_cfg = edict({
-    # dataset
-    'data_file': '/data/workspace/mindspore_dataset/FCN/FCN/dataset/MINDRECORED_NAME.mindrecord',
-    'batch_size': 32,
-    'crop_size': 512,
-    'image_mean': [103.53, 116.28, 123.675],
-    'image_std': [57.375, 57.120, 58.395],
-    'min_scale': 0.5,
-    'max_scale': 2.0,
-    'ignore_label': 255,
-    'num_classes': 21,
-
-    # optimizer
-    'train_epochs': 500,
-    'base_lr': 0.015,
-    'loss_scale': 1024.0,
-
-    # model
-    'model': 'FCN8s',
-    'ckpt_vgg16': '',
-    'ckpt_pre_trained': '',
-
-    # train
-    'save_steps': 330,
-    'keep_checkpoint_max': 5,
-    'ckpt_dir': './ckpt',
-})

model_zoo/official/cv/FCN8s/src/data/build_seg_data.py

@@ -54,10 +54,8 @@ if __name__ == '__main__':
     cnt = 0
 
     for l in lines:
-        img_name = l.strip('\n')
-
-        img_path = 'img/' + str(img_name) + '.jpg'
-        label_path = 'cls_png/' + str(img_name) + '.png'
+        img_path = l.split(' ')[0].strip('\n')
+        label_path = l.split(' ')[1].strip('\n')
 
         sample_ = {"file_name": img_path.split('/')[-1]}
 

model_zoo/official/cv/FCN8s/src/model_utils/config.py

@@ -21,6 +21,9 @@ from pprint import pprint, pformat
 import yaml
 
 
+global_yaml = '../../default_config.yaml'
+
+
 class Config:
     """
     Configuration namespace. Convert dictionary to members
@@ -115,7 +118,7 @@ def get_config():
     """
     parser = argparse.ArgumentParser(description='default name', add_help=False)
     current_dir = os.path.dirname(os.path.abspath(__file__))
-    parser.add_argument('--config_path', type=str, default=os.path.join(current_dir, '../../default_config.yaml'),
+    parser.add_argument('--config_path', type=str, default=os.path.join(current_dir, global_yaml),
                         help='Config file path')
     path_args, _ = parser.parse_known_args()
     default, helper, choices = parse_yaml(path_args.config_path)

model_zoo/official/cv/FCN8s/train.py

@@ -14,7 +14,7 @@
 # ============================================================================
 """train FCN8s."""
 
-import os
+
 from mindspore import context, Tensor
 from mindspore.train.model import Model
 from mindspore.context import ParallelMode
@@ -38,7 +38,7 @@ set_seed(1)
 
 
 def modelarts_pre_process():
-    config.checkpoint_path = os.path.join(config.output_path, str(get_rank_id()), config.checkpoint_path)
+    pass
 
 
 @moxing_wrapper(pre_process=modelarts_pre_process)
@@ -59,7 +59,7 @@ def train():
     # dataset
     dataset = data_generator.SegDataset(image_mean=config.image_mean,
                                         image_std=config.image_std,
-                                        data_file=os.path.join(config.data_path, config.data_file),
+                                        data_file=config.data_file,
                                         batch_size=config.train_batch_size,
                                         crop_size=config.crop_size,
                                         max_scale=config.max_scale,
@@ -77,7 +77,6 @@ def train():
 
     # load pretrained vgg16 parameters to init FCN8s
     if config.ckpt_vgg16:
-        config.ckpt_vgg16 = os.path.join(config.data_path, config.ckpt_vgg16)
         param_vgg = load_checkpoint(config.ckpt_vgg16)
         param_dict = {}
         for layer_id in range(1, 6):
@@ -97,7 +96,6 @@ def train():
         load_param_into_net(net, param_dict)
     # load pretrained FCN8s
     elif config.ckpt_pre_trained:
-        config.ckpt_pre_trained = os.path.join(config.data_path, config.ckpt_pre_trained)
         param_dict = load_checkpoint(config.ckpt_pre_trained)
         load_param_into_net(net, param_dict)
 
@@ -117,7 +115,6 @@ def train():
 
     optimizer = nn.Momentum(params=net.trainable_params(), learning_rate=lr, momentum=0.9, weight_decay=0.0001,
                             loss_scale=config.loss_scale)
-    print(optimizer.get_lr())
     model = Model(net, loss_fn=loss_, loss_scale_manager=manager_loss_scale, optimizer=optimizer, amp_level="O3")
 
     # callback for saving ckpts
@@ -128,7 +125,7 @@ def train():
     if config.rank == 0:
         config_ck = CheckpointConfig(save_checkpoint_steps=config.save_steps,
                                      keep_checkpoint_max=config.keep_checkpoint_max)
-        ckpoint_cb = ModelCheckpoint(prefix=config.model, directory=config.checkpoint_path, config=config_ck)
+        ckpoint_cb = ModelCheckpoint(prefix=config.model, directory=config.ckpt_dir, config=config_ck)
         cbs.append(ckpoint_cb)
 
     model.train(config.train_epochs, dataset, callbacks=cbs)