Col_lin
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix seq2seq percision problem

This commit is contained in:
zhaojichen 2021-08-11 10:19:52 +08:00
parent 14ed247eeb
commit 11fde3f6cd
10 changed files with 113 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
model_zoo/official/nlp/fasttext/README.md
View File

@ -68,10 +68,10 @@ After dataset preparation, you can start training and evaluation as follows:
```bash
# run training example
cd ./scripts
bash run_standalone_train.sh [TRAIN_DATASET] [DEVICEID]
bash run_standalone_train.sh [TRAIN_DATASET] [DEVICEID] [DATANAME]
# run distributed training example
bash run_distribute_train.sh [TRAIN_DATASET] [RANK_TABLE_PATH]
bash run_distribute_train.sh [TRAIN_DATASET] [RANK_TABLE_PATH] [DATANAME]
# run evaluation example
bash run_eval.sh [EVAL_DATASET_PATH] [DATASET_NAME] [MODEL_CKPT] [DEVICEID]
@ -219,14 +219,14 @@ Parameters for both training and evaluation can be set in config.py. All the dat
```bash
cd ./scripts
bash run_standalone_train.sh [DATASET_PATH]
bash run_standalone_train.sh [DATASET_PATH] [DEVICE_ID] [DATANAME]
```
- Running scripts for distributed training of FastText. Task training on multiple device and run the following command in bash to be executed in `scripts/`:
```bash
cd ./scripts
bash run_distributed_train.sh [DATASET_PATH] [RANK_TABLE_PATH]
bash run_distributed_train.sh [DATASET_PATH] [RANK_TABLE_PATH] [DATANAME]
```
- Running on GPU

24
model_zoo/official/nlp/fasttext/scripts/run_distribute_train_8p.sh
View File

@ -17,8 +17,9 @@
echo "=============================================================================================================="
echo "Please run the script as: "
echo "sh run_distributed_train.sh DATASET_PATH RANK_TABLE_PATH"
echo "for example: sh run_distributed_train.sh /home/workspace/ag /home/workspace/rank_table_file.json"
echo "for example: sh run_distributed_train.sh /home/workspace/ag /home/workspace/rank_table_file.json ag"
echo "It is better to use absolute path."
echo "Please pay attention that the dataset should corresponds to dataset_name"
echo "=============================================================================================================="
get_real_path(){
if [ "${1:0:1}" == "/" ]; then
@ -28,11 +29,15 @@ get_real_path(){
fi
}
if [ $3 != "ag" ] && [ $3 != "dbpedia" ] && [ $3 != "yelp_p" ]
then
echo "Unrecognized dataset name, the name can choose from [ag, dbpedia, yelp_p]"
exit 1
fi
DATASET=$(get_real_path $1)
echo $DATASET
DATANAME=$(basename $DATASET)
RANK_TABLE_PATH=$(get_real_path $2)
echo $DATANAME
if [ ! -d $DATASET ]
then
echo "Error: DATA_PATH=$DATASET is not a file"
@ -48,6 +53,19 @@ echo $RANK_TABLE_FILE
export RANK_SIZE=8
export DEVICE_NUM=8
if [ $# -ge 1 ]; then
if [ $3 == 'ag' ]; then
DATANAME='ag'
elif [ $3 == 'dbpedia' ]; then
DATANAME='dbpedia'
elif [ $3 == 'yelp_p' ]; then
DATANAME='yelp_p'
else
echo "Unrecognized dataset name,he name can choose from [ag, dbpedia, yelp_p]"
exit 1
fi
fi
config_path="./${DATANAME}_config.yaml"
echo "config path is : ${config_path}"

28
model_zoo/official/nlp/fasttext/scripts/run_standalone_train.sh
View File

@ -16,9 +16,21 @@
echo "=============================================================================================================="
echo "Please run the script as: "
echo "sh run_standalone_train.sh DATASET_PATH"
echo "for example: sh run_standalone_train.sh /home/workspace/ag"
echo "for example: sh run_standalone_train.sh /home/workspace/ag 0 ag"
echo "It is better to use absolute path."
echo "Please pay attention that the dataset should corresponds to dataset_name"
echo "=============================================================================================================="
if [[ $# -lt 3 ]]; then
echo "Usage: bash run_standalone_train.sh [DATA_PATH] [DEVICE_ID] [DATANAME]
DATANAME can choose from [ag, dbpedia, yelp_p]"
exit 1
fi
if [ $3 != "ag" ] && [ $3 != "dbpedia" ] && [ $3 != "yelp_p" ]
then
echo "Unrecognized dataset name, the name can choose from [ag, dbpedia, yelp_p]"
exit 1
fi
get_real_path(){
if [ "${1:0:1}" == "/" ]; then
@ -29,10 +41,20 @@ get_real_path(){
}
DATASET=$(get_real_path $1)
echo $DATASET
DATANAME=$(basename $DATASET)
echo $DATANAME
DEVICEID=$2
if [ $# -ge 1 ]; then
if [ $3 == 'ag' ]; then
DATANAME='ag'
elif [ $3 == 'dbpedia' ]; then
DATANAME='dbpedia'
elif [ $3 == 'yelp_p' ]; then
DATANAME='yelp_p'
else
echo "Unrecognized dataset name"
exit 1
fi
fi
config_path="./${DATANAME}_config.yaml"
echo "config path is : ${config_path}"

4
model_zoo/research/nlp/seq2seq/config/config.json
View File

@ -22,9 +22,9 @@
"max_decode_length": 50
},
"loss_scale_config": {
"init_loss_scale": 65536,
"init_loss_scale": 64,
"loss_scale_factor": 2,
"scale_window": 1000
"scale_window": 5000
},
"learn_rate_config": {
"optimizer": "adam",

26
model_zoo/research/nlp/seq2seq/eval.py
View File

@ -16,10 +16,9 @@
import os
# os.system("pip3 install subword-nmt")
# os.system("pip3 install sacremoses")
import ast
import argparse
import pickle
import moxing as mox
from mindspore.common import dtype as mstype
from mindspore import context
@ -30,19 +29,14 @@ from src.dataset.tokenizer import Tokenizer
is_modelarts = False
if is_modelarts:
parser = argparse.ArgumentParser(description='seq2seq')
parser.add_argument("--config", type=str, required=True,
help="model config json file path.")
parser.add_argument("--data_url", type=str, required=True,
help="data address.")
parser.add_argument("--train_url", type=str, required=True,
help="output address.")
parser = argparse.ArgumentParser(description='seq2seq')
parser.add_argument("--config", type=str, required=True,
help="model config json file path.")
parser.add_argument("--data_url", type=str, default=None,
help="data address.")
parser.add_argument("--train_url", type=str, default=None,
help="output address.")
parser.add_argument("--test_dataset", type=str, required=True,
help="test dataset address.")
parser.add_argument("--existed_ckpt", type=str, required=True,
@ -57,6 +51,11 @@ parser.add_argument("--test_tgt", type=str, required=True,
parser.add_argument("--output", type=str, required=False,
default="./output.npz",
help="result file path.")
parser.add_argument("--is_modelarts", type=ast.literal_eval, default=False,
help="running on modelarts")
args, _ = parser.parse_known_args()
if args.is_modelarts:
import moxing as mox
context.set_context(
mode=context.GRAPH_MODE,
@ -78,11 +77,10 @@ def _check_args(config):
if __name__ == '__main__':
args, _ = parser.parse_known_args()
_check_args(args.config)
_config = get_config(args.config)
if is_modelarts:
if args.is_modelarts:
mox.file.copy_parallel(src_url=args.data_url, dst_url='/cache/dataset_menu/')
_config.test_dataset = '/cache/dataset_menu/newstest2014.en.mindrecord'
_config.existed_ckpt = '/cache/dataset_menu/seq2seq-7_1642.ckpt'
@ -103,7 +101,7 @@ if __name__ == '__main__':
scores = bleu_calculate(tokenizer, result_npy_addr, test_tgt)
print(f"BLEU scores is :{scores}")
if is_modelarts:
if args.is_modelarts:
result_npy_addr = output
vocab = '/cache/dataset_menu/vocab.bpe.32000'
bpe_codes = '/cache/dataset_menu/bpe.32000'

4
model_zoo/research/nlp/seq2seq/src/seq2seq_model/beam_search.py
View File

@ -34,7 +34,7 @@ class LengthPenalty(nn.Cell):
def __init__(self, weight=1.0, compute_type=mstype.float32):
super(LengthPenalty, self).__init__()
self.weight = weight
self.add = P.TensorAdd()
self.add = P.Add()
self.pow = P.Pow()
self.div = P.RealDiv()
self.five = Tensor(5.0, mstype.float32)
@ -183,7 +183,7 @@ class BeamSearchDecoder(nn.Cell):
self.decoder = decoder
self.is_using_while = is_using_while
self.add = P.TensorAdd()
self.add = P.Add()
self.expand = P.ExpandDims()
self.reshape = P.Reshape()
self.shape_flat = (-1,)

1
model_zoo/research/nlp/seq2seq/src/seq2seq_model/dynamic_rnn.py
View File

@ -90,7 +90,6 @@ class DynamicRNNNet(nn.Cell):
self.cast = P.Cast()
self.concat = P.Concat(axis=0)
self.get_shape = P.Shape()
self.print = P.Print()
self.net = DynamicRNNCell(num_setp=seq_length,
batch_size=batchsize,
word_embed_dim=word_embed_dim,

2
model_zoo/research/nlp/seq2seq/src/seq2seq_model/embedding.py
View File

@ -49,7 +49,7 @@ class EmbeddingLookup(nn.Cell):
init_weight = np.random.normal(-initializer_range, initializer_range, size=[vocab_size, embed_dim])
self.embedding_table = Parameter(Tensor(init_weight, mstype.float32), name="embedding_table")
self.expand = P.ExpandDims()
self.gather = P.GatherV2()
self.gather = P.Gather()
self.one_hot = P.OneHot()
self.on_value = Tensor(1.0, mstype.float32)
self.off_value = Tensor(0.0, mstype.float32)

137
model_zoo/research/nlp/seq2seq/src/seq2seq_model/seq2seq_for_train.py
View File

@ -23,8 +23,7 @@ from mindspore.common.tensor import Tensor
from mindspore import Parameter
from mindspore.common import dtype as mstype
from mindspore.nn.wrap.grad_reducer import DistributedGradReducer
from mindspore.context import ParallelMode
from mindspore.parallel._utils import _get_device_num, _get_parallel_mode, _get_gradients_mean
from mindspore.communication.management import get_group_size
from .seq2seq import Seq2seqModel
@ -32,43 +31,31 @@ from .seq2seq import Seq2seqModel
GRADIENT_CLIP_TYPE = 1
GRADIENT_CLIP_VALUE = 5.0
class ClipGradients(nn.Cell):
clip_grad = C.MultitypeFuncGraph("clip_grad")
@clip_grad.register("Number", "Number", "Tensor")
def _clip_grad(clip_type, clip_value, grad):
"""
Clip gradients.
Args:
grads (list): List of gradient tuples.
clip_type (Tensor): The way to clip, 'value' or 'norm'.
clip_value (Tensor): Specifies how much to clip.
Inputs:
clip_type (int): The way to clip, 0 for 'value', 1 for 'norm'.
clip_value (float): Specifies how much to clip.
grad (tuple[Tensor]): Gradients.
Returns:
List, a list of clipped_grad tuples.
Outputs:
tuple[Tensor], clipped gradients.
"""
def __init__(self):
super(ClipGradients, self).__init__()
self.clip_by_norm = nn.ClipByNorm()
self.cast = P.Cast()
self.dtype = P.DType()
def construct(self,
grads,
clip_type,
clip_value):
"""Defines the gradients clip."""
if clip_type not in (0, 1):
return grads
new_grads = ()
for grad in grads:
dt = self.dtype(grad)
if clip_type == 0:
t = C.clip_by_value(grad, self.cast(F.tuple_to_array((-clip_value,)), dt),
self.cast(F.tuple_to_array((clip_value,)), dt))
else:
t = self.clip_by_norm(grad, self.cast(F.tuple_to_array((clip_value,)), dt))
new_grads = new_grads + (t,)
return new_grads
if clip_type not in (0, 1):
return grad
dt = F.dtype(grad)
if clip_type == 0:
new_grad = C.clip_by_value(grad, F.cast(F.tuple_to_array((-clip_value,)), dt),
F.cast(F.tuple_to_array((clip_value,)), dt))
else:
new_grad = nn.ClipByNorm()(grad, F.cast(F.tuple_to_array((clip_value,)), dt))
return new_grad
class PredLogProbs(nn.Cell):
"""
@ -238,8 +225,7 @@ grad_overflow = P.FloatStatus()
def _tensor_grad_overflow(grad):
return grad_overflow(grad)
class Seq2seqTrainOneStepWithLossScaleCell(nn.Cell):
class Seq2seqTrainOneStepWithLossScaleCell(nn.TrainOneStepWithLossScaleCell):
"""
Encapsulation class of seq2seq network training.
@ -254,48 +240,18 @@ class Seq2seqTrainOneStepWithLossScaleCell(nn.Cell):
Returns:
Tuple[Tensor, Tensor, Tensor], loss, overflow, sen.
"""
def __init__(self, network, optimizer, scale_update_cell=None):
super(Seq2seqTrainOneStepWithLossScaleCell, self).__init__(auto_prefix=False)
self.network = network
self.network.set_grad()
self.network.add_flags(defer_inline=True)
self.weights = optimizer.parameters
self.optimizer = optimizer
self.grad = C.GradOperation(get_by_list=True,
sens_param=True)
self.reducer_flag = False
self.all_reduce = P.AllReduce()
self.parallel_mode = _get_parallel_mode()
if self.parallel_mode not in ParallelMode.MODE_LIST:
raise ValueError("Parallel mode does not support: ", self.parallel_mode)
if self.parallel_mode in [ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL]:
self.reducer_flag = True
self.grad_reducer = None
if self.reducer_flag:
mean = _get_gradients_mean()
degree = _get_device_num()
self.grad_reducer = DistributedGradReducer(optimizer.parameters, mean, degree)
self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE)
self.clip_gradients = ClipGradients()
super(Seq2seqTrainOneStepWithLossScaleCell, self).__init__(network, optimizer, scale_update_cell)
self.cast = P.Cast()
self.alloc_status = P.NPUAllocFloatStatus()
self.get_status = P.NPUGetFloatStatus()
self.clear_before_grad = P.NPUClearFloatStatus()
self.reduce_sum = P.ReduceSum(keep_dims=False)
self.base = Tensor(1, mstype.float32)
self.less_equal = P.LessEqual()
self.hyper_map = C.HyperMap()
self.degree = 1
if self.reducer_flag:
self.degree = get_group_size()
self.grad_reducer = DistributedGradReducer(optimizer.parameters, False, self.degree)
self.loss_scale = None
self.loss_scaling_manager = scale_update_cell
if scale_update_cell:
self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(),
dtype=mstype.float32), name="loss_scale")
self.add_flags(has_effect=True)
self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32))
def construct(self,
source_eos_ids,
@ -330,14 +286,13 @@ class Seq2seqTrainOneStepWithLossScaleCell(nn.Cell):
target_ids,
label_ids,
label_weights)
# Alloc status.
init = self.alloc_status()
# Clear overflow buffer.
self.clear_before_grad(init)
if sens is None:
scaling_sens = self.loss_scale
else:
scaling_sens = sens
status, scaling_sens = self.start_overflow_check(loss, scaling_sens)
grads = self.grad(self.network, weights)(source_ids,
source_mask,
target_ids,
@ -345,26 +300,18 @@ class Seq2seqTrainOneStepWithLossScaleCell(nn.Cell):
label_weights,
self.cast(scaling_sens,
mstype.float32))
# apply grad reducer on grads
grads = self.grad_reducer(grads)
grads = self.hyper_map(F.partial(grad_scale, scaling_sens * self.degree), grads)
grads = self.hyper_map(F.partial(clip_grad, GRADIENT_CLIP_TYPE, GRADIENT_CLIP_VALUE), grads)
grads = self.hyper_map(F.partial(grad_scale, scaling_sens), grads)
grads = self.clip_gradients(grads, GRADIENT_CLIP_TYPE, GRADIENT_CLIP_VALUE)
if self.reducer_flag:
# Apply grad reducer on grads.
grads = self.grad_reducer(grads)
self.get_status(init)
flag_sum = self.reduce_sum(init, (0,))
if self.is_distributed:
# Sum overflow flag over devices.
flag_reduce = self.all_reduce(flag_sum)
cond = self.less_equal(self.base, flag_reduce)
else:
cond = self.less_equal(self.base, flag_sum)
cond = self.get_overflow_status(status, grads)
overflow = cond
if sens is None:
overflow = self.loss_scaling_manager(self.loss_scale, cond)
if not overflow:
self.optimizer(grads)
return (loss, cond, scaling_sens)
if overflow:
succ = False
else:
succ = self.optimizer(grads)
ret = (loss, cond, scaling_sens)
return F.depend(ret, succ)

12
model_zoo/research/nlp/seq2seq/train.py
View File

@ -25,7 +25,7 @@ from mindspore.nn.optim import Lamb
from mindspore.train.model import Model
from mindspore.train.loss_scale_manager import DynamicLossScaleManager
from mindspore.train.callback import CheckpointConfig, ModelCheckpoint, TimeMonitor
from mindspore.train.callback import LossMonitor, SummaryCollector
from mindspore.train.callback import LossMonitor
from mindspore import context, Parameter
from mindspore.context import ParallelMode
from mindspore.communication import management as MultiAscend
@ -52,7 +52,7 @@ if args.is_modelarts:
import moxing as mox
context.set_context(
mode=context.GRAPH_MODE,
save_graphs=True,
save_graphs=False,
device_target="Ascend",
reserve_class_name_in_scope=True)
@ -221,12 +221,12 @@ def _build_training_pipeline(config: Seq2seqConfig,
loss_monitor = LossCallBack(config)
dataset_size = dataset.get_dataset_size()
time_cb = TimeMonitor(data_size=dataset_size)
ckpt_config = CheckpointConfig(save_checkpoint_steps=config.save_ckpt_steps,
ckpt_config = CheckpointConfig(save_checkpoint_steps=dataset.get_dataset_size(),
keep_checkpoint_max=config.keep_ckpt_max)
rank_size = os.getenv('RANK_SIZE')
callbacks = [time_cb, loss_monitor]
callbacks.append(LossMonitor(1642))
callbacks.append(LossMonitor())
if rank_size is not None and int(rank_size) > 1 and MultiAscend.get_rank() % 8 == 0:
ckpt_callback = ModelCheckpoint(
@ -234,8 +234,6 @@ def _build_training_pipeline(config: Seq2seqConfig,
directory=os.path.join(config.ckpt_path, 'ckpt_{}'.format(os.getenv('DEVICE_ID'))),
config=ckpt_config)
callbacks.append(ckpt_callback)
summary_callback = SummaryCollector(summary_dir="./summary", collect_freq=50)
callbacks.append(summary_callback)
if rank_size is None or int(rank_size) == 1:
ckpt_callback = ModelCheckpoint(
@ -243,8 +241,6 @@ def _build_training_pipeline(config: Seq2seqConfig,
directory=os.path.join(config.ckpt_path, 'ckpt_{}'.format(os.getenv('DEVICE_ID'))),
config=ckpt_config)
callbacks.append(ckpt_callback)
summary_callback = SummaryCollector(summary_dir="./summary", collect_freq=50)
callbacks.append(summary_callback)
print(f" | ALL SET, PREPARE TO TRAIN.")
_train(model=model, config=config,