diff --git a/model_zoo/official/nlp/fasttext/README.md b/model_zoo/official/nlp/fasttext/README.md index ab109a88178..479f762dc29 100644 --- a/model_zoo/official/nlp/fasttext/README.md +++ b/model_zoo/official/nlp/fasttext/README.md @@ -68,10 +68,10 @@ After dataset preparation, you can start training and evaluation as follows: ```bash # run training example cd ./scripts - bash run_standalone_train.sh [TRAIN_DATASET] [DEVICEID] + bash run_standalone_train.sh [TRAIN_DATASET] [DEVICEID] [DATANAME] # run distributed training example - bash run_distribute_train.sh [TRAIN_DATASET] [RANK_TABLE_PATH] + bash run_distribute_train.sh [TRAIN_DATASET] [RANK_TABLE_PATH] [DATANAME] # run evaluation example bash run_eval.sh [EVAL_DATASET_PATH] [DATASET_NAME] [MODEL_CKPT] [DEVICEID] @@ -219,14 +219,14 @@ Parameters for both training and evaluation can be set in config.py. All the dat ```bash cd ./scripts - bash run_standalone_train.sh [DATASET_PATH] + bash run_standalone_train.sh [DATASET_PATH] [DEVICE_ID] [DATANAME] ``` - Running scripts for distributed training of FastText. Task training on multiple device and run the following command in bash to be executed in `scripts/`: ```bash cd ./scripts - bash run_distributed_train.sh [DATASET_PATH] [RANK_TABLE_PATH] + bash run_distributed_train.sh [DATASET_PATH] [RANK_TABLE_PATH] [DATANAME] ``` - Running on GPU diff --git a/model_zoo/official/nlp/fasttext/scripts/run_distribute_train_8p.sh b/model_zoo/official/nlp/fasttext/scripts/run_distribute_train_8p.sh index a8a44296f06..ea0d2183db7 100644 --- a/model_zoo/official/nlp/fasttext/scripts/run_distribute_train_8p.sh +++ b/model_zoo/official/nlp/fasttext/scripts/run_distribute_train_8p.sh @@ -17,8 +17,9 @@ echo "==============================================================================================================" echo "Please run the script as: " echo "sh run_distributed_train.sh DATASET_PATH RANK_TABLE_PATH" -echo "for example: sh run_distributed_train.sh /home/workspace/ag /home/workspace/rank_table_file.json" +echo "for example: sh run_distributed_train.sh /home/workspace/ag /home/workspace/rank_table_file.json ag" echo "It is better to use absolute path." +echo "Please pay attention that the dataset should corresponds to dataset_name" echo "==============================================================================================================" get_real_path(){ if [ "${1:0:1}" == "/" ]; then @@ -28,11 +29,15 @@ get_real_path(){ fi } +if [ $3 != "ag" ] && [ $3 != "dbpedia" ] && [ $3 != "yelp_p" ] +then + echo "Unrecognized dataset name, the name can choose from [ag, dbpedia, yelp_p]" +exit 1 +fi + DATASET=$(get_real_path $1) echo $DATASET -DATANAME=$(basename $DATASET) RANK_TABLE_PATH=$(get_real_path $2) -echo $DATANAME if [ ! -d $DATASET ] then echo "Error: DATA_PATH=$DATASET is not a file" @@ -48,6 +53,19 @@ echo $RANK_TABLE_FILE export RANK_SIZE=8 export DEVICE_NUM=8 +if [ $# -ge 1 ]; then + if [ $3 == 'ag' ]; then + DATANAME='ag' + elif [ $3 == 'dbpedia' ]; then + DATANAME='dbpedia' + elif [ $3 == 'yelp_p' ]; then + DATANAME='yelp_p' + else + echo "Unrecognized dataset name,he name can choose from [ag, dbpedia, yelp_p]" + exit 1 + fi +fi + config_path="./${DATANAME}_config.yaml" echo "config path is : ${config_path}" diff --git a/model_zoo/official/nlp/fasttext/scripts/run_standalone_train.sh b/model_zoo/official/nlp/fasttext/scripts/run_standalone_train.sh index 9b3ef21733c..908d5453719 100644 --- a/model_zoo/official/nlp/fasttext/scripts/run_standalone_train.sh +++ b/model_zoo/official/nlp/fasttext/scripts/run_standalone_train.sh @@ -16,9 +16,21 @@ echo "==============================================================================================================" echo "Please run the script as: " echo "sh run_standalone_train.sh DATASET_PATH" -echo "for example: sh run_standalone_train.sh /home/workspace/ag" +echo "for example: sh run_standalone_train.sh /home/workspace/ag 0 ag" echo "It is better to use absolute path." +echo "Please pay attention that the dataset should corresponds to dataset_name" echo "==============================================================================================================" +if [[ $# -lt 3 ]]; then + echo "Usage: bash run_standalone_train.sh [DATA_PATH] [DEVICE_ID] [DATANAME] + DATANAME can choose from [ag, dbpedia, yelp_p]" +exit 1 +fi + +if [ $3 != "ag" ] && [ $3 != "dbpedia" ] && [ $3 != "yelp_p" ] +then + echo "Unrecognized dataset name, the name can choose from [ag, dbpedia, yelp_p]" +exit 1 +fi get_real_path(){ if [ "${1:0:1}" == "/" ]; then @@ -29,10 +41,20 @@ get_real_path(){ } DATASET=$(get_real_path $1) -echo $DATASET DATANAME=$(basename $DATASET) -echo $DATANAME DEVICEID=$2 +if [ $# -ge 1 ]; then + if [ $3 == 'ag' ]; then + DATANAME='ag' + elif [ $3 == 'dbpedia' ]; then + DATANAME='dbpedia' + elif [ $3 == 'yelp_p' ]; then + DATANAME='yelp_p' + else + echo "Unrecognized dataset name" + exit 1 + fi +fi config_path="./${DATANAME}_config.yaml" echo "config path is : ${config_path}" diff --git a/model_zoo/research/nlp/seq2seq/config/config.json b/model_zoo/research/nlp/seq2seq/config/config.json index 6a7d92f63c9..61c1455d13d 100644 --- a/model_zoo/research/nlp/seq2seq/config/config.json +++ b/model_zoo/research/nlp/seq2seq/config/config.json @@ -22,9 +22,9 @@ "max_decode_length": 50 }, "loss_scale_config": { - "init_loss_scale": 65536, + "init_loss_scale": 64, "loss_scale_factor": 2, - "scale_window": 1000 + "scale_window": 5000 }, "learn_rate_config": { "optimizer": "adam", diff --git a/model_zoo/research/nlp/seq2seq/eval.py b/model_zoo/research/nlp/seq2seq/eval.py index 6f10eeb5a0c..060d75c9314 100644 --- a/model_zoo/research/nlp/seq2seq/eval.py +++ b/model_zoo/research/nlp/seq2seq/eval.py @@ -16,10 +16,9 @@ import os # os.system("pip3 install subword-nmt") # os.system("pip3 install sacremoses") - +import ast import argparse import pickle -import moxing as mox from mindspore.common import dtype as mstype from mindspore import context @@ -30,19 +29,14 @@ from src.dataset.tokenizer import Tokenizer is_modelarts = False -if is_modelarts: - parser = argparse.ArgumentParser(description='seq2seq') - parser.add_argument("--config", type=str, required=True, - help="model config json file path.") - parser.add_argument("--data_url", type=str, required=True, - help="data address.") - parser.add_argument("--train_url", type=str, required=True, - help="output address.") - parser = argparse.ArgumentParser(description='seq2seq') parser.add_argument("--config", type=str, required=True, help="model config json file path.") +parser.add_argument("--data_url", type=str, default=None, + help="data address.") +parser.add_argument("--train_url", type=str, default=None, + help="output address.") parser.add_argument("--test_dataset", type=str, required=True, help="test dataset address.") parser.add_argument("--existed_ckpt", type=str, required=True, @@ -57,6 +51,11 @@ parser.add_argument("--test_tgt", type=str, required=True, parser.add_argument("--output", type=str, required=False, default="./output.npz", help="result file path.") +parser.add_argument("--is_modelarts", type=ast.literal_eval, default=False, + help="running on modelarts") +args, _ = parser.parse_known_args() +if args.is_modelarts: + import moxing as mox context.set_context( mode=context.GRAPH_MODE, @@ -78,11 +77,10 @@ def _check_args(config): if __name__ == '__main__': - args, _ = parser.parse_known_args() _check_args(args.config) _config = get_config(args.config) - if is_modelarts: + if args.is_modelarts: mox.file.copy_parallel(src_url=args.data_url, dst_url='/cache/dataset_menu/') _config.test_dataset = '/cache/dataset_menu/newstest2014.en.mindrecord' _config.existed_ckpt = '/cache/dataset_menu/seq2seq-7_1642.ckpt' @@ -103,7 +101,7 @@ if __name__ == '__main__': scores = bleu_calculate(tokenizer, result_npy_addr, test_tgt) print(f"BLEU scores is :{scores}") - if is_modelarts: + if args.is_modelarts: result_npy_addr = output vocab = '/cache/dataset_menu/vocab.bpe.32000' bpe_codes = '/cache/dataset_menu/bpe.32000' diff --git a/model_zoo/research/nlp/seq2seq/src/seq2seq_model/beam_search.py b/model_zoo/research/nlp/seq2seq/src/seq2seq_model/beam_search.py index 7d142cec666..14ac445c7e8 100644 --- a/model_zoo/research/nlp/seq2seq/src/seq2seq_model/beam_search.py +++ b/model_zoo/research/nlp/seq2seq/src/seq2seq_model/beam_search.py @@ -34,7 +34,7 @@ class LengthPenalty(nn.Cell): def __init__(self, weight=1.0, compute_type=mstype.float32): super(LengthPenalty, self).__init__() self.weight = weight - self.add = P.TensorAdd() + self.add = P.Add() self.pow = P.Pow() self.div = P.RealDiv() self.five = Tensor(5.0, mstype.float32) @@ -183,7 +183,7 @@ class BeamSearchDecoder(nn.Cell): self.decoder = decoder self.is_using_while = is_using_while - self.add = P.TensorAdd() + self.add = P.Add() self.expand = P.ExpandDims() self.reshape = P.Reshape() self.shape_flat = (-1,) diff --git a/model_zoo/research/nlp/seq2seq/src/seq2seq_model/dynamic_rnn.py b/model_zoo/research/nlp/seq2seq/src/seq2seq_model/dynamic_rnn.py index 9d956816109..014c40287ee 100644 --- a/model_zoo/research/nlp/seq2seq/src/seq2seq_model/dynamic_rnn.py +++ b/model_zoo/research/nlp/seq2seq/src/seq2seq_model/dynamic_rnn.py @@ -90,7 +90,6 @@ class DynamicRNNNet(nn.Cell): self.cast = P.Cast() self.concat = P.Concat(axis=0) self.get_shape = P.Shape() - self.print = P.Print() self.net = DynamicRNNCell(num_setp=seq_length, batch_size=batchsize, word_embed_dim=word_embed_dim, diff --git a/model_zoo/research/nlp/seq2seq/src/seq2seq_model/embedding.py b/model_zoo/research/nlp/seq2seq/src/seq2seq_model/embedding.py index 68202c18b41..a56ba2a3c89 100644 --- a/model_zoo/research/nlp/seq2seq/src/seq2seq_model/embedding.py +++ b/model_zoo/research/nlp/seq2seq/src/seq2seq_model/embedding.py @@ -49,7 +49,7 @@ class EmbeddingLookup(nn.Cell): init_weight = np.random.normal(-initializer_range, initializer_range, size=[vocab_size, embed_dim]) self.embedding_table = Parameter(Tensor(init_weight, mstype.float32), name="embedding_table") self.expand = P.ExpandDims() - self.gather = P.GatherV2() + self.gather = P.Gather() self.one_hot = P.OneHot() self.on_value = Tensor(1.0, mstype.float32) self.off_value = Tensor(0.0, mstype.float32) diff --git a/model_zoo/research/nlp/seq2seq/src/seq2seq_model/seq2seq_for_train.py b/model_zoo/research/nlp/seq2seq/src/seq2seq_model/seq2seq_for_train.py index c1edff1ada2..3f6bd3b9a01 100644 --- a/model_zoo/research/nlp/seq2seq/src/seq2seq_model/seq2seq_for_train.py +++ b/model_zoo/research/nlp/seq2seq/src/seq2seq_model/seq2seq_for_train.py @@ -23,8 +23,7 @@ from mindspore.common.tensor import Tensor from mindspore import Parameter from mindspore.common import dtype as mstype from mindspore.nn.wrap.grad_reducer import DistributedGradReducer -from mindspore.context import ParallelMode -from mindspore.parallel._utils import _get_device_num, _get_parallel_mode, _get_gradients_mean +from mindspore.communication.management import get_group_size from .seq2seq import Seq2seqModel @@ -32,43 +31,31 @@ from .seq2seq import Seq2seqModel GRADIENT_CLIP_TYPE = 1 GRADIENT_CLIP_VALUE = 5.0 -class ClipGradients(nn.Cell): +clip_grad = C.MultitypeFuncGraph("clip_grad") + + +@clip_grad.register("Number", "Number", "Tensor") +def _clip_grad(clip_type, clip_value, grad): """ Clip gradients. - Args: - grads (list): List of gradient tuples. - clip_type (Tensor): The way to clip, 'value' or 'norm'. - clip_value (Tensor): Specifies how much to clip. + Inputs: + clip_type (int): The way to clip, 0 for 'value', 1 for 'norm'. + clip_value (float): Specifies how much to clip. + grad (tuple[Tensor]): Gradients. - Returns: - List, a list of clipped_grad tuples. + Outputs: + tuple[Tensor], clipped gradients. """ - def __init__(self): - super(ClipGradients, self).__init__() - self.clip_by_norm = nn.ClipByNorm() - self.cast = P.Cast() - self.dtype = P.DType() - - def construct(self, - grads, - clip_type, - clip_value): - """Defines the gradients clip.""" - if clip_type not in (0, 1): - return grads - - new_grads = () - for grad in grads: - dt = self.dtype(grad) - if clip_type == 0: - t = C.clip_by_value(grad, self.cast(F.tuple_to_array((-clip_value,)), dt), - self.cast(F.tuple_to_array((clip_value,)), dt)) - else: - t = self.clip_by_norm(grad, self.cast(F.tuple_to_array((clip_value,)), dt)) - new_grads = new_grads + (t,) - - return new_grads + if clip_type not in (0, 1): + return grad + dt = F.dtype(grad) + if clip_type == 0: + new_grad = C.clip_by_value(grad, F.cast(F.tuple_to_array((-clip_value,)), dt), + F.cast(F.tuple_to_array((clip_value,)), dt)) + else: + new_grad = nn.ClipByNorm()(grad, F.cast(F.tuple_to_array((clip_value,)), dt)) + return new_grad class PredLogProbs(nn.Cell): """ @@ -238,8 +225,7 @@ grad_overflow = P.FloatStatus() def _tensor_grad_overflow(grad): return grad_overflow(grad) - -class Seq2seqTrainOneStepWithLossScaleCell(nn.Cell): +class Seq2seqTrainOneStepWithLossScaleCell(nn.TrainOneStepWithLossScaleCell): """ Encapsulation class of seq2seq network training. @@ -254,48 +240,18 @@ class Seq2seqTrainOneStepWithLossScaleCell(nn.Cell): Returns: Tuple[Tensor, Tensor, Tensor], loss, overflow, sen. """ - def __init__(self, network, optimizer, scale_update_cell=None): - - super(Seq2seqTrainOneStepWithLossScaleCell, self).__init__(auto_prefix=False) - self.network = network - self.network.set_grad() - self.network.add_flags(defer_inline=True) - self.weights = optimizer.parameters - self.optimizer = optimizer - self.grad = C.GradOperation(get_by_list=True, - sens_param=True) - self.reducer_flag = False - self.all_reduce = P.AllReduce() - - self.parallel_mode = _get_parallel_mode() - if self.parallel_mode not in ParallelMode.MODE_LIST: - raise ValueError("Parallel mode does not support: ", self.parallel_mode) - if self.parallel_mode in [ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL]: - self.reducer_flag = True - self.grad_reducer = None - if self.reducer_flag: - mean = _get_gradients_mean() - degree = _get_device_num() - self.grad_reducer = DistributedGradReducer(optimizer.parameters, mean, degree) - self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE) - self.clip_gradients = ClipGradients() + super(Seq2seqTrainOneStepWithLossScaleCell, self).__init__(network, optimizer, scale_update_cell) self.cast = P.Cast() - self.alloc_status = P.NPUAllocFloatStatus() - self.get_status = P.NPUGetFloatStatus() - self.clear_before_grad = P.NPUClearFloatStatus() - self.reduce_sum = P.ReduceSum(keep_dims=False) - self.base = Tensor(1, mstype.float32) - self.less_equal = P.LessEqual() - self.hyper_map = C.HyperMap() + self.degree = 1 + if self.reducer_flag: + self.degree = get_group_size() + self.grad_reducer = DistributedGradReducer(optimizer.parameters, False, self.degree) self.loss_scale = None self.loss_scaling_manager = scale_update_cell if scale_update_cell: - self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), - dtype=mstype.float32), name="loss_scale") - self.add_flags(has_effect=True) - + self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32)) def construct(self, source_eos_ids, @@ -330,14 +286,13 @@ class Seq2seqTrainOneStepWithLossScaleCell(nn.Cell): target_ids, label_ids, label_weights) - # Alloc status. - init = self.alloc_status() - # Clear overflow buffer. - self.clear_before_grad(init) if sens is None: scaling_sens = self.loss_scale else: scaling_sens = sens + + status, scaling_sens = self.start_overflow_check(loss, scaling_sens) + grads = self.grad(self.network, weights)(source_ids, source_mask, target_ids, @@ -345,26 +300,18 @@ class Seq2seqTrainOneStepWithLossScaleCell(nn.Cell): label_weights, self.cast(scaling_sens, mstype.float32)) + # apply grad reducer on grads + grads = self.grad_reducer(grads) + grads = self.hyper_map(F.partial(grad_scale, scaling_sens * self.degree), grads) + grads = self.hyper_map(F.partial(clip_grad, GRADIENT_CLIP_TYPE, GRADIENT_CLIP_VALUE), grads) - grads = self.hyper_map(F.partial(grad_scale, scaling_sens), grads) - grads = self.clip_gradients(grads, GRADIENT_CLIP_TYPE, GRADIENT_CLIP_VALUE) - if self.reducer_flag: - # Apply grad reducer on grads. - grads = self.grad_reducer(grads) - self.get_status(init) - flag_sum = self.reduce_sum(init, (0,)) - - if self.is_distributed: - # Sum overflow flag over devices. - flag_reduce = self.all_reduce(flag_sum) - cond = self.less_equal(self.base, flag_reduce) - else: - cond = self.less_equal(self.base, flag_sum) - + cond = self.get_overflow_status(status, grads) overflow = cond if sens is None: overflow = self.loss_scaling_manager(self.loss_scale, cond) - if not overflow: - self.optimizer(grads) - - return (loss, cond, scaling_sens) + if overflow: + succ = False + else: + succ = self.optimizer(grads) + ret = (loss, cond, scaling_sens) + return F.depend(ret, succ) diff --git a/model_zoo/research/nlp/seq2seq/train.py b/model_zoo/research/nlp/seq2seq/train.py index 23c9e0fb9d8..ef39d22e8fe 100644 --- a/model_zoo/research/nlp/seq2seq/train.py +++ b/model_zoo/research/nlp/seq2seq/train.py @@ -25,7 +25,7 @@ from mindspore.nn.optim import Lamb from mindspore.train.model import Model from mindspore.train.loss_scale_manager import DynamicLossScaleManager from mindspore.train.callback import CheckpointConfig, ModelCheckpoint, TimeMonitor -from mindspore.train.callback import LossMonitor, SummaryCollector +from mindspore.train.callback import LossMonitor from mindspore import context, Parameter from mindspore.context import ParallelMode from mindspore.communication import management as MultiAscend @@ -52,7 +52,7 @@ if args.is_modelarts: import moxing as mox context.set_context( mode=context.GRAPH_MODE, - save_graphs=True, + save_graphs=False, device_target="Ascend", reserve_class_name_in_scope=True) @@ -221,12 +221,12 @@ def _build_training_pipeline(config: Seq2seqConfig, loss_monitor = LossCallBack(config) dataset_size = dataset.get_dataset_size() time_cb = TimeMonitor(data_size=dataset_size) - ckpt_config = CheckpointConfig(save_checkpoint_steps=config.save_ckpt_steps, + ckpt_config = CheckpointConfig(save_checkpoint_steps=dataset.get_dataset_size(), keep_checkpoint_max=config.keep_ckpt_max) rank_size = os.getenv('RANK_SIZE') callbacks = [time_cb, loss_monitor] - callbacks.append(LossMonitor(1642)) + callbacks.append(LossMonitor()) if rank_size is not None and int(rank_size) > 1 and MultiAscend.get_rank() % 8 == 0: ckpt_callback = ModelCheckpoint( @@ -234,8 +234,6 @@ def _build_training_pipeline(config: Seq2seqConfig, directory=os.path.join(config.ckpt_path, 'ckpt_{}'.format(os.getenv('DEVICE_ID'))), config=ckpt_config) callbacks.append(ckpt_callback) - summary_callback = SummaryCollector(summary_dir="./summary", collect_freq=50) - callbacks.append(summary_callback) if rank_size is None or int(rank_size) == 1: ckpt_callback = ModelCheckpoint( @@ -243,8 +241,6 @@ def _build_training_pipeline(config: Seq2seqConfig, directory=os.path.join(config.ckpt_path, 'ckpt_{}'.format(os.getenv('DEVICE_ID'))), config=ckpt_config) callbacks.append(ckpt_callback) - summary_callback = SummaryCollector(summary_dir="./summary", collect_freq=50) - callbacks.append(summary_callback) print(f" | ALL SET, PREPARE TO TRAIN.") _train(model=model, config=config,