From f71d127c310e9b6244d914546bf6447a1bbe55c9 Mon Sep 17 00:00:00 2001 From: gaojing Date: Tue, 15 Jun 2021 13:54:37 -0400 Subject: [PATCH] cpm model added --- model_zoo/README.md | 1 + model_zoo/README_CN.md | 1 + model_zoo/official/nlp/cpm/README.md | 433 +++++++++++++++++ model_zoo/official/nlp/cpm/README_CN.md | 435 ++++++++++++++++++ .../data_process/make_finetune_mindrecord.py | 125 +++++ .../data_process/make_zero_shot_mindrecord.py | 140 ++++++ .../nlp/cpm/data_process/tokenizer_cpm.py | 71 +++ model_zoo/official/nlp/cpm/eval.py | 296 ++++++++++++ model_zoo/official/nlp/cpm/export.py | 87 ++++ .../official/nlp/cpm/gpt_ckpt_2_mindspore.py | 122 +++++ model_zoo/official/nlp/cpm/requirements.txt | 2 + ...n_distribute_train_ascend_multi_machine.sh | 89 ++++ ..._distribute_train_ascend_single_machine.sh | 85 ++++ .../cpm/scripts/run_eval_distribute_ascend.sh | 65 +++ .../cpm/scripts/run_test_distribute_ascend.sh | 77 ++++ ...n_zero-shot_inference_distribute_ascend.sh | 64 +++ ...n_zero-shot_inference_standalone_ascend.sh | 59 +++ model_zoo/official/nlp/cpm/src/attention.py | 283 ++++++++++++ model_zoo/official/nlp/cpm/src/config.py | 132 ++++++ model_zoo/official/nlp/cpm/src/cpm.py | 369 +++++++++++++++ model_zoo/official/nlp/cpm/src/cpm_loss.py | 168 +++++++ model_zoo/official/nlp/cpm/src/cpm_train.py | 386 ++++++++++++++++ model_zoo/official/nlp/cpm/src/embedding.py | 166 +++++++ .../official/nlp/cpm/src/loss_monitor.py | 121 +++++ model_zoo/official/nlp/cpm/src/lr_schedule.py | 73 +++ model_zoo/official/nlp/cpm/src/model_cpm.py | 172 +++++++ model_zoo/official/nlp/cpm/src/util.py | 230 +++++++++ model_zoo/official/nlp/cpm/src/weight_init.py | 63 +++ model_zoo/official/nlp/cpm/test.py | 115 +++++ model_zoo/official/nlp/cpm/train.py | 269 +++++++++++ model_zoo/official/nlp/cpm/zero-shot.py | 289 ++++++++++++ 31 files changed, 4988 insertions(+) create mode 100644 model_zoo/official/nlp/cpm/README.md create mode 100644 model_zoo/official/nlp/cpm/README_CN.md create mode 100644 model_zoo/official/nlp/cpm/data_process/make_finetune_mindrecord.py create mode 100644 model_zoo/official/nlp/cpm/data_process/make_zero_shot_mindrecord.py create mode 100644 model_zoo/official/nlp/cpm/data_process/tokenizer_cpm.py create mode 100644 model_zoo/official/nlp/cpm/eval.py create mode 100644 model_zoo/official/nlp/cpm/export.py create mode 100644 model_zoo/official/nlp/cpm/gpt_ckpt_2_mindspore.py create mode 100644 model_zoo/official/nlp/cpm/requirements.txt create mode 100644 model_zoo/official/nlp/cpm/scripts/run_distribute_train_ascend_multi_machine.sh create mode 100644 model_zoo/official/nlp/cpm/scripts/run_distribute_train_ascend_single_machine.sh create mode 100644 model_zoo/official/nlp/cpm/scripts/run_eval_distribute_ascend.sh create mode 100644 model_zoo/official/nlp/cpm/scripts/run_test_distribute_ascend.sh create mode 100644 model_zoo/official/nlp/cpm/scripts/run_zero-shot_inference_distribute_ascend.sh create mode 100644 model_zoo/official/nlp/cpm/scripts/run_zero-shot_inference_standalone_ascend.sh create mode 100644 model_zoo/official/nlp/cpm/src/attention.py create mode 100644 model_zoo/official/nlp/cpm/src/config.py create mode 100644 model_zoo/official/nlp/cpm/src/cpm.py create mode 100644 model_zoo/official/nlp/cpm/src/cpm_loss.py create mode 100644 model_zoo/official/nlp/cpm/src/cpm_train.py create mode 100644 model_zoo/official/nlp/cpm/src/embedding.py create mode 100644 model_zoo/official/nlp/cpm/src/loss_monitor.py create mode 100644 model_zoo/official/nlp/cpm/src/lr_schedule.py create mode 100644 model_zoo/official/nlp/cpm/src/model_cpm.py create mode 100644 model_zoo/official/nlp/cpm/src/util.py create mode 100644 model_zoo/official/nlp/cpm/src/weight_init.py create mode 100644 model_zoo/official/nlp/cpm/test.py create mode 100644 model_zoo/official/nlp/cpm/train.py create mode 100644 model_zoo/official/nlp/cpm/zero-shot.py diff --git a/model_zoo/README.md b/model_zoo/README.md index 64b030b7967..ab683b63149 100644 --- a/model_zoo/README.md +++ b/model_zoo/README.md @@ -51,6 +51,7 @@ In order to facilitate developers to enjoy the benefits of MindSpore framework, - [LSTM](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/nlp/lstm/README.md) - [MASS](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/nlp/mass/README.md) - [Transformer](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/nlp/transformer/README.md) + - [CPM](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/nlp/cpm/README.md) - [Recommender Systems](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/recommend) - [DeepFM](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/recommend/deepfm/README.md) - [NAML](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/recommend/naml/README.md) diff --git a/model_zoo/README_CN.md b/model_zoo/README_CN.md index 1be1b214bb8..07bcba8648c 100644 --- a/model_zoo/README_CN.md +++ b/model_zoo/README_CN.md @@ -51,6 +51,7 @@ - [LSTM](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/nlp/lstm/README.md) - [MASS](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/nlp/mass/README.md) - [Transformer](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/nlp/transformer/README.md) + - [CPM](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/nlp/cpm/README.md) - [推荐系统](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/recommend) - [DeepFM](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/recommend/deepfm/README.md) - [NAML](https://gitee.com/mindspore/mindspore/tree/master/model_zoo/official/recommend/naml/README.md) diff --git a/model_zoo/official/nlp/cpm/README.md b/model_zoo/official/nlp/cpm/README.md new file mode 100644 index 00000000000..d73311619ca --- /dev/null +++ b/model_zoo/official/nlp/cpm/README.md @@ -0,0 +1,433 @@ +# Contents + +[查看中文](./README_CN.md) + + + +- [CPM Description](#CPM-Description) +- [Model Architecture](#Model-Architecture) +- [Dataset](#Dataset) +- [Environment Requirements](#Environment-Requirements) +- [Quick Start](#Quick Start) +- [Script Description](#Script-Description) + - [Script and Sample Code](#Script and Sample Code) + - [Script Parameters](#Script Parameters) + - [Zero-shot Inference](#zero-shot inference) + - [Pre-training Model Download](#Pre-training Model Download) + - [Dataset Preparation](#Dataset Preparation) + - [Zero-shot Inference Process](#Zero-shot Inference Process) + - [Finetune](#Finetune) + - [Dataset Preparation](#Dataset Preparation) + - [Finetune Training Process](#Finetune Training Process) + - [Evaluation Process](#Evaluation Process) +- [Performance](#Performance) + - [Zero-shot Performance](#Zero-shot Performance) + - [Finetune Performance](#Finetune Performance) +- [Description of Random Situation](#Description of Random Situation) +- [Other](#Other) +- [ModelZoo Homepage](#ModelZoo Homepage) + + + +# CPM CPM-Description + +This is the fine-tune code warehouse of CPM model, which can be used for multi-machine and multi-card training/testing of model finetune. CPM[Project Home Page](https://cpm.baai.ac.cn/) was proposed in 2020 and a large-scale model based on Chinese processing. CPM is mainly used in the field of Chinese natural language processing (NLP) and generating tasks, such as machine translation, word selection and text summarization. + +[Paper](https://arxiv.org/abs/2012.00413): Zhang Z, Han X, Zhou H, et al. CPM: A Large-scale Generative Chinese Pre-trained Language Model[J]. arXiv preprint arXiv:2012.00413, 2020. + +# Model Architecture + +CPM is implemented by GPT, which includes multi-layer decoder module. + +# Dataset + +- Training dataset*ChID* +- Training dataset*ChID* + ChID is a large-scale dataset of Chinese idioms for cloze, and it comes from the paper [ChID: A Large-scale Chinese IDiom Dataset for Cloze Test](https://www.aclweb.org/anthology/P19-1075/). This warehouse uses [Json format](https://drive.google.com/file/d/1KkwLSLgrV9JknO8rxxfmU5Iql-D4O_-6/view). + +# Environment Requirements + +- Hardware（Ascend) + - Prepare hardware environment with Ascend processor. +- Framework + - [MindSpore](https://gitee.com/mindspore/mindspore) +- For more information, please check the resources below: + - [MindSpore Tutorials](https://www.mindspore.cn/tutorial/training/zh-CN/master/index.html) + - [MindSpore Python API](https://www.mindspore.cn/doc/api_python/zh-CN/master/index.html) + +# Quick Start + +After dataset preparation, you can start zero-shot inference and finetune, evaluation as follows: + +```bash +# run zero-shot inference example +cd scripts +sh run_zero-shot_inference_distribute_ascend.sh /path/test.mindrecord /path/true_labels.txt /path/cpm_mindspore_1p_fp32.ckpt /path/rank_table_2p.json + +# run distributed finetune example +cd scripts +sh run_distribute_train_ascend_single_machine.sh /path/train.mindrecord /path/cpm_mindspore_1p_fp32.ckpt /path/rank_table_8p.json + +# run evaluation example +cd scripts +bash run_eval_distribute_ascend.sh /path/finetune_test.mindrecord /path/test.json /path/ckpt_dictionary/ 8 /path/rank_table_2p.json + +# Selects the best model on the dev dataset, and then tests the example on the test dataset +cd scripts +bash run_test_distribute_ascend.sh /path/finetune_dev.mindrecord /path/dev.json /path/finetune_test.mindrecord /path/test.json /path/ckpt_dictionary/ 8 /path/rank_table_2p.json +``` + +# Script Description + +## Script and Sample Code + +```shell +. +└─CPM + ├─README.md // Introduction of CPM model. + ├─scripts + ├─run_zero-shot_inference_standalone_ascend.sh // Shell script for standalone zero-shot on ascend. + ├─run_zero-shot_inference_distribute_ascend.sh // Shell script for distributed zero-shot on ascend. + ├─run_distribute_train_ascend_single_machine.sh // Shell script for distributed finetune on ascend with single machine. + ├─run_distribute_train_ascend_multi_machine.sh // Shell script for distributed finetune on ascend with multi-machine. + ├─run_test_distribute_ascend.sh // Shell script for distributed evaluation and test on ascend. + └─run_eval_distribute_ascend.sh // Shell script for distributed evaluation on ascend. + ├─data_process + ├─make_zero_shot_mindrecord.py // Make dataset for zero-shot. + ├─make_finetune_mindrecord.py // Make dataset for finetune. + └─tokenizer_cpm.py // Tokenization. + ├─src + ├─attention.py // attention mechanism. + ├─config.py // Configuration file for zero-shot or finetune. + ├─cpm_loss.py // Loss function. + ├─cpm.py // CPM model. + ├─cpm_train.py // Use CPM to train. + ├─embedding.py // Embedding component. + ├─loss_monitor.py // Callback of monitering loss during training step. + ├─lr_schedule.py // Learning rate scheduler. + ├─model_cpm.py // Model use for gradient cumulative. + ├─util.py // User interface. + └─weight_init.py // Weight init. + ├─gpt_ckpt_2_mindspore.py // Transform the model that MindSpore can load. + ├─requirements.txt // Requirements of third party package. + ├─zero-shot.py // Zero-shot api entry. + ├─export.py // Export model. + ├─train.py // Train api entry. + ├─test.py // Evaluation and test api entry. + └─eval.py // Infer api entry. + +``` + +## Script Parameters + +The CPM network configuration parameters are in `src/config.py`, and the main parameters are described as follows: + +```text +Parameters for dataset and network (Training/Evaluation): + mp Number of Model parallel. + batch_size Global batch size of input dataset. + seq_length max length of input sequence. + vocab_size size of each embedding vector. + hidden_size size of Transformer encoder layers. + num_hidden_layers number of hidden layers. + num_attention_heads number of attention heads. + lr init learning rate. + end_learning_rate end of learning rate. + weight_decay weight decay. + warmup_steps_rate rate of warmup steps. + dropout dropout probability. + grad_accumulation_step gradient cumulative steps. + sink_size control the amount of data in each sink. + epoch total number of iterations on the data per epoch. +``` + +## Zero-shot Inference + +### Pre-training Model Download + +- The CPM network pre training model can be downloaded here: [Model Download](https://cpm.baai.ac.cn/download.html). + Suppose you have the following documents: + - CPM-large/latest_checkpointed_iteration.txt + - CPM-large/80000/mp_rank_00_model_states.pt + - CPM-large/80000/mp_rank_01_model_states.pt + Next, the model integration script[change_mp.py](https://github.com/TsinghuaAI/CPM-Generate/blob/main/change_mp.py) is used to synthesize the above two fragment models into a complete single model. + +```[bash] + python change_mp.py /path/to/CPM 1 +``` + + The complete single model is as follows: + - CPM-large_MP1/latest_checkpointed_iteration.txt + - CPM-large_MP1/iter_0080000/mp_rank_01_model_states.pt + Then run the file`gpt_ckpt_2_mindspore.py` in the warehouse to convert the model into the model that can be loaded directly by Mindstore in the warehouse. Pay attention to modify the input and output file address in the file. + We get the model that mindpool can load, such as:`cpm_mindspore_1p_fp32.ckpt`. + +- Word segmentation Download: [Model Download](https://github.com/TsinghuaAI/CPM-Finetune/tree/main/bpe_3w_new). + Suppose you have the following documents: + - bpe_3w_new/chinese_vocab.model + - bpe_3w_new/chinese_vocab.vocab + - bpe_3w_new/vocab.json + +### Dataset Preparation + +- The original dataset download address is [ChiD-Dataset](https://drive.google.com/drive/folders/1gL01xbFBcrgP0TmgOhJ_uplkeG-BCwvM)，and we can refer to [ChiD-Dataset](https://github.com/chujiezheng/ChID-Dataset). + Suppose you have the following documents: + - chid_json/train.json + - chid_json/train_answer.json + - chid_json/dev.json + - chid_json/dev_answer.json + - chid_json/test.json + - chid_json/test_answer.json + +- Data preprocessing: you may use [preprocess_chid_zeroshot.py](https://github.com/TsinghuaAI/CPM-Finetune/blob/main/preprocess_chid_zeroshot.py)to process the original data into the corresponding JSON format. + +```[bash] + python preprocess_chid_zeroshot.py --data_dir ${PATH_TO_DATA_DIR} --tokenizer_path ${PATH_TO_TOKENIZER VOCAB} --output_dir ${PATH_TO_OUTPUT_JSON} +``` + +Mainly, `data_dir` is the address of the json data, such as `/home/dataset/chid_json`. + `tokenizer_path` is the address folder for the dictionary, such as `/home/bpe_3w_new/`. + `output_dir` is the preprocessing output address, such as`/home/dataset/test_dataset`. + +The file will fill each candidate idiom into the corresponding blank of the article, and each blank will generate 10 new candidate articles. Finally, the data format generated by the file is as follows: + +```[python] +{ + "contents": [ + [8, 15, ....], + .... + ], # After BPE word segmentation, all samples have the ID corresponding to the token. + "sids": [ + 0, + 0, + ... + 1, + 1, + ... + ], # Each generated candidate article corresponds to the number of the original sample. + "cids": [ + 0, + 1, + 2, + ... + 9, + 0, + 1, + ... + ], # The number of idioms corresponding to each generated candidate article. + "labels": [ + 3, + 2, + ... + ], # Correct answer number of each original sample (integer between 0 and 9). +} +``` + +After the data preprocessing,the `test.json` file will be generated in the `--output_dir` directory. + +- Then the file `test.json` obtained by data preprocessing is converted to mindrecord format: + +```[bash] + python make_zero_shot_mindrecord.py --data_file ${PATH_TO_DATA_FILE} --vocab_path ${PATH_TO_TOKENIZER VOCAB} --output_path ${PATH_TO_OUTPUT FILE} +``` + +Mainly, `data_file` is the data address, such as `/home/dataset/test_dataset/test.json`. + `vocab_path` is the address folder directory of the dictionary, its definition is the same as before. + `output_path` is the output result file of the generated mindrecord, such as `/home/dataset/test_dataset/test.mindrecord`. + +After processing, the specified directory `--output_path` will generate the inference mindrecord file and the ground file `true_labels.txt` in the same directory. + +### Zero-shot Inference Process + +- Set parameters in file `src/config.py`. +- Run`run_zero-shot_inference_distribute_ascend.sh` to zero shot inference. + +```bash + cd scripts + bash run_zero-shot_inference_distribute_ascend.sh Test_MindRecord_addr test_json_addr model_addr rank_table_addr +``` + +Mainly, `Test_MindRecord_addr` is the address of the dataset, such as `/home/dataset/test_dataset/test.mindrecord`. + `test_json_addr` is the groundtruth file obtained from the data preprocessing, such as `/home/dataset/test_dataset/true_labels.txt`. + `model_addr` is the pre training model address, such as `/home/cpm_ckpt_ms/cpm_mindspore_1p.ckpt`. + `rank_table_addr` is a rank address for distributed reasoning, such as `/home/rank_table_2p.json`. +After reasoning, the accuracy rate will be generated. Please refer to the`zero-shot.py` file of thie warehouse for details. + +## Finetune + +In addition to zero shot reasoning, the pre training model can also be trained by finetune. + +### Dataset Preparation + +- The original data set is downloaded as above. + Suppose you have the following documents: + - chid_json/train.json + - chid_json/train_answer.json + - chid_json/dev.json + - chid_json/dev_answer.json + - chid_json/test.json + - chid_json/test_answer.json + +- Data preprocessing: you may use [preprocess_chid_finetune.py](https://github.com/TsinghuaAI/CPM-Finetune/blob/main/preprocess_chid_finetune.py) scripts process the original data into the corresponding JSON format. + +```[bash] + python preprocess_chid_finetune.py --data_dir ${PATH_TO_DATA_DIR} --tokenizer_path ${PATH_TO_TOKENIZER VOCAB} --output_dir ${PATH_TO_OUTPUT_JSON} +``` + +Mainly, `data_dir` is the address of the json data, such as `/home/dataset/chid_json`. + `tokenizer_path` is the address folder for the dictionary, such as `/home/vocab/`. + `output_dir` is the preprocessing output address, such as `/home/dataset/finetune_dataset`. + +The template is defined and implemented in `process_sample` function of the `preprocess_chid_finetune.py` file. Finally, the data format generated by the file is as follows: + +```[python] +[ + { + "sent": [8, 15, ....], # The ID corresponding to the token after BPE segmentation. + "truth": 3 # The number of the correct answer idiom (an integer between 0 and 9). + } + ... +] +``` + +After processing, three files, namely `train.json`, `valid.json` and `test.json` will be generated in the output directory `--output_dir`. + +- After data preprocessing, the JSON data is transformed into mindrecord data set. + +```[bash] + cd ./data_process/ + python3 make_finetune_mindrecord.py --data_file ${PATH_TO_OUTPUT_JSON} --vocab_path ${PATH_TO_TOKENIZER VOCAB} --output_path ${PATH_TO_OUTPUT FILE} --num_patitions ${NUMBER_OF_MINDRECORD_PARTITIONS} +``` + +Mainly, `data_file` is the JSON data address, such as`/home/dataset/finetune_dataset/train.json` and `/home/dataset/finetune_dataset/test.json`. + `vocab_path` is the address folder directory of the dictionary, its definition is the same as before. + `output_path` is the preprocessing output address, such as`/home/dataset/finetune_dataset/`. + +After processing, the mindrecord file of training and reasoning is generated in the specified directory `--output_path`, such as`train.mindrecord` and`test.mindrecord`. + +### Finetune Training Process + +- Set options in `src/config.py`, including loss_scale, learning rate and network hyperparameters. Click [here](https://www.mindspore.cn/tutorial/training/zh-CN/master/use/data_preparation.html) for more information about dataset. + +- Run `run_distribute_train_ascend_single_machine.sh` for distributed and single machine training of CPM model. + +``` bash + cd scripts + bash run_distribute_train_ascend_single_machine.sh Dataset_addr PreTrain_ckpt_addr Rank_table_addr +``` + +- Run `run_distribute_train_ascend_multi_machine.sh`，for distributed and multi-machines training of CPM model. + +``` bash + cd scripts + bash run_distribute_train_ascend_multi_machine.sh Dataset_addr PreTrain_ckpt_addr Rank_table_addr SERVER_ID +``` + +Mainly, `Dataset_addr` is the address of the dataset, such as `/home/dataset/finetune_dataset/train.mindrecord`. + `PreTrain_ckpt_addr` is the address for the pre training model, such as `/home/cpm_mindspore_1p_fp32.ckpt`. + `Rank_table_addr` is a rank address for distributed training, such as `/home/rank_table_8p.json`. + `SERVER_ID` is the sequence of the machine numbers from 0 in the multi machine process, such as: 0. + +**Attention**：Because the CPM model is too large to train on one card, distributed training is needed, including model parallel and data parallel. + In distributed parallel training, the device of the machine is the device of the device_ ID is numbered from 1 and incremented by 1. + Run a stand-alone 8 card, the index of rank is numbered from 0,2,4,6,1,3,5,7. When running multiple machines, the rank of the first machine's ID is 0,2,4,6,1,3,5,7. Rank of the second machine_ ID is 8,10,12,14,9,11,13,15. The rank of other machines and so on. + +### Evaluation Process + +- Set options in `src/config.py`. +- After finetune training, place the partition model with a specified epoch number to the same directory, including `train_strategy.ckpt`, `cpm_rank_1-*.ckpt`, where `train_strategy.ckpt` is the distributed policy file. +- Run `run_eval_distribute_ascend.sh` to evaluate. + +```bash + cd scripts + bash run_eval_distribute_ascend.sh Test_MindRecord_addr Test_json_addr Model_addr Model_patition_number Rank_table_addr +``` + +In general, we select the model with the highest accuracy on the dev dataset, then infer on the test dataset, and finally generate the accuracy on the test dataset. Model selection can refer to the `run_test_distribute_ascend.sh` and `test.py` files for details. + +```bash + cd scripts + bash run_test_distribute_ascend.sh Dev_MindRecord_addr Dev_json_addr Test_MindRecord_addr Test_json_addr Model_addr Model_patition_number Rank_table_addr +``` + +Mainly, `Test_MindRecord_addr` is the address of the test dataset, such as `/home/dataset/finetune_dataset/test.mindrecord`. + `Test_json_addr` is the test JSON file after data preprocessing, such as `/home/dataset/finetune_dataset/test.json`. + `Dev_MindRecord_addr` is the address of the dev dataset, such as `/home/dataset/finetune_dataset/dev.mindrecord`. + `Dev_json_addr` is the dev JSON file after data preprocessing, such as `/home/dataset/finetune_dataset/dev.json`. + `Model_addr` is used to infer the partition model of the folder, such as `/home/finetune_model/`. + `Model_patition_number`is the number of fragmentation models，excluding policy file`train_strategy.ckpt`. For example, after 8-card training, the number of partitioned models is 8. + `Rank_table_addr` is a rank address for distributed evaluation, such as `/home/rank_table_2p.json`. + +**Attention**: the dataset preprocessing methods of zero-shot and finetuene are different. + +# Performance + +## Zero-shot Performance + +The inference performance and accuracy of zero-shot single machine and dual cards are as follows: + +| Parameters | Ascend | +| -------------------------- | --------------------------- | +| Resource |Ascend 910；CPU 2.60GHz，192 cores；Memory 755GB；OS Euler2.8 | +| MindSpore Version | 1.3.0 | +| Dataset | ChID | +| Number of parallel models | 2 | +| Speed | 152ms/step (2pcs) | +| batch_size | 2 | +| Output | Accuracy | +| Accuracy | accuracy=67.94% | + +## Finetune Performance + +The finetune performance and accuracy of single machine and 8 cards are as follows: + +| Parameters | Ascend | +| -------------------------- | -------------------------------------------------------------- | +| Resource |Ascend 910；CPU 2.60GHz，192 cores；Memory 755GB；OS Euler2.8 | +| uploaded Date | 2021-06-07 | +| MindSpore Version | 1.3.0 | +| Dataset | ChID | +| Training Parameters | epoch=10, global_batch_size=16 | +| Number of parallel models | 2 | +| Optimizer | Adam | +| Accuracy | 80.4% | +| Speed | 1683ms/step (8pcs) | +| Loss | 0.7 | +| Params (M) | 2597.1 | +| Checkpoint for inference | 76G （.ckpt file） | +| Scripts | | + +The finetune performance and accuracy of 4 machines and 32 cards are as follows: + +| Parameters | Ascend | +| -------------------------- | -------------------------------------------------------------- | +| Resource |Ascend 910；CPU 2.60GHz，192 cores；Memory 755GB；OS Euler2.8 | +| uploaded Date | 2021-06-07 | +| MindSpore Version | 1.3.0 | +| Dataset | ChID | +| Training Parameters | epoch=10, global_batch_size=128 | +| Number of parallel models | 2 | +| Optimizer | Adam | +| Accuracy | 81.4% | +| Speed | 2740ms/step (32pcs) | +| Loss | 0.008 | +| Params (M) | 2597.1 | +| Checkpoint for inference | 57G （.ckpt file） | +| Scripts | | + +# Description of Random Situation + +There are two random situations: + +- Shuffle of the dataset. +- Dropout operations. + +Some seeds have already been set in train.py to avoid the randomness of dataset shuffle and weight initialization. If you want to disable dropout, please set the corresponding dropout_prob parameter to 0 in src/config.py. + +# Other + +The accuracy and performance of this model have been verified in Ascend environment, but not in CPU and GPU. + +# ModelZoo Homepage + +Please check the official [homepage](https://gitee.com/mindspore/mindspore/tree/master/model_zoo). diff --git a/model_zoo/official/nlp/cpm/README_CN.md b/model_zoo/official/nlp/cpm/README_CN.md new file mode 100644 index 00000000000..81cf67ba90e --- /dev/null +++ b/model_zoo/official/nlp/cpm/README_CN.md @@ -0,0 +1,435 @@ +# 目录 + +[view English](./README.md) + + + +- [目录](#目录) +- [CPM 概述](#CPM-概述) +- [模型架构](#模型架构) +- [数据集](#数据集) +- [环境要求](#环境要求) +- [快速入门](#快速入门) +- [脚本说明](#脚本说明) + - [脚本和样例代码](#脚本和样例代码) + - [网络参数说明](#网络参数说明) + - [Zero-shot推理](#Zero-shot推理) + - [预训练模型下载](#预训练模型下载) + - [Zero-shot准备数据集](#Zero-shot准备数据集) + - [Zero-shot推理过程](#Zero-shot推理过程) + - [Finetune微调训练](#Finetune微调训练) + - [Finetune准备数据集](#准备数据集) + - [Finetune训练过程](#训练过程) + - [Finetune评估过程](#评估过程) +- [性能和精度](#性能和精度) + - [Zero-shot评估性能和精度](#Zero-shot评估性能) + - [Finetune训练性能和精度](#Finetune训练性能) +- [随机情况说明](#随机情况说明) +- [其他](#其他) +- [ModelZoo主页](#modelzoo主页) + + + +# CPM 概述 + +本仓库为CPM模型的fine-tune代码仓库，可用于模型finetune的多机多卡训练/测试。CPM网络[项目首页](https://cpm.baai.ac.cn/)于2020年提出，是一种以中文处理为核心的大规模模型。CPM主要应用于中文自然语言处理（NLP）领域、生成任务等,如机器翻译、选词填空或文本摘要等任务。 + +[论文](https://arxiv.org/abs/2012.00413): Zhang Z, Han X, Zhou H, et al. CPM: A Large-scale Generative Chinese Pre-trained Language Model[J]. arXiv preprint arXiv:2012.00413, 2020. + +# 模型架构 + +CPM网络由GPT实现，GPT包括多层解码器模块。 + +# 数据集 + +- 训练数据集*ChID* +- 评估数据集*ChID* + ChID 是一种面向完形填空的大规模汉语成语数据集，其来源于论文 [ChID: A Large-scale Chinese IDiom Dataset for Cloze Test](https://www.aclweb.org/anthology/P19-1075/). 本仓库中使用 [Json 格式](https://drive.google.com/file/d/1KkwLSLgrV9JknO8rxxfmU5Iql-D4O_-6/view). + +# 环境要求 + +- 硬件（Ascend处理器） + - 使用Ascend处理器准备硬件环境。 +- 框架 + - [MindSpore](https://gitee.com/mindspore/mindspore) +- 如需查看详情，请参见如下资源： + - [MindSpore教程](https://www.mindspore.cn/tutorial/training/zh-CN/master/index.html) + - [MindSpore Python API](https://www.mindspore.cn/doc/api_python/zh-CN/master/index.html) + +# 快速入门 + +数据集准备完成后，请按照如下步骤开始Zero-shot推理、Finetune训练和评估： + +```bash +# zero-shot推理示例 +cd scripts +sh run_zero-shot_inference_distribute_ascend.sh /path/test.mindrecord /path/true_labels.txt /path/cpm_mindspore_1p_fp32.ckpt /path/rank_table_2p.json + +# 运行分布式训练Finetune示例 +cd scripts +sh run_distribute_train_ascend_single_machine.sh /path/train.mindrecord /path/cpm_mindspore_1p_fp32.ckpt /path/rank_table_8p.json + +# Finetune模型评估示例 +cd scripts +bash run_eval_distribute_ascend.sh /path/finetune_test.mindrecord /path/test.json /path/ckpt_dictionary/ 8 /path/rank_table_2p.json + +# Finetune模型在dev数据集上选最优再在test数据集上测试示例 +cd scripts +bash run_test_distribute_ascend.sh /path/finetune_dev.mindrecord /path/dev.json /path/finetune_test.mindrecord /path/test.json /path/ckpt_dictionary/ 8 /path/rank_table_2p.json +``` + +# 脚本说明 + +## 脚本和样例代码 + +```shell +. +└─CPM + ├─README.md // Introduction of CPM model. + ├─scripts + ├─run_zero-shot_inference_standalone_ascend.sh // Shell script for standalone zero-shot on ascend. + ├─run_zero-shot_inference_distribute_ascend.sh // Shell script for distributed zero-shot on ascend. + ├─run_distribute_train_ascend_single_machine.sh // Shell script for distributed finetune on ascend with single machine. + ├─run_distribute_train_ascend_multi_machine.sh // Shell script for distributed finetune on ascend with multi-machine. + ├─run_test_distribute_ascend.sh // Shell script for distributed evaluation and test on ascend. + └─run_eval_distribute_ascend.sh // Shell script for distributed evaluation on ascend. + ├─data_process + ├─make_zero_shot_mindrecord.py // Make dataset for zero-shot. + ├─make_finetune_mindrecord.py // Make dataset for finetune. + └─tokenizer_cpm.py // Tokenization. + ├─src + ├─attention.py // attention mechanism. + ├─config.py // Configuration file for zero-shot or finetune. + ├─cpm_loss.py // Loss function. + ├─cpm.py // CPM model. + ├─cpm_train.py // Use CPM to train. + ├─embedding.py // Embedding component. + ├─loss_monitor.py // Callback of monitering loss during training step. + ├─lr_schedule.py // Learning rate scheduler. + ├─model_cpm.py // Model use for gradient cumulative. + ├─util.py // User interface. + └─weight_init.py // Weight init. + ├─gpt_ckpt_2_mindspore.py // Transform the model that MindSpore can load. + ├─requirements.txt // Requirements of third party package. + ├─zero-shot.py // Zero-shot api entry. + ├─export.py // Export model. + ├─train.py // Train api entry. + ├─test.py // Evaluation and test api entry. + └─eval.py // Infer api entry. + +``` + +## 网络参数说明 + +网络配置参数在src/config.py中，将主要参数说明： + +```text +Parameters for dataset and network (Training/Evaluation): + mp Number of Model parallel. + batch_size Global batch size of input dataset. + seq_length max length of input sequence. + vocab_size size of each embedding vector. + hidden_size size of Transformer encoder layers. + num_hidden_layers number of hidden layers. + num_attention_heads number of attention heads. + lr init learning rate. + end_learning_rate end of learning rate. + weight_decay weight decay. + warmup_steps_rate rate of warmup steps. + dropout dropout probability. + grad_accumulation_step gradient cumulative steps. + sink_size control the amount of data in each sink. + epoch total number of iterations on the data per epoch. +``` + +## Zero-shot推理 + +### 预训练模型下载 + +- CPM网络预训练模型下载：[模型下载](https://cpm.baai.ac.cn/download.html)。 + 假设您已获得下列文件： + - CPM-large/latest_checkpointed_iteration.txt + - CPM-large/80000/mp_rank_00_model_states.pt + - CPM-large/80000/mp_rank_01_model_states.pt + 接下来，您可能会使用模型合并脚本[change_mp.py](https://github.com/TsinghuaAI/CPM-Generate/blob/main/change_mp.py)将上述两个分片模型合成完整的单个模型： + +```[bash] + python change_mp.py /path/to/CPM 1 +``` + + 上述，得到完整的单个模型： + - CPM-large_MP1/latest_checkpointed_iteration.txt + - CPM-large_MP1/iter_0080000/mp_rank_01_model_states.pt + 再运行本仓库中的`gpt_ckpt_2_mindspore.py`文件将模型转化为本仓库中mindspore能直接加载的模型，注意修改该文件中的输入输出文件地址。 + 由此得到mindspore可加载的模型，如：`cpm_mindspore_1p_fp32.ckpt`。 + +- 分词器下载：[模型下载](https://github.com/TsinghuaAI/CPM-Finetune/tree/main/bpe_3w_new)。 + 假设您已获得下列文件： + - bpe_3w_new/chinese_vocab.model + - bpe_3w_new/chinese_vocab.vocab + - bpe_3w_new/vocab.json + +### Zero-shot准备数据集 + +- 原始数据集下载地址[ChiD-Dataset](https://drive.google.com/drive/folders/1gL01xbFBcrgP0TmgOhJ_uplkeG-BCwvM)，可参考[ChiD-Dataset说明](https://github.com/chujiezheng/ChID-Dataset)。 + 假设您已获得下列文件： + - chid_json/train.json + - chid_json/train_answer.json + - chid_json/dev.json + - chid_json/dev_answer.json + - chid_json/test.json + - chid_json/test_answer.json + +- 数据预处理：您可能会使用脚本[preprocess_chid_zeroshot.py](https://github.com/TsinghuaAI/CPM-Finetune/blob/main/preprocess_chid_zeroshot.py)将原始数据处理成相应的json格式。 + +```[bash] + python preprocess_chid_zeroshot.py --data_dir ${PATH_TO_DATA_DIR} --tokenizer_path ${PATH_TO_TOKENIZER VOCAB} --output_dir ${PATH_TO_OUTPUT_JSON} +``` + +主要地，`data_dir`是数据集的地址，如`/home/dataset/chid_json`； + `tokenizer_path`为字典的地址文件夹，如`/home/bpe_3w_new/`； + `output_dir`为预处理输出结果地址,如`/home/dataset/test_dataset`。 + +该文件会将每个候选的成语填入文章相应的空白中，每个空白生成10个新的候选文章。最终，该文件生成的数据格式为： + +```[python] +{ + "contents": [ + [8, 15, ....], + .... + ], # 所有样本经过 bpe 分词之后 token 对应的 id。 + "sids": [ + 0, + 0, + ... + 1, + 1, + ... + ], # 每个生成出的候选文章对应原来样本的编号 + "cids": [ + 0, + 1, + 2, + ... + 9, + 0, + 1, + ... + ], # 每个生成出的候选文章对应的成语的编号 + "labels": [ + 3, + 2, + ... + ], # 每个原样本的正确答案编号（0~9之间的整数） +} +``` + +预处理完成后，在上述指定的`--output_dir`输出目录下会生成`test.json`文件。 + +- 将上一步得到的`--output_dir`路径下产生的json数据转换为MindRecord数据格式： + +```[bash] + python make_zero_shot_mindrecord.py --data_file ${PATH_TO_DATA_FILE} --vocab_path ${PATH_TO_TOKENIZER VOCAB} --output_path ${PATH_TO_OUTPUT FILE} +``` + +主要地，`data_file`是数据地址，如`/home/dataset/test_dataset/test.json`； + `vocab_path`为字典的地址文件夹目录，同上； + `output_path`为生成的mindrecord的输出结果文件,如`/home/dataset/test_dataset/test.mindrecord`。 + +处理完成后，指定的`--output_path`目录下生成推理的mindrecord文件，以及同目录下生成ground_truth文件`true_labels.txt`。 + +### Zero-shot推理过程 + +- 在`src/config.py`中设置参数； +- 运行`run_zero-shot_inference_distribute_ascend.sh`，进行zero-shot推理。 + +```bash + cd scripts + bash run_zero-shot_inference_distribute_ascend.sh Test_MindRecord_addr test_json_addr model_addr rank_table_addr +``` + +主要地， `Test_MindRecord_addr`为推理数据集mindrecord，如`/home/dataset/test_dataset/test.mindrecord`； + `test_json_addr`为预处理后的数据集的groundtruth文件，如`/home/dataset/test_dataset/true_labels.txt`； + `model_addr`为预训练模型地址，如`/home/cpm_ckpt_ms/cpm_mindspore_1p.ckpt`； + `rank_table_addr`为进行推理的时候的分布式推理的rank_table地址，如`/home/rank_table_2p.json`。 + +推理完后，会生成准确率，具体可参考本仓库`zero-shot.py`文件。 + +## Finetune微调训练 + +上述预训练模型除了可以进行zero-shot推理外，还可以进行Finetune训练。 + +### Finetune准备数据集 + +- 原始数据集下载同上。 + 假设您已获得下列文件： + - chid_json/train.json + - chid_json/train_answer.json + - chid_json/dev.json + - chid_json/dev_answer.json + - chid_json/test.json + - chid_json/test_answer.json + +- 数据预处理：您可能会使用脚本[preprocess_chid_finetune.py](https://github.com/TsinghuaAI/CPM-Finetune/blob/main/preprocess_chid_finetune.py)将原始数据处理成相应的json格式。 + +```[bash] + python preprocess_chid_finetune.py --data_dir ${PATH_TO_DATA_DIR} --tokenizer_path ${PATH_TO_TOKENIZER VOCAB} --output_dir ${PATH_TO_OUTPUT_JSON} +``` + +主要地，`--data_dir`是数据集的地址，如`/home/dataset/chid_json`；`--tokenizer_path`为字典的地址文件夹，如`/home/vocab/`； + `--output_dir`为预处理输出结果地址,如`/home/dataset/finetune_dataset`。 + +其中，模板定义与实现在 `preprocess_chid_finetune.py` 文件 `process_sample` 函数中。最终，该文件生成的数据格式为： + +```[python] +[ + { + "sent": [8, 15, ....], # 经过 bpe 分词之后 token 对应的 id + "truth": 3 # 正确答案成语的编号（0~9之间的整数） + } + ... +] +``` + +处理完成后，在上述指定的`--output_dir`输出目录下会生成 `train.json`, `valid.json`, `test.json` 三个文件。 + +- 将上一步得到的`--output_dir`路径下产生的json数据转换为MindRecord数据格式进行训练： + +```[bash] + cd ./data_process/ + python3 make_finetune_mindrecord.py --data_file ${PATH_TO_OUTPUT_JSON} --vocab_path ${PATH_TO_TOKENIZER VOCAB} --output_path ${PATH_TO_OUTPUT FILE} --num_patitions ${NUMBER_OF_MINDRECORD_PARTITIONS} +``` + +主要地，`--data_file`是数据地址，如`/home/dataset/finetune_dataset/train.json`；`--vocab_path`为字典的地址文件夹目录，同上； + `--output_path`为生成的mindrecord的输出结果文件夹目录,如`/home/dataset/finetune_dataset/`； + +处理完成后，指定的`--output_path`目录下生成训练和推理的mindrecord文件，如`train.mindrecord`和`test.mindrecord`。 + +### Finetune训练过程 + +- 在`src/config.py`中设置，包括模型并行、batchsize、学习率和网络超参数。点击[这里](https://www.mindspore.cn/tutorial/training/zh-CN/master/use/data_preparation.html)查看更多数据集信息。 + +- 运行`run_distribute_train_ascend_single_machine.sh`，进行CPM模型的单机8卡分布式训练。 + +``` bash + cd scripts + bash run_distribute_train_ascend_single_machine.sh Dataset_addr PreTrain_ckpt_addr Rank_table_addr +``` + +- 运行`run_distribute_train_ascend_multi_machine.sh`，进行CPM模型的多机多卡分布式训练。 + +``` bash + cd scripts + bash run_distribute_train_ascend_multi_machine.sh Dataset_addr PreTrain_ckpt_addr Rank_table_addr SERVER_ID +``` + +主要地，`Dataset_addr` 是数据地址，如`/home/dataset/finetune_dataset/train.mindrecord`； + `PreTrain_ckpt_addr` 为预训练模型的地址，如`/home/cpm_mindspore_1p_fp32.ckpt`； + `Rank_table_addr` 为Rank_table的地址,如`/home/rank_table_8p.json`； + `SERVER_ID` 为多机过程中，机器从0开始编号的的依次顺序，如：0。 + +**注意**：由于本CPM模型较大，无法在一张卡上训练，需要进行分布式训练，包括：模型并行和数据并行。 + 分布式并行训练时，机器的device的device_id从1开始编号，依次递增1。 + 运行单机8卡，rank_table里的rank_id从0,2,4,6,1,3,5,7编号； + 运行多机多卡时，第一台机器的rank_id分别为0,2,4,6,1,3,5,7；第2台机器的rank_id是8,10,12,14,9,11,13,15；后面的机器的rank_id依次类推。 + +### Finetune评估过程 + +- 在`src/config.py`中设置参数； +- 上述Finetune训练结束，将指定某个epoch数的分片模型放置到同指定目录下，包括：`train_strategy.ckpt`, `cpm_rank_1-*.ckpt`等，其中`train_strategy.ckpt`为分布式训练的策略文件。 +- 运行`run_eval_distribute_ascend.sh`，评估某个CPM模型。 + +```bash + cd scripts + bash run_eval_distribute_ascend.sh Test_MindRecord_addr Test_json_addr Model_addr Model_patition_number Rank_table_addr +``` + +通常我们会选择在dev数据集上精度最高的模型，再在test数据集上进行推理，最后会生成测试集上的准确率，模型选择可参考`run_test_distribute_ascend.sh`或`test.py`文件。 + +```bash + cd scripts + bash run_test_distribute_ascend.sh Dev_MindRecord_addr Dev_json_addr Test_MindRecord_addr Test_json_addr Model_addr Model_patition_number Rank_table_addr +``` + +主要地， `Test_MindRecord_addr`为test数据集mindrecord，如`/home/dataset/finetune_dataset/test.mindrecord`； + `Test_json_addr`为预处理后的test数据集的json文件，如`/home/dataset/finetune_dataset/test.json`； + `Dev_MindRecord_addr`为dev数据集mindrecord，如`/home/dataset/finetune_dataset/dev.mindrecord`； + `Dev_json_addr`为预处理后的dev数据集的json文件，如`/home/dataset/finetune_dataset/dev.json`； + `Model_addr`为Finetune得到的模型文件夹，如`/home/finetune_model/`； + `Model_patition_number`为Finetune得到的模型的分片数量，不包括策略文件`train_strategy.ckpt`, 如单机8卡得到的为8； + `Rank_table_addr`为进行推理的时候的分布式推理的rank_table地址，如`/home/rank_table_2p.json`。 + +注意：Finetune的推理的数据集预处理和zero-shot的数据集预处理方式不一样。 + +# 性能和精度 + +## Zero-shot评估性能和精度 + +Zero-shot单机双卡推理性能和精度如下： + +| 参数 | Ascend | +| ------------------- | --------------------------- | +| 资源 |Ascend 910；CPU 2.60GHz，192核；内存 755GB；系统 Euler2.8 | +| MindSpore版本 | 1.3.0 | +| 数据集 | ChID数据集 | +| 模型并行数 | 2 | +| 速度 | 152毫秒/步 | +| Ascend芯片使用数量 | 2 | +| batch_size | 2 | +| 输出 | 准确率 | +| 准确率 | accuracy=67.94% | + +## Finetune训练性能和精度 + +单机8卡Finetune性能和精度如下： + +| 参数 | Ascend | +| -------------------------- | -------------------------------------------------------------- | +| 资源 |Ascend 910；CPU 2.60GHz，192核；内存 755GB；系统 Euler2.8 | +| 上传日期 | 2021-06-07 | +| MindSpore版本 | 1.3.0 | +| 数据集 | ChID | +| 训练参数 | epoch=10, global_batch_size=16 | +| 模型并行数 | 2 | +| 优化器 | Adam | +| 准确率 | 80.4% | +| 速度 | 1683毫秒/步(8卡) | +| 损失 | 0.7 | +| 参数 (M) | 2597.1 | +| 推理检查点 | 76G （.ckpt文件） | +| 脚本 | | + +四机32卡Finetune性能和精度如下： + +| 参数 | Ascend | +| -------------------------- | -------------------------------------------------------------- | +| 资源 |Ascend 910；CPU 2.60GHz，192核；内存 755GB；系统 Euler2.8 | +| 上传日期 | 2021-06-07 | +| MindSpore版本 | 1.3.0 | +| 数据集 | ChID | +| 训练参数 | epoch=10, global_batch_size=128 | +| 模型并行数 | 2 | +| 优化器 | Adam | +| 准确率 | 81.4% | +| 速度 | 2740毫秒/步(32卡) | +| 损失 | 0.08 | +| 参数 (M) | 2597.1 | +| 推理检查点 | 57G （.ckpt文件） | +| 脚本 | | + +# 随机情况说明 + +以下两种随机情况： + +- 数据集Shuffle +- Dropout随机丢弃 + +train.py已经设置了一些种子，避免数据集轮换和权重初始化的随机性。 + +# 其他 + +本模型已经在Ascend环境上验证了精度和性能，还没有在CPU和GPU上验证. + +# ModelZoo主页 + +请浏览官网[主页](https://gitee.com/mindspore/mindspore/tree/master/model_zoo)。 diff --git a/model_zoo/official/nlp/cpm/data_process/make_finetune_mindrecord.py b/model_zoo/official/nlp/cpm/data_process/make_finetune_mindrecord.py new file mode 100644 index 00000000000..6a7e9c9c55d --- /dev/null +++ b/model_zoo/official/nlp/cpm/data_process/make_finetune_mindrecord.py @@ -0,0 +1,125 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""make finetune-mindrecord dataset.""" +import os +import json +import argparse +import numpy as np +from tqdm import trange + +from mindspore.mindrecord import FileWriter + +from tokenizer_cpm import CPMTokenizer + + +class CHIDDataset(): + """Dataset define for ChId.""" + def __init__(self, data_path, tokenizer): + self.pad_id = tokenizer.pad_id + with open(data_path, "r") as f: + self.cand_ids, data_cpm = json.load(f) + + self.samples, self.sizes = self.process(data_cpm) + self.max_size = max(self.sizes) + + def process(self, process_data, num_samples=10): + """Dataset process.""" + sizes = [] + samples = [] + for d in process_data: + loss_mask = [0] * (len(d["sent"]) - 2) + [1] + + samples.append(( + d["sent"][:-1], # ids for the tokenized sentence + loss_mask, # mask of the loss + d["sent"][1:], # token labels of each sentence + d["truth"], # labels if each sentence, should be an integer in [0, 9] + )) + sizes.append(len(d["sent"]) - 1) + return samples, sizes + + def _pad_process(self, input_ids, loss_mask, labels): + """Dataset padding process.""" + pad_input_ids = np.ones(shape=(self.max_size), dtype=np.int64) * self.pad_id + pad_loss_mask = np.zeros(shape=(self.max_size)) * 1.0 + pad_labels = np.ones(shape=(self.max_size), dtype=np.int64) * self.pad_id + + pad_input_ids[:len(input_ids)] = input_ids + pad_loss_mask[:len(loss_mask)] = loss_mask + pad_labels[:len(labels)] = labels + + return pad_input_ids, pad_loss_mask, pad_labels + + def __len__(self): + return len(self.sizes) + + def __getitem__(self, idx): + input_ids, loss_mask, labels, truth = self.samples[idx] + pad_input_ids, pad_loss_mask, pad_labels = self._pad_process(input_ids, loss_mask, labels) + sample = { + "truth": truth, + "input_ids": pad_input_ids, + "loss_mask": pad_loss_mask, + "labels": pad_labels, + "size": self.sizes[idx] + } + return sample + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='CPM dataset.') + parser.add_argument("--vocab_path", type=str, required=False, + default="./vocab", + help="the tokenizer vocab path.") + + parser.add_argument("--data_file", type=str, required=False, + default="./preprocessed/train.json", + help="finetune train dataset files.") + + parser.add_argument("--output_path", type=str, required=False, + default="./output/train.mindrecord", + help="mindrecord dataset output path.") + parser.add_argument("--num_partitions", type=int, required=False, + default=1, help="the number of mindrecord partitions.") + + args = parser.parse_args() + + # get the tokenizer + tokenizer_cpm = CPMTokenizer(os.path.join(args.vocab_path, 'vocab.json'), + os.path.join(args.vocab_path, 'chinese_vocab.model')) + + os.makedirs(os.path.dirname(args.output_path), exist_ok=True) + + chidDataset = CHIDDataset(args.data_file, tokenizer_cpm) + chid_schema = {"truth": {"type": "int64"}, + "input_ids": {"type": "int64", "shape": [-1]}, + "loss_mask": {"type": "float64", "shape": [-1]}, + "labels": {"type": "int64", "shape": [-1]}, + "size": {"type": "int64"}} + + writer = FileWriter(file_name=args.output_path, shard_num=args.num_partitions) + writer.add_schema(chid_schema, "preprocessed chid dataset") + data = [] + for i in trange(len(chidDataset)): + data.append(chidDataset[i]) + if i % 100 == 0: + writer.write_raw_data(data) + data = [] + +if data: + writer.write_raw_data(data) + +writer.commit() +print("transform mindrecord successfully, refer: {}".format(args.output_path)) diff --git a/model_zoo/official/nlp/cpm/data_process/make_zero_shot_mindrecord.py b/model_zoo/official/nlp/cpm/data_process/make_zero_shot_mindrecord.py new file mode 100644 index 00000000000..91d8323b989 --- /dev/null +++ b/model_zoo/official/nlp/cpm/data_process/make_zero_shot_mindrecord.py @@ -0,0 +1,140 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""make zero-shot mindrecord dataset.""" +import os +import json +import argparse +import numpy as np +from tqdm import tqdm + +from mindspore.mindrecord import FileWriter + +from tokenizer_cpm import CPMTokenizer + + +class CHIDDataset(): + """Dataset define for ChId.""" + def __init__(self, data_path, tokenizer): + self.pad_id = tokenizer.pad_id + self.eod_token = tokenizer.eod_id + with open(data_path, "r") as f: + data_cpm = json.load(f) + self.samples, self.sizes, self.truth_labels = self.process(data_cpm) + + with open(os.path.join(os.path.dirname(args.output_path), "true_labels.txt"), "w") as f: + for it in range(0, len(self.truth_labels)): + f.write(str(self.truth_labels[it]) + "\n") + + self.max_size = max(self.sizes) + + def process(self, process_data): + """Dataset process.""" + contents = process_data["contents"] + sids = process_data["sids"] + truth_labels = process_data["labels"] + cids = process_data["cids"] + sizes = [] + samples = [] + for content, sid, cid in zip(contents, sids, cids): + input_ids = content + input_ids = input_ids + [self.eod_token] + length = len(input_ids) - 1 + sizes.append(length) + samples.append(( + sid, + cid, + input_ids[:-1], + [1.0] * length, + input_ids[1:] + )) + + return samples, sizes, truth_labels + + def _pad_process(self, input_ids, loss_mask, labels): + """Dataset padding process.""" + pad_input_ids = np.ones(shape=(self.max_size), dtype=np.int64) * self.pad_id + pad_loss_mask = np.zeros(shape=(self.max_size)) * 1.0 + pad_labels = np.ones(shape=(self.max_size), dtype=np.int64) * self.pad_id + + pad_input_ids[:len(input_ids)] = input_ids + pad_loss_mask[:len(loss_mask)] = loss_mask + pad_labels[:len(labels)] = labels + + return pad_input_ids, pad_loss_mask, pad_labels + + def __len__(self): + return len(self.sizes) + + def __getitem__(self, idx): + sid, cid, input_ids, loss_mask, labels = self.samples[idx] + pad_input_ids, pad_loss_mask, pad_labels = self._pad_process(input_ids, loss_mask, labels) + sample = { + "sid": sid, + "cid": cid, + "input_ids": pad_input_ids, + "loss_mask": pad_loss_mask, + "labels": pad_labels, + "size": self.sizes[idx] + } + return sample + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='CPM dataset.') + parser.add_argument("--vocab_path", type=str, required=False, + default="./vocab", + help="the tokenizer vocab path.") + + parser.add_argument("--data_file", type=str, required=False, + default="./preprocessed/zeroshot/test.json", + help="zero-shot test dataset files.") + + parser.add_argument("--output_path", type=str, required=False, + default="./output/test.mindrecord", + help="mindrecord dataset output path.") + + parser.add_argument("--num_partitions", type=int, required=False, + default=4, help="the number of mindrecord partitions.") + + args = parser.parse_args() + + # get the tokenizer + tokenizer_cpm = CPMTokenizer(os.path.join(args.vocab_path, 'vocab.json'), + os.path.join(args.vocab_path, 'chinese_vocab.model')) + os.makedirs(os.path.dirname(args.output_path), exist_ok=True) + + chidDataset = CHIDDataset(args.data_file, tokenizer_cpm) + + chid_schema = {"sid": {"type": "int64"}, + "cid": {"type": "int64"}, + "input_ids": {"type": "int64", "shape": [-1]}, + "loss_mask": {"type": "float64", "shape": [-1]}, + "labels": {"type": "int64", "shape": [-1]}, + "size": {"type": "int64"}} + + writer = FileWriter(file_name=args.output_path, shard_num=args.num_partitions) + writer.add_schema(chid_schema, "preprocessed chid dataset") + data = [] + for i in tqdm(range(len(chidDataset))): + data.append(chidDataset[i]) + if i % 100 == 0: + writer.write_raw_data(data) + data = [] + +if data: + writer.write_raw_data(data) + +writer.commit() +print("transform mindrecord successfully, refer: {}".format(args.output_path)) diff --git a/model_zoo/official/nlp/cpm/data_process/tokenizer_cpm.py b/model_zoo/official/nlp/cpm/data_process/tokenizer_cpm.py new file mode 100644 index 00000000000..dfa8b7d20b4 --- /dev/null +++ b/model_zoo/official/nlp/cpm/data_process/tokenizer_cpm.py @@ -0,0 +1,71 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Tokenization classes for CPM.""" + +from functools import lru_cache +from io import open +import logging +import json +import jieba +import sentencepiece as spm + +jieba.setLogLevel(logging.INFO) + + +class CPMTokenizer(): + """Tokenizer for CPM network.""" + + def __init__(self, vocab_json, chinese_vocab_file): + self.tok2idx = json.load(open(vocab_json)) + self.idx2tok = {token_idx: token_key for token_key, token_idx in self.tok2idx.items()} + self.spp = spm.SentencePieceProcessor(model_file=chinese_vocab_file) + self.translator = str.maketrans(" \n", "\u2582\u2583") + self.eod_id = self.tok2idx[''] + self.unk_id = self.tok2idx[''] + self.mask_id = self.tok2idx[''] + self.pad_id = self.tok2idx[''] + + @property + def vocab_size(self): + return len(self.tok2idx) + + def __len__(self): + return len(self.tok2idx) + + @property + def unk(self): + return self.unk_id + + @property + def mask(self): + return self.mask_id + + @property + def eod(self): + return self.eod_id + + @lru_cache() + def tokenize_op(self, input_text): + input_tokens = [x_item.translate(self.translator) for x_item in jieba.cut(input_text, cut_all=False)] + return self.spp.encode(" ".join(input_tokens)) + + def encode(self, input_text): + result_encode = self.tokenize_op(input_text) + return result_encode + + def decode(self, input_tokens): + text_decoder = self.spp.decode(input_tokens) + text_decoder = text_decoder.replace(' ', '').replace('\u2582', ' ').replace('\u2583', '\n') + return text_decoder diff --git a/model_zoo/official/nlp/cpm/eval.py b/model_zoo/official/nlp/cpm/eval.py new file mode 100644 index 00000000000..4e0b16bd8d4 --- /dev/null +++ b/model_zoo/official/nlp/cpm/eval.py @@ -0,0 +1,296 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Eval.""" +import os +import ast +import argparse +import json +import numpy as np + +from mindspore import context, load_distributed_checkpoint +import mindspore.nn as nn +from mindspore.common.tensor import Tensor +from mindspore.train.model import Model +import mindspore.common.dtype as mstype +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.communication import management as MultiAscend +from mindspore.context import ParallelMode +from mindspore.parallel import set_algo_parameters + +from src.cpm import CPMModel +from src.cpm_train import VirtualDatasetOneInputCell +from src.cpm_loss import Cross_entropy_eval +from src.config import finetune_test_distrubute, finetune_test_standalone +from train import load_dataset + +device_id = int(os.getenv("DEVICE_ID")) +rank_size = os.getenv('RANK_SIZE') +context.set_context(mode=context.GRAPH_MODE, + save_graphs=False, + device_target="Ascend", + device_id=device_id) + + +class CPMForInfer(nn.Cell): + """ + Encapsulation class of CPM network infer. + + Args: + network (nn.Cell): CPM model. + batch_size (int): Batch size of input dataset. + seq_length (int): Length of input tensor sequence. + vocab_size (int): Size of the dictionary of embeddings. + config: The config of networks. + + Returns: + Tensor, losses. + """ + def __init__(self, network, batch_size, seq_length, vocab_size, config): + super(CPMForInfer, self).__init__(auto_prefix=False) + self.network = network + self.batch_size = batch_size + self.seq_length = seq_length + self.vocab_size = vocab_size + self.loss_net = Cross_entropy_eval(batch_size=self.batch_size, + seq_length=self.seq_length, + vocab_size=self.vocab_size, + config=config) + + def construct(self, input_ids, position_ids, attention_mask, loss_mask): + logits = self.network(input_ids, position_ids, attention_mask) + loss = self.loss_net(logits, loss_mask) + return loss + + +class CPM_LAYER(nn.Cell): + """ + CPM model training with loss function. + """ + + def __init__(self, config_eval): + super(CPM_LAYER, self).__init__() + self.cpm_model = CPMModel(batch_size=config_eval.batch_size, + seq_length=config_eval.seq_length, + vocab_size=config_eval.vocab_size, + hidden_size=config_eval.hidden_size, + num_hidden_layers=config_eval.num_hidden_layers, + num_attention_heads=config_eval.num_attention_heads, + config=config_eval) + + def construct(self, input_ids, position_ids=None, attention_mask=None): + output = self.cpm_model(input_ids, position_ids, attention_mask) + return output + + +def run_eval(args, config_eval, ckpt_file_list=None): + """ + Building infer pipeline + """ + with open(args.data_path, "r") as f: + # cand_ids, data + cand_ids, _ = json.load(f) + print("++++ cand_ids: ", cand_ids) + + if args.distribute: + dataset = load_dataset(args.dataset, config_eval.batch_size, + rank_size=MultiAscend.get_group_size(), + rank_id=MultiAscend.get_rank(), + drop_remainder=False, + is_training=False, + shuffle=False) + else: + dataset = load_dataset(args.dataset, + config_eval.batch_size, + drop_remainder=False, + is_training=False, + shuffle=False) + + cpm_model = CPM_LAYER(config_eval) + + if args.distribute: + cpm_model = VirtualDatasetOneInputCell(cpm_model) + params = cpm_model.trainable_params() + print("+++++++current network parameter+++++") + for pas in params: + print(pas.name) + print("++++++++++++") + if not args.has_train_strategy: + # load the checkpoint without train strategy. + weights = load_checkpoint(args.ckpt_path_doc) + can_be_loaded = {} + print("+++++++loading weights+++++") + for name, _ in weights.items(): + print('oldname: ' + name) + if 'cpm_model.' not in name: + can_be_loaded['cpm_model.' + name] = weights[name] + + print('newname: cpm_model.' + name) + else: + can_be_loaded[name] = weights[name] + print("+++++++loaded weights+++++") + load_param_into_net(cpm_model, parameter_dict=can_be_loaded) + + infer_net = CPMForInfer(network=cpm_model, + batch_size=config_eval.batch_size, + seq_length=config_eval.seq_length, + vocab_size=config_eval.vocab_size, + config=config_eval) + + model = Model(infer_net) + + if args.has_train_strategy and not args.distribute: + # load sliced checkpoint with train strategy, but will run standalone inference without model parallel. + load_distributed_checkpoint(infer_net, ckpt_file_list, None) + + if args.has_train_strategy and args.distribute: + # load sliced checkpoint with train strategy, will run distribute inference with model parallel. + fake_input_ids = Tensor(np.ones((config_eval.batch_size, config_eval.seq_length)), mstype.int64) + fake_position_ids = Tensor(np.random.randint(0, 10, [config_eval.batch_size, config_eval.seq_length]), + mstype.int64) + fake_attention_mask = Tensor( + np.random.randn(config_eval.batch_size, config_eval.seq_length, config_eval.seq_length), mstype.float16) + fake_loss_mask = Tensor(np.random.randn(config_eval.batch_size, config_eval.seq_length), mstype.float16) + predict_layout = model.infer_predict_layout(fake_input_ids, + fake_position_ids, + fake_attention_mask, + fake_loss_mask) + print("Loaded sliced checkpoint, will run distribute inference with model parallel.", flush=True) + load_distributed_checkpoint(infer_net, ckpt_file_list, predict_layout) + + all_losses = [] + truth_labels = [] + + steps_per_epoch = dataset.get_dataset_size() + print("++++++Dataset size", steps_per_epoch, flush=True) + + for batch in dataset.create_dict_iterator(output_numpy=True, num_epochs=1): + print("++++ start") + ms_truth = batch['truth'] + + input_ids = Tensor(batch['input_ids'], mstype.int64) + position_ids = Tensor(batch['position_ids'], mstype.int64) + attention_mask = Tensor(batch['attention_mask'], mstype.float16) + loss_mask = Tensor(batch['loss_mask'], mstype.float16) + + pred_id_tensor = model.predict(input_ids, position_ids, attention_mask, loss_mask) + # numpy do it. + pred_id_np = pred_id_tensor.asnumpy() + pred_id_np = pred_id_np[:, cand_ids] + pred_id = pred_id_np.argmax(axis=-1) + print("++++ pred_id_np: ", pred_id_np) + print("++++ ms_truth: ", ms_truth) + + all_losses.append(pred_id) + truth_labels.append(ms_truth) + + all_losses = np.stack(all_losses).reshape(-1) + truth_labels = np.stack(truth_labels).reshape(-1) + print("++++ all_losses= \n", all_losses) + print("++++ truthlabel= \n", truth_labels) + result = sum([int(p == l) for p, l in zip(all_losses, truth_labels)]) / len(truth_labels) + print("RESULT: ", result) + return result + + +def set_parallel_env(): + r""" + Parallel environment. + """ + context.reset_auto_parallel_context() + MultiAscend.init() + + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, + device_num=MultiAscend.get_group_size(), + gradients_mean=True, + full_batch=True) + set_algo_parameters(elementwise_op_strategy_follow=True) + + +def find_file(path, qianzhui): + r''' + Find the file address according to the prefix. + ''' + result_addr = None + for i, _, k in os.walk(path): + for file in k: + if file.startswith(qianzhui): + result_addr = os.path.join(i, file) + break + return result_addr + + +def create_ckpt_file_list(args, max_index=None, train_strategy=None, steps_per_epoch=4509): + """user-defined ckpt file list""" + ckpt_file_list = [] + # train_strategy + if train_strategy is not None: + true_path = find_file(args.ckpt_path_doc, train_strategy) + if true_path is not None: + ckpt_file_list.append(true_path) + else: + raise Exception("+++ ckpt not found!!! +++") + return ckpt_file_list + + # order in rank_id + for i in range(0, args.ckpt_partition): + path_name = "cpm_rank_" + str(i) + "-" + if max_index is not None: + path_name = path_name + str(max_index) + "_" + true_path = find_file(args.ckpt_path_doc, path_name) + if true_path is not None: + ckpt_file_list.append(true_path) + else: + path_name = "cpm_rank_" + str(i) + "-" + str(max_index * steps_per_epoch) + true_path = find_file(args.ckpt_path_doc, path_name) + if true_path is not None: + ckpt_file_list.append(true_path) + else: + raise Exception("+++ ckpt not found!!! +++") + return ckpt_file_list + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="CPM inference") + parser.add_argument('--dataset', type=str, default="", help="dataset path.") + parser.add_argument("--data_path", type=str, default="/disk0/dataset/finetune_dataset/test.json", + help='test_json path.') + parser.add_argument('--ckpt_path_doc', type=str, default="", help="Checkpoint path document.") + parser.add_argument('--ckpt_partition', type=int, default=8, help="Number of checkpoint partition.") + parser.add_argument("--distribute", type=ast.literal_eval, default=False, + help='Distribute evaluating with model parallel.') + parser.add_argument("--has_train_strategy", type=ast.literal_eval, default=True, + help='Model has distributed training strategy.') + args_eval = parser.parse_args() + + ckpt_file_list_test = None + if args_eval.has_train_strategy: + # Get the checkpoint with train strategy. + train_strategy_list = create_ckpt_file_list(args_eval, train_strategy="train_strategy.ckpt") + context.set_auto_parallel_context( + strategy_ckpt_load_file=train_strategy_list[0] + ) + ckpt_file_list_test = create_ckpt_file_list(args_eval) + print("++++ Get sliced checkpoint file, lists: ", ckpt_file_list_test, flush=True) + + result_accuracy = 0.0 + if args_eval.distribute: + set_parallel_env() + print("Start validation on 2 devices with model parallel.") + result_accuracy = run_eval(args_eval, finetune_test_distrubute, ckpt_file_list_test) + else: + print("Start validation on 1 device without model parallel.") + result_accuracy = run_eval(args_eval, finetune_test_standalone, ckpt_file_list_test) + + print("++++ Accuracy=", result_accuracy) diff --git a/model_zoo/official/nlp/cpm/export.py b/model_zoo/official/nlp/cpm/export.py new file mode 100644 index 00000000000..0528d280d06 --- /dev/null +++ b/model_zoo/official/nlp/cpm/export.py @@ -0,0 +1,87 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""export checkpoint file into air models""" +import ast +import argparse +from easydict import EasyDict as ed +import numpy as np + +from mindspore import context, load_distributed_checkpoint +from mindspore.common.tensor import Tensor +import mindspore.common.dtype as mstype +from mindspore.train.serialization import load_checkpoint, load_param_into_net, export + +from eval import CPM_LAYER, create_ckpt_file_list + + +parser = argparse.ArgumentParser(description="CPM export") +parser.add_argument('--ckpt_path_doc', type=str, default="", help="checkpoint path document.") +parser.add_argument('--ckpt_partition', type=int, default=8, help="Number of checkpoint partition.") +parser.add_argument("--has_train_strategy", type=ast.literal_eval, default=True, + help='has distributed training strategy') +parser.add_argument("--file_name", type=str, default="cpm", help="output file name.") +parser.add_argument('--file_format', type=str, choices=["AIR", "MINDIR"], default='AIR', help='file format') +args = parser.parse_args() + +context.set_context(mode=context.GRAPH_MODE, + save_graphs=False, + device_target="Ascend") + + +finetune_eval_single = ed({ + "dp": 1, + "mp": 1, + "batch_size": 1, + "rank_size": 1, + "vocab_size": 30000, + 'seq_length': 666, + "hidden_size": 2560, + "num_hidden_layers": 32, + "num_attention_heads": 32 +}) + +if __name__ == '__main__': + config_eval = finetune_eval_single + cpm_model = CPM_LAYER(config_eval) + + if not args.has_train_strategy: + weights = load_checkpoint(args.ckpt_path) + can_be_loaded = {} + print("+++++++loading weights+++++") + for name, _ in weights.items(): + print('oldname: ' + name) + if 'cpm_model.' not in name: + can_be_loaded['cpm_model.' + name] = weights[name] + print('newname: cpm_model.' + name) + else: + can_be_loaded[name] = weights[name] + print("+++++++loaded weights+++++") + load_param_into_net(cpm_model, parameter_dict=can_be_loaded) + else: + context.set_auto_parallel_context( + strategy_ckpt_load_file=args.ckpt_path_doc + "/train_strategy.ckpt" + ) + ckpt_file_list = create_ckpt_file_list(args) + print("Get checkpoint file lists++++", ckpt_file_list, flush=True) + load_distributed_checkpoint(cpm_model, ckpt_file_list, None) + + input_ids = Tensor(np.ones((config_eval.batch_size, config_eval.seq_length)), mstype.int64) + position_ids = Tensor(np.random.randint(0, 10, [config_eval.batch_size, config_eval.seq_length]), + mstype.int64) + attention_mask = Tensor(np.random.randn(config_eval.batch_size, config_eval.seq_length, config_eval.seq_length), + mstype.float16) + + export(cpm_model, input_ids, position_ids, attention_mask, file_name=args.file_name, + file_format=args.file_format) diff --git a/model_zoo/official/nlp/cpm/gpt_ckpt_2_mindspore.py b/model_zoo/official/nlp/cpm/gpt_ckpt_2_mindspore.py new file mode 100644 index 00000000000..b89888888df --- /dev/null +++ b/model_zoo/official/nlp/cpm/gpt_ckpt_2_mindspore.py @@ -0,0 +1,122 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Checkpoint.""" +import numpy as np +import torch + +from mindspore.train.serialization import save_checkpoint +from mindspore import Tensor + +param_names = { + "word_embeddings.weight": "word_embedding.embedding_table", + "position_embeddings.weight": "position_embedding.position_embedding_table", + "transformer.final_layernorm.weight": "transformer.final_layernorm.gamma", + "transformer.final_layernorm.bias": "transformer.final_layernorm.beta", +} +for i in range(0, 32): + param_names["transformer.layers." + str(i) + ".attention.query_key_value.weight"] = "" + param_names["transformer.layers." + str(i) + ".attention.query_key_value.bias"] = "" + param_names["transformer.layers." + str(i) + ".attention.dense.weight"] = "transformer.layers." + str( + i) + ".masked_multi_head_attention.masked_self_attention.dense.weight" + param_names["transformer.layers." + str(i) + ".attention.dense.bias"] = "transformer.layers." + str( + i) + ".masked_multi_head_attention.masked_self_attention.dense.bias" + param_names["transformer.layers." + str(i) + ".input_layernorm.weight"] = "transformer.layers." + str( + i) + ".masked_multi_head_attention.layernorm.gamma" + param_names["transformer.layers." + str(i) + ".input_layernorm.bias"] = "transformer.layers." + str( + i) + ".masked_multi_head_attention.layernorm.beta" + param_names["transformer.layers." + str(i) + ".mlp.dense_h_to_4h.weight"] = "transformer.layers." + str( + i) + ".mlp.dense_fc.weight" + param_names["transformer.layers." + str(i) + ".mlp.dense_h_to_4h.bias"] = "transformer.layers." + str( + i) + ".mlp.dense_fc.bias" + param_names["transformer.layers." + str(i) + ".mlp.dense_4h_to_h.weight"] = "transformer.layers." + str( + i) + ".mlp.dense_proj.weight" + param_names["transformer.layers." + str(i) + ".mlp.dense_4h_to_h.bias"] = "transformer.layers." + str( + i) + ".mlp.dense_proj.bias" + param_names["transformer.layers." + str(i) + ".post_attention_layernorm.weight"] = "transformer.layers." + str( + i) + ".mlp.layernorm.gamma" + param_names["transformer.layers." + str(i) + ".post_attention_layernorm.bias"] = "transformer.layers." + str( + i) + ".mlp.layernorm.beta" + + +def torch2ms(torch_ckpt): + """Translate the model to mindspore checkpoint.""" + torch_param_dict = torch.load(torch_ckpt, map_location=torch.device('cpu'))['module'] + + with open("weight_torch.txt", "w") as f: + for key, value in torch_param_dict.items(): + print(f'torch key = {key}') + f.write(key + ' ' + 'dtype=' + str(value.dtype) + "\n") + print(f'value = {value}') + print("-----------------------------------------") + new_params_list = [] + for torch_name in torch_param_dict: + ms_param_dict = {} + + torch_value = torch_param_dict[torch_name] + ms_name = param_names[torch_name] + + if "word_embeddings.weight" in torch_name: + ms_param_dict['name'] = ms_name + ms_param_dict['data'] = Tensor(torch_value.numpy().astype(np.float32)) + new_params_list.append(ms_param_dict) + print(f'torch_name = {torch_name}, ms_name = {ms_name}, fp32') + elif "layernorm" in torch_name: + ms_param_dict['name'] = ms_name + ms_param_dict['data'] = Tensor(torch_value.numpy().astype(np.float32)) + new_params_list.append(ms_param_dict) + print(f'torch_name = {torch_name}, ms_name = {ms_name}, fp32') + elif "query_key_value" not in torch_name: + ms_param_dict['name'] = ms_name + ms_param_dict['data'] = Tensor(torch_value.numpy().astype(np.float32)) + new_params_list.append(ms_param_dict) + print(f'torch_name = {torch_name}, ms_name = {ms_name}, fp16') + else: + _, _, index, _, _, end = torch_name.split(".") + + prefix = "transformer.layers." + q_mid = ".masked_multi_head_attention.masked_self_attention.dense1." + k_mid = ".masked_multi_head_attention.masked_self_attention.dense2." + v_mid = ".masked_multi_head_attention.masked_self_attention.dense3." + + q_name = prefix + str(index) + q_mid + end + k_name = prefix + str(index) + k_mid + end + v_name = prefix + str(index) + v_mid + end + print(f'q_name = {q_name}') + print(f'k_name = {k_name}') + print(f'v_name = {v_name}') + + query, key, value = torch_param_dict[torch_name].chunk(3, dim=0) + print(f"query shape = {query.shape}, key shape = {key.shape}, value shape = {value.shape}") + q_param_dict = {} + k_param_dict = {} + v_param_dict = {} + q_param_dict['name'] = q_name + k_param_dict['name'] = k_name + v_param_dict['name'] = v_name + q_param_dict['data'] = Tensor(query.numpy().astype(np.float32)) + k_param_dict['data'] = Tensor(key.numpy().astype(np.float32)) + v_param_dict['data'] = Tensor(value.numpy().astype(np.float32)) + new_params_list.append(q_param_dict) + new_params_list.append(k_param_dict) + new_params_list.append(v_param_dict) + print(f'torch_name = {torch_name}, ms_name = {ms_name}, fp16') + + save_checkpoint(new_params_list, '/home/cpm_mindspore_1p_fp32.ckpt') + + +if __name__ == '__main__': + original_ckpt = "/home/CPM-large_MP1/iter_0080000/mp_rank_00_model_states.pt" + + torch2ms(original_ckpt) diff --git a/model_zoo/official/nlp/cpm/requirements.txt b/model_zoo/official/nlp/cpm/requirements.txt new file mode 100644 index 00000000000..f0f0c8cdb20 --- /dev/null +++ b/model_zoo/official/nlp/cpm/requirements.txt @@ -0,0 +1,2 @@ +jieba +numpy diff --git a/model_zoo/official/nlp/cpm/scripts/run_distribute_train_ascend_multi_machine.sh b/model_zoo/official/nlp/cpm/scripts/run_distribute_train_ascend_multi_machine.sh new file mode 100644 index 00000000000..5e1ab396aa6 --- /dev/null +++ b/model_zoo/official/nlp/cpm/scripts/run_distribute_train_ascend_multi_machine.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +if [ $# != 4 ] ; then +echo "==============================================================================================================" +echo "Please run the script as: " +echo "sh run_distribute_train_ascend_multi_machine.sh DATASET_PATH CKPT_PATH RANK_TABLE_PATH SERVER_ID" +echo "for example:" +echo "sh run_distribute_train_ascend_multi_machine.sh /disk0/dataset/finetune_dataset/train.mindrecord /disk0/cpm_ckpt_ms/cpm_mindspore_1p_fp32.ckpt /disk0/rank_table_32p.json 0" +echo "It is better to use absolute path." +echo "==============================================================================================================" +exit 1; +fi + +get_real_path(){ + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} + +DATASET=$(get_real_path $1) +echo $DATASET +PRECKPT=$(get_real_path $2) +RANK_TABLE_PATH=$(get_real_path $3) +SERVER_ID=$4 + +echo $DATANAME + +current_exec_path=$(pwd) +echo ${current_exec_path} + +export RANK_TABLE_FILE=$RANK_TABLE_PATH + + +echo $RANK_TABLE_FILE +export RANK_SIZE=8 +export DEVICE_NUM=8 + +RANK_START=$(($DEVICE_NUM * $SERVER_ID)) +GROUP_RANK=2 +GROUP_DIFF=7 + +for((i=0;i<=3;i++)); +do + rm -rf ${current_exec_path}/device$i + mkdir ${current_exec_path}/device$i + cd ${current_exec_path}/device$i + cp ../../*.py ./ + cp -r ../../src ./ + cp -r ../*.sh ./ + export RANK_ID=$(((i*GROUP_RANK)+RANK_START)) + export DEVICE_ID=$i + echo "start training for device $DEVICE_ID, rank $RANK_ID" + python ../../train.py --dataset $DATASET --pretrain_ckpt_path $PRECKPT --multi_machine True > log_cpm.log 2>&1 & + cd ${current_exec_path} +done +cd ${current_exec_path} + + +for((i=4;i<=7;i++)); +do + rm -rf ${current_exec_path}/device$i + mkdir ${current_exec_path}/device$i + cd ${current_exec_path}/device$i + cp ../../*.py ./ + cp -r ../../src ./ + cp -r ../*.sh ./ + export RANK_ID=$(((i*GROUP_RANK)-GROUP_DIFF+RANK_START)) + export DEVICE_ID=$i + echo "start training for device $DEVICE_ID, rank $RANK_ID" + python ../../train.py --dataset $DATASET --pretrain_ckpt_path $PRECKPT --multi_machine True > log_cpm.log 2>&1 & + cd ${current_exec_path} +done +cd ${current_exec_path} + diff --git a/model_zoo/official/nlp/cpm/scripts/run_distribute_train_ascend_single_machine.sh b/model_zoo/official/nlp/cpm/scripts/run_distribute_train_ascend_single_machine.sh new file mode 100644 index 00000000000..1c748427c88 --- /dev/null +++ b/model_zoo/official/nlp/cpm/scripts/run_distribute_train_ascend_single_machine.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +if [ $# != 3 ] ; then +echo "==============================================================================================================" +echo "Please run the script as: " +echo "sh run_distribute_train_ascend_single_machine.sh DATASET_PATH CKPT_PATH RANK_TABLE_PATH" +echo "for example:" +echo "sh run_distribute_train_ascend_single_machine.sh /disk0/dataset/finetune_dataset/train.mindrecord /disk0/cpm_ckpt_ms/cpm_mindspore_1p_fp32.ckpt /disk0/rank_table_8p.json" +echo "It is better to use absolute path." +echo "==============================================================================================================" +exit 1; +fi + +get_real_path(){ + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} + +DATASET=$(get_real_path $1) +echo $DATASET +PRECKPT=$(get_real_path $2) +RANK_TABLE_PATH=$(get_real_path $3) +echo $DATANAME + +current_exec_path=$(pwd) +echo ${current_exec_path} + +export RANK_TABLE_FILE=$RANK_TABLE_PATH + + +echo $RANK_TABLE_FILE +export RANK_SIZE=8 +export DEVICE_NUM=8 + +GROUP_RANK=2 +GROUP_DIFF=7 + +for((i=0;i<=3;i++)); +do + rm -rf ${current_exec_path}/device$i + mkdir ${current_exec_path}/device$i + cd ${current_exec_path}/device$i + cp ../../*.py ./ + cp -r ../../src ./ + cp -r ../*.sh ./ + export RANK_ID=$((i*GROUP_RANK)) + export DEVICE_ID=$i + echo "start training for device $DEVICE_ID, rank $RANK_ID" + python ../../train.py --dataset $DATASET --pretrain_ckpt_path $PRECKPT --multi_machine False > log_cpm.log 2>&1 & + cd ${current_exec_path} +done +cd ${current_exec_path} + + +for((i=4;i<=7;i++)); +do + rm -rf ${current_exec_path}/device$i + mkdir ${current_exec_path}/device$i + cd ${current_exec_path}/device$i + cp ../../*.py ./ + cp -r ../../src ./ + cp -r ../*.sh ./ + export RANK_ID=$(((i*GROUP_RANK)-GROUP_DIFF)) + export DEVICE_ID=$i + echo "start training for device $DEVICE_ID, rank $RANK_ID" + python ../../train.py --dataset $DATASET --pretrain_ckpt_path $PRECKPT --multi_machine False > log_cpm.log 2>&1 & + cd ${current_exec_path} +done +cd ${current_exec_path} diff --git a/model_zoo/official/nlp/cpm/scripts/run_eval_distribute_ascend.sh b/model_zoo/official/nlp/cpm/scripts/run_eval_distribute_ascend.sh new file mode 100644 index 00000000000..34050e3db47 --- /dev/null +++ b/model_zoo/official/nlp/cpm/scripts/run_eval_distribute_ascend.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +if [ $# != 5 ] ; then +echo "==============================================================================================================" +echo "Please run the script as: " +echo "sh run_eval_distribute_ascend.sh DATASET_PATH LABEL_PATH MODEL_CKPT CKPT_NUMBER RANK_TABLE_PATH" +echo "for example:" +echo "sh run_eval_distribute_ascend.sh /disk0/dataset/finetune_dataset/finetune_test.mindrecord /disk0/dataset/finetune_dataset/test.json /disk2/ckpt_32p_0602 32 /disk0/rank_table_2p.json" +echo "It is better to use absolute path." +echo "==============================================================================================================" +exit 1; +fi + +get_real_path(){ + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} + +DATASET=$(get_real_path $1) +echo $DATASET +LABEL=$(get_real_path $2) +MODEL_CKPT=$(get_real_path $3) +CKPT_NUMBER=$4 +RANK_TABLE_PATH=$(get_real_path $5) + +current_exec_path=$(pwd) +echo ${current_exec_path} + +export RANK_SIZE=2 +export DEVICE_NUM=2 +export RANK_TABLE_FILE=$RANK_TABLE_PATH + +for((i=0;i<=1;i++)); +do + rm -rf ${current_exec_path}/eval$i + mkdir ${current_exec_path}/eval$i + cd ${current_exec_path}/eval$i + cp -r ../../*.py ./ + cp -r ../../src ./ + cp -r ../../scripts/*.sh ./ + + export RANK_ID=$i + export DEVICE_ID=$i + echo "start eval for rank $RANK_ID, device $DEVICE_ID" + env > env.log + python ../../eval.py --dataset $DATASET --data_path $LABEL --ckpt_path $MODEL_CKPT --ckpt_partition $CKPT_NUMBER --distribute True --has_train_strategy True> log_cpm.log 2>&1 & + cd ${current_exec_path} +done +cd ${current_exec_path} diff --git a/model_zoo/official/nlp/cpm/scripts/run_test_distribute_ascend.sh b/model_zoo/official/nlp/cpm/scripts/run_test_distribute_ascend.sh new file mode 100644 index 00000000000..f520a014d30 --- /dev/null +++ b/model_zoo/official/nlp/cpm/scripts/run_test_distribute_ascend.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +if [ $# != 7 ] ; then +echo "==============================================================================================================" +echo "Please run the script as: " +echo "sh run_test_distribute_ascend.sh DEV_DATASET_PATH DEV_JSON_PATH TEST_DATASET_PATH TEST_JSON_PATH MODEL_CKPT CKPT_NUMBER RANK_TABLE_PATH" +echo "for example:" +echo "sh run_test_distribute_ascend.sh /disk0/dataset/finetune_dataset/finetune_dev.mindrecord /disk0/dataset/finetune_dataset/dev.json /disk0/dataset/finetune_dataset/finetune_test.mindrecord /disk0/dataset/finetune_dataset/test.json /disk2/ckpt_8p 8 /disk0/rank_table_2p.json" +echo "It is better to use absolute path." +echo "==============================================================================================================" +exit 1; +fi + +get_real_path(){ + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} + +DEV_DATASET=$(get_real_path $1) +echo $DEV_DATASET +DEV_LABEL=$(get_real_path $2) +echo $DEV_LABEL +TEST_DATASET=$(get_real_path $3) +echo $TEST_DATASET +TEST_LABEL=$(get_real_path $4) +echo $TEST_LABEL +MODEL_CKPT=$(get_real_path $5) +echo $MODEL_CKPT +CKPT_NUMBER=$6 +echo $CKPT_NUMBER +RANK_TABLE_PATH=$(get_real_path $7) +echo $RANK_TABLE_PATH + +current_exec_path=$(pwd) +echo ${current_exec_path} + +export RANK_SIZE=2 +export DEVICE_NUM=2 +export RANK_TABLE_FILE=$RANK_TABLE_PATH + +for((i=0;i<=1;i++)); +do + rm -rf ${current_exec_path}/eval$i + mkdir ${current_exec_path}/eval$i + cd ${current_exec_path}/eval$i + cp -r ../../*.py ./ + cp -r ../../src ./ + cp -r ../../scripts/*.sh ./ + + export RANK_ID=$i + export DEVICE_ID=$i + echo "start eval for rank $RANK_ID, device $DEVICE_ID" + env > env.log + python ../../test.py --dev_dataset $DEV_DATASET --dev_data_path $DEV_LABEL \ + --test_dataset $TEST_DATASET --test_data_path $TEST_LABEL \ + --ckpt_path $MODEL_CKPT --ckpt_partition $CKPT_NUMBER \ + --distribute True --has_train_strategy True> log_cpm.log 2>&1 & + + cd ${current_exec_path} +done +cd ${current_exec_path} diff --git a/model_zoo/official/nlp/cpm/scripts/run_zero-shot_inference_distribute_ascend.sh b/model_zoo/official/nlp/cpm/scripts/run_zero-shot_inference_distribute_ascend.sh new file mode 100644 index 00000000000..8d243614d99 --- /dev/null +++ b/model_zoo/official/nlp/cpm/scripts/run_zero-shot_inference_distribute_ascend.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +if [ $# != 4 ] ; then +echo "==============================================================================================================" +echo "Please run the script as: " +echo "sh run_zero-shot_inference_distribute_ascend.sh DATASET_PATH LABEL_PATH MODEL_CKPT RANK_TABLE_PATH" +echo "for example:" +echo "sh run_zero-shot_inference_distribute_ascend.sh /disk0/dataset/zero_shot_dataset_infer/test.mindrecord /disk0/dataset/zero_shot_dataset_infer/true_labels.txt /disk0/cpm_ckpt_ms/cpm_mindspore_1p_fp32.ckpt /disk0/rank_table_2p.json" +echo "It is better to use absolute path." +echo "==============================================================================================================" +exit 1; +fi + +get_real_path(){ + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} + +DATASET=$(get_real_path $1) +echo $DATASET +LABEL=$(get_real_path $2) +MODEL_CKPT=$(get_real_path $3) +RANK_TABLE_PATH=$(get_real_path $4) + +current_exec_path=$(pwd) +echo ${current_exec_path} + +export RANK_SIZE=2 +export DEVICE_NUM=2 +export RANK_TABLE_FILE=$RANK_TABLE_PATH + +for((i=0;i<=1;i++)); +do + rm -rf ${current_exec_path}/eval$i + mkdir ${current_exec_path}/eval$i + cd ${current_exec_path}/eval$i + cp -r ../../*.py ./ + cp -r ../../src ./ + cp -r ../../scripts/*.sh ./ + + export RANK_ID=$i + export DEVICE_ID=$i + echo "start eval for rank $RANK_ID, device $DEVICE_ID" + env > env.log + python ../../zero-shot.py --dataset $DATASET --truth_labels_path $LABEL --ckpt_path $MODEL_CKPT --distribute True --has_train_strategy False> log_cpm.log 2>&1 & + cd ${current_exec_path} +done +cd ${current_exec_path} diff --git a/model_zoo/official/nlp/cpm/scripts/run_zero-shot_inference_standalone_ascend.sh b/model_zoo/official/nlp/cpm/scripts/run_zero-shot_inference_standalone_ascend.sh new file mode 100644 index 00000000000..15bfd6064cb --- /dev/null +++ b/model_zoo/official/nlp/cpm/scripts/run_zero-shot_inference_standalone_ascend.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +if [ $# != 4 ] ; then +echo "==============================================================================================================" +echo "Please run the script as: " +echo "sh run_zero-shot_inference_standalone_ascend.sh DATASET_PATH LABEL_PATH MODEL_CKPT DEVICE_ID" +echo "for example: " +echo "sh run_zero-shot_inference_standalone_ascend.sh /disk0/dataset/zero_shot_dataset_infer/test.mindrecord /disk0/dataset/zero_shot_dataset_infer/true_labels.txt /disk0/cpm_ckpt_ms/cpm_mindspore_1p_fp32.ckpt 5" +echo "It is better to use absolute path." +echo "==============================================================================================================" +exit 1; +fi + +get_real_path(){ + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} + +DATASET=$(get_real_path $1) +echo $DATASET +LABEL=$(get_real_path $2) +MODEL_CKPT=$(get_real_path $3) +DEVICEID=$4 +export DEVICE_NUM=1 +export DEVICE_ID=$DEVICEID +export RANK_ID=0 +export RANK_SIZE=1 + + +if [ -d "eval" ]; +then + rm -rf ./eval +fi +mkdir ./eval +cp ../*.py ./eval +cp -r ../src ./eval +cp -r ../scripts/*.sh ./eval +cd ./eval || exit +echo "start training for device $DEVICE_ID" +env > env.log +python ../../zero-shot.py --dataset $DATASET --truth_labels_path $LABEL --ckpt_path $MODEL_CKPT --has_train_strategy False > log_cpm.log 2>&1 & +cd .. + diff --git a/model_zoo/official/nlp/cpm/src/attention.py b/model_zoo/official/nlp/cpm/src/attention.py new file mode 100644 index 00000000000..36fc2651380 --- /dev/null +++ b/model_zoo/official/nlp/cpm/src/attention.py @@ -0,0 +1,283 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Attention module""" +import math +import numpy as np + +import mindspore.common.dtype as mstype +import mindspore.nn as nn +import mindspore.ops.functional as F +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor + +from src.util import LayerNorm +from src.util import LinearLayer, ResidualConnection, Dropout + + +class MaskedSelfAttention(nn.Cell): + """ + Self-Attention module for each layer. + + Args: + batch_size (int): Batch size of input dataset. + hidden_size (int): Length of last dim of hidden layer. + seq_length (int): Length of input tensor sequence. + num_attention_heads (int): Number of attention heads. + dim_per_head (int): Size of each attention head. + config: The config of networks. + has_attention_mask (bool): Specifies whether to use attention mask. + do_return_2d_tensor (bool): Whether use 2-dimension. + attention_dropout (float): The dropout probability for attention. + is_training (bool): Whether is training. + compute_type (:class:`mindspore.dtype`): Compute type in attention. + + Returns: + Tensor, with the shape [batch_size, hidden_size] + """ + + def __init__(self, + batch_size, + hidden_size, + seq_length, + num_attention_heads, + dim_per_head, + config=None, + has_attention_mask=True, + do_return_2d_tensor=True, + attention_dropout=0.0, + is_training=False, + compute_type=mstype.float16): + super(MaskedSelfAttention, self).__init__() + + self.hidden_size = hidden_size + self.batch_size = batch_size + self.seq_length = seq_length + self.num_heads = num_attention_heads + self.dim_per_head = dim_per_head + self.has_attention_mask = has_attention_mask + self.compute_type = compute_type + self.is_training = is_training + + self.scale = Tensor(math.sqrt(float(self.dim_per_head)), dtype=compute_type) + self.mask_data = Tensor([-10000.0], dtype=mstype.float32) + self.split_head_shape = (self.batch_size, self.seq_length, self.num_heads, self.dim_per_head) + + self.dense = LinearLayer(hidden_size, hidden_size) + self.dense.matmul.shard(((config.dp, config.mp), (1, config.mp))) + self.dense.bias_add.shard(((config.dp, 1), (1,))) + self.dense.bias.parallel_optimizer = False + + self.reshape = P.Reshape() + self.transpose = P.Transpose().shard(((config.dp, 1, config.mp, 1),)) + self.merge_transpose = P.Transpose().shard(((config.dp, config.mp, 1, 1),)) + self.trans_shape = (0, 2, 1, 3) + self.trans_shape2 = (0, 2, 3, 1) + self.matmul_trans_b = P.BatchMatMul().shard(((config.dp, config.mp, 1, 1), (config.dp, config.mp, 1, 1))) + self.matmul = P.BatchMatMul().shard(((config.dp, config.mp, 1, 1), (config.dp, config.mp, 1, 1))) + self.multiply = P.Mul().shard(((config.dp, 1, 1, 1), (1,))).add_prim_attr("_side_effect", True) + self.realdiv = P.RealDiv().shard(((config.dp, config.mp, 1, 1), ())) + + if self.has_attention_mask: + self.expand_dims = P.ExpandDims().shard(((config.dp, 1, 1),)) + self.sub = P.Sub().shard(((1,), (config.dp, 1, 1, 1))).add_prim_attr("_side_effect", True) + self.add = P.TensorAdd().shard(((config.dp, 1, 1, 1), (config.dp, config.mp, 1, 1))) + self.cast = P.Cast() + self.get_dtype = P.DType() + + if do_return_2d_tensor: + self.shape_return = (-1, hidden_size) + else: + self.shape_return = (-1, seq_length, hidden_size) + + self.softmax = nn.Softmax() + self.softmax.softmax.shard(((config.dp, config.mp, 1, 1),)) + self.softmax_cast = P.Cast() + self.shape = P.Shape() + + self.dropout = Dropout(1 - attention_dropout) + self.dropout.dropout_gen_mask.shard(((config.dp, 1),)) + self.dropout.dropout_do_mask.shard(((config.dp, 1),)) + + self.dropout_probs = Dropout(1 - attention_dropout) + self.dropout_probs.dropout_gen_mask.shard(((config.dp, config.mp, 1, 1),)) + self.dropout_probs.dropout_do_mask.shard(((config.dp, config.mp, 1, 1),)) + + self.use_attention_dropout = is_training + self.dense1 = LinearLayer(self.hidden_size, self.hidden_size) + self.dense1.matmul.shard(((config.dp, 1), (config.mp, 1))) + self.dense1.bias_add.shard(((config.dp, config.mp), (config.mp,))) + self.dense2 = LinearLayer(self.hidden_size, self.hidden_size) + self.dense2.matmul.shard(((config.dp, 1), (config.mp, 1))) + self.dense2.bias_add.shard(((config.dp, config.mp), (config.mp,))) + self.dense3 = LinearLayer(self.hidden_size, self.hidden_size) + self.dense3.matmul.shard(((config.dp, 1), (config.mp, 1))) + self.dense3.bias_add.shard(((config.dp, config.mp), (config.mp,))) + + attention_mask = np.tril(np.ones(shape=(config.seq_length, config.seq_length),)) + attention_mask = np.expand_dims(attention_mask, 0) + attention_mask = np.tile(attention_mask, (config.batch_size, 1, 1)) + attention_mask = np.expand_dims(attention_mask, 1) + self.attention_mask = Tensor(attention_mask, dtype=compute_type) + + def construct(self, input_tensor, attention_mask=None): + """do masked self-attention""" + # input_tensor [batch_size, seq_length, hidden_size], eg:[1,571,2560]. + query = self.dense1(input_tensor) + key = self.dense2(input_tensor) + value = self.dense3(input_tensor) + + # split head + query = self.reshape(query, self.split_head_shape) + # query shape [2, 571, 32, 80] -> [2, 32, 571, 80] + query = self.transpose(query, self.trans_shape) + + key = self.reshape(key, self.split_head_shape) + # key shape [batch_size, num_heads, dim_per_head, seq_len] + key = self.transpose(key, self.trans_shape2) + + value = self.reshape(value, self.split_head_shape) + + # value shape [batch_size, num_heads, seq_len, dim_per_head] + value = self.transpose(value, self.trans_shape) + + # precision transition fp32 -> fp16 + query = self.cast(query, self.compute_type) + key = self.cast(key, self.compute_type) + # 8, 32, 725, 80|8, 32, 80, 725 -> 8, 32, 725, 725 + attention_scores = self.matmul_trans_b(query, key) + + attention_scores = self.cast(attention_scores, mstype.float32) + attention_scores = self.realdiv(attention_scores, self.cast(self.scale, self.get_dtype(attention_scores))) + attention_scores = P.Cast()(attention_scores, mstype.float32) + if self.has_attention_mask: + if attention_mask is None: + attention_mask = self.attention_mask + else: + attention_mask = self.expand_dims(attention_mask, 1) + multiply_out = self.sub(self.cast(F.tuple_to_array((1.0,)), self.get_dtype(attention_scores)), + self.cast(attention_mask, + self.get_dtype(attention_scores))) + # 1, 1, 725, 725 + adder = self.multiply(multiply_out, self.mask_data) + adder = self.cast(adder, mstype.float32) + attention_scores = self.cast(attention_scores, mstype.float32) + # 1, 1, 725, 725|8, 32, 725, 725-》8, 32, 725, 725 + attention_scores = self.add(adder, attention_scores) + + attention_scores = self.softmax_cast(attention_scores, mstype.float32) + # [8, 32, 725, 725] ->8, 32, 725, 725 + attention_probs = self.softmax(attention_scores) + attention_probs = self.softmax_cast(attention_probs, self.get_dtype(key)) + + if self.use_attention_dropout: + attention_probs = self.dropout_probs(attention_probs) + + value = self.cast(value, self.compute_type) + attention_probs = self.cast(attention_probs, self.compute_type) + if self.is_training: + # 1, 8, 2, 725, 725 -> 8, 2, 725, 725 + attention_probs = self.reshape(attention_probs, ( + self.batch_size, self.num_heads, self.seq_length, + self.seq_length)) + # 8, 32, 725, 725 | 8, 32, 725, 80 -> 8, 2, 725, 1280 + outputs = self.matmul(attention_probs, value) + + outputs = self.cast(outputs, mstype.float32) + + # merge heads, [8, 2, 725, 1280]->8, 725, 2, 1280 + outputs = self.merge_transpose(outputs, self.trans_shape) + # 8, 725, 2, 1280->5800, 2560 + outputs = self.reshape(outputs, + self.shape_return) + # project + outputs = self.dense(outputs) + if self.is_training: + outputs = self.dropout(outputs) + + return outputs + + +class MaskedMultiHeadAttention(nn.Cell): + """ + Constructor for the MaskedMultiHeadAttention. + + Args: + batch_size (int): Batch size of input dataset. + seq_length (int): Length of input tensor sequence. + hidden_size (int): Length of last dim of hidden layer. + config: The config of networks. + num_attention_heads (int): Number of attention heads. + attention_dropout (float): The dropout probability for attention. + hidden_dropout (float): The dropout probability for hidden layer. + has_attention_mask (bool): Specifies whether to use attention mask. + is_training (bool): Whether to train. + compute_type (:class:`mindspore.dtype`): Compute type in attention. + + Returns: + Tensor, shape (N, T). + """ + + def __init__(self, + batch_size, + seq_length, + hidden_size, + config=None, + num_attention_heads=12, + attention_dropout=0.02, + hidden_dropout=0.1, + has_attention_mask=True, + is_training=False, + compute_type=mstype.float16 + ): + super(MaskedMultiHeadAttention, self).__init__() + if hidden_size % num_attention_heads != 0: + raise ValueError("The hidden size (%d) is not a multiple of the number " + "of attention heads (%d)" % (hidden_size, num_attention_heads)) + + self.dim_per_head = int(hidden_size / num_attention_heads) + + self.masked_self_attention = MaskedSelfAttention( + batch_size=batch_size, + hidden_size=hidden_size, + seq_length=seq_length, + config=config, + num_attention_heads=num_attention_heads, + dim_per_head=self.dim_per_head, + has_attention_mask=has_attention_mask, + do_return_2d_tensor=True, + attention_dropout=attention_dropout, + is_training=is_training, + compute_type=compute_type + ) + + self.layernorm = LayerNorm((hidden_size,), config, epsilon=1e-5).to_float(mstype.float32) + self.layernorm.gamma.parallel_optimizer = False + self.layernorm.beta.parallel_optimizer = False + + self.residual_connection = ResidualConnection(dropout_prob=0.1) + self.residual_connection.add.shard(((config.dp, 1), (config.dp, 1))) + + self.reshape = P.Reshape() + self.shape = P.Shape() + self.new_shape = (-1, hidden_size) + + def construct(self, input_tensor, attention_mask=None): + # input tensor shape[batch_size*sen_length, hidden_size] + output_tensor = self.layernorm(input_tensor) + # attention_output shape [batch_size * sen_length, hidden_size] + attention_output = self.masked_self_attention(output_tensor, attention_mask) + # residual connection, [5800, 2560] | [5800, 2560] -> [5800, 2560] + output = self.residual_connection(attention_output, input_tensor) + return output diff --git a/model_zoo/official/nlp/cpm/src/config.py b/model_zoo/official/nlp/cpm/src/config.py new file mode 100644 index 00000000000..f9695f1cb0a --- /dev/null +++ b/model_zoo/official/nlp/cpm/src/config.py @@ -0,0 +1,132 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Configure""" +from easydict import EasyDict as ed + +config_zero_shot_standalone = ed({ + "dp": 1, + "mp": 1, + "batch_size": 1, + "rank_size": 1, + "vocab_size": 30000, + 'seq_length': 571, + "hidden_size": 2560, + "num_hidden_layers": 32, + "num_attention_heads": 32 +}) + +config_zero_shot_distrubute = ed({ + "dp": 1, + "mp": 2, + "batch_size": 2, + "rank_size": 2, + "vocab_size": 30000, + 'seq_length': 571, + "hidden_size": 2560, + "num_hidden_layers": 32, + "num_attention_heads": 32 +}) + +finetune_dev_standalone = ed({ + "dp": 1, + "mp": 1, + "batch_size": 1, + "rank_size": 1, + "vocab_size": 30000, + 'seq_length': 696, + "hidden_size": 2560, + "num_hidden_layers": 32, + "num_attention_heads": 32 +}) + +finetune_dev_distrubute = ed({ + "dp": 1, + "mp": 2, + "batch_size": 1, + "rank_size": 2, + "vocab_size": 30000, + 'seq_length': 696, + "hidden_size": 2560, + "num_hidden_layers": 32, + "num_attention_heads": 32 +}) + +finetune_test_standalone = ed({ + "dp": 1, + "mp": 1, + "batch_size": 1, + "rank_size": 1, + "vocab_size": 30000, + 'seq_length': 666, + "hidden_size": 2560, + "num_hidden_layers": 32, + "num_attention_heads": 32 +}) + +finetune_test_distrubute = ed({ + "dp": 1, + "mp": 2, + "batch_size": 1, + "rank_size": 2, + "vocab_size": 30000, + 'seq_length': 666, + "hidden_size": 2560, + "num_hidden_layers": 32, + "num_attention_heads": 32 +}) + +config_train_8p = ed({ + "dp": 4, + "mp": 2, + "epoch": 10, + "batch_size": 16, + "rank_size": 8, + "vocab_size": 30000, + 'seq_length': 725, + "hidden_size": 2560, + "num_hidden_layers": 32, + "num_attention_heads": 32, + "lr": 1e-5, + "eps": 1e-8, + "dropout": 0.2, + "end_learning_rate": 1e-7, + "weight_decay": 1e-2, + "warmup_steps": 0.05, + "power": 1.0, + "grad_accumulation_step": 4, + "sink_size": 1 +}) + +config_train_32p = ed({ + "dp": 16, + "mp": 2, + "epoch": 10, + "batch_size": 128, + "rank_size": 32, + "vocab_size": 30000, + 'seq_length': 725, + "hidden_size": 2560, + "num_hidden_layers": 32, + "num_attention_heads": 32, + "lr": 2e-5, + "eps": 1e-8, + "dropout": 0.1, + "end_learning_rate": 1e-7, + "weight_decay": 1e-2, + "warmup_steps": 0.1, + "power": 1.0, + "grad_accumulation_step": 1, + "sink_size": 1 +}) diff --git a/model_zoo/official/nlp/cpm/src/cpm.py b/model_zoo/official/nlp/cpm/src/cpm.py new file mode 100644 index 00000000000..0988c0802b6 --- /dev/null +++ b/model_zoo/official/nlp/cpm/src/cpm.py @@ -0,0 +1,369 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""CPM model.""" +import mindspore.common.dtype as mstype +import mindspore.nn as nn +from mindspore.ops import operations as P + +from src.embedding import EmbeddingPostprocessor, EmbeddingLookup +from src.attention import MaskedMultiHeadAttention +from src.util import LinearLayer, ResidualConnection, LayerNorm, Dropout + + +class MLPLayer(nn.Cell): + """ + The output mapping module for each layer. + + Args: + hidden_size (int): Length of last dim of hidden layer. + config: The config of networks. + dropout_prob (float): The dropout probability for network. + is_training (bool): Whether is training. + Inputs: + x: output of the self-attention module. + Returns: + output: Tensor, the output of this layer after mapping. + """ + + def __init__(self, hidden_size, config=None, dropout_prob=0.1, is_training=False): + super(MLPLayer, self).__init__() + self.hidden_size = hidden_size + self.dense_fc = LinearLayer(hidden_size, 4 * hidden_size) + + self.dense_fc.bias_add.shard(((config.dp, config.mp), (config.mp,))) + self.dense_fc.matmul.shard(((config.dp, 1), (config.mp, 1))) + + self.dense_proj = LinearLayer(4 * hidden_size, hidden_size) + self.dense_proj.bias_add.shard(((config.dp, 1), (1,))) + self.dense_proj.matmul.shard(((config.dp, config.mp), (1, config.mp))) + self.dense_proj.matmul.add_prim_attr("recompute_comm_op", False) + + self.layernorm = LayerNorm((hidden_size,), config, epsilon=1e-5).to_float(mstype.float32) + # parallel optimizer + self.dense_proj.bias.parallel_optimizer = False + self.layernorm.gamma.parallel_optimizer = False + self.layernorm.beta.parallel_optimizer = False + + self.residual_connect = ResidualConnection() + self.residual_connect.add.shard(((config.dp, 1), (config.dp, 1))).add_prim_attr("recompute", False) + self.gelu = P.Gelu().shard(((config.dp, config.mp),)) + self.dropout = Dropout(1 - dropout_prob) + self.dropout.dropout_gen_mask.shard(((config.dp, 1),)) + self.dropout.dropout_do_mask.shard(((config.dp, 1),)) + + self.use_dropout = is_training + self.reshape = P.Reshape() + + def construct(self, input_tensor): + """FeedForward construct function.""" + # LayerNorm, eg: [5800, 2560]. + output = self.layernorm(input_tensor) + + # Feed Forward + output = self.dense_fc(output) + # eg: 5800, 10240 + output = self.gelu(output) + output = self.dense_proj(output) + if self.use_dropout: + output = self.dropout(output) + output = self.residual_connect(output, input_tensor) + return output + + +class CPMTransformerLayer(nn.Cell): + """ + The basic block of GPT network. + + Args: + batch_size (int): Batch size of input dataset. + seq_length (int): Length of input tensor sequence. + hidden_size (int): Length of last dim of hidden layer. + config: The config of networks. + num_attention_heads (int): Number of attention heads. + attention_dropout (float): The dropout probability for attention. + hidden_dropout (float): The dropout probability for hidden layers. + has_attention_mask (bool): Specifies whether to use attention mask. + is_training (bool): Whether is training. + compute_type (:class:`mindspore.dtype`): Compute type in attention. + + Inputs: + input_tensor: the output of previous layer(input_ids for the first layer). + attention_mask: the attention mask matrix with shape (batch_size, seq_length, seq_length). + + Returns: + output: Tensor, the output logit of this layer. + """ + + def __init__(self, + batch_size, + seq_length, + hidden_size, + config=None, + num_attention_heads=32, + attention_dropout=0.1, + hidden_dropout=0.1, + has_attention_mask=True, + is_training=False, + compute_type=mstype.float16 + ): + super(CPMTransformerLayer, self).__init__() + if hidden_size % num_attention_heads != 0: + raise ValueError("The hidden size (%d) is not a multiple of the number " + "of attention heads (%d)" % (hidden_size, num_attention_heads)) + + self.dim_per_head = int(hidden_size / num_attention_heads) + + self.masked_multi_head_attention = MaskedMultiHeadAttention( + batch_size=batch_size, + seq_length=seq_length, + hidden_size=hidden_size, + config=config, + num_attention_heads=num_attention_heads, + attention_dropout=attention_dropout, + hidden_dropout=hidden_dropout, + has_attention_mask=has_attention_mask, + is_training=is_training, + compute_type=compute_type + ) + self.mlp = MLPLayer(hidden_size=hidden_size, + config=config, + dropout_prob=hidden_dropout, + is_training=is_training) + + self.reshape = P.Reshape() + self.new_shape = (-1, hidden_size) + + def construct(self, input_tensor, attention_mask=None): + # input tensor shape[batch_size, seq_length, hidden_size] + input_tensor = self.reshape(input_tensor, self.new_shape) + # masked multi head attention with ln, res + attention_output = self.masked_multi_head_attention(input_tensor, attention_mask) + # feed forward, [batch_size * seq_length, hidden_size] + output = self.mlp(attention_output) + + return output + + +class CPMTransformer(nn.Cell): + """ + Implements of gpt module. + + Args: + batch_size (int): Batch size of input dataset. + hidden_size (int): Length of last dim of hidden layer. + seq_length (int): Length of input tensor sequence. + config: The config of networks. + num_hidden_layers (int): Numbers of hidden layers. + num_attention_heads (int): Number of attention heads. + has_attention_mask (bool): Specifies whether to use attention mask. + attention_dropout (float): The dropout probability for attention. + hidden_dropout (float): The dropout probability for hidden layers. + is_training (bool): Whether is training. + compute_type (:class:`mindspore.dtype`): Compute type in attention. + + Returns: + Tensor, shape of (N, T'). + """ + + def __init__(self, + batch_size, + hidden_size, + seq_length, + config=None, + num_hidden_layers=12, + num_attention_heads=12, + has_attention_mask=True, + attention_dropout=0.1, + hidden_dropout=0.1, + is_training=False, + compute_type=mstype.float16): + super(CPMTransformer, self).__init__() + + fusion_group_num = 4 + fusion_group_size = num_hidden_layers // fusion_group_num + fusion_group_size = max(fusion_group_size, 1) + + layers = [] + for i in range(num_hidden_layers): + layer = CPMTransformerLayer(batch_size=batch_size, + seq_length=seq_length, + hidden_size=hidden_size, + config=config, + num_attention_heads=num_attention_heads, + attention_dropout=attention_dropout, + hidden_dropout=hidden_dropout, + has_attention_mask=has_attention_mask, + is_training=is_training, + compute_type=compute_type).set_comm_fusion(int(i / fusion_group_size) + 2) + layer.recompute() + layer.masked_multi_head_attention.masked_self_attention.dropout.dropout_gen_mask.recompute(False) + layer.masked_multi_head_attention.masked_self_attention.dropout_probs.dropout_gen_mask.recompute(False) + layer.mlp.dropout.dropout_gen_mask.recompute(False) + + layer.masked_multi_head_attention.masked_self_attention.dropout.dropout_do_mask.recompute(False) + layer.masked_multi_head_attention.masked_self_attention.dropout_probs.dropout_do_mask.recompute(False) + layer.mlp.dropout.dropout_do_mask.recompute(False) + + layer.masked_multi_head_attention.masked_self_attention.dropout.dropout_gen_mask.add_prim_attr( + "_side_effect", True) + layer.masked_multi_head_attention.masked_self_attention.dropout_probs.dropout_gen_mask.add_prim_attr( + "_side_effect", True) + layer.mlp.dropout.dropout_gen_mask.add_prim_attr("_side_effect", True) + layers.append(layer) + + self.layers = nn.CellList(layers) + + self.reshape = P.Reshape() + self.final_layernorm = LayerNorm((hidden_size,), config, epsilon=1e-5).to_float(mstype.float32).set_comm_fusion( + int((num_hidden_layers - 1) / fusion_group_size) + 2) + + self.final_layernorm.gamma.parallel_optimizer = False + self.final_layernorm.beta.parallel_optimizer = False + self.new_shape = (-1, hidden_size) + + def construct(self, input_tensor, attention_mask=None): + """gpt module.""" + prev_output = self.reshape(input_tensor, self.new_shape) + for layer_module in self.layers: + layer_output = layer_module(prev_output, attention_mask) + prev_output = layer_output + + prev_output = self.final_layernorm(prev_output) + return prev_output + + +class CPMModel(nn.Cell): + """ + Implements of CPM model. + + Args: + batch_size (int): Batch size of input dataset. + seq_length (int): Length of input tensor sequence. + vocab_size (int): Size of the dictionary of embeddings. + hidden_size (int): Length of last dim of hidden layer. + num_hidden_layers (int): Numbers of hidden layers. + num_attention_heads (int): Number of attention heads. + config: The config of networks. + use_one_hot_embedding (bool): Whether use one-hot embedding. Default: False. + hidden_dropout (float): The dropout probability for hidden layers. + attention_dropout (float): The dropout probability for attention. + max_position_embeddings (int): The max length of position embedding. + initializer_range (int): The initialize range of parameters. + input_mask_from_dataset (bool): Specifies whether to use input mask. + is_training (bool): Whether is training. + compute_type (:class:`mindspore.dtype`): Compute type in attention. + + Returns: + Tensor, shape of (N, T'). + """ + + def __init__(self, + batch_size, + seq_length, + vocab_size, + hidden_size, + num_hidden_layers, + num_attention_heads, + config=None, + use_one_hot_embedding=False, + hidden_dropout=0.1, + attention_dropout=0.1, + max_position_embeddings=1024, + initializer_range=0.02, + input_mask_from_dataset=True, + is_training=False, + compute_type=mstype.float16): + super(CPMModel, self).__init__() + self.is_training = is_training + self.batch_size = batch_size + self.seq_length = seq_length + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.embedding_dim = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_dropout = hidden_dropout + self.attention_dropout = attention_dropout + self.max_position_embeddings = max_position_embeddings + self.initializer_range = initializer_range + self.input_mask_from_dataset = input_mask_from_dataset + self.compute_type = compute_type + + self.last_idx = self.num_hidden_layers - 1 + self.word_embedding = EmbeddingLookup( + batch_size=self.batch_size, + seq_length=self.seq_length, + vocab_size=self.vocab_size, + embedding_dim=self.hidden_size, + config=config, + use_one_hot_embeddings=use_one_hot_embedding, + compute_type=self.compute_type + ).set_comm_fusion(1) + self.position_embedding = EmbeddingPostprocessor( + max_seq_length=self.max_position_embeddings, + embedding_dim=self.embedding_dim, + config=config, + use_one_hot_embeddings=use_one_hot_embedding, + compute_type=self.compute_type + ).set_comm_fusion(1) + self.transformer = CPMTransformer(batch_size=self.batch_size, + hidden_size=self.hidden_size, + seq_length=self.seq_length, + config=config, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + has_attention_mask=True, + attention_dropout=self.attention_dropout, + hidden_dropout=self.hidden_dropout, + is_training=self.is_training, + compute_type=self.compute_type) + self.dropout = Dropout(1 - self.hidden_dropout) + self.dropout.dropout_gen_mask.shard(((config.dp, 1, 1),)) + self.dropout.dropout_do_mask.shard(((config.dp, 1, 1),)) + + self.matmul = P.MatMul(transpose_b=True).shard(((config.dp, 1), (1, 1))) + self.reshape = P.Reshape() + self.add = P.TensorAdd().shard(((config.dp, 1, config.mp), (config.dp, 1, config.mp))) + self.cast = P.Cast() + self.out_shape = (-1, self.seq_length, self.vocab_size) + + def construct(self, input_ids, position_ids=None, attention_mask=None): + """ + Construct network. + + Args: + input_ids (Tensor): Input sentences with shape (N, T). + position_ids (Tensor): Target of input sentences with shape (N, T). + attention_mask (Tensor): Source sentences padding mask with shape (N, T, T). + + Returns: + Tensor, network outputs. + """ + words_embeddings, embedding_tab = self.word_embedding(input_ids) + position_embedding = self.position_embedding(position_ids) + embedding = self.add(words_embeddings, position_embedding) + + if self.is_training: + embedding = self.dropout(embedding) + + transformer_output = self.transformer(embedding, attention_mask) + + transformer_output = self.cast(transformer_output, self.compute_type) + logits_output = self.matmul(transformer_output, + self.cast(embedding_tab, self.compute_type)) + logits_output = self.reshape(logits_output, self.out_shape) + logits_output = self.cast(logits_output, mstype.float32) + + return logits_output diff --git a/model_zoo/official/nlp/cpm/src/cpm_loss.py b/model_zoo/official/nlp/cpm/src/cpm_loss.py new file mode 100644 index 00000000000..5f3214e27e3 --- /dev/null +++ b/model_zoo/official/nlp/cpm/src/cpm_loss.py @@ -0,0 +1,168 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Loss.""" +import numpy as np + +import mindspore.common.dtype as mstype +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor + + +class Cross_entropy(nn.Cell): + """ + Calculate loss of Training mode and zero-shot mode. + + Args: + batch_size (int): Batch size of input dataset. + seq_length (int): Length of input tensor sequence. + vocab_size (int): Size of the dictionary of embeddings. + config: The config of networks. + is_training (bool): Whether is training. + Returns: + Tensor, shape of (batch_size,). + """ + + def __init__(self, batch_size, seq_length, vocab_size, config, is_training=False): + super(Cross_entropy, self).__init__() + self.batch_size = batch_size + self.vocab_size = vocab_size + self.argmax = P.ArgMaxWithValue(axis=-1).shard(((config.dp, 1, 1),)) + self.expanddim = P.ExpandDims().shard(((config.dp, 1),)) + self.sub = P.Sub().shard(((config.dp, 1), (config.dp, 1))) + self.sub_logist = P.Sub().shard(((config.dp, 1, 1), (config.dp, 1, 1))) + self.exp = P.Exp().shard(((config.dp, 1, 1),)) + self.till = P.Tile().shard(((1, 1),)) + self.till_expand = P.Tile().shard(((config.dp, 1, 1),)) + self.reduce_sum = P.ReduceSum().shard(((config.dp, 1, 1),)) + self.reshape = P.Reshape() + self.is_training = is_training + if self.is_training: + self.seq_length = 1 + else: + self.seq_length = seq_length + + self.first_dim = self.batch_size * self.seq_length + self.start = Tensor(np.zeros((self.batch_size, self.seq_length), dtype=np.int32)) + self.zero = Tensor(np.zeros((self.batch_size, self.seq_length), dtype=np.float32)) + self.end = Tensor(np.array([[self.vocab_size]], dtype=np.int32)) + self.arange = Tensor(np.expand_dims(np.arange(0, self.first_dim), -1), mstype.int32) + + self.greater = P.GreaterEqual().shard(((config.dp, 1), (config.dp, 1))) + self.logicalor = P.LogicalOr().shard(((config.dp, 1), (config.dp, 1))) + self.less = P.Less().shard(((config.dp, 1), (config.dp, 1))) + self.squeeze = P.Squeeze(axis=0) + self.log = P.Log().shard(((config.dp, 1),)) + self.cast = P.Cast() + self.select = P.Select().shard(((config.dp, 1), (config.dp, 1), (config.dp, 1))) + self.select_target = P.Select().shard(((config.dp, 1), (config.dp, 1), (config.dp, 1))) + self.concat = P.Concat(axis=-1).shard(((config.dp, 1), (config.dp, 1))) + self.gathernd = P.GatherNd().shard(((1, 1), (1, 1))) + self.sub_last = P.Sub().shard(((config.dp, 1), (config.dp, 1))) + + self.realdiv = P.RealDiv().shard(((1,), (1,))) + self.mul = P.Mul().shard(((1, 1), (1, 1))) + self.reduce_sum2 = P.ReduceSum().shard(((1, 1),)) + self.reduce_sum3 = P.ReduceSum().shard(((1, 1),)) + + def construct(self, logits, target, loss_mask=None): + r""" + Compute loss using logits, target and loss mask. + """ + # [8, 1, 30000] + _, logits_max = self.argmax(logits) + # [8 1] + logits_max_expand = self.expanddim(logits_max, -1) + logits_max_expand = self.till_expand(logits_max_expand, (1, 1, self.vocab_size)) + # [8, 1, 30000] | [8, 1, 30000] + logits_sub = self.sub_logist(logits, logits_max_expand) + logits_exp = self.exp(logits_sub) + # [8, 1, 30000] ->[8,30000] + sum_exp_logits = self.reduce_sum(logits_exp, -1) + # create a mask of a valid vocab ids + ends = self.end + ends = self.till(ends, (self.batch_size, self.seq_length)) + + vocab_start_res = self.less(target, self.start) + vocab_end_res = self.greater(target, ends) + # training mode: [batch 1]. + target_mask = self.logicalor(vocab_start_res, vocab_end_res) + masked_target = self.sub(target, self.start) + masked_target = self.select_target(target_mask, self.zero, self.cast(masked_target, mstype.float32)) + # [batch, vocab] + logits_2d = self.reshape(logits_sub, (-1, self.vocab_size)) + masked_target_1d = self.reshape(masked_target, (-1, 1)) + masked_target_1d = self.cast(masked_target_1d, mstype.int32) + + zeros = self.zero + # the next stack/concat means: predicted_logits_1d, logits_2d[self.arange, masked_target_1d] + stack_out = self.concat((self.arange, masked_target_1d)) + + predicted_logits_1d = self.gathernd(logits_2d, stack_out) + predicted_logits = self.reshape(predicted_logits_1d, (self.batch_size, -1)) + predicted_logits_masked = self.select(target_mask, zeros, self.cast(predicted_logits, mstype.float32)) + losses = self.sub_last(self.log(sum_exp_logits), predicted_logits_masked) + + if (not self.is_training) and (loss_mask is not None): + # loss calculate. + loss_mask_sum = self.reduce_sum2(loss_mask, -1) + loss_with_mask = self.mul(losses, loss_mask) + loss_with_mask_sum = self.reduce_sum3(loss_with_mask, -1) + loss = self.realdiv(loss_with_mask_sum, loss_mask_sum) + return loss + return losses + + +class Cross_entropy_eval(nn.Cell): + """ + Calculate loss of validation mode. + + Args: + batch_size (int): Batch size of input dataset. + seq_length (int): Length of input tensor sequence. + vocab_size (int): Size of the dictionary of embeddings. + config: The config of networks. + + Returns: + Tensor, shape of (batch_size,). + """ + + def __init__(self, batch_size, seq_length, vocab_size, config): + super(Cross_entropy_eval, self).__init__() + self.batch_size = batch_size + self.vocab_size = vocab_size + self.argmax = P.ArgMaxWithValue(axis=-1).shard(((config.dp, 1),)) + self.squeeze = P.Squeeze() + self.expanddims = P.ExpandDims().shard(((config.dp, 1),)) + self.expanddims1 = P.ExpandDims().shard(((config.dp,),)) + self.tile = P.Tile().shard(((config.dp, 1, 1),)) + self.reducesum = P.ReduceSum().shard(((config.dp, 1, 1),)) + self.reducesum2 = P.ReduceSum().shard(((config.dp, 1),)) + self.readdiv = P.RealDiv().shard(((config.dp, 1), (config.dp, 1))) + self.mul = P.Mul().shard(((config.dp, 1, 1), (config.dp, 1, 1))) + self.reshape = P.Reshape() + + def construct(self, logist, loss_mask): + r""" + Compute loss using logits and loss mask. + """ + loss_mask_expand = self.expanddims(loss_mask, -1) + loss_masks = self.tile(loss_mask_expand, (1, 1, self.vocab_size)) + loss_mask_sum = self.expanddims1(self.reducesum2(loss_mask, -1), -1) + logist_mask_mul = self.mul(logist, loss_masks) + logist_mask_sum = self.reducesum(logist_mask_mul, 1) + output = self.readdiv(logist_mask_sum, loss_mask_sum) + + return output diff --git a/model_zoo/official/nlp/cpm/src/cpm_train.py b/model_zoo/official/nlp/cpm/src/cpm_train.py new file mode 100644 index 00000000000..3087c3979a0 --- /dev/null +++ b/model_zoo/official/nlp/cpm/src/cpm_train.py @@ -0,0 +1,386 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""CPM train module""" +import numpy as np + +import mindspore.nn as nn +from mindspore.common.initializer import initializer +from mindspore.common.tensor import Tensor +from mindspore.ops import operations as P +from mindspore.ops import composite as C +from mindspore import context +import mindspore.common.dtype as mstype +from mindspore.common.parameter import Parameter +from mindspore.ops.operations.comm_ops import _VirtualDataset +from mindspore.ops import functional as F +from mindspore.nn.wrap.loss_scale import TrainOneStepWithLossScaleCell + +from src.cpm_loss import Cross_entropy +from src.cpm import CPMModel +from src.util import ClipByGlobalNorm + + +class CPMWithLoss(nn.Cell): + """ + Provide CPM training loss through network. + + Args: + batch_size (int): Batch size of input dataset. + seq_length (int): Length of input tensor sequence. + vocab_size (int): Size of the vocabulary list. + hidden_size (int): Internal feature dimension. + config: The config of CPM network. + num_hidden_layers (int): Number of hidden layers. + num_attention_heads (int): Number of attention heads. + + Returns: + Tensor, the loss of the network. + """ + + def __init__(self, batch_size, seq_length, vocab_size, hidden_size, + config, num_hidden_layers, num_attention_heads): + super(CPMWithLoss, self).__init__() + self.batch_size = batch_size + self.seq_length = seq_length + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.squeeze = P.Squeeze() + self.expanddims = P.ExpandDims().shard(((config.dp, 1),)) + self.expanddims1 = P.ExpandDims().shard(((config.dp,),)) + self.tile = P.Tile().shard(((config.dp, 1, 1),)) + self.reducesum = P.ReduceSum().shard(((config.dp, 1, 1),)) + self.reducesum2 = P.ReduceSum().shard(((config.dp, 1),)) + self.reducemean = P.ReduceMean().shard(((1, 1),)) + self.cast = P.Cast() + self.readdiv = P.RealDiv().shard(((config.dp, 1), (config.dp, 1))) + self.readdiv2 = P.RealDiv().shard(((1,), (1,))) + self.mul = P.Mul().shard(((config.dp, 1, 1), (config.dp, 1, 1))) + self.mul2 = P.Mul().shard(((config.dp, 1), (config.dp, 1))) + + self.cpm_model = CPMModel(batch_size=self.batch_size, + seq_length=self.seq_length, + vocab_size=self.vocab_size, + hidden_size=self.hidden_size, + config=config, + hidden_dropout=config.dropout, + attention_dropout=config.dropout, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + is_training=True) + + self.loss_net = Cross_entropy(batch_size=self.batch_size, + seq_length=self.seq_length, + vocab_size=self.vocab_size, + config=config, + is_training=True) + self.slice = P.StridedSlice().shard(((config.dp, 1),)) + self.slice_mask = P.StridedSlice().shard(((config.dp, 1, 1),)) + + def construct(self, input_ids, attention_mask=None, position_ids=None, loss_mask=None, labels=None): + r""" + CPM model with loss. + """ + input_ids = self.slice(input_ids, (0, 0), + (self.batch_size, self.seq_length), + (1, 1)) + position_ids = self.slice(position_ids, (0, 0), + (self.batch_size, self.seq_length), + (1, 1)) + attention_mask_1 = self.slice_mask(attention_mask, (0, 0, 0), + (self.batch_size, self.seq_length, self.seq_length), + (1, 1, 1)) + logist = self.cpm_model(input_ids, position_ids, attention_mask_1) + loss_mask_expand = self.expanddims(loss_mask, -1) + # 8 725 -> 8, 725, 1 + loss_masks = self.tile(loss_mask_expand, (1, 1, self.vocab_size)) + # 8 725 30000 + loss_mask_sum = self.expanddims1(self.reducesum2(loss_mask, -1), -1) + # [8, 725, 30000|8, 725, 30000 + logist_mask_mul = self.mul(logist, loss_masks) + # 8, 725, 30000->8, 30000 + logist_mask_sum = self.reducesum(logist_mask_mul, 1) + # 8, 30000| 8 1 + output = self.readdiv(logist_mask_sum, loss_mask_sum) + # 8 725 | 8 725 + label_mul_mask = self.mul2(labels, loss_mask) + # 8 725 -> 8 + label_mask = self.reducesum2(label_mul_mask, 1) + # 8 725 -> 8 + loss_mask_for_label = self.reducesum2(loss_mask, -1) + # 8 / 8 + label_final = self.readdiv2(label_mask, loss_mask_for_label) + # batch 1 vocabe_size + output = self.expanddims(output, 1) + # batchsize 1 + label_final = self.expanddims1(label_final, 1) + # batchsize 1 + losses = self.loss_net(output, self.cast(label_final, mstype.float32)) + loss = self.reducemean(losses, 0) + return loss + + +GRADIENT_CLIP_TYPE = 1 +GRADIENT_CLIP_VALUE = 1.0 +clip_grad = C.MultitypeFuncGraph("clip_grad") + + +@clip_grad.register("Number", "Number", "Tensor") +def _clip_grad(clip_type, clip_value, grad): + """ + Clip gradients. + + Inputs: + clip_type (int): The way to clip, 0 for 'value', 1 for 'norm'. + clip_value (float): Specifies how much to clip. + grad (tuple[Tensor]): Gradients. + + Outputs: + tuple[Tensor], clipped gradients. + """ + if clip_type not in [0, 1]: + return grad + dt = F.dtype(grad) + if clip_type == 0: + new_grad = C.clip_by_value( + grad, F.cast(F.tuple_to_array((-clip_value,)), dt), + F.cast(F.tuple_to_array((clip_value,)), dt)) + else: + new_grad = nn.ClipByNorm()(grad, + F.cast(F.tuple_to_array((clip_value,)), + dt)) + return new_grad + + +class VirtualDatasetOneInputCell(nn.Cell): + def __init__(self, backbone): + super(VirtualDatasetOneInputCell, self).__init__(auto_prefix=False) + self._backbone = backbone + self._virtual_dataset = _VirtualDataset() + + def construct(self, *data): + data_ = self._virtual_dataset(*data) + return self._backbone(*data_) + + +grad_scale = C.MultitypeFuncGraph("grad_scale") +reciprocal = P.Reciprocal() + + +@grad_scale.register("Tensor", "Tensor") +def tensor_grad_scale(scale, grad): + return grad * reciprocal(scale) + + +class CPMTrainOneStepWithLossScaleCell(TrainOneStepWithLossScaleCell): + """ + Encapsulation class of CPM network training. + + Append an optimizer to the training network after that the construct + function can be called to create the backward graph. + + Args: + network (Cell): The training network. Note that loss function should have been added. + optimizer (Optimizer): Optimizer for updating the weights. + scale_update_cell (Cell): Cell to do the loss scale. Default: None. + enable_global_norm (Bool): Whether using global normalization. + """ + + def __init__(self, + network, + optimizer, + scale_update_cell=None, + enable_global_norm=True): + super(CPMTrainOneStepWithLossScaleCell, + self).__init__(network, optimizer, scale_update_cell) + self.network = network + self.weights = optimizer.parameters + self.optimizer = optimizer + self.default_lr = Tensor([0.0], dtype=mstype.float32) + self.enable_global_norm = enable_global_norm + self.cast = P.Cast() + self.clip = ClipByGlobalNorm(self.weights) + + def construct(self, + input_ids, + attention_mask, + position_ids, + loss_mask, + labels, + sens=None): + """Defines the computation performed.""" + weights = self.weights + loss = self.network(input_ids, + attention_mask, + position_ids, + loss_mask, + labels) + + scaling_sens = self.scale_sense + # alloc status and clear should be right before grad operation. + status, scaling_sens = self.start_overflow_check(loss, scaling_sens) + scaling_sens_filled = C.ones_like(loss) * F.cast(scaling_sens, F.dtype(loss)) + grads = self.grad(self.network, + weights)(input_ids, + attention_mask, + position_ids, + loss_mask, + labels, + scaling_sens_filled) + # apply grad reducer on grads. + grads = self.grad_reducer(grads) + grads = self.hyper_map( + F.partial(grad_scale, scaling_sens), grads) + + if self.enable_global_norm: + grads, _ = self.clip(grads) + else: + grads = self.hyper_map( + F.partial(clip_grad, GRADIENT_CLIP_TYPE, GRADIENT_CLIP_VALUE), + grads) + + cond = self.get_overflow_status(status, grads) + overflow = self.process_loss_scale(cond) + if overflow: + succ = False + else: + succ = self.optimizer(grads) + return F.depend(loss, succ), cond, scaling_sens + + +cast = P.Cast() +update_accu_grads = C.MultitypeFuncGraph("update_accu_grads") + + +@update_accu_grads.register("Tensor", "Tensor") +def _update_accu_grads(accu_grad, grad): + succ = True + return F.depend(succ, F.assign_add(accu_grad, cast(grad, mstype.float32))) + + +zeroslike = P.ZerosLike() +reset_accu_grads = C.MultitypeFuncGraph("reset_accu_grads") + + +@reset_accu_grads.register("Tensor") +def _reset_accu_grads(accu_grad): + succ = True + return F.depend(succ, F.assign(accu_grad, zeroslike(accu_grad))) + + +class CPMTrainAccuStepsWithLossScaleCell(TrainOneStepWithLossScaleCell): + """ + Encapsulation class of CPM network training with loss scale. + + Append an optimizer to the training network after that the construct + function can be called to create the backward graph. + + Args: + network (Cell): The training network. Note that loss function should have been added. + optimizer (Optimizer): Optimizer for updating the weights. + scale_update_cell (Cell): Cell to do the loss scale. Default: None. + enable_global_norm (Bool): Whether using global normalization. + """ + + def __init__(self, + network, + optimizer, + scale_update_cell=None, + enable_global_norm=True): + super(CPMTrainAccuStepsWithLossScaleCell, self).__init__(network, optimizer, scale_update_cell) + self.accumulation = False + self.accumulation_steps = context.get_auto_parallel_context("grad_accumulation_step") + self.one = Tensor(np.array([1]).astype(np.int32)) + self.zero = Tensor(np.array([0]).astype(np.int32)) + + self.accu_grads = self.weights.clone(prefix="accu_grads", init='zeros') + self.accu_overflow = Parameter(initializer(0, [1], mstype.int32)) + self.accu_loss = Parameter(initializer(0, [1], mstype.float32)) + + self.cast = P.Cast() + self.logical_or = P.LogicalOr() + self.not_equal = P.NotEqual() + self.select = P.Select() + self.reshape = P.Reshape() + self.enable_global_norm = enable_global_norm + self.clip = ClipByGlobalNorm(self.weights) + + def construct(self, + input_ids, + attention_mask, + position_ids, + loss_mask, + labels, + sens=None): + """Defines the computation performed.""" + weights = self.weights + loss = self.network(input_ids, + attention_mask, + position_ids, + loss_mask, + labels) + scaling_sens = self.scale_sense + status, scaling_sens = self.start_overflow_check(loss, scaling_sens) + scaling_sens_filled = C.ones_like(loss) * F.cast(scaling_sens, F.dtype(loss)) + + grads = self.grad(self.network, + weights)(input_ids, + attention_mask, + position_ids, + loss_mask, + labels, scaling_sens_filled) + + if self.accumulation and self.accumulation_steps > 1: + accu_succ = self.hyper_map(update_accu_grads, self.accu_grads, grads) + loss = F.depend(loss, accu_succ) + + overflow = self.get_overflow_status(status, grads) + overflow = self.logical_or(self.not_equal(self.accu_overflow, self.zero), overflow) + accu_overflow = self.select(overflow, self.one, self.zero) + + if self.accumulation: + succ = False + self.accu_overflow = accu_overflow + else: + my_zero = F.depend(self.zero, accu_overflow) + initialize = P.Assign()(self.accu_overflow, my_zero) + grads1 = F.depend(grads, initialize) + # apply grad reducer on grads + grads = self.grad_reducer(grads1) + + scaling = scaling_sens * self.accumulation_steps + grads = self.hyper_map(F.partial(grad_scale, scaling), grads) + if self.enable_global_norm: + grads, _ = self.clip(grads) + else: + grads = self.hyper_map(F.partial(clip_grad, GRADIENT_CLIP_TYPE, GRADIENT_CLIP_VALUE), grads) + accu_overflow = self.allreduce(accu_overflow) + + overflow = self.less_equal(self.base, accu_overflow) + accu_grads = F.depend(self.accu_grads, grads) + + accu_succ = self.hyper_map(reset_accu_grads, accu_grads) + overflow = F.depend(overflow, accu_succ) + + overflow = self.reshape(overflow, (())) + overflow = self.process_loss_scale(overflow) + + if overflow: + succ = False + else: + succ = self.optimizer(grads) + + return F.depend(loss, succ), overflow, scaling_sens diff --git a/model_zoo/official/nlp/cpm/src/embedding.py b/model_zoo/official/nlp/cpm/src/embedding.py new file mode 100644 index 00000000000..36bb04b3e49 --- /dev/null +++ b/model_zoo/official/nlp/cpm/src/embedding.py @@ -0,0 +1,166 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Embedding.""" +import numpy as np + +import mindspore.common.dtype as mstype +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter + +from src.weight_init import normal_weight + + +class EmbeddingLookup(nn.Cell): + """ + A embeddings lookup table with a fixed dictionary and size. + + Args: + batch_size (int): Batch size of input dataset. + seq_length (int): Length of input tensor sequence. + vocab_size (int): Size of the dictionary of embeddings. + embedding_dim (int): The size of each embedding vector. + use_one_hot_embeddings (bool): Specifies whether to use one hot encoding form. + config: The config of networks. + compute_type (:class:`mindspore.dtype`): Compute type. + """ + + def __init__(self, + batch_size, + seq_length, + vocab_size, + embedding_dim, + config=None, + use_one_hot_embeddings=True, + compute_type=mstype.float16): + super(EmbeddingLookup, self).__init__() + self.vocab_size = vocab_size + self.embedding_dim = embedding_dim + self.use_one_hot_embeddings = use_one_hot_embeddings + self.compute_type = compute_type + self.batch_size = batch_size + self.seq_length = seq_length + self.embedding_table = Parameter(normal_weight([vocab_size, embedding_dim], embedding_dim), + name='embedding_table') + self.embedding_table.parallel_optimizer = False + self.shape_flat = (-1,) + self.gather = P.GatherV2().shard(((1, 1), (config.dp,))) + + self.reshape = P.Reshape() + self.shape = P.Shape() + self.cast = P.Cast() + + self.less = P.Less().shard(((config.dp, 1), (config.dp, 1))) + self.greaterequal = P.GreaterEqual().shard(((config.dp, 1), (config.dp, 1))) + self.logicalor = P.LogicalOr().shard(((config.dp, 1), (config.dp, 1))) + self.zero = Tensor(np.zeros((self.batch_size, self.seq_length), dtype=np.int32)) + self.zero_2 = Tensor(np.zeros((self.batch_size, self.seq_length, 1), dtype=np.int32)) + self.start = Tensor(np.zeros((self.batch_size, self.seq_length), dtype=np.int32)) + self.end = Tensor(np.array([[self.vocab_size]], dtype=np.int32)) + self.expanddim_first = P.ExpandDims().shard(((config.dp, 1),)) + self.expanddim = P.ExpandDims().shard(((config.dp, 1),)) + self.tile_in_mask = P.Tile().shard(((config.dp, 1, 1),)) + + self.tile_2 = P.Tile().shard(((1, 1),)) + self.tile = P.Tile().shard(((config.dp, 1, 1),)) + self.select = P.Select().shard( + ((config.dp, 1, config.mp), (config.dp, 1, config.mp), (config.dp, 1, config.mp))) + self.mask_select = P.Select().shard(((config.dp, 1), (config.dp, 1), (config.dp, 1))) + self.sub = P.Sub().shard(((config.dp, 1), (config.dp, 1))) + self.get_dtype = P.DType() + + def construct(self, input_ids): + """ + get embedding according to input_ids. + """ + input_less = self.less(input_ids, self.start) + ends = self.end + ends = self.tile_2(ends, (self.batch_size, self.seq_length)) + input_greater = self.greaterequal(input_ids, ends) + input_mask = self.logicalor(input_less, input_greater) + masked_input = self.sub(input_ids, self.start) + # [batchsize, seq_length] + masked_input = self.mask_select(input_mask, self.zero, self.cast(masked_input, mstype.int32)) + + input_shape = self.shape(masked_input) + flat_ids = self.reshape(masked_input, self.shape_flat) + + flat_ides = self.cast(flat_ids, mstype.int32) + output_for_reshape = self.gather(self.embedding_table, flat_ides, 0) + + out_shape = input_shape + (self.embedding_dim,) + output = self.reshape(output_for_reshape, out_shape) + # [batchsize, seq_length] + input_masks = self.expanddim(input_mask, -1) + input_mask_tile = self.tile_in_mask(self.cast(input_masks, mstype.int32), (1, 1, self.embedding_dim)) + zero_expand = self.zero_2 + zero_tiled = self.tile(zero_expand, (1, 1, self.embedding_dim)) + zero_tiled = self.cast(zero_tiled, self.get_dtype(output)) + output = self.select(self.cast(input_mask_tile, mstype.bool_), zero_tiled, output) + output = self.cast(output, mstype.float32) + return output, self.embedding_table + + +class EmbeddingPostprocessor(nn.Cell): + """ + Positional embeddings. + + Args: + max_seq_length (int): the length of input sequence. + embedding_dim (int): The size of each embedding vector. + config: The config of networks. + use_one_hot_embeddings (bool): Whether use one-hot embedding. + compute_type (:class:`mindspore.dtype`): Compute type. Default: mstype.float16. + """ + + def __init__(self, + max_seq_length, + embedding_dim, + config=None, + use_one_hot_embeddings=True, + compute_type=mstype.float16): + super(EmbeddingPostprocessor, self).__init__() + self.max_seq_length = max_seq_length + self.embedding_dim = embedding_dim + self.use_one_hot_embeddings = use_one_hot_embeddings + self.compute_type = compute_type + self.position_embedding_table = Parameter(normal_weight([max_seq_length, embedding_dim], embedding_dim), + name='position_embeddings') + self.position_embedding_table.parallel_optimizer = False + self.shape_flat = (-1,) + self.gather = P.GatherV2().shard(((1, 1), (config.dp,))) + + self.reshape = P.Reshape() + self.shape = P.Shape() + self.cast = P.Cast() + position_ids = np.expand_dims(np.arange(config.seq_length * 1), 0) + self.position_ids = Tensor(np.tile(position_ids, (config.batch_size, 1)), dtype=mstype.int64) + + def construct(self, input_ids=None): + r""" + get embedding according to input_ids. + """ + if input_ids is None: + input_ids = self.position_ids + + input_shape = self.shape(input_ids) + flat_ids = self.reshape(input_ids, self.shape_flat) + + output_for_reshape = self.gather(self.position_embedding_table, flat_ids, 0) + + out_shape = input_shape + (self.embedding_dim,) + output = self.reshape(output_for_reshape, out_shape) + return output diff --git a/model_zoo/official/nlp/cpm/src/loss_monitor.py b/model_zoo/official/nlp/cpm/src/loss_monitor.py new file mode 100644 index 00000000000..5dc9a92e2b8 --- /dev/null +++ b/model_zoo/official/nlp/cpm/src/loss_monitor.py @@ -0,0 +1,121 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Loss monitor.""" +import time +import math + +from mindspore.train.callback import Callback + + +class LossCallBack(Callback): + """ + Monitor the loss in training. + If the loss is NAN or INF terminating training. + """ + time_stamp_init = False + time_stamp_first = 0 + + def __init__(self, per_print_times=-1): + super(LossCallBack, self).__init__() + self._per_print_times = per_print_times + + if not self.time_stamp_init: + self.time_stamp_first = self._get_ms_timestamp() + self.time_stamp_init = True + + def step_end(self, run_context): + """Print loss after each step.""" + cb_params = run_context.original_args() + file_name = "./loss.log" + with open(file_name, "a+") as f: + time_stamp_current = self._get_ms_timestamp() + if self._per_print_times > 0: + _, epoch_num = math.modf(cb_params.cur_step_num / self._per_print_times) + f.write("time: {},epoch: {},step: {},outputs:[loss: {},overflow: {}, loss scale value: {} ].\n".format( + time_stamp_current - self.time_stamp_first, + int(epoch_num), + cb_params.cur_step_num, + str(cb_params.net_outputs[0].asnumpy()), + str(cb_params.net_outputs[1].asnumpy()), + str(cb_params.net_outputs[2].asnumpy()) + )) + else: + f.write("time: {},epoch: {},step: {},outputs: [loss: {},overflow: {},loss scale value: {} ].\n".format( + time_stamp_current - self.time_stamp_first, + cb_params.cur_epoch_num, + cb_params.cur_step_num, + str(cb_params.net_outputs[0].asnumpy()), + str(cb_params.net_outputs[1].asnumpy()), + str(cb_params.net_outputs[2].asnumpy()) + )) + + @staticmethod + def _get_ms_timestamp(): + """Get timestamp.""" + t = time.time() + return int(round(t * 1000)) + + +class TimeCallBack(Callback): + """ + Monitor the time in training. + + Args: + data_size (int): Dataset size. Default: None. + """ + + def __init__(self, data_size=None): + super(TimeCallBack, self).__init__() + self.data_size = data_size + + def step_begin(self, run_context): + """Step begin.""" + self.epoch_time = time.time() + + def step_end(self, run_context): + """Step end.""" + epoch_seconds = (time.time() - self.epoch_time) * 1000 + step_size = self.data_size + cb_params = run_context.original_args() + if hasattr(cb_params, "batch_num"): + batch_num = cb_params.batch_num + if isinstance(batch_num, int) and batch_num > 0: + step_size = cb_params.batch_num + + if not isinstance(step_size, int) or step_size < 1: + logger.error("data_size must be positive int.") + return + + print("epoch time: {:5.3f} ms".format(epoch_seconds), flush=True) + + def epoch_begin(self, run_context): + """Epoch begin.""" + self.epoch_time = time.time() + + def epoch_end(self, run_context): + """Epoch end.""" + epoch_seconds = (time.time() - self.epoch_time) * 1000 + step_size = self.data_size + cb_params = run_context.original_args() + if hasattr(cb_params, "batch_num"): + batch_num = cb_params.batch_num + if isinstance(batch_num, int) and batch_num > 0: + step_size = cb_params.batch_num + + if not isinstance(step_size, int) or step_size < 1: + logger.error("data_size must be positive int.") + return + + print("epoch time: {:5.3f} ms".format(epoch_seconds), flush=True) diff --git a/model_zoo/official/nlp/cpm/src/lr_schedule.py b/model_zoo/official/nlp/cpm/src/lr_schedule.py new file mode 100644 index 00000000000..7948d7caab6 --- /dev/null +++ b/model_zoo/official/nlp/cpm/src/lr_schedule.py @@ -0,0 +1,73 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Learning rate schedule.""" +import numpy as np +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from mindspore.common import dtype as mstype +from mindspore.nn.learning_rate_schedule import LearningRateSchedule, WarmUpLR + + +class DecayLR(LearningRateSchedule): + """ + Implements of decay learning rate scheduler. + + Args: + learning_rate (float): Initial learning rate. + warmup_steps (int): Warmup steps. + end_steps (int): A value used to calculate decayed learning rate. + + Returns: + np.ndarray, learning rate of each step. + """ + + def __init__(self, learning_rate, warmup_steps, end_iter): + super(DecayLR, self).__init__() + self.learning_rate = learning_rate + self.warmup_steps = warmup_steps + self.end_iter = end_iter + self.cast = P.Cast() + + def construct(self, global_step): + warmup_percent = self.cast((self.end_iter - (global_step - self.warmup_steps)), mstype.float32) / self.end_iter + + return self.learning_rate * warmup_percent + + +class CPMLearningRate(LearningRateSchedule): + """ + Implements of warmup-polynomial decay learning rate scheduler. + + Args: + learning_rate (float): The initial value of learning rate. + warmup_steps (int): The warm up steps of learning rate. + end_steps (int): A value used to calculate decayed learning rate. + + Returns: + Tensor. The learning rate value for the current step. + """ + + def __init__(self, learning_rate, warmup_steps, end_steps): + super(CPMLearningRate, self).__init__() + self.warmup_lr = WarmUpLR(learning_rate, warmup_steps) + self.decay_lr = DecayLR(learning_rate, warmup_steps, end_steps) + self.warmup_steps = Tensor(np.array([warmup_steps]).astype(np.float32)) + + def construct(self, global_step): + if global_step < self.warmup_steps: + lr = self.warmup_lr(global_step) + else: + lr = self.decay_lr(global_step) + return lr diff --git a/model_zoo/official/nlp/cpm/src/model_cpm.py b/model_zoo/official/nlp/cpm/src/model_cpm.py new file mode 100644 index 00000000000..6275e29907b --- /dev/null +++ b/model_zoo/official/nlp/cpm/src/model_cpm.py @@ -0,0 +1,172 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Model CPM""" +import math + +from mindspore.train.callback import RunContext +from mindspore import context +from mindspore.context import ParallelMode +from mindspore import Model, connect_network_with_dataset +from mindspore.common.dtype import pytype_to_dtype +from mindspore._c_expression import init_exec_dataset +from mindspore.train.train_thor.dataset_helper import DatasetHelper + + +def _convert_type(types): + """ + Convert from numpy type to tensor type. + + Args: + types (list): Numpy type list of element in dataset. + + Returns: + list, list of element in dataset. + """ + ms_types = [] + for np_type in types: + ms_type = pytype_to_dtype(np_type) + ms_types.append(ms_type) + return ms_types + + +def _get_types_and_shapes(dataset): + """Get dataset types and shapes.""" + dataset_types = _convert_type(dataset.output_types()) + dataset_shapes = dataset.output_shapes() + return dataset_types, dataset_shapes + + +def _exec_datagraph(exec_dataset, dataset_size, phase='dataset'): + """Initialize and execute the dataset graph.""" + batch_size = exec_dataset.get_batch_size() + input_indexs = exec_dataset.input_indexs + + # transform data format + dataset_types, dataset_shapes = _get_types_and_shapes(exec_dataset) + init_exec_dataset(exec_dataset.__transfer_dataset__.queue_name, + dataset_size, + batch_size, + dataset_types, + dataset_shapes, + input_indexs, + phase=phase, + need_run=False) + + +class Model_ACCU(Model): + r""" + Overwrite Model class for gradient accumulation, docking dataset interface. + """ + def __init__(self, network, loss_fn=None, optimizer=None, metrics=None, eval_network=None, + eval_indexes=None, amp_level="O0", **kwargs): + super(Model_ACCU, self).__init__(network, loss_fn, optimizer, metrics, eval_network, + eval_indexes, amp_level, **kwargs) + self._frequency = context.get_auto_parallel_context("grad_accumulation_step") + self._train_network = self._build_train_network() + + def _exec_preprocess(self, network, is_train, phase, dataset, dataset_sink_mode, sink_size=-1, + epoch_num=1, iter_first_order=1): + """Initializes dataset.""" + if dataset_sink_mode and not is_train: + dataset.__loop_size__ = 1 + dataset_helper = DatasetHelper(dataset, dataset_sink_mode, sink_size, epoch_num, iter_first_order) + + if dataset_sink_mode and context.get_context("device_target") != "GPU": + network = connect_network_with_dataset(network, dataset_helper) + network.set_train(is_train) + network.phase = phase + + if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): + network.set_auto_parallel() + + return dataset_helper, network + + def _train_dataset_sink_process(self, epoch, train_dataset, list_callback=None, cb_params=None, sink_size=-1): + """ + Training process. The data would be passed to network through dataset channel. + + Args: + epoch (int): Total number of iterations on the data. + train_dataset (Dataset): A training dataset iterator. If there is no + loss_fn, a tuple with multiple data (data1, data2, data3, ...) should be + returned and passed to the network. Otherwise, a tuple (data, label) should + be returned. The data and label would be passed to the network and loss + function respectively. + list_callback (Callback): Executor of callback list. Default: None. + cb_params (_InternalCallbackParam): Callback parameters. Default: None. + sink_size (int): Control the amount of data in each sink. Default: -1. + """ + if sink_size == -1: + epoch_num = epoch + else: + epoch_num = math.ceil(epoch * sink_size / train_dataset.get_dataset_size()) + + iter_first_order = 1 + iter_second_order = self._frequency - 1 + train_dataset.__loop_size__ = iter_second_order + dataset_helper, train_network = self._exec_preprocess(self._train_network, + is_train=True, + phase='train', + dataset=train_dataset, + dataset_sink_mode=True, + sink_size=sink_size, + epoch_num=epoch_num, + iter_first_order=iter_first_order) + + self._train_network = train_network + cb_params.train_network = self._train_network + cb_params.cur_step_num = 0 + + run_context = RunContext(cb_params) + list_callback.begin(run_context) + + # used to stop training for early stop, such as stopAtTIme or stopATStep + should_stop = False + switch_branch_one = True + train_network_init_flag = True + has_do_dataset_init = False + + for i in range(epoch): + cb_params.cur_epoch_num = i + 1 + list_callback.epoch_begin(run_context) + # for data sink dataset_helper only iter once, other wise iter epoch_size times. + for inputs in dataset_helper: + list_callback.step_begin(run_context) + if switch_branch_one: + cb_params.cur_step_num += iter_second_order + if train_network_init_flag: + self._train_network.add_flags_recursive(accumulation=True) + self._train_network.phase = 'train0' + else: + cb_params.cur_step_num += iter_first_order + if train_network_init_flag: + self._train_network.add_flags_recursive(accumulation=False) + train_network_init_flag = False + self._train_network.phase = 'train1' + if not has_do_dataset_init: + _exec_datagraph(train_dataset, iter_first_order, phase='train1_dataset') + has_do_dataset_init = True + switch_branch_one = not switch_branch_one + outputs = self._train_network(*inputs) + cb_params.net_outputs = outputs + list_callback.step_end(run_context) + + list_callback.epoch_end(run_context) + should_stop = should_stop or run_context.get_stop_requested() + if should_stop: + break + dataset_helper.stop_send() + + list_callback.end(run_context) diff --git a/model_zoo/official/nlp/cpm/src/util.py b/model_zoo/official/nlp/cpm/src/util.py new file mode 100644 index 00000000000..8a5023a3e2b --- /dev/null +++ b/model_zoo/official/nlp/cpm/src/util.py @@ -0,0 +1,230 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Utils.""" +import mindspore.common.dtype as mstype +import mindspore.nn as nn +import mindspore.ops.functional as F +from mindspore.ops import operations as P +from mindspore.ops import composite as C +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter +from mindspore.communication.management import get_group_size +from mindspore.common.seed import _get_graph_seed +from mindspore._checkparam import Validator +from mindspore import context + +from src.weight_init import normal_weight, zero_weight, one_weight + + +class ResidualConnection(nn.Cell): + """ + Add residual to output. + + Args: + dropout_prob (float): Dropout rate. + + Returns: + Tensor, with the same shape of hidden_tensor + """ + + def __init__(self, dropout_prob=0.0): + super(ResidualConnection, self).__init__() + self.add = P.TensorAdd() + + def construct(self, hidden_tensor, input_tensor): + # hidden_tensor is the output of sublayer + output = hidden_tensor + output = self.add(output, input_tensor) + return output + + +class LinearLayer(nn.Cell): + """ + Args: + input_size (int): The number of input features. + output_size (int): The number of output features. + """ + + def __init__(self, + input_size, + output_size): + super(LinearLayer, self).__init__() + self.input_size = input_size + self.output_size = output_size + self.weight = Parameter(normal_weight([output_size, input_size], output_size), name='projection_weight') + self.bias = Parameter(zero_weight(output_size), name='projection_bias') + self.matmul = P.MatMul(transpose_b=True) + self.bias_add = P.BiasAdd() + self.cast = P.Cast() + + def construct(self, input_tensor): + input_tensor = self.cast(input_tensor, mstype.float16) + fp16_weight = self.cast(self.weight, mstype.float16) + + output_tensor = self.matmul(input_tensor, fp16_weight) + fp16_bias = self.cast(self.bias, mstype.float16) + output_tensor = self.bias_add(output_tensor, fp16_bias) + output_tensor = self.cast(output_tensor, mstype.float32) + return output_tensor + + +class LayerNorm(nn.Cell): + r""" + A self-defined layer norm operation using reduce sum and reduce mean + """ + + def __init__(self, normalized_shape, config=None, epsilon=1e-5, scale=1e-3): + super(LayerNorm, self).__init__() + self.gamma = Parameter(one_weight(normalized_shape, mstype.float32), name="gamma") + self.beta = Parameter(zero_weight(normalized_shape, mstype.float32), name="beta") + self.mean = P.ReduceMean(keep_dims=True).shard(((config.dp, 1),)) + self.square = P.Square().shard(((config.dp, 1),)) + self.sqrt = P.Sqrt().shard(((config.dp, 1),)) + self.sub1 = P.Sub().shard(((config.dp, 1), (config.dp, 1))) + self.add = P.TensorAdd().shard(((config.dp, 1), ())) + self.eps = epsilon + self.mul = P.Mul().shard(((config.dp, 1), (1,))) + self.add2 = P.TensorAdd().shard(((config.dp, 1), (1,))) + self.real_div = P.RealDiv().shard(((config.dp, 1), (config.dp, 1))) + + def construct(self, x): + mean = self.mean(x, -1) + diff = self.sub1(x, mean) + variance = self.mean(self.square(diff), -1) + variance_eps = self.sqrt(self.add(variance, self.eps)) + output = self.real_div(diff, variance_eps) + output = self.add2(self.mul(output, self.gamma), self.beta) + return output + + +get_square_sum = C.MultitypeFuncGraph("get_square_sum") + +@get_square_sum.register("Tensor", "Tensor") +def _get_square_sum(grad, value): + norm = P.ReduceSum(False)(F.square(grad) / value, ()) + norm = F.expand_dims(F.cast(norm, mstype.float32), 0) + return norm + +@get_square_sum.register("Tensor", "Number") +def _get_square_sum_number(grad, value): + norm = P.ReduceSum(False)(F.square(grad), ()) / value + norm = F.expand_dims(F.cast(norm, mstype.float32), 0) + return norm + + +apply_global_norm = C.MultitypeFuncGraph("apply_global_norm") + +@apply_global_norm.register("Tensor", "Tensor", "Tensor") +def _apply_global_norm(clip_norm, global_norm, grad): + grad = grad * clip_norm / global_norm + return grad + + +class GlobalNorm(nn.Cell): + r""" + Calculate the global norm value of given tensors + """ + + def __init__(self, params): + super(GlobalNorm, self).__init__() + self.norm = nn.Norm() + self.hyper_map = C.HyperMap() + + self.allreduce_filter = tuple( + "layernorm" not in x.name and + "dense_proj.bias" not in x.name and + "embedding_table" not in x.name for x in params) + + self.values = [] + self.group_size = get_group_size() + for item in self.allreduce_filter: + if item: + self.values.append(Tensor([1.0], mstype.float32)) + else: + self.values.append(Tensor([self.group_size * 1.0], mstype.float32)) + self.values = tuple(self.values) + + def construct(self, grads): + square_sum_dp = self.hyper_map(get_square_sum, grads, self.values) + global_norms = F.sqrt(P.AllReduce()(F.addn(square_sum_dp))) + return global_norms + + +class ClipByGlobalNorm(nn.Cell): + r""" + Clip grads by global norm + """ + + def __init__(self, params, clip_norm=1.0): + super(ClipByGlobalNorm, self).__init__() + self.global_norm = GlobalNorm(params) + self.clip_norm = Tensor([clip_norm], mstype.float32) + self.hyper_map = C.HyperMap() + + def construct(self, grads): + global_norm_value = self.global_norm(grads) + cond = P.GreaterEqual()(global_norm_value, self.clip_norm) + # P.Print()("do clip ", cond, ", global norm is ", global_norm_value) + global_norm = F.select(cond, global_norm_value, self.clip_norm) + grads = self.hyper_map(F.partial(apply_global_norm, self.clip_norm, global_norm), grads) + return grads, global_norm_value + + +class Dropout(nn.Cell): + r""" + A Dropout Implements with P.DropoutGenMask and P.DropoutDoMask for parallel training. + """ + + def __init__(self, keep_prob=0.5, dtype=mstype.float32): + super(Dropout, self).__init__() + if keep_prob <= 0 or keep_prob > 1: + raise ValueError("dropout probability should be a number in range (0, 1], but got {}".format(keep_prob)) + Validator.check_subclass("dtype", dtype, mstype.number_type, self.cls_name) + Validator.check_value_type('keep_prob', keep_prob, [float], self.cls_name) + self.keep_prob = keep_prob + seed0, seed1 = _get_graph_seed(0, "dropout") + self.seed0 = seed0 + self.seed1 = seed1 + self.dtype = dtype + self.get_shape = P.Shape() + self.dropout_gen_mask = P.DropoutGenMask(Seed0=self.seed0, Seed1=self.seed1) + self.dropout_do_mask = P.DropoutDoMask() + self.cast = P.Cast() + self.is_ascend = context.get_context('device_target') in ["Ascend"] + self.dropout = P.Dropout(keep_prob) + + def construct(self, x): + r""" + Input: a tensor + Returns: a tensor + """ + if not self.training: + return x + + if not self.is_ascend: + out, _ = self.dropout(x) + return out + + if self.keep_prob == 1: + return x + + shape = self.get_shape(x) + dtype = P.DType()(x) + keep_prob = self.cast(self.keep_prob, dtype) + output = self.dropout_gen_mask(shape, keep_prob) + return self.dropout_do_mask(x, output, keep_prob) + + def extend_repr(self): + return 'keep_prob={}, dtype={}'.format(self.keep_prob, self.dtype) diff --git a/model_zoo/official/nlp/cpm/src/weight_init.py b/model_zoo/official/nlp/cpm/src/weight_init.py new file mode 100644 index 00000000000..1f3ec8e73c9 --- /dev/null +++ b/model_zoo/official/nlp/cpm/src/weight_init.py @@ -0,0 +1,63 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Initializer.""" +import math +import numpy as np +from mindspore.common.tensor import Tensor +import mindspore.common.dtype as mstype + + +def _average_units(shape): + if not shape: + return 1 + if len(shape) == 1: + return float(shape[0]) + if len(shape) == 2: + return float(shape[0] + shape[1]) / 2. + raise RuntimeError("not support shape.") + + +def weight_variable(shape): + scale_shape = shape + avg_units = _average_units(scale_shape) + scale = 1.0 / max(1., avg_units) + limit = math.sqrt(3.0 * scale) + values = np.random.uniform(-limit, limit, shape).astype(np.float32) + return Tensor(values) + + +def one_weight(shape, dtype=mstype.float32): + ones = np.ones(shape).astype(np.float32) + return Tensor(ones, dtype=dtype) + + +def zero_weight(shape, dtype=mstype.float32): + zeros = np.zeros(shape).astype(np.float32) + return Tensor(zeros, dtype=dtype) + + +def zero_weight_fp32(shape, dtype=mstype.float32): + zeros = np.zeros(shape).astype(np.float32) + return Tensor(zeros, dtype=dtype) + + +def normal_weight(shape, num_units, dtype=mstype.float32): + norm = np.random.normal(0.0, num_units ** -0.5, shape).astype(np.float32) + return Tensor(norm, dtype=dtype) + + +def normal_weightfp32(shape, num_units, dtype=mstype.float32): + norm = np.random.normal(0.0, num_units ** -0.5, shape).astype(np.float32) + return Tensor(norm, dtype=dtype) diff --git a/model_zoo/official/nlp/cpm/test.py b/model_zoo/official/nlp/cpm/test.py new file mode 100644 index 00000000000..f2f8e6d8a67 --- /dev/null +++ b/model_zoo/official/nlp/cpm/test.py @@ -0,0 +1,115 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test.""" +import os +import ast +import argparse + +from mindspore import context +from mindspore.communication import management as MultiAscend +from mindspore.context import ParallelMode +from mindspore.parallel import set_algo_parameters + +from src.config import finetune_test_standalone, finetune_test_distrubute, \ + finetune_dev_distrubute, finetune_dev_standalone +from eval import run_eval, create_ckpt_file_list + +device_id = int(os.getenv("DEVICE_ID")) +rank_size = os.getenv('RANK_SIZE') +context.set_context(mode=context.GRAPH_MODE, + save_graphs=False, + device_target="Ascend", + device_id=device_id) + + + +def set_parallel_env(): + r""" + Parallel environment. + """ + context.reset_auto_parallel_context() + MultiAscend.init() + + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, + device_num=MultiAscend.get_group_size(), + gradients_mean=True, + full_batch=True) + set_algo_parameters(elementwise_op_strategy_follow=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="CPM inference") + parser.add_argument('--dev_dataset', type=str, default="", help="dev_dataset path.") + parser.add_argument("--dev_data_path", type=str, default="/disk0/dataset/finetune_dataset/dev.json", + help='dev_json path.') + parser.add_argument('--test_dataset', type=str, default="", help="test_dataset path.") + parser.add_argument("--test_data_path", type=str, default="/disk0/dataset/finetune_dataset/test.json", + help='test_json path.') + parser.add_argument('--ckpt_path_doc', type=str, default="", help="checkpoint path document.") + parser.add_argument('--ckpt_partition', type=int, default=8, help="Number of checkpoint partition.") + parser.add_argument("--distribute", type=ast.literal_eval, default=False, + help='Whether distributed evaluation with model parallel.') + parser.add_argument("--has_train_strategy", type=ast.literal_eval, default=True, + help='Whether the loaded checkpoints have distributed training strategy.') + + args_eval = parser.parse_args() + if args_eval.distribute: + set_parallel_env() + print("Start validation on 2 devices.") + else: + print("Start validation on 1 device.") + + args_eval.dataset = args_eval.dev_dataset + args_eval.data_path = args_eval.dev_data_path + if args_eval.has_train_strategy: + # Get the checkpoint with train strategy. + train_strategy_list = create_ckpt_file_list(args_eval, train_strategy="train_strategy.ckpt") + context.set_auto_parallel_context( + strategy_ckpt_load_file=train_strategy_list[0] + ) + # start run in dev dataset. + result_dev = [] + for i in range(4, 11): + ckpt_file_list_dev = None + if args_eval.has_train_strategy: + # Get the checkpoint slice. + ckpt_file_list_dev = create_ckpt_file_list(args_eval, i) + print("++++ Get sliced checkpoint file, lists: ", ckpt_file_list_dev, flush=True) + result_i = 0.0 + if args_eval.distribute: + result_i = run_eval(args_eval, finetune_dev_distrubute, ckpt_file_list_dev) + else: + result_i = run_eval(args_eval, finetune_dev_standalone, ckpt_file_list_dev) + print("+++++ i=", i, ", dev_dataset Accuracy: ", result_i) + result_dev.append(result_i) + + print("++++ The accuracy of each checkpoint on the validation dataset is:", result_dev) + print("++++ Then we take the model with the highest accuracy ") + print(" on the validation dataset to predict on the test dataset.") + index_max_dev = result_dev.index(max(result_dev)) + 4 + ckpt_file_list_test = None + if args_eval.has_train_strategy: + # Get the best precision checkpoint slice. + ckpt_file_list_test = create_ckpt_file_list(args_eval, index_max_dev) + + args_eval.dataset = args_eval.test_dataset + args_eval.data_path = args_eval.test_data_path + # start run in test dataset. + result_last = 0.0 + if args_eval.distribute: + result_last = run_eval(args_eval, finetune_test_distrubute, ckpt_file_list_test) + else: + result_last = run_eval(args_eval, finetune_test_standalone, ckpt_file_list_test) + print("++++ Accuracy on test dataset is: ", result_last) diff --git a/model_zoo/official/nlp/cpm/train.py b/model_zoo/official/nlp/cpm/train.py new file mode 100644 index 00000000000..5645b2c7b3b --- /dev/null +++ b/model_zoo/official/nlp/cpm/train.py @@ -0,0 +1,269 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Train.""" +import os +import ast +import argparse +import math +import numpy as np + +import mindspore.dataset as ds +from mindspore import context +from mindspore.train.model import Model +from mindspore.nn.optim import AdamWeightDecay +from mindspore.communication import management as MultiAscend +from mindspore.context import ParallelMode +from mindspore.common import set_seed + +from mindspore.train.callback import TimeMonitor, ModelCheckpoint, CheckpointConfig +from mindspore.nn.wrap.loss_scale import DynamicLossScaleUpdateCell +from mindspore.train.serialization import load_checkpoint, load_param_into_net +import mindspore.common.dtype as mstype +import mindspore.dataset.transforms.c_transforms as C +from mindspore.parallel import set_algo_parameters + +from src.config import config_train_8p, config_train_32p +from src.cpm_train import CPMWithLoss, CPMTrainOneStepWithLossScaleCell, VirtualDatasetOneInputCell, \ + CPMTrainAccuStepsWithLossScaleCell +from src.lr_schedule import CPMLearningRate +from src.loss_monitor import LossCallBack, TimeCallBack +from src.model_cpm import Model_ACCU as Model_CPM + +device_id = int(os.getenv("DEVICE_ID")) + +set_seed(23333) +context.set_context(mode=context.GRAPH_MODE, + save_graphs=False, + device_target="Ascend", + device_id=device_id) +context.set_context(variable_memory_max_size="30GB") + + +def collate(truth, input_ids, BatchInfo): + """Collate operation for dataset.""" + bs = len(truth) + max_size = np.size(input_ids, 1) + + attn_mask = np.tril(np.ones(shape=(max_size, max_size),)) + attention_mask = np.expand_dims(attn_mask, 0) + attention_mask = np.tile(attention_mask, (bs, 1, 1)).astype(np.float32) + + position_ids = np.expand_dims(np.arange(max_size * 1), 0) + position_ids = np.tile(position_ids, (bs, 1)).astype(np.int32) + + truth_list = np.zeros(bs, dtype=np.int32) + + for i in range(bs): + truth_list[i] = truth[i] + + return input_ids, attention_mask, position_ids, truth_list + + +def _load_dataset(dataset_path, batch_size, rank_size=None, rank_id=None, shuffle=True, drop_remainder=True, + is_training=True): + """Loader for data.""" + ds.config.set_seed(1) + data = ds.MindDataset(dataset_file=dataset_path, + columns_list=["truth", "input_ids", "loss_mask", "labels", "size"], + shuffle=shuffle) + + type_cast_op = C.TypeCast(mstype.float32) + type_cast_op_int = C.TypeCast(mstype.int32) + data = data.map(input_columns="input_ids", operations=type_cast_op_int) + data = data.map(input_columns="labels", operations=type_cast_op_int) + data = data.map(input_columns="loss_mask", operations=type_cast_op) + if is_training: + data = data.batch(batch_size, + per_batch_map=collate, + input_columns=["truth", "input_ids"], + output_columns=["input_ids", "attention_mask", "position_ids", "truth"], + column_order=["input_ids", "attention_mask", "position_ids", "loss_mask", "labels"], + num_parallel_workers=4, + drop_remainder=drop_remainder) + else: + data = data.batch(batch_size, + per_batch_map=collate, + input_columns=["truth", "input_ids"], + output_columns=["input_ids", "attention_mask", "position_ids", "truth"], + column_order=["input_ids", "attention_mask", "position_ids", "loss_mask", "labels", "truth"], + num_parallel_workers=4, + drop_remainder=drop_remainder) + + return data + + +def load_dataset(dataset, batch_size, + rank_size=None, rank_id=None, + shuffle=True, + drop_remainder=True, + is_training=True): + """ + Load dataset. + + Args: + dataset (class): Dataset. + batch_size (int): Batch size. + rank_size (int): Rank size. + rank_id (int): Rank id. + shuffle (bool): Whether shuffle dataset. + drop_remainder (bool): Determines whether or not to drop the last possibly incomplete batch. + is_training (bool): Whether training mode. + + Returns: + Dataset, dataset instance. + """ + return _load_dataset(dataset, + batch_size, rank_size=rank_size, + rank_id=rank_id, shuffle=shuffle, + drop_remainder=drop_remainder, + is_training=is_training) + + +def _build_training_pipeline(datasets, pretrain_ckpt_path, config_train): + """ + Building training pipeline + """ + net_with_loss = CPMWithLoss(batch_size=config_train.batch_size, + seq_length=config_train.seq_length, + vocab_size=config_train.vocab_size, + hidden_size=config_train.hidden_size, + config=config_train, + num_hidden_layers=config_train.num_hidden_layers, + num_attention_heads=config_train.num_attention_heads) + + net_with_loss = VirtualDatasetOneInputCell(net_with_loss) + + param_dict = load_checkpoint(pretrain_ckpt_path) + + can_be_loaded = {} + for name, _ in param_dict.items(): + if 'cpm_model.' not in name: + can_be_loaded['cpm_model.' + name] = param_dict[name] + else: + can_be_loaded[name] = param_dict[name] + load_param_into_net(net_with_loss, parameter_dict=can_be_loaded) + print("------->Load pretrained parameter successfully<------------") + + steps_per_epoch = datasets.get_dataset_size() + print("++++++Dataset size= ", steps_per_epoch, flush=True) + print("++++++MP= ", str(config_train.mp), flush=True) + print("++++++DP= ", str(config_train.dp), flush=True) + print("++++++Global_batch_size= ", str(config_train.batch_size), flush=True) + lr_schedule = CPMLearningRate(learning_rate=config_train.lr, + warmup_steps=int(steps_per_epoch * config_train.epoch * config_train.warmup_steps), + end_steps=steps_per_epoch * config_train.epoch) + params = net_with_loss.trainable_params() + + decay_filter = lambda x: 'layernorm' not in x.name.lower() and "bias" not in x.name.lower() + decay_params = list(filter(decay_filter, params)) + other_params = list(filter(lambda x: not decay_filter(x), params)) + group_params = [{'params': decay_params, 'weight_decay': config_train.weight_decay}, + {'params': other_params, 'weight_decay': 0.0}, + {'order_params': params}] + optimizer = AdamWeightDecay(group_params, lr_schedule, eps=config_train.eps, beta1=0.9, beta2=0.95) + + callback_size = config_train.grad_accumulation_step if config_train.grad_accumulation_step > 1 \ + else config_train.sink_size + actual_epoch_num = int(config_train.epoch * steps_per_epoch // callback_size) + print("++++++actual_epoch_num= ", str(actual_epoch_num), flush=True) + + if config_train.grad_accumulation_step > 1: + callback = [TimeCallBack(), LossCallBack(steps_per_epoch)] + else: + callback = [TimeMonitor(), LossCallBack(steps_per_epoch)] + + ckpt_config = CheckpointConfig(save_checkpoint_steps=steps_per_epoch, + integrated_save=False, + keep_checkpoint_max=config_train.epoch) + ckpt_model = ModelCheckpoint(prefix='cpm_rank_{}'.format(os.getenv("RANK_ID")), + directory=os.path.join('./', 'ckpt_rank_{}'.format(os.getenv("RANK_ID"))), + config=ckpt_config) + callback.append(ckpt_model) + + dynamic_loss_cale = DynamicLossScaleUpdateCell(loss_scale_value=math.pow(2, 32), + scale_factor=2, + scale_window=1000) + print(dynamic_loss_cale) + print(" | Start pre-training job.") + if config_train.grad_accumulation_step > 1: + cpm_with_grads = CPMTrainAccuStepsWithLossScaleCell(net_with_loss, optimizer=optimizer, + scale_update_cell=dynamic_loss_cale) + model = Model_CPM(cpm_with_grads) + model.train(config_train.epoch, datasets, callbacks=callback, + dataset_sink_mode=True) + else: + cpm_with_grads = CPMTrainOneStepWithLossScaleCell(net_with_loss, optimizer, dynamic_loss_cale) + + model = Model(cpm_with_grads) + model.train(epoch=actual_epoch_num, + train_dataset=datasets, + callbacks=callback, + sink_size=callback_size, + dataset_sink_mode=True) + + +def set_parallel_env(config_train): + r""" + Parallel environment. + """ + context.reset_auto_parallel_context() + MultiAscend.init() + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, + device_num=MultiAscend.get_group_size(), + gradients_mean=True, + grad_accumulation_step=config_train.grad_accumulation_step, + full_batch=True) + context.set_auto_parallel_context(enable_parallel_optimizer=True) + context.set_auto_parallel_context(strategy_ckpt_save_file='./train_strategy.ckpt') + set_algo_parameters(elementwise_op_strategy_follow=True) + + +def train_single(input_file, pretrain_ckpt_path, config_train): + """ + Training on single device + """ + print("Staring training on single device") + preprocessed_data = load_dataset(dataset=input_file, + batch_size=config_train.batch_size) + _build_training_pipeline(preprocessed_data, pretrain_ckpt_path, config_train) + + +def train_paralle(input_file, pretrain_ckpt_path, config_train): + """ + Training on multi device + """ + set_parallel_env(config_train) + print("Staring training on multiple device") + processed_data = load_dataset(dataset=input_file, + batch_size=config_train.batch_size, + rank_size=MultiAscend.get_group_size(), + rank_id=MultiAscend.get_rank()) + _build_training_pipeline(processed_data, pretrain_ckpt_path, config_train) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="CPM training.") + parser.add_argument("--dataset", type=str, default="", help="CPM dataset path") + parser.add_argument("--pretrain_ckpt_path", type=str, default="", + help="Load the checkpoint file path for train.") + parser.add_argument("--multi_machine", type=ast.literal_eval, default=False, help='distributed training') + + args = parser.parse_args() + if args.multi_machine: + print("Training on multiple machines") + train_paralle(args.dataset, args.pretrain_ckpt_path, config_train_32p) + else: + print("Training on single machine and using 8 cards.") + train_paralle(args.dataset, args.pretrain_ckpt_path, config_train_8p) diff --git a/model_zoo/official/nlp/cpm/zero-shot.py b/model_zoo/official/nlp/cpm/zero-shot.py new file mode 100644 index 00000000000..de018ab6b58 --- /dev/null +++ b/model_zoo/official/nlp/cpm/zero-shot.py @@ -0,0 +1,289 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Zero-shot.""" +import os +import ast +import argparse +import time +import numpy as np + +from mindspore import context, load_distributed_checkpoint +import mindspore.nn as nn +import mindspore.dataset as ds +from mindspore.common.tensor import Tensor +from mindspore.train.model import Model +import mindspore.common.dtype as mstype +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.communication import management as MultiAscend +from mindspore.context import ParallelMode +from mindspore.parallel import set_algo_parameters + +from src.cpm import CPMModel +from src.cpm_train import VirtualDatasetOneInputCell +from src.cpm_loss import Cross_entropy +from src.config import config_zero_shot_standalone, config_zero_shot_distrubute +from eval import create_ckpt_file_list + +device_id = int(os.getenv("DEVICE_ID")) +context.set_context(mode=context.GRAPH_MODE, + save_graphs=False, + device_target="Ascend", + device_id=device_id) + + +class CPMForInfer(nn.Cell): + """ + Encapsulation class of CPM network infer. + + Args: + network (nn.Cell): CPM model. + batch_size (int): Batch size of input dataset. + seq_length (int): Length of input tensor sequence. + vocab_size (int): Size of the dictionary of embeddings. + config: The config of networks. + + Returns: + Tensor, losses. + """ + def __init__(self, network, batch_size, seq_length, vocab_size, config): + super(CPMForInfer, self).__init__(auto_prefix=False) + self.network = network + self.batch_size = batch_size + self.seq_length = seq_length + self.vocab_size = vocab_size + self.loss_net = Cross_entropy(batch_size=self.batch_size, + seq_length=self.seq_length, + vocab_size=self.vocab_size, + config=config) + + def construct(self, input_ids, target, loss_mask): + """Defines the computation performed.""" + logist = self.network(input_ids) + loss = self.loss_net(logist, target, loss_mask) + return loss + + +def collate(sid, cid, input_ids, BatchInfo): + """Collate operation for dataset.""" + bs = len(sid) + max_size = np.size(input_ids, 1) + + attn_mask = np.tril(np.ones(shape=(max_size, max_size),)) + attention_mask = np.expand_dims(attn_mask, 0) + attention_mask = np.tile(attention_mask, (bs, 1, 1)) + + position_ids = np.expand_dims(np.arange(max_size * 1), 0) + position_ids = np.tile(position_ids, (bs, 1)) + + sids_list = np.zeros(bs, dtype=np.int64) + cids_list = np.zeros(bs, dtype=np.int64) + + for i in range(bs): + sids_list[i] = sid[i] + cids_list[i] = cid[i] + + return input_ids, attention_mask, position_ids, sids_list, cids_list + + +def _load_dataset(dataset_path, batch_size, rank_size=None, rank_id=None, shuffle=True, drop_remainder=True): + """Loader for data.""" + data = ds.MindDataset(dataset_file=dataset_path, + columns_list=["sid", "cid", "input_ids", "loss_mask", "labels", "size"], + shuffle=shuffle, + num_shards=rank_size, + shard_id=rank_id, + ) + data = data.batch(batch_size, + num_parallel_workers=4, + drop_remainder=drop_remainder) + return data + + +def load_dataset(dataset, batch_size, + rank_size=None, rank_id=None, + shuffle=True, + drop_remainder=True): + """ + Load dataset. + + Args: + dataset (class): Dataset. + batch_size (int): Batch size. + rank_size (int): Rank size. + rank_id (int): Rank index. + shuffle (bool): Whether shuffle dataset. + drop_remainder (bool): Determines whether or not to drop the last possibly incomplete batch. + + Returns: + Dataset, dataset instance. + """ + return _load_dataset(dataset, + batch_size, + shuffle=shuffle, + drop_remainder=drop_remainder) + + +class CPM_LAYER(nn.Cell): + """ + CPM model training with loss function. + """ + + def __init__(self, config_eval): + super(CPM_LAYER, self).__init__() + self.cpm_model = CPMModel(batch_size=config_eval.batch_size, + seq_length=config_eval.seq_length, + vocab_size=config_eval.vocab_size, + hidden_size=config_eval.hidden_size, + num_hidden_layers=config_eval.num_hidden_layers, + num_attention_heads=config_eval.num_attention_heads, + config=config_eval) + + def construct(self, input_ids, position_ids=None, attention_mask=None): + output = self.cpm_model(input_ids, position_ids, attention_mask) + return output + + +def run_eval(args, config_eval, ckpt_file_list=None): + """ + Building infer pipeline + """ + truth_labels = np.loadtxt(args.truth_labels_path) + + if args.distribute: + dataset = load_dataset(args.dataset, config_eval.batch_size, + rank_size=MultiAscend.get_group_size(), + rank_id=MultiAscend.get_rank(), + drop_remainder=False, + shuffle=False) + else: + dataset = load_dataset(args.dataset, + config_eval.batch_size, + drop_remainder=False, + shuffle=False) + + cpm_model = CPM_LAYER(config_eval) + + if args.distribute: + cpm_model = VirtualDatasetOneInputCell(cpm_model) + if not args.has_train_strategy: + weights = load_checkpoint(args.ckpt_path_doc) + can_be_loaded = {} + print("+++++++loading weights without train_strategy+++++") + for name, _ in weights.items(): + if 'cpm_model.' not in name: + can_be_loaded['cpm_model.' + name] = weights[name] + else: + can_be_loaded[name] = weights[name] + load_param_into_net(cpm_model, parameter_dict=can_be_loaded) + + infer_net = CPMForInfer(network=cpm_model, + batch_size=config_eval.batch_size, + seq_length=config_eval.seq_length, + vocab_size=config_eval.vocab_size, + config=config_eval) + + model = Model(infer_net) + + if args.has_train_strategy and not args.distribute: + load_distributed_checkpoint(infer_net, ckpt_file_list, None) + + if args.has_train_strategy and args.distribute: + fake_input_ids = Tensor(np.ones((config_eval.batch_size, config_eval.seq_length)), mstype.int64) + fake_target = Tensor(np.random.randint(0, 10, [config_eval.batch_size, config_eval.seq_length]), mstype.int64) + fake_loss_mask = Tensor(np.random.randn(config_eval.batch_size, config_eval.seq_length), mstype.float16) + predict_layout = model.infer_predict_layout(fake_input_ids, + fake_target, + fake_loss_mask) + + print("Start to load distributed checkpoint with train strategy.", flush=True) + load_distributed_checkpoint(infer_net, ckpt_file_list, predict_layout) + + all_sids = [] + all_losses = [] + all_cids = [] + + steps_per_epoch = dataset.get_dataset_size() + print("++++++Dataset size", steps_per_epoch, flush=True) + + for batch in dataset.create_dict_iterator(output_numpy=True, num_epochs=1): + input_ids = Tensor(batch['input_ids'], mstype.int64) + target = Tensor(batch['labels'], mstype.int64) + loss_mask = Tensor(batch['loss_mask'], mstype.float16) + sids = batch['sid'] + cids = batch['cid'] + + loss = model.predict(input_ids, + target, loss_mask) + + all_losses.append(loss.asnumpy()) + all_cids.append(cids) + all_sids.append(sids) + print("+++++ ", int(round(time.time() * 1000))) + + all_losses = np.stack(all_losses).reshape(-1) + all_sids = np.stack(all_sids).reshape(-1) + all_cids = np.stack(all_cids).reshape(-1) + + print("++++ all_losses= \n", all_losses) + + preds = [[] for _ in truth_labels] + for sid, cid, loss in zip(all_sids, all_cids, all_losses): + preds[sid].append((cid, loss)) + preds = [min(p, key=lambda x: x[1])[0] for p in preds if len(p) > 0] + result = sum([int(p == l) for p, l in zip(preds, truth_labels)]) / len(truth_labels) + print("RESULT: ", result) + + +def set_parallel_env(): + r""" + Parallel environment. + """ + context.reset_auto_parallel_context() + MultiAscend.init() + + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, + device_num=MultiAscend.get_group_size(), + gradients_mean=True, + full_batch=True) + set_algo_parameters(elementwise_op_strategy_follow=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="CPM inference") + parser.add_argument('--dataset', type=str, default="", help="dataset path.") + parser.add_argument('--truth_labels_path', type=str, default="", help="truth_labels path.") + parser.add_argument('--ckpt_path_doc', type=str, default="", help="checkpoint path doc or checkpoint path.") + parser.add_argument("--distribute", type=ast.literal_eval, default=False, help='Whether distributed evaluation' + ' with model parallel.') + parser.add_argument("--has_train_strategy", type=ast.literal_eval, default=False, + help='Whether the loaded checkpoints have distributed training strategy.') + parser.add_argument('--ckpt_partition', type=int, default=1, help="Number of checkpoint partition.") + args_parse = parser.parse_args() + ckpt_file_list_test = None + if args_parse.has_train_strategy: + # Get the checkpoint with train strategy. + train_strategy_list = create_ckpt_file_list(args_parse, train_strategy="train_strategy.ckpt") + context.set_auto_parallel_context( + strategy_ckpt_load_file=train_strategy_list[0] + ) + ckpt_file_list_test = create_ckpt_file_list(args_parse) + print("Get checkpoint file lists++++", ckpt_file_list_test, flush=True) + if args_parse.distribute: + set_parallel_env() + print("Staring evaluating on 2 devices with model parallel.") + run_eval(args_parse, config_zero_shot_distrubute, ckpt_file_list_test) + else: + print("Staring evaluating on 1 device without model parallel.") + run_eval(args_parse, config_zero_shot_standalone, ckpt_file_list_test)