Col_lin
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!26556 modify parallel api note for master 

Merge pull request !26556 from lilei/modify_parallel_API_note
This commit is contained in:
i-robot 2021-11-25 03:02:55 +00:00 committed by Gitee
parent 5048d3823b f8827a09fb
commit dfd71f668b
12 changed files with 244 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
mindspore/ccsrc/frontend/parallel/device_manager.cc
View File

@ -30,32 +30,43 @@ DeviceManagerPtr g_device_manager = nullptr;
bool InitDevice(int64_t device_num, int64_t global_rank, const std::string &backend,
const std::vector<int64_t> &stage) {
if (device_num <= 0) {
MS_LOG(ERROR) << "'device_num' must be positive.";
MS_LOG(ERROR) << "The context configuration parameter 'device_num' must be positive, "
"but got the value of device_num: "
<< device_num;
return false;
}
if (global_rank < 0) {
MS_LOG(ERROR) << "'global_rank' must be nonnegative.";
MS_LOG(ERROR) << "The context configuration parameter 'global_rank' must be nonnegative, "
"but got the value of global_rank: "
<< global_rank;
return false;
}
if (device_num > MAX_DEVICE_NUM) {
MS_LOG(ERROR) << "'device_num' must be no more than " << MAX_DEVICE_NUM << ".";
MS_LOG(ERROR) << "The context configuration parameter 'device_num' must be no more than " << MAX_DEVICE_NUM
<< ", but got the value of device_num: " << device_num;
return false;
}
// 'device_num_converted' must be the power of 2
if ((LongToUlong(device_num) & LongToUlong(device_num - 1)) != 0) {
MS_LOG(ERROR) << "'device_num' must be the power of 2.";
MS_LOG(ERROR) << "The context configuration parameter device_num' must be the power of 2, "
"but got the value of device_num: "
<< device_num;
return false;
}
if (global_rank >= device_num) {
MS_LOG(ERROR) << "'global_rank' must be less than 'device_num'.";
MS_LOG(ERROR) << "The context configuration parameter 'global_rank' must be less than 'device_num', "
"but got the value of global_rank: "
<< global_rank << ", and the value of device_num: " << device_num;
return false;
}
if ((backend != HCCL_BACKEND) && (backend != NCCL_BACKEND) && (backend != UNDEFINED_BACKEND)) {
MS_LOG(ERROR) << "Invalid backend: " << backend;
MS_LOG(ERROR) << "The context configuration parameter 'backend' must be hccl, nccl "
"or undefined_backend, but got invalid backend: "
<< backend;
return false;
}
if (stage.empty()) {
MS_LOG(ERROR) << "The size of stage must be positive";
MS_LOG(ERROR) << "The size of parameter 'stage' must be positive, but got the size of stage is empty.";
return false;
}
@ -67,7 +78,7 @@ bool InitDevice(int64_t device_num, int64_t global_rank, const std::string &back
int64_t summed_value = 0;
for (auto begin = stage.begin(); begin != stage.end(); ++begin) {
if (*begin <= 0) {
MS_LOG(ERROR) << "The value in the pipeline stages should be positive value";
MS_LOG(ERROR) << "The value in the pipeline stages should be positive value, but got the value: " << *begin;
return false;
}
summed_value += *begin;
@ -75,8 +86,9 @@ bool InitDevice(int64_t device_num, int64_t global_rank, const std::string &back
}
if (summed_value != device_num) {
MS_LOG(ERROR) << "The sum of the pipeline stage :" << summed_value << " is not equal to the device_num "
<< device_num;
MS_LOG(ERROR) << "The sum of the pipeline stage must be equal to the device_num, "
"but got sum of the pipeline stage :"
<< summed_value << " and the device_num : " << device_num;
return false;
}
@ -144,12 +156,15 @@ std::shared_ptr<Device> GetListMemberByIndex(size_t index, const std::vector<std
Status DeviceManager::Init(const RankList &devices, int64_t global_device_rank, const RankList &stage_map,
const std::string &backend) {
if ((backend != HCCL_BACKEND) && (backend != NCCL_BACKEND) && (backend != UNDEFINED_BACKEND)) {
MS_LOG(ERROR) << "Invalid backend: " << backend;
MS_LOG(ERROR) << "The context configuration parameter 'backend' must be hccl, nccl "
"or undefined_backend, but got invalid backend: "
<< backend;
return FAILED;
}
if (stage_map.empty() || devices.empty()) {
MS_LOG(ERROR) << "The size of stage_map and devices must be positive";
MS_LOG(ERROR) << "The size of stage_map and devices must be positive, but got the size of stage_map: "
<< stage_map.size() << ", and the size of devices : " << devices.size();
return FAILED;
}
@ -162,11 +177,12 @@ Status DeviceManager::Init(const RankList &devices, int64_t global_device_rank,
for (auto &stage : stage_map) {
int64_t num_device = stage;
if (num_device > MAX_DEVICE_NUM) {
MS_LOG(ERROR) << "The number of 'devices' in a stage must not be greater than " << MAX_DEVICE_NUM;
MS_LOG(ERROR) << "The number of 'devices' in a stage must not be greater than " << MAX_DEVICE_NUM
<< ", but got the number of 'devices' in a stage: " << num_device;
return FAILED;
}
if (num_device <= 0) {
MS_LOG(ERROR) << "The number of 'devices' in a stage must be positive";
MS_LOG(ERROR) << "The number of 'devices' in a stage must be positive, but got the num_device: " << num_device;
return FAILED;
}
RankList curr_dev_list;

16
mindspore/ccsrc/frontend/parallel/step_parallel.cc
View File

@ -2997,25 +2997,31 @@ Status ParallelInit() {
int32_t split_stage_num = ParallelContext::GetInstance()->pipeline_stage_split_num();
std::string parallel_mode = ParallelContext::GetInstance()->parallel_mode();
if (split_stage_num <= 0) {
MS_LOG(ERROR) << "Invalid stage num " << split_stage_num << ", expected a positive stage number";
MS_LOG(ERROR) << "The parameter 'split_stage_num' must be a positive number, but got the value : "
<< split_stage_num;
return FAILED;
}
auto comm_info = GetCommInfo();
int64_t device_num = comm_info.device_num;
int64_t global_rank = comm_info.global_rank;
if ((device_num <= 0) || (device_num > MAX_DEVICE_NUM)) {
MS_LOG(ERROR) << "Invalid device num " << device_num;
MS_LOG(ERROR) << "The context configuration parameter 'device_num' must be positive, "
"but got the value of device_num: "
<< device_num;
return FAILED;
}
// the device_num maybe get from communication interface
if (device_num % split_stage_num != 0) {
MS_LOG(ERROR) << "Device num " << device_num << " can't be divided by stage num " << split_stage_num;
MS_LOG(ERROR) << "The parameter 'device_num' must be divided by 'split_stage_num', but got the device_num : "
<< device_num << "and the split_stage_num : " << split_stage_num;
return FAILED;
}
if ((global_rank < 0) || (global_rank >= device_num)) {
MS_LOG(ERROR) << "Global rank " << global_rank << " is out of range, the device num is " << device_num;
MS_LOG(ERROR) << "The parameter 'global_rank' must be greater than 0 and less equal 'device num', "
"but got the global_rank : "
<< global_rank << "and the device_num : " << device_num;
return FAILED;
}
@ -3034,7 +3040,7 @@ Status ParallelInit() {
return FAILED;
}
MS_LOG(INFO) << "The parallel context: dev num: " << device_num << ", global rank: " << global_rank
MS_LOG(INFO) << "The parallel context: device_num: " << device_num << ", global_rank: " << global_rank
<< ", communication_backend: " << comm_info.communication_backend
<< ", gradients_mean: " << ParallelContext::GetInstance()->gradients_mean()
<< ", gradient_fp32_sync: " << ParallelContext::GetInstance()->gradient_fp32_sync();

10
mindspore/ccsrc/pipeline/jit/pipeline_split.cc
View File

@ -77,7 +77,7 @@ bool PipelineSplit(const ResourcePtr &res) {
}
auto stage_num = parallel::ParallelContext::GetInstance()->pipeline_stage_split_num();
if (stage_num <= 1) {
MS_LOG(INFO) << "stage num is: " << stage_num << ". No need Pipeline split.";
MS_LOG(INFO) << "The parameter 'stage_num' is: " << stage_num << ". No need Pipeline split.";
return true;
}
auto manager = res->manager();
@ -96,10 +96,14 @@ bool PipelineSplit(const ResourcePtr &res) {
device_num = parallel::ParallelContext::GetInstance()->device_num();
}
if (device_num < 1) {
MS_LOG(EXCEPTION) << "Invalid device num: " << device_num;
MS_LOG(ERROR) << "The context configuration parameter 'device_num' must be positive, "
"but got the value of device_num: "
<< device_num;
}
if (global_rank < 0) {
MS_LOG(EXCEPTION) << "Invalid global rank: " << global_rank;
MS_LOG(ERROR) << "The context configuration parameter 'global_rank' must be nonnegative, "
"but got the value of global_rank: "
<< global_rank;
}
auto stage = InferStage(global_rank, stage_num, device_num);
auto per_stage_rank_num = device_num / stage_num;

48
mindspore/communication/_comm_helper.py
View File

@ -82,10 +82,12 @@ class Backend:
def __new__(cls, name):
"""Create instance object of Backend."""
if not isinstance(name, str):
raise TypeError("Backend name must be a string, but got {}".format(type(name)))
raise TypeError("The context configuration parameter 'name' must be a string, "
"but got the type : {}".format(type(name)))
value = getattr(Backend, name.upper(), Backend.UNDEFINED)
if value == Backend.UNDEFINED:
raise ValueError("Invalid backend: '{}'".format(name))
raise ValueError("The context configuration parameter 'name' {} is not supported, "
"please use hccl or nccl.".format(name))
return value
DEFAULT_BACKEND = Backend("hccl")
@ -160,8 +162,8 @@ def check_parameter_available(func):
if "group" in kargs.keys():
group = kargs.get("group")
if group is not None and not isinstance(group, str):
raise TypeError("Group should be str or None, "
"but got group {}".format(type(group)))
raise TypeError("The parameter 'group' should be str or None, "
"but got the type : {}".format(type(group)))
if "backend" in kargs.keys():
backend = kargs.get("backend")
@ -210,7 +212,8 @@ def _get_rank_helper(group, backend):
elif backend == Backend.NCCL:
rank_id = mpi.get_rank_id(group)
else:
raise ValueError("Invalid backend: '{}'".format(backend))
raise ValueError("The context configuration parameter 'backend' {} is not supported, "
"please use hccl_mpi, hccl or nccl.".format(backend))
return rank_id
@ -240,7 +243,8 @@ def _get_local_rank_helper(group, backend):
elif backend == Backend.NCCL:
raise RuntimeError("Nccl doesn't support get_local_rank_id now.")
else:
raise ValueError("Invalid backend: '{}'".format(backend))
raise ValueError("The context configuration parameter 'backend' {} is not supported, "
"please use hccl_mpi or hccl.".format(backend))
return rank_id
@ -273,7 +277,8 @@ def _get_size_helper(group, backend):
elif backend == Backend.NCCL:
size = mpi.get_rank_size(group)
else:
raise ValueError("Invalid backend: '{}'".format(backend))
raise ValueError("The context configuration parameter 'backend' {} is not supported, "
"please use hccl or nccl.".format(backend))
return size
@ -301,7 +306,8 @@ def _get_local_size_helper(group, backend):
elif backend == Backend.NCCL:
raise RuntimeError("Nccl doesn't support get_local_rank_size now.")
else:
raise ValueError("Invalid backend: '{}'".format(backend))
raise ValueError("The context configuration parameter 'backend' {} is not supported, "
"please use hccl.".format(backend))
return size
@ -324,15 +330,16 @@ def _get_world_rank_from_group_rank_helper(group, group_rank_id, backend):
"""
world_rank_id = None
if not isinstance(group_rank_id, int):
raise TypeError("group_rank_id should be int, but got type {}".format(type(group_rank_id)))
raise TypeError("The parameter 'group_rank_id' must be int, but got type {}".format(type(group_rank_id)))
if backend == Backend.HCCL:
if group == HCCL_WORLD_COMM_GROUP:
raise ValueError("Group cannot be 'hccl_world_group'. ")
raise ValueError("The parameter 'group' cannot be 'hccl_world_group'. ")
world_rank_id = hccl.get_world_rank_from_group_rank(group, group_rank_id)
elif backend == Backend.NCCL:
raise RuntimeError("Nccl doesn't support get_world_rank_from_group_rank now.")
else:
raise ValueError("Invalid backend: '{}'".format(backend))
raise ValueError("The context configuration parameter 'backend' {} is not supported, "
"please use hccl.".format(backend))
return world_rank_id
@ -355,15 +362,16 @@ def _get_group_rank_from_world_rank_helper(world_rank_id, group, backend):
"""
group_rank_id = None
if not isinstance(world_rank_id, int):
raise TypeError("world_rank_id should be int, but got type {}".format(type(world_rank_id)))
raise TypeError("The parameter 'world_rank_id' should be int, but got type {}".format(type(world_rank_id)))
if backend == Backend.HCCL:
if group == HCCL_WORLD_COMM_GROUP:
raise ValueError("Group cannot be 'hccl_world_group'. ")
raise ValueError("The parameter group cannot be 'hccl_world_group'. ")
group_rank_id = hccl.get_group_rank_from_world_rank(world_rank_id, group)
elif backend == Backend.NCCL:
raise RuntimeError("Nccl doesn't support get_group_rank_from_world_rank now.")
else:
raise ValueError("Invalid backend: '{}'".format(backend))
raise ValueError("The context configuration parameter 'backend' {} is not supported, "
"please use hccl.".format(backend))
return group_rank_id
@ -390,10 +398,12 @@ def _create_group_helper(group, rank_ids, backend):
return
if backend == Backend.HCCL:
if not isinstance(rank_ids, list):
raise TypeError("Rank_ids {} should be list".format(rank_ids))
raise TypeError("The type of parameter 'rank_ids' should be list, but got the type : {}."
.format(type(rank_ids)))
rank_size = len(rank_ids)
if rank_size < 1:
raise ValueError("Rank_ids size {} should be large than 0".format(rank_size))
raise ValueError("The parameter 'rank_ids' size should be large than 0, "
"but got the value : {}.".format(rank_size))
if len(rank_ids) - len(list(set(rank_ids))) > 0:
raise ValueError("List rank_ids in Group {} has duplicate data!".format(group))
hccl.create_group(group, rank_size, rank_ids)
@ -402,7 +412,8 @@ def _create_group_helper(group, rank_ids, backend):
elif backend == Backend.NCCL:
raise RuntimeError("Nccl doesn't support create_group now.")
else:
raise ValueError("Invalid backend: '{}'".format(backend))
raise ValueError("The context configuration parameter 'backend' {} is not supported, "
"please use hccl.".format(backend))
_ExistingGroup.ITEMS[group] = rank_ids
@ -425,4 +436,5 @@ def _destroy_group_helper(group, backend):
elif backend == Backend.NCCL:
raise RuntimeError("Nccl doesn't support destroy_group now.")
else:
raise ValueError("Invalid backend: '{}'".format(backend))
raise ValueError("The context configuration parameter 'backend' {} is not supported, "
"please use hccl.".format(backend))

34
mindspore/communication/_hccl_management.py
View File

@ -35,9 +35,10 @@ def check_group(group):
if isinstance(group, (str)):
group_len = len(group)
if group_len > MAX_GROUP_NAME_LEN or group_len == 0:
raise ValueError('Group name is invalid.')
raise ValueError("The length of parameter 'group' should in range [1, {}], but got the value : {}"
.format(MAX_GROUP_NAME_LEN, group_len))
else:
raise TypeError('Group must be a python str.')
raise TypeError("The context configuration parameter 'group' must be a string, but got {}".format(type(group)))
def check_rank_num(rank_num):
@ -49,9 +50,10 @@ def check_rank_num(rank_num):
"""
if isinstance(rank_num, (int)):
if rank_num > MAX_RANK_NUM or rank_num <= 0:
raise ValueError('Rank number is out of range.')
raise ValueError("The parameter 'rank_num' should in range [1, {}], but got the value : {}"
.format(MAX_RANK_NUM, rank_num))
else:
raise TypeError('Rank number must be a python int.')
raise TypeError("The parameter 'rank_num' must be a python int, but got {}".format(type(rank_num)))
def check_rank_id(rank_id):
@ -63,9 +65,10 @@ def check_rank_id(rank_id):
"""
if isinstance(rank_id, (int)):
if rank_id >= MAX_RANK_NUM or rank_id < 0:
raise ValueError('Rank id is out of range.')
raise ValueError("The parameter 'rank_id' should in range [1, {}], but got the value : {}"
.format(MAX_RANK_NUM, rank_id))
else:
raise TypeError('Rank id must be a python int.')
raise TypeError("The parameter 'rank_id' must be a python int, but got {}".format(type(rank_id)))
def load_lib():
@ -110,10 +113,11 @@ def create_group(group, rank_num, rank_ids):
check_rank_num(rank_num)
if isinstance(rank_ids, (list)):
if rank_num != len(rank_ids):
raise ValueError('Rank number is not equal to the length of rank_ids.')
raise ValueError("The parameter 'rank_num' number is not equal to the length of rank_ids, "
"but got 'rank_num' : {} and 'rank_ids' : {}.".format(rank_num, rank_ids))
for rank_id in rank_ids:
if not isinstance(rank_id, (int)) or rank_id < 0:
raise ValueError('Rank id must be unsigned integer!')
raise ValueError("The parameter 'rank_id' must be unsigned integer, but got {}".format(type(rank_id)))
c_array_rank_ids = c_array(ctypes.c_uint, rank_ids)
c_rank_num = ctypes.c_uint(rank_num)
c_group = c_str(group)
@ -121,7 +125,7 @@ def create_group(group, rank_num, rank_ids):
if ret != 0:
raise RuntimeError('Create group error, the error code is ' + str(ret))
else:
raise TypeError('Rank ids must be a python list.')
raise TypeError("The parameter 'rank_id' must be a python list, but got {}".format(type(rank_ids)))
def destroy_group(group):
@ -198,7 +202,8 @@ def get_local_rank_size(group="hccl_world_group"):
An integer scalar with the num of local ranks.
"""
if context.get_context("mode") is context.PYNATIVE_MODE:
raise RuntimeError("get_local_rank_size is not supported in PYNATIVE_MODE.")
raise RuntimeError("The function 'get_local_rank_size' is not supported in PYNATIVE_MODE, "
"'get_local_rank_size' only support GRAPH_MODE")
check_group(group)
c_group = c_str(group)
c_local_rank_size = ctypes.c_uint()
@ -220,7 +225,8 @@ def get_local_rank_id(group="hccl_world_group"):
"""
if context.get_context("mode") is context.PYNATIVE_MODE:
raise RuntimeError("get_local_rank_id is not supported in PYNATIVE_MODE.")
raise RuntimeError("The function 'get_local_rank_id' is not supported in PYNATIVE_MODE, "
"'get_local_rank_id' only support GRAPH_MODE")
check_group(group)
c_group = c_str(group)
c_local_rank_id = ctypes.c_uint()
@ -242,7 +248,8 @@ def get_world_rank_from_group_rank(group, group_rank_id):
An integer scalar with the rank id in the world group.
"""
if context.get_context("mode") is context.PYNATIVE_MODE:
raise RuntimeError("get_world_rank_from_group_rank is not supported in PYNATIVE_MODE.")
raise RuntimeError("The function 'get_world_rank_from_group_rank' is not supported in PYNATIVE_MODE, "
"'get_world_rank_from_group_rank' only support GRAPH_MODE")
check_group(group)
check_rank_id(group_rank_id)
c_group = c_str(group)
@ -266,7 +273,8 @@ def get_group_rank_from_world_rank(world_rank_id, group):
An integer scalar with the rank id in the user group.
"""
if context.get_context("mode") is context.PYNATIVE_MODE:
raise RuntimeError("get_group_rank_from_world_rank is not supported in PYNATIVE_MODE.")
raise RuntimeError("The function 'get_group_rank_from_world_rank' is not supported in PYNATIVE_MODE, "
"'get_group_rank_from_world_rank' only support GRAPH_MODE")
check_group(group)
check_rank_id(world_rank_id)
c_group = c_str(group)

41
mindspore/communication/management.py
View File

@ -68,11 +68,11 @@ def _check_parallel_envs():
import os
rank_id_str = os.getenv("RANK_ID")
if not rank_id_str:
raise RuntimeError("Environment variables RANK_ID has not been exported")
raise RuntimeError("Environment variables RANK_ID has not been exported, please export variables 'RANK_ID'.")
try:
int(rank_id_str)
except ValueError:
print("RANK_ID should be number")
print("The parameter 'RANK_ID' should be number, but got {}".format(type(rank_id_str)))
finally:
pass
rank_table_file_str = os.getenv("MINDSPORE_HCCL_CONFIG_PATH")
@ -120,14 +120,16 @@ def init(backend_name=None):
elif device_target == "GPU":
backend_name = "nccl"
else:
raise RuntimeError("Device target {} is not supported in parallel initialization, "
"please use Ascend or GPU.".format(device_target))
raise RuntimeError("The context configuration parameter 'device_target' {} is not supported in "
"parallel initialization, please use Ascend or GPU.".format(device_target))
if not isinstance(backend_name, str):
raise TypeError("Backend name must be a string, but got {}".format(type(backend_name)))
raise TypeError("The context configuration parameter 'backend_name' must be a string, "
"but got the type : {}".format(type(backend_name)))
if backend_name == "hccl":
if device_target != "Ascend":
raise RuntimeError("Device target should be 'Ascend' to init hccl, but got {}".format(device_target))
raise RuntimeError("The context configuration parameter 'device_target' should be 'Ascend' to init hccl, "
"but got {}".format(device_target))
if not mpi_init:
_check_parallel_envs()
GlobalComm.BACKEND = Backend("hccl")
@ -142,7 +144,8 @@ def init(backend_name=None):
GlobalComm.WORLD_COMM_GROUP = NCCL_WORLD_COMM_GROUP
GlobalComm.INITED = True
else:
raise RuntimeError("Backend name {} is not supported.".format(backend_name))
raise RuntimeError("The context configuration parameter 'backend_name' {} is not supported, "
"please use hccl or nccl.".format(backend_name))
def release():
@ -189,7 +192,8 @@ def get_rank(group=GlobalComm.WORLD_COMM_GROUP):
>>> # the result is the rank_id in world_group
"""
if not isinstance(group, str):
raise TypeError("Group name must be a string, but got {}".format(type(group)))
raise TypeError("The context configuration parameter 'group' must be a string, "
"but got the type : {}".format(type(group)))
return _get_rank_helper(group=_get_group(group), backend=GlobalComm.BACKEND)
@ -223,7 +227,8 @@ def get_local_rank(group=GlobalComm.WORLD_COMM_GROUP):
local_rank is: 1, world_rank is 9
"""
if not isinstance(group, str):
raise TypeError("Group name must be a string, but got {}".format(type(group)))
raise TypeError("The context configuration parameter 'group' must be a string, "
"but got the type : {}".format(type(group)))
return _get_local_rank_helper(group=_get_group(group), backend=GlobalComm.BACKEND)
@ -256,7 +261,8 @@ def get_group_size(group=GlobalComm.WORLD_COMM_GROUP):
group_size is: 8
"""
if not isinstance(group, str):
raise TypeError("Group name must be a string, but got {}".format(type(group)))
raise TypeError("The context configuration parameter 'group' must be a string, "
"but got the type : {}".format(type(group)))
return _get_size_helper(group=_get_group(group), backend=GlobalComm.BACKEND)
@ -289,7 +295,8 @@ def get_local_rank_size(group=GlobalComm.WORLD_COMM_GROUP):
local_rank_size is: 8
"""
if not isinstance(group, str):
raise TypeError("Group name must be a string, but got {}".format(type(group)))
raise TypeError("The context configuration parameter 'group' must be a string, "
"but got the type : {}".format(type(group)))
return _get_local_size_helper(group=_get_group(group), backend=GlobalComm.BACKEND)
@ -328,7 +335,8 @@ def get_world_rank_from_group_rank(group, group_rank_id):
world_rank_id is: 4
"""
if not isinstance(group, str):
raise TypeError("Group name must be a string, but got {}".format(type(group)))
raise TypeError("The context configuration parameter 'group' must be a string, "
"but got the type : {}".format(type(group)))
return _get_world_rank_from_group_rank_helper(group=group, group_rank_id=group_rank_id, backend=GlobalComm.BACKEND)
@ -367,7 +375,8 @@ def get_group_rank_from_world_rank(world_rank_id, group):
group_rank_id is: 1
"""
if not isinstance(group, str):
raise TypeError("Group name must be a string, but got {}".format(type(group)))
raise TypeError("The context configuration parameter 'group' must be a string, "
"but got the type : {}".format(type(group)))
return _get_group_rank_from_world_rank_helper(world_rank_id=world_rank_id, group=group, backend=GlobalComm.BACKEND)
@ -405,7 +414,8 @@ def create_group(group, rank_ids):
>>> allreduce = ops.AllReduce(group)
"""
if not isinstance(group, str):
raise TypeError("Group name must be a string, but got {}".format(type(group)))
raise TypeError("The context configuration parameter 'group' must be a string, "
"but got the value : {}".format(type(group)))
_create_group_helper(group, rank_ids, backend=GlobalComm.BACKEND)
@ -427,5 +437,6 @@ def destroy_group(group):
RuntimeError: If HCCL is not available or MindSpore is GPU version.
"""
if not isinstance(group, str):
raise TypeError("Group name must be a string, but got {}".format(type(group)))
raise TypeError("The context configuration parameter 'group' must be a string, "
"but got the type : {}".format(type(group)))
_destroy_group_helper(group, backend=GlobalComm.BACKEND)

19
mindspore/nn/cell.py
View File

@ -243,7 +243,8 @@ class Cell(Cell_):
@parameter_layout_dict.setter
def parameter_layout_dict(self, value):
if not isinstance(value, dict):
raise TypeError("The 'parameter_layout_dict' must be a dict type.")
raise TypeError("The type of parameter 'value' must be a dict type, "
"but got the type : {}.".format(type(value)))
self._parameter_layout_dict = value
@property
@ -253,7 +254,8 @@ class Cell(Cell_):
@parallel_parameter_name_list.setter
def parallel_parameter_name_list(self, value):
if not isinstance(value, list):
raise TypeError("The 'parallel_parameter_name_list' must be a list type.")
raise TypeError("The type of parameter 'parallel_parameter_name_list' must be a list type, "
"but got the type : {}.".format(type(value)))
self._parallel_parameter_name_list = value
@property
@ -262,13 +264,13 @@ class Cell(Cell_):
@pipeline_stage.setter
def pipeline_stage(self, value):
if isinstance(value, bool):
raise TypeError("'pipeline_stage' must be an int type, but got bool.")
if not isinstance(value, int):
raise TypeError("'pipeline_stage' must be an int type, but got {}".format(value))
if not isinstance(value, int) or isinstance(value, bool):
raise TypeError("The parameter 'pipeline_stage' must be an int type, "
"but got the type : {}.".format(type(value)))
if value < 0:
raise TypeError("'pipeline_stage' can not be less than 0 but got {}".format(value))
raise TypeError("The parameter 'pipeline_stage' can not be less than 0, "
"but got the value : {}".format(value))
self._pipeline_stage = value
for item in self.trainable_params():
item.add_pipeline_stage(value)
@ -280,7 +282,8 @@ class Cell(Cell_):
@parallel_parameter_merge_net_dict.setter
def parallel_parameter_merge_net_dict(self, value):
if not isinstance(value, dict):
raise TypeError("The 'parallel_parameter_merge_net_dict' must be a dict type.")
raise TypeError("The parameter 'parallel_parameter_merge_net_dict' must be a dict type, "
"but got the type : {}".format(type(value)))
self._parallel_parameter_merge_net_dict = value
def get_func_graph_proto(self):

75
mindspore/parallel/_auto_parallel_context.py
View File

@ -78,7 +78,8 @@ class _AutoParallelContext:
"""
self.check_context_handle()
if device_num < 1 or device_num > 4096:
raise ValueError("Device num must be in [1, 4096], but got {}".format(device_num))
raise ValueError("The context configuration parameter 'device_num' must be in [1, 4096], "
"but got the value of device_num : {}.".format(device_num))
from mindspore.communication._comm_helper import _HCCL_TEST_AVAILABLE
self._context_handle.set_hccl_test_avaible(_HCCL_TEST_AVAILABLE)
self._context_handle.set_device_num(device_num)
@ -100,7 +101,8 @@ class _AutoParallelContext:
"""
self.check_context_handle()
if global_rank < 0 or global_rank > 4095:
raise ValueError("Global rank must be in [0, 4095], but got {}".format(global_rank))
raise ValueError("The context configuration parameter 'global_rank' must be in [0, 4095], "
"but got the value of global_rank : {}.".format(global_rank))
self._context_handle.set_global_rank(global_rank)
def get_global_rank(self):
@ -110,12 +112,11 @@ class _AutoParallelContext:
def set_pipeline_stages(self, stages):
"""Set the stages of the pipeline"""
if isinstance(stages, bool):
raise TypeError("The type of pipeline_stage_num must be int, but got bool.")
if not isinstance(stages, int):
raise TypeError("The type of pipeline_stage_num must be int.")
if isinstance(stages, bool) or not isinstance(stages, int):
raise TypeError("The type of pipeline_stage_num must be int, but got the type : {}.".format(type(stages)))
if stages < 1:
raise ValueError("pipeline_stage_num can't be less than 1.")
raise ValueError("The parameter pipeline_stage_num be greater or equal 1, "
"but got the value of stages : {}.".format(stages))
self.check_context_handle()
self._context_handle.set_pipeline_stage_split_num(stages)
@ -174,7 +175,8 @@ class _AutoParallelContext:
loss_repeated_mean (bool): The loss_repeated_mean flag.
"""
if not isinstance(loss_repeated_mean, bool):
raise TypeError(f"The type of loss_repeated_mean must be bool, but got {type(loss_repeated_mean)}.")
raise TypeError("The type of context configuration parameter 'loss_repeated_mean' must be bool, "
"but got the type : {}.".format(type(loss_repeated_mean)))
self.check_context_handle()
self._context_handle.set_loss_repeated_mean(loss_repeated_mean)
@ -201,7 +203,9 @@ class _AutoParallelContext:
f"but got {parallel_mode.upper()}.")
ret = self._context_handle.set_parallel_mode(parallel_mode)
if ret is False:
raise ValueError("Parallel mode does not support {}".format(parallel_mode))
raise ValueError("The context configuration parameter 'parallel_mode' only support 'stand_alone', "
"'data_parallel', 'hybrid_parallel', 'semi_auto_parallel' and 'auto_parallel', "
"but got the value : {}.".format(parallel_mode))
def get_parallel_mode(self):
"""Get parallel mode."""
@ -220,7 +224,9 @@ class _AutoParallelContext:
self.check_context_handle()
ret = self._context_handle.set_strategy_search_mode(auto_parallel_search_mode)
if ret is False:
raise ValueError("Strategy search mode does not support {}".format(auto_parallel_search_mode))
raise ValueError("The context configuration parameter 'auto_parallel_search_mode' only support "
"'recursive_programming' and 'dynamic_programming', but got the value : {}."
.format(auto_parallel_search_mode))
def get_strategy_search_mode(self):
"""Get search mode of strategy."""
@ -284,19 +290,21 @@ class _AutoParallelContext:
self.check_context_handle()
if isinstance(dataset_strategy, str):
if dataset_strategy not in ("full_batch", "data_parallel"):
raise ValueError("The dataset_strategy string should be 'full_batch' or 'data_parallel', "
"otherwise, incoming tuple(tuple) type strategy")
raise ValueError("The context configuration parameter 'dataset_strategy' must be "
"'full_batch' or 'data_parallel', but got the value : {}.".format(dataset_strategy))
self._context_handle.set_full_batch(dataset_strategy == "full_batch")
self._dataset_strategy_using_str = True
return
if not isinstance(dataset_strategy, tuple):
raise TypeError(f'strategy must be str or tuple type, but got:{type(dataset_strategy)}')
raise TypeError("The type of context configuration parameter 'strategy' must be str or tuple type, "
"but got the type : {}.".format(type(dataset_strategy)))
for ele in dataset_strategy:
if not isinstance(ele, tuple):
raise TypeError(f'The element of strategy must be tuple type, but got:{type(ele)}')
raise TypeError("The element of strategy must be tuple, but got the type : {} .".format(type(ele)))
for dim in ele:
if not isinstance(dim, int):
raise TypeError(f'The dim of each strategy value must be int type, but got:{type(dim)}')
raise TypeError("The dim of each strategy value must be int type, "
"but got the type : {} .".format(type(dim)))
self._dataset_strategy_using_str = False
self._context_handle.set_dataset_strategy(dataset_strategy)
@ -370,20 +378,22 @@ class _AutoParallelContext:
"""
self.check_context_handle()
if not indices:
raise ValueError('indices can not be empty')
raise ValueError("The parameter 'indices' can not be empty")
if isinstance(indices, (list)):
for index in indices:
if not isinstance(index, int) or isinstance(index, bool):
raise TypeError(f"The type of index must be int, but got {type(index)}.")
raise TypeError("The type of parameter 'index' must be int, but got the type : {} ."
.format(type(index)))
else:
raise TypeError('indices must be a python list')
raise TypeError("The type of parameter 'indices' must be a python list, but got the type : {} ."
.format(type(indices)))
if len(set(indices)) != len(indices):
raise ValueError('indices has duplicate elements')
raise ValueError("The indices has duplicate elements")
if sorted(indices) != indices:
raise ValueError('elements in indices must be sorted in ascending order')
raise ValueError("The elements in indices must be sorted in ascending order")
new_group = self._check_and_default_group(group)
@ -424,9 +434,10 @@ class _AutoParallelContext:
if isinstance(sizes, (list)):
for size in sizes:
if not isinstance(size, int) or isinstance(size, bool):
raise TypeError(f"The type of size must be int, but got {type(size)}.")
raise TypeError("The type of size must be int, but got the type : {}.".format(type(size)))
else:
raise TypeError('sizes must be a python list')
raise TypeError("The type of parameter 'sizes' must be a python list, but got the type : {}."
.format(type(sizes)))
new_group = self._check_and_default_group(group)
self._context_handle.set_all_reduce_fusion_split_sizes(sizes, new_group)
@ -459,7 +470,8 @@ class _AutoParallelContext:
"""
self.check_context_handle()
if not isinstance(enable_all_reduce_fusion, bool):
raise TypeError('enable_all_reduce_fusion is invalid type')
raise TypeError("The type of parameter 'enable_all_reduce_fusion' must be bool, "
"but got the type : {}.".format(type(enable_all_reduce_fusion)))
self._context_handle.set_enable_all_reduce_fusion(enable_all_reduce_fusion)
def get_enable_all_reduce_fusion(self):
@ -486,7 +498,8 @@ class _AutoParallelContext:
"""
self.check_context_handle()
if not isinstance(enable_parallel_optimizer, bool):
raise TypeError('enable_parallel_optimizer is invalid type')
raise TypeError("The type of parameter 'enable_parallel_optimizer' must be bool, "
"but got the type : {}.".format(type(enable_parallel_optimizer)))
self._context_handle.set_enable_parallel_optimizer(enable_parallel_optimizer)
def get_enable_parallel_optimizer(self):
@ -541,7 +554,8 @@ class _AutoParallelContext:
"""
self.check_context_handle()
if not isinstance(sharding_propagation, bool):
raise TypeError("'sharding_propagation' is an invalid type.")
raise TypeError("The type of parameter 'sharding_propagation' must be bool, "
"but got the type : {}.".format(type(sharding_propagation)))
self._context_handle.set_sharding_propagation(sharding_propagation)
def get_sharding_propagation(self):
@ -559,7 +573,8 @@ class _AutoParallelContext:
"""
self.check_context_handle()
if not isinstance(enable_a2a, bool):
raise TypeError("'enable_a2a' is an invalid type.")
raise TypeError("The type of parameter 'enable_a2a' must be bool, "
"but got the type : {}.".format(type(enable_a2a)))
self._context_handle.set_enable_alltoall(enable_a2a)
def get_enable_alltoall(self):
@ -578,12 +593,14 @@ class _AutoParallelContext:
ValueError: If parallel mode is not supported.
"""
if not isinstance(communi_parallel_mode, str):
raise TypeError(f"The type of communi_parallel_mode must be str, \
but got {type(communi_parallel_mode)}.")
raise TypeError("The type of parameter 'communi_parallel_mode' must be str, "
"but got the type : {}.".format(type(communi_parallel_mode)))
self.check_context_handle()
ret = self._context_handle.set_communi_parallel_mode(communi_parallel_mode)
if ret is False:
raise ValueError("Communication parallel mode does not support {}".format(communi_parallel_mode))
raise ValueError("The parameter 'communi_parallel_mode' only support 'ALL_GROUP_PARALLEL', "
"'SAME_SEVER_GROUP_PARALLEL' and 'NO_GROUP_PARALLEL', but got the value : {}."
.format(communi_parallel_mode))
def get_communi_parallel_mode(self):
"""Get communication parallel mode."""

6
mindspore/parallel/_cost_model_context.py
View File

@ -250,11 +250,11 @@ class _CostModelContext:
ValueError: If context handle is none, or phase is not in {0, 1}.
"""
if not isinstance(phase, int) or isinstance(phase, bool):
raise TypeError(f"The type of communi_const must be int, but got {type(phase)}.")
raise TypeError(f"The type of parameter 'communi_const' must be int, but got {type(phase)}.")
if self._context_handle is None:
raise ValueError("Context handle is none in context!!!")
if phase not in (0, 1):
raise ValueError("The argument of set_run_phase() must be '0' or '1', but got {}".format(phase))
raise ValueError("The parameter of 'phase' must be '0' or '1', but got {}".format(phase))
self._context_handle.set_run_phase(phase)
def get_run_phase(self):
@ -279,7 +279,7 @@ class _CostModelContext:
ValueError: If context handle is none.
"""
if not isinstance(single_loop, bool):
raise TypeError(f"The type of single_loop must be bool, but got {type(single_loop)}.")
raise TypeError(f"The type of parameter 'single_loop' must be bool, but got {type(single_loop)}.")
if self._context_handle is None:
raise ValueError("Context handle is none in context!!!")
self._context_handle.set_dp_algo_single_loop(single_loop)

30
mindspore/parallel/nn/layers.py
View File

@ -204,7 +204,8 @@ class _LayerNorm(Cell):
def __init__(self, normalized_shape, eps=1e-5, param_init_type=mstype.float32):
super(_LayerNorm, self).__init__()
if param_init_type not in [mstype.float32, mstype.float16]:
raise TypeError(f"param type should in [float32, float16], but found type {type(param_init_type)}")
raise TypeError("The type of parameter 'param_init_type' should in [float32, float16], "
"but got the type : {}.".format(type(param_init_type)))
if normalized_shape[0] <= 1024:
self.layer_norm = P.LayerNorm(begin_norm_axis=-1,
begin_params_axis=-1,
@ -335,12 +336,14 @@ class _Linear(Cell):
self.in_channels = Validator.check_positive_int(in_channels)
self.out_channels = Validator.check_positive_int(out_channels)
if param_init_type not in [mstype.float32, mstype.float16]:
raise TypeError(f"param type should in [float32, float16], but found type {type(param_init_type)}")
raise TypeError("The type of parameter 'param_init_type' should in [float32, float16], "
"but got the type : {}.".format(type(param_init_type)))
if activation and not isinstance(activation, str):
raise ValueError("Activation can only be str, but found type {}".format(activation))
raise TypeError("The type of parameter 'activation' must be str, but got type {}".format(type(activation)))
if isinstance(weight_init, Tensor) and (weight_init.ndim != 2 or weight_init.shape[0] != out_channels or \
weight_init.shape[1] != in_channels):
raise ValueError("Weight init shape error.")
raise ValueError("The shape of parameter 'weight_init' is error, please check shape of 'weight_init'.")
weight_shape = [out_channels, in_channels] if transpose_b else [in_channels, out_channels]
self.expert_num = expert_num
if self.expert_num > 1:
@ -356,13 +359,14 @@ class _Linear(Cell):
self.has_bias = has_bias
if self.has_bias:
if isinstance(bias_init, Tensor) and (bias_init.ndim != 1 or bias_init.shape[0] != out_channels):
raise ValueError("Bias init shape error.")
raise ValueError("The shape of parameter 'bias_init' is error, please check shape of 'bias_init'.")
self.bias = Parameter(initializer(bias_init, [out_channels], param_init_type), name="bias")
self.bias_add = P.Add()
self.act_name = activation
self.activation = get_activation(activation) if isinstance(activation, str) else activation
if activation is not None and not isinstance(self.activation, (Cell, Primitive)):
raise TypeError("The activation must be str or Cell or Primitive,"" but got {}.".format(activation))
raise TypeError("The type of parameter 'activation' must be str or Cell or Primitive, "
"but got the type {}".format(type(activation)))
self.activation_flag = self.activation is not None
self.dtype = compute_dtype
self.cast = P.Cast()
@ -409,7 +413,8 @@ class _Linear(Cell):
self.activation.rec.shard(strategy_activation)
self.activation.log.shard(strategy_activation)
elif self.act_name.lower() == "logsoftmax":
raise ValueError("logsoftmax is not supported.")
raise ValueError("The 'LogSoftmax' function is not supported in semi auto parallel "
"or auto parallel mode.")
else:
getattr(self.activation, self.act_name).shard(strategy_activation)
@ -516,14 +521,15 @@ class FixedSparseAttention(nn.Cell):
self.parallel_config = parallel_config
size_per_head_list = [64, 128]
if self.seq_length != 1024:
raise ValueError("seq_length only supports 1024 for now.")
raise ValueError("The parameter of 'seq_length' must be 1024, but got the value : {}.".format(seq_length))
if self.block_size != 64:
raise ValueError("block_size only supports 64 for now.")
raise ValueError("The parameter of 'block_size' must be 64, but got the value : {}.".format(block_size))
if num_different_global_patterns != 4:
raise ValueError("num_different_global_patterns only supports 4 for now.")
raise ValueError("The parameter of 'num_different_global_patterns' must be 4, "
"but got the value : {}".format(num_different_global_patterns))
if self.size_per_head not in size_per_head_list:
raise ValueError(f"size_per_head only supports {size_per_head_list} for now, "
f"but found {self.size_per_head}")
raise ValueError("The parameter of 'size_per_head' only supports {}, "
"but got the value : {}.".format(size_per_head_list, self.size_per_head))
local_ones = np.ones((self.block_size, self.block_size),
dtype=np.float16)
global_mask_original = np.ones((self.seq_length, self.global_size), dtype=np.float16)

3
mindspore/parallel/nn/loss.py
View File

@ -66,7 +66,8 @@ class CrossEntropyLoss(Cell):
def __init__(self, parallel_config=default_dpmp_config):
super(CrossEntropyLoss, self).__init__()
if not isinstance(parallel_config, OpParallelConfig):
raise TypeError("Input args parallel_config must be the type OpParallelConfig.")
raise TypeError("The type of parameter 'parallel_config' must be OpParallelConfig, "
"but got the type: {}.".format(type(parallel_config)))
dp = parallel_config.data_parallel
mp = parallel_config.model_parallel
self.sum = P.ReduceSum().shard(((dp, mp),))

70
mindspore/parallel/nn/transformer.py
View File

@ -407,12 +407,16 @@ class FeedForward(Cell):
dp = parallel_config.data_parallel
mp = parallel_config.model_parallel
if ffn_hidden_size % mp != 0:
raise ValueError(f"ffn_hidden_size {ffn_hidden_size} should be a multiple of the model parallel way {mp}")
raise ValueError("The parameter of 'ffn_hidden_size' must be a multiple of the model parallel way, "
"but got the ffn_hidden_size is {} and the num of model parallel is {}."
.format(ffn_hidden_size, mp))
if hidden_size % mp != 0:
raise ValueError(f"hidden_size {hidden_size} should be a multiple of the model parallel way {mp}")
raise ValueError("The parameter of 'hidden_size' must be a multiple of the model parallel way, "
"but got the hidden_size is {} and the num of model parallel is {}."
.format(hidden_size, mp))
if dropout_rate < 0 or dropout_rate >= 1:
raise ValueError(f"dropout_rate probability should be a number in range [0, 1.0), "
f"but got {dropout_rate}")
raise ValueError("The parameter of 'dropout_rate' must be in the range [0, 1.0), "
"but got the value : {}.".format(dropout_rate))
input_size = hidden_size
output_size = ffn_hidden_size
# Here, 'ep' stands for expert parallel number, which is equal to data parallel number.
@ -779,19 +783,22 @@ class MultiHeadAttention(Cell):
self.hidden_size = hidden_size
self.batch_size = batch_size
if hidden_dropout_rate < 0 or hidden_dropout_rate >= 1:
raise ValueError(f"hidden_dropout_rate probability should be a number in range [0, 1.0), "
f"but got {hidden_dropout_rate}")
raise ValueError("The parameter 'hidden_dropout_rate' must be in range [0, 1.0), "
"but got the value : {}.".format(hidden_dropout_rate))
if attention_dropout_rate < 0 or attention_dropout_rate >= 1:
raise ValueError(f"attention_dropout_rate probability should be a number in range [0, 1.0), "
f"but got {attention_dropout_rate}")
raise ValueError("The parameter 'attention_dropout_rate' must be in range [0, 1.0), "
"but got the value : {}.".format(attention_dropout_rate))
if hidden_size % num_heads != 0:
raise ValueError(f"The hidden size {hidden_size} should be a multiple of num_heads {num_heads}")
raise ValueError("The parameter 'hidden_size' should be a multiple of 'num_heads', "
"but got the hidden_size is {} and the num_heads is {}.".format(hidden_size, num_heads))
if num_heads % parallel_config.model_parallel != 0:
raise ValueError(f"The number of heads {num_heads} must be a "
f"multiple of parallel_config.model_parallel {parallel_config.model_parallel}.")
raise ValueError("The parameter 'num_heads' must be a multiple of 'parallel_config.model_parallel', "
"but got the num_heads is {} and the parallel_config.model_parallel is {}."
.format(num_heads, parallel_config.model_parallel))
if self.is_parallel_mode and batch_size % parallel_config.data_parallel != 0:
raise ValueError(f"The batch size {batch_size} must be a "
f"multiple of parallel_config.data_parallel {parallel_config.data_parallel}.")
raise ValueError("The parameter 'batch_size' must be a multiple of 'parallel_config.data_parallel', "
"but got the batch_size is {} and the parallel_config.data_parallel is {}."
.format(batch_size, parallel_config.data_parallel))
self.is_first_iteration = True
# Output layer
self.projection = _Linear(in_channels=hidden_size,
@ -1244,17 +1251,17 @@ class TransformerEncoderLayer(Cell):
super(TransformerEncoderLayer, self).__init__()
_check_config(parallel_config)
if num_heads % parallel_config.model_parallel != 0:
raise ValueError(
f"num heads must be divisibled by the model parallel way {parallel_config.model_parallel}, "
f"but found {num_heads}")
raise ValueError("The parameter of 'num_heads' must be divisibled by the "
"'parallel_config.model_parallel', but got the num_heads is {} and "
"parallel_config.model_parallel is {}.".format(num_heads, parallel_config.model_parallel))
if hidden_size % parallel_config.model_parallel != 0:
raise ValueError(
f"hidden_size must be divisibled by the model parallel way {parallel_config.model_parallel}, "
f"but found {hidden_size}")
raise ValueError("The parameter of 'hidden_size' must be divisibled by the "
"'parallel_config.model_parallel', but got the hidden_size is {} and parallel_config. "
"model_parallel is {}.".format(hidden_size, parallel_config.model_parallel))
if ffn_hidden_size % parallel_config.model_parallel != 0:
raise ValueError(
f"ffn_hidden_size must be divisibled by the model parallel way {parallel_config.model_parallel}, "
f"but found {ffn_hidden_size}")
raise ValueError("The parameter of 'ffn_hidden_size' must be divisibled by the "
"'parallel_config.model_parallel', but got the ffn_hidden_size is {} and parallel_config. "
"model_parallel is {}.".format(ffn_hidden_size, parallel_config.model_parallel))
self.use_past = use_past
self.seq_length = seq_length
self.hidden_size = hidden_size
@ -1541,17 +1548,18 @@ class TransformerDecoderLayer(Cell):
super(TransformerDecoderLayer, self).__init__()
_check_config(parallel_config)
if num_heads % parallel_config.model_parallel != 0:
raise ValueError(
f"num heads must be divisibled by the model parallel way {parallel_config.model_parallel}, "
f"but found {num_heads}")
raise ValueError("The parameter of 'num_heads' must be divisibled by 'parallel_config.model_parallel', "
"but got the num_heads is {} and parallel_config.model_parallel is {}."
.format(num_heads, parallel_config.model_parallel))
if hidden_size % parallel_config.model_parallel != 0:
raise ValueError(
f"hidden_size must be divisibled by the model parallel way {parallel_config.model_parallel}, "
f"but found {hidden_size}")
raise ValueError("The parameter of 'hidden_size' must be divisibled by 'parallel_config.model_parallel', "
"but got the hidden_size is {} and parallel_config.model_parallel is {}."
.format(hidden_size, parallel_config.model_parallel))
if ffn_hidden_size % parallel_config.model_parallel != 0:
raise ValueError(
f"ffn_hidden_size must be divisibled by the model parallel way {parallel_config.model_parallel}, "
f"but found {ffn_hidden_size}")
raise ValueError("The parameter of 'ffn_hidden_size' must be divisibled by "
"'parallel_config.model_parallel', but got the ffn_hidden_size is {} "
"and parallel_config.model_parallel is {}."
.format(ffn_hidden_size, parallel_config.model_parallel))
if use_past is True:
raise ValueError(f"The {self.cls_name} does not support use_past=True.")
self.batch_size = batch_size