ssk015
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!21248 clean code 

Merge pull request !21248 from zhoufeng/code-check
This commit is contained in:
i-robot 2021-08-03 11:14:50 +00:00 committed by Gitee
parent 4c3c3ceee6 67a3558c0e
commit b917ceca36
18 changed files with 106 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
mindspore/ccsrc/backend/kernel_compiler/hccl/hccl_kernel.cc
View File

@ -47,11 +47,11 @@ std::string MsOpNameToHcomOpType(const std::string &ms_op_type) {
namespace mindspore {
namespace kernel {
void HcclKernelFactory::Register(const std::string &name, HcclKernelCreater &&fun) {
hcclKernelMap_.emplace(name, std::move(fun));
hccl_kernel_map_.emplace(name, fun);
}
std::shared_ptr<HcclKernel> HcclKernelFactory::Get(const std::string &name) {
const auto &map = Get().hcclKernelMap_;
const auto &map = Get().hccl_kernel_map_;
auto it = map.find(name);
if (it != map.end() && it->second) {
return (it->second)();
@ -64,14 +64,14 @@ HcclKernelFactory &HcclKernelFactory::Get() {
return _this;
}
HcclKernel::HcclKernel() : hccl_count_(0), op_type_(HCCL_REDUCE_SUM), root_id_(0) {}
HcclKernel::HcclKernel() : hccl_count_(0), op_type_(::HcclReduceOp::HCCL_REDUCE_SUM), root_id_(0) {}
HcclKernel::~HcclKernel() {
hccl_kernel_input_shape_list_.clear();
hccl_kernel_output_shape_list_.clear();
hccl_data_type_list_.clear();
hccl_count_ = 0;
op_type_ = HCCL_REDUCE_SUM;
op_type_ = ::HcclReduceOp::HCCL_REDUCE_SUM;
root_id_ = 0;
input_size_list_.clear();
output_size_list_.clear();
@ -126,6 +126,10 @@ const std::vector<size_t> &HcclKernel::GetInputSizeList() const {
if (!input_size_list_.empty()) {
return input_size_list_;
}
if (hccl_data_type_list_.size() != hccl_kernel_input_shape_list_.size()) {
MS_LOG(EXCEPTION) << "Invalid data type size " << hccl_data_type_list_.size() << " diff shape size "
<< hccl_kernel_input_shape_list_.size();
}
for (ulong i = 0; i < hccl_data_type_list_.size(); ++i) {
if (!HcomUtil::GetHcclOpSize(hccl_data_type_list_[i], hccl_kernel_input_shape_list_[i], &size)) {
MS_LOG(ERROR) << "GetHcclOpInputSize failed";
@ -145,6 +149,7 @@ const std::vector<size_t> &HcclKernel::GetOutputSizeList() const {
return output_size_list_;
}
auto cnode = anf_node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(cnode);
auto op_name = AnfAlgo::GetCNodeName(cnode);
int64_t rank_size = 1;
if (AnfAlgo::HasNodeAttr(kAttrRankSize, cnode)) {
@ -154,6 +159,10 @@ const std::vector<size_t> &HcclKernel::GetOutputSizeList() const {
if (AnfAlgo::HasNodeAttr(kAttrFusion, cnode)) {
fusion = AnfAlgo::GetNodeAttr<int64_t>(cnode, kAttrFusion);
}
if (hccl_data_type_list_.size() != hccl_kernel_input_shape_list_.size()) {
MS_LOG(EXCEPTION) << "Invalid data type size " << hccl_data_type_list_.size() << " diff shape size "
<< hccl_kernel_input_shape_list_.size();
}
ulong loop_size = hccl_data_type_list_.size();
if (AnfAlgo::GetInputTensorNum(anf_node) > 1 && op_name == kAllGatherOpName && fusion >= 1) {
loop_size *= static_cast<ulong>(rank_size);
@ -204,6 +213,9 @@ std::vector<TaskInfoPtr> HcclKernel::GenTask(const std::vector<AddressPtr> &inpu
MS_EXCEPTION_IF_NULL(outputs.at(0));
auto output_data_addr = outputs.at(0)->addr;
std::vector<uint8_t> private_def;
if (hccl_data_type_list_.empty()) {
MS_LOG(EXCEPTION) << "Hccl data type list is empty";
}
HcclDataType data_type = hccl_data_type_list_[0];
std::vector<hccl::HcclTaskInfo> task_info;
bool ret = hccl::HcclAdapter::GetInstance().GenTask(anf_node, data_type, &task_info);

2
mindspore/ccsrc/backend/kernel_compiler/hccl/hccl_kernel.h
View File

@ -71,7 +71,7 @@ class HcclKernelFactory {
static std::shared_ptr<HcclKernel> Get(const string &name);
private:
std::map<string, HcclKernelCreater> hcclKernelMap_;
std::map<string, HcclKernelCreater> hccl_kernel_map_;
};
class _HcclKernelRegister {

10
mindspore/ccsrc/backend/kernel_compiler/hccl/hccl_kernel_metadata.cc
View File

@ -30,7 +30,9 @@ namespace {
constexpr size_t N_nchw = 0;
constexpr size_t C_nchw = 1;
std::string GetKernelFormat(const CNodePtr &kernel_node, size_t index) {
const std::set<std::string> kReduceNoSupportedSet = {kOpFormat_FRAC_Z, kOpFormat_FRACTAL_Z_C04, kOpFormat_C1HWNCoC0};
static const std::set<std::string> kReduceNoSupportedSet = {kOpFormat_FRAC_Z, kOpFormat_FRACTAL_Z_C04,
kOpFormat_C1HWNCoC0};
MS_EXCEPTION_IF_NULL(kernel_node);
auto op_name = AnfAlgo::GetCNodeName(kernel_node);
auto parallel_context_instance = parallel::ParallelContext::GetInstance();
MS_EXCEPTION_IF_NULL(parallel_context_instance);
@ -61,8 +63,8 @@ std::string GetKernelFormat(const CNodePtr &kernel_node, size_t index) {
}
} // namespace
void HcclMetadataInfo(const CNodePtr &kernel_node, std::vector<std::shared_ptr<KernelBuildInfo>> *kernel_info_list) {
const std::vector<TypeId> kHcclSupportTypes = {kNumberTypeInt8, kNumberTypeInt32, kNumberTypeFloat16,
kNumberTypeFloat32, kNumberTypeInt16};
static const std::vector<TypeId> kHcclSupportTypes = {kNumberTypeInt8, kNumberTypeInt32, kNumberTypeFloat16,
kNumberTypeFloat32, kNumberTypeInt16};
MS_EXCEPTION_IF_NULL(kernel_info_list);
MS_EXCEPTION_IF_NULL(kernel_node);
std::string op_name = AnfAlgo::GetCNodeName(kernel_node);
@ -76,7 +78,7 @@ void HcclMetadataInfo(const CNodePtr &kernel_node, std::vector<std::shared_ptr<K
if (!HcomUtil::GetHcomReceiveType(kernel_node, &recv_type)) {
MS_LOG(EXCEPTION) << "GetHcomReceiveType fail!";
}
auto res = find(kHcclSupportTypes.begin(), kHcclSupportTypes.end(), recv_type);
auto res = std::find(kHcclSupportTypes.begin(), kHcclSupportTypes.end(), recv_type);
if (res == kHcclSupportTypes.end()) {
MS_LOG(EXCEPTION) << "HcclReceive cannot support data type: " << TypeIdToType(recv_type);
}

5
mindspore/ccsrc/backend/kernel_compiler/hccl/hcom_all_reduce.cc
View File

@ -23,8 +23,9 @@ namespace kernel {
bool HcomAllReduceKernel::Launch(const std::vector<AddressPtr> &inputs, const std::vector<AddressPtr> &,
const std::vector<AddressPtr> &outputs, void *stream_ptr) {
MS_LOG(INFO) << "HcclAllReduce launch";
if (inputs.empty() || outputs.empty()) {
MS_LOG(ERROR) << "Invalid AllReduce input output size(" << inputs.size() << ", " << outputs.size() << ").";
if (inputs.empty() || outputs.empty() || hccl_data_type_list_.empty()) {
MS_LOG(ERROR) << "Invalid AllReduce input, output or data type size(" << inputs.size() << ", " << outputs.size()
<< ", " << hccl_data_type_list_.size() << ").";
return false;
}
MS_EXCEPTION_IF_NULL(inputs[0]);

25
mindspore/ccsrc/backend/kernel_compiler/hccl/hcom_util.cc
View File

@ -22,11 +22,13 @@
#include "utils/utils.h"
namespace mindspore {
namespace {
bool IsPyNativeMode() {
auto ms_context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(ms_context);
return ms_context->get_param<int>(MS_CTX_EXECUTION_MODE) == kPynativeMode;
}
} // namespace
bool HcomUtil::GetKernelInputShape(const AnfNodePtr &anf_node, vector<vector<size_t>> *hccl_kernel_intput_shape_list) {
MS_EXCEPTION_IF_NULL(anf_node);
@ -67,8 +69,8 @@ bool HcomUtil::GetHcomDataType(const AnfNodePtr &anf_node, vector<HcclDataType>
} else {
type_ptr = AnfAlgo::GetInputDeviceDataType(anf_node, i);
}
auto iter = CONST_OP_HCOM_DATA_TYPE_MAP.find(type_ptr);
if (iter == CONST_OP_HCOM_DATA_TYPE_MAP.end()) {
auto iter = kConstOpHcomDataTypeMap.find(type_ptr);
if (iter == kConstOpHcomDataTypeMap.end()) {
MS_LOG(EXCEPTION) << "HcomDataType can't support Current Ascend Data Type : " << type_ptr;
}
data_type_list->emplace_back(iter->second);
@ -102,8 +104,8 @@ bool HcomUtil::GetHcclOpSize(const HcclDataType &data_type, const vector<size_t>
bool HcomUtil::GetHcomTypeSize(const HcclDataType &data_type, uint32_t *size) {
MS_EXCEPTION_IF_NULL(size);
auto iter = CONST_OP_HCOM_DATA_TYPE_SIZE_MAP.find(data_type);
if (iter == CONST_OP_HCOM_DATA_TYPE_SIZE_MAP.end()) {
auto iter = kConstOpHcomDataTypeSizeMap.find(data_type);
if (iter == kConstOpHcomDataTypeSizeMap.end()) {
MS_LOG(ERROR) << "HcomUtil::HcomDataTypeSize, No DataTypeSize!";
return false;
}
@ -123,6 +125,7 @@ bool HcomUtil::GetHcomCount(const AnfNodePtr &anf_node, const vector<HcclDataTyp
uint32_t type_size = 4;
size_t size = AnfAlgo::GetInputTensorNum(anf_node);
auto cnode = anf_node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(cnode);
if (AnfAlgo::GetCNodeName(anf_node) == kReceiveOpName) {
size = AnfAlgo::GetOutputTensorNum(anf_node);
}
@ -140,8 +143,8 @@ bool HcomUtil::GetHcomCount(const AnfNodePtr &anf_node, const vector<HcclDataTyp
int64_t rank_size;
auto primitive = AnfAlgo::GetCNodePrimitive(anf_node);
MS_EXCEPTION_IF_NULL(primitive);
if (primitive->GetAttr("rank_size") != nullptr) {
rank_size = GetValue<int64_t>(primitive->GetAttr("rank_size"));
if (primitive->GetAttr(kAttrRankSize) != nullptr) {
rank_size = GetValue<int64_t>(primitive->GetAttr(kAttrRankSize));
} else {
MS_LOG(ERROR) << "Get rank size failed";
return false;
@ -181,11 +184,11 @@ bool HcomUtil::GetHcomOperationType(const AnfNodePtr &anf_node, HcclReduceOp *op
MS_EXCEPTION_IF_NULL(op_type);
auto primitive = AnfAlgo::GetCNodePrimitive(anf_node);
MS_EXCEPTION_IF_NULL(primitive);
if (primitive->GetAttr("op") == nullptr) {
if (primitive->GetAttr(kAttrOp) == nullptr) {
MS_LOG(ERROR) << "Get HCOM_ATTR_REDUCE_TYPE fail, not support!";
return false;
}
auto hcom_op_type = GetValue<std::string>(primitive->GetAttr("op"));
auto hcom_op_type = GetValue<std::string>(primitive->GetAttr(kAttrOp));
if (hcom_op_type == "min") {
*op_type = HCCL_REDUCE_MIN;
} else if (hcom_op_type == "max") {
@ -206,8 +209,8 @@ bool HcomUtil::GetHcomRootId(const AnfNodePtr &anf_node, uint32_t *root_id) {
MS_EXCEPTION_IF_NULL(root_id);
auto primitive = AnfAlgo::GetCNodePrimitive(anf_node);
MS_EXCEPTION_IF_NULL(primitive);
if (primitive->GetAttr("root_rank") != nullptr) {
*root_id = (uint32_t)GetValue<int64_t>(primitive->GetAttr("root_rank"));
if (primitive->GetAttr(kAttrRootRank) != nullptr) {
*root_id = (uint32_t)GetValue<int64_t>(primitive->GetAttr(kAttrRootRank));
} else {
MS_LOG(ERROR) << "HcomUtil::Get HCOM_ATTR_ROOT_INDEX fail, not support!";
return false;
@ -232,7 +235,7 @@ bool HcomUtil::GetHcomReceiveType(const AnfNodePtr &anf_node, TypeId *receive_ty
void HcomUtil::GetHcomGroup(NotNull<const AnfNodePtr &> anf_node, NotNull<std::string *> group) {
auto primitive = AnfAlgo::GetCNodePrimitive(anf_node);
MS_EXCEPTION_IF_NULL(primitive);
auto attr = primitive->GetAttr("group");
auto attr = primitive->GetAttr(kAttrGroup);
if (attr != nullptr) {
*group = GetValue<std::string>(attr);
} else {

4
mindspore/ccsrc/backend/kernel_compiler/hccl/hcom_util.h
View File

@ -40,7 +40,7 @@ constexpr auto kReduceScatter = "ReduceScatter";
constexpr auto kAllToAllv = "AllToAllv";
/* Correspondence between data_type and hcom data type in Ascend */
static map<int64_t, HcclDataType> CONST_OP_HCOM_DATA_TYPE_MAP = {
static map<int64_t, HcclDataType> kConstOpHcomDataTypeMap = {
{TypeId::kNumberTypeFloat32, HCCL_DATA_TYPE_FP32},
{TypeId::kNumberTypeFloat16, HCCL_DATA_TYPE_FP16},
{TypeId::kNumberTypeInt8, HCCL_DATA_TYPE_INT8},
@ -48,7 +48,7 @@ static map<int64_t, HcclDataType> CONST_OP_HCOM_DATA_TYPE_MAP = {
};
/* Correspondence between data_type and occupied byte size in hcom */
static map<HcclDataType, uint32_t> CONST_OP_HCOM_DATA_TYPE_SIZE_MAP = {
static map<HcclDataType, uint32_t> kConstOpHcomDataTypeSizeMap = {
{HCCL_DATA_TYPE_FP32, sizeof(float)},
{HCCL_DATA_TYPE_FP16, sizeof(float) / 2},
{HCCL_DATA_TYPE_INT8, sizeof(int8_t)},

2
mindspore/ccsrc/backend/optimizer/ascend/mindir/all_to_all_unify_mindir.cc
View File

@ -69,7 +69,7 @@ CNodePtr CreateSplitNode(const FuncGraphPtr &graph, const CNodePtr &all_to_all)
if (SizeToLong(shape.size()) <= split_dim) {
MS_LOG(EXCEPTION) << "Invalid split dim " << split_dim << " is over the shape size " << shape.size();
}
if (shape[LongToSize(split_dim)] % split_count != 0) {
if (split_count == 0 || shape[LongToSize(split_dim)] % split_count != 0) {
MS_LOG(EXCEPTION) << "Invalid split count " << split_count << " cannot be divisible by shape[" << split_dim
<< "] = " << shape[LongToSize(split_dim)];
}

9
mindspore/ccsrc/cxx_api/graph/acl/acl_graph_impl.cc
View File

@ -17,6 +17,7 @@
#include "include/api/context.h"
#include "cxx_api/model/acl/model_converter.h"
#include "utils/log_adapter.h"
#include "mindspore/core/utils/convert_utils_base.h"
namespace mindspore {
API_FACTORY_REG(GraphCell::GraphImpl, Ascend310, AclGraphImpl);
@ -33,7 +34,7 @@ AclGraphImpl::~AclGraphImpl() { (void)FinalizeEnv(); }
Status AclGraphImpl::Run(const std::vector<MSTensor> &inputs, std::vector<MSTensor> *outputs) {
MS_EXCEPTION_IF_NULL(outputs);
Status ret = Load(device_id_);
Status ret = Load(IntToUint(device_id_));
if (ret != kSuccess) {
MS_LOG(ERROR) << "Prepare model resource failed.";
return ret;
@ -43,7 +44,7 @@ Status AclGraphImpl::Run(const std::vector<MSTensor> &inputs, std::vector<MSTens
}
std::vector<MSTensor> AclGraphImpl::GetInputs() {
Status ret = Load(device_id_);
Status ret = Load(IntToUint(device_id_));
if (ret != kSuccess) {
MS_LOG(ERROR) << "Prepare model resource failed.";
return {};
@ -53,7 +54,7 @@ std::vector<MSTensor> AclGraphImpl::GetInputs() {
}
std::vector<MSTensor> AclGraphImpl::GetOutputs() {
Status ret = Load(device_id_);
Status ret = Load(IntToUint(device_id_));
if (ret != kSuccess) {
MS_LOG(ERROR) << "Prepare model resource failed.";
return {};
@ -176,7 +177,7 @@ Status AclGraphImpl::Load(uint32_t device_id) {
auto om_data = graph_data->GetOMData();
// init
device_id_ = device_id;
device_id_ = UintToInt(device_id);
Status ret = InitEnv();
if (ret != kSuccess) {
MS_LOG(ERROR) << "InitEnv failed.";

17
mindspore/ccsrc/cxx_api/graph/acl/model_process.cc
View File

@ -19,6 +19,7 @@
#include <algorithm>
#include <map>
#include "utils/utils.h"
#include "mindspore/core/utils/convert_utils_base.h"
namespace mindspore {
static DataType TransToApiType(aclDataType data_type) {
@ -157,13 +158,14 @@ Status ModelProcess::InitInputsBuffer() {
if (ret != ACL_ERROR_NONE) {
MS_LOG(ERROR) << "Get input shape failed";
if (!is_run_on_device_) {
aclrtFree(data_mem_buffer);
(void)aclrtFree(data_mem_buffer);
}
return kMCDeviceError;
}
aclDataType data_type = aclmdlGetInputDataType(model_desc_, i);
std::vector<int64_t> shape(dims.dims, dims.dims + dims.dimCount);
std::string input_name = aclmdlGetInputNameByIndex(model_desc_, i);
const char *input_name_char = aclmdlGetInputNameByIndex(model_desc_, i);
std::string input_name = (input_name_char == nullptr) ? input_name_char : std::string();
if (input_name.empty()) {
MS_LOG(WARNING) << "Get name of input " << i << " failed.";
}
@ -175,7 +177,7 @@ Status ModelProcess::InitInputsBuffer() {
return kSuccess;
}
Status ModelProcess::CreateDataBuffer(void **data_mem_buffer, size_t buffer_size, aclmdlDataset *dataset) {
Status ModelProcess::CreateDataBuffer(void **data_mem_buffer, size_t buffer_size, aclmdlDataset *dataset) const {
MS_EXCEPTION_IF_NULL(data_mem_buffer);
aclError ret;
auto free_data_buffer = [this](void *dataMemBuffer) {
@ -246,7 +248,8 @@ Status ModelProcess::InitOutputsBuffer() {
}
aclDataType data_type = aclmdlGetOutputDataType(model_desc_, i);
std::vector<int64_t> shape(dims.dims, dims.dims + dims.dimCount);
std::string output_name = aclmdlGetOutputNameByIndex(model_desc_, i);
const char *output_name_char = aclmdlGetOutputNameByIndex(model_desc_, i);
std::string output_name = (output_name_char == nullptr) ? output_name_char : std::string();
if (output_name.empty()) {
MS_LOG(WARNING) << "Get name of output " << i << " failed.";
}
@ -344,7 +347,7 @@ Status ModelProcess::SetBatchSize(const std::vector<MSTensor> &inputs) {
}
auto *p = reinterpret_cast<const float *>(inputs[inputs.size() - 1].Data().get());
MS_EXCEPTION_IF_NULL(p);
auto dynamicBatchSize = p[0];
size_t dynamicBatchSize = FloatToSize(p[0]);
ret = aclmdlGetInputIndexByName(model_desc_, ACL_DYNAMIC_TENSOR_NAME, &index);
if (ret != ACL_ERROR_NONE) {
MS_LOG(ERROR) << "get index failed";
@ -442,7 +445,7 @@ Status ModelProcess::ResetOutputSize() {
aclError ret;
size_t output_size = aclmdlGetNumOutputs(model_desc_);
for (size_t index = 0; index < output_size; index++) {
size_t dims = 1;
int64_t dims = 1;
struct aclmdlIODims output_dims;
ret = aclmdlGetCurOutputDims(model_desc_, index, &output_dims);
if (ret != ACL_ERROR_NONE) {
@ -453,7 +456,7 @@ Status ModelProcess::ResetOutputSize() {
dims *= output_dims.dims[i];
}
output_type = aclmdlGetOutputDataType(model_desc_, index);
output_infos_[index].buffer_size = dims * aclDataTypeSize(output_type);
output_infos_[index].buffer_size = LongToSize(dims) * aclDataTypeSize(output_type);
}
return kSuccess;
}

2
mindspore/ccsrc/cxx_api/graph/acl/model_process.h
View File

@ -60,7 +60,7 @@ class ModelProcess {
uint32_t model_id() const { return model_id_; }
private:
Status CreateDataBuffer(void **data_mem_buffer, size_t buffer_size, aclmdlDataset *dataset);
Status CreateDataBuffer(void **data_mem_buffer, size_t buffer_size, aclmdlDataset *dataset) const;
Status CheckAndInitInput(const std::vector<MSTensor> &inputs);
Status ConstructTensors(const std::vector<AclTensorInfo> &acl_tensor_list, std::vector<MSTensor> *tensor_list);
Status BuildOutputs(std::vector<MSTensor> *outputs);

4
mindspore/ccsrc/cxx_api/model/acl/model_converter.cc
View File

@ -164,7 +164,7 @@ Buffer ModelConverter::LoadMindIR(const FuncGraphPtr &func_graph) {
}
// receive convert model result from child
CreateBufferCall call = [&buffer_ret](size_t msg_len) -> uint8_t * {
buffer_ret.ResizeData(msg_len);
(void)buffer_ret.ResizeData(msg_len);
return reinterpret_cast<uint8_t *>(buffer_ret.MutableData());
};
status = multi_process->ReceiveMsg(call);
@ -179,7 +179,7 @@ Buffer ModelConverter::LoadMindIR(const FuncGraphPtr &func_graph) {
// receive original model from parent
Buffer model;
CreateBufferCall call = [&model](size_t msg_len) -> uint8_t * {
model.ResizeData(msg_len);
(void)model.ResizeData(msg_len);
return reinterpret_cast<uint8_t *>(model.MutableData());
};
auto status = multi_process->ReceiveMsg(call);

5
mindspore/ccsrc/cxx_api/model/acl/model_converter.h
View File

@ -31,6 +31,7 @@ namespace mindspore {
class ModelConverter {
public:
ModelConverter() : options_(nullptr) {}
~ModelConverter() = default;
Buffer LoadMindIR(const FuncGraphPtr &func_graph);
@ -40,9 +41,9 @@ class ModelConverter {
transform::DfGraphPtr ConvertFuncGraphToAIR(const FuncGraphPtr &anf_graph);
Buffer BuildAirModel(const transform::DfGraphPtr &graph, const std::map<std::string, std::string> &init_options,
const std::map<std::string, std::string> &build_options);
AclModelOptions *options_;
Buffer LoadAscendIRInner(const Buffer &model_data);
AclModelOptions *options_;
};
} // namespace mindspore
#endif // MINDSPORE_CCSRC_CXXAPI_SESSION_ACL_MODEL_CONVERTER_H

8
mindspore/ccsrc/cxx_api/model/model.cc
View File

@ -65,14 +65,14 @@ Status Model::Build(GraphCell graph_cell, const std::shared_ptr<Context> &model_
return impl_->Build();
}
Status Model::Build(const void *model_data, size_t data_size, ModelType model_type,
const std::shared_ptr<Context> &model_context, const Key &dec_key, const std::string &dec_mode) {
Status Model::Build(const void *, size_t, ModelType, const std::shared_ptr<Context> &, const Key &,
const std::string &) {
MS_LOG(ERROR) << "Unsupported Feature.";
return kMCFailed;
}
Status Model::Build(const std::string &model_path, ModelType model_type, const std::shared_ptr<Context> &model_context,
const Key &dec_key, const std::string &dec_mode) {
Status Model::Build(const std::string &, ModelType, const std::shared_ptr<Context> &, const Key &,
const std::string &) {
MS_LOG(ERROR) << "Unsupported Feature.";
return kMCFailed;
}

39
mindspore/ccsrc/cxx_api/model/model_converter_utils/multi_process.cc
View File

@ -25,14 +25,14 @@
namespace mindspore {
namespace {
uint64_t kSharedMemorySize = 100ull << 20; // 100 MB
constexpr uint64_t kSharedMemorySize = 100ull << 20; // 100 MB
}
MultiProcess::MultiProcess() = default;
MultiProcess::~MultiProcess() = default;
Status MultiProcess::MainProcess(ProcessFuncCall parent_process, ProcessFuncCall child_process) {
Status MultiProcess::MainProcess(const ProcessFuncCall &parent_process, const ProcessFuncCall &child_process) {
MS_EXCEPTION_IF_NULL(parent_process);
MS_EXCEPTION_IF_NULL(child_process);
Status ret;
@ -61,7 +61,8 @@ Status MultiProcess::MainProcess(ProcessFuncCall parent_process, ProcessFuncCall
}
constexpr size_t kMsgStructNum = 2;
shmat_data_addr_ = shmat_addr_ + sizeof(MessageFlag) * kMsgStructNum;
shmat_data_max_size_ = memory_size_ - (shmat_data_addr_ - shmat_addr_);
shmat_data_max_size_ =
memory_size_ - (reinterpret_cast<uintptr_t>(shmat_data_addr_) - reinterpret_cast<uintptr_t>(shmat_addr_));
MS_LOG_INFO << "Shm addr " << (uint64_t)shmat_addr_;
if (pid == 0) {
ChildProcess(child_process);
@ -85,7 +86,7 @@ Status MultiProcess::MainProcess(ProcessFuncCall parent_process, ProcessFuncCall
child_exited = true;
break;
}
sleep(1);
(void)sleep(1);
}
if (!child_exited) {
MS_LOG(WARNING) << "Child process " << pid << " has been killed but waitpid failed.";
@ -95,7 +96,7 @@ Status MultiProcess::MainProcess(ProcessFuncCall parent_process, ProcessFuncCall
return ret;
}
Status MultiProcess::ParentProcess(ProcessFuncCall parent_process) {
Status MultiProcess::ParentProcess(const ProcessFuncCall &parent_process) {
auto parent_msg = reinterpret_cast<MessageFlag *>(shmat_addr_);
auto child_msg = reinterpret_cast<MessageFlag *>(shmat_addr_ + sizeof(MessageFlag));
send_msg_ = parent_msg;
@ -112,12 +113,12 @@ Status MultiProcess::ParentProcess(ProcessFuncCall parent_process) {
ret = kMEFailed;
}
stopped_ = true;
send_msg_->stop = true;
send_msg_->stop = 1;
heartbeat_thread.join();
return ret;
}
void MultiProcess::ChildProcess(ProcessFuncCall child_process) {
void MultiProcess::ChildProcess(const ProcessFuncCall &child_process) {
auto parent_msg = reinterpret_cast<MessageFlag *>(shmat_addr_);
auto child_msg = reinterpret_cast<MessageFlag *>(shmat_addr_ + sizeof(MessageFlag));
send_msg_ = child_msg;
@ -138,26 +139,30 @@ void MultiProcess::ChildProcess(ProcessFuncCall child_process) {
}
Status MultiProcess::SendMsg(const void *buffer, uint64_t msg_len) {
MS_EXCEPTION_IF_NULL(buffer);
MS_LOG_INFO << "Start to send message to peer process, msg len " << msg_len;
send_msg_->msg_total_len = msg_len;
uint64_t cur_offset = 0;
while (msg_len > cur_offset) {
uint64_t sub_msg_len = std::min(msg_len - cur_offset, shmat_data_max_size_);
if (sub_msg_len == 0) {
MS_LOG(ERROR) << "Invalid message len " << sub_msg_len;
return kMEFailed;
}
auto ret =
memcpy_s(shmat_data_addr_, shmat_data_max_size_, static_cast<const uint8_t *>(buffer) + cur_offset, sub_msg_len);
if (ret != EOK) {
MS_LOG(INFO) << "memcpy_s failed, ret = " << ret;
MS_LOG(ERROR) << "memcpy_s failed, ret = " << ret;
return kMEFailed;
}
cur_offset += sub_msg_len;
send_msg_->msg_len = sub_msg_len;
send_msg_->read_finish_flag = false;
send_msg_->read_ready_flag = true;
send_msg_->read_finish_flag = 0;
send_msg_->read_ready_flag = 1;
MS_LOG_INFO << "Send start " << cur_offset << ", msg len " << sub_msg_len << ", total len " << msg_len;
while (!send_msg_->read_finish_flag && !peer_stopped_) {
usleep(1000); // 1ms
(void)usleep(1000); // 1ms
}
if (peer_stopped_) {
if (!send_msg_->read_finish_flag) {
@ -171,14 +176,14 @@ Status MultiProcess::SendMsg(const void *buffer, uint64_t msg_len) {
return kSuccess;
}
Status MultiProcess::ReceiveMsg(CreateBufferCall create_buffer_call) {
Status MultiProcess::ReceiveMsg(const CreateBufferCall &create_buffer_call) {
uint64_t cur_offset = 0;
uint8_t *msg_buffer = nullptr;
uint64_t msg_len = 0;
do {
MS_LOG_INFO << "Receive start from " << cur_offset;
while (!receive_msg_->read_ready_flag && !peer_stopped_) {
usleep(1000); // 1ms
(void)usleep(1000); // 1ms
}
if (peer_stopped_) {
return kMEFailed;
@ -193,8 +198,8 @@ Status MultiProcess::ReceiveMsg(CreateBufferCall create_buffer_call) {
return kMEFailed;
}
cur_offset += receive_msg_->msg_len;
receive_msg_->read_ready_flag = false;
receive_msg_->read_finish_flag = true;
receive_msg_->read_ready_flag = 0;
receive_msg_->read_finish_flag = 1;
MS_LOG_INFO << "Receive end, current length " << cur_offset << ", total length " << msg_len << std::endl;
} while (msg_len > cur_offset);
return kSuccess;
@ -225,7 +230,7 @@ void MultiProcess::HeartbeatThreadFuncInner() {
}
}
send_msg_->heartbeat += 1;
usleep(100000); // sleep 100 ms
(void)usleep(100000); // sleep 100 ms
}
}
} // namespace mindspore

8
mindspore/ccsrc/cxx_api/model/model_converter_utils/multi_process.h
View File

@ -39,9 +39,9 @@ class MultiProcess {
MultiProcess();
~MultiProcess();
Status MainProcess(ProcessFuncCall parent_process, ProcessFuncCall child_process);
Status MainProcess(const ProcessFuncCall &parent_process, const ProcessFuncCall &child_process);
Status SendMsg(const void *buffer, uint64_t msg_len);
Status ReceiveMsg(CreateBufferCall create_buffer_call);
Status ReceiveMsg(const CreateBufferCall &create_buffer_call);
private:
uint8_t *shmat_addr_ = nullptr;
@ -56,8 +56,8 @@ class MultiProcess {
static void HeartbeatThreadFunc(MultiProcess *multi_process);
void HeartbeatThreadFuncInner();
Status ParentProcess(ProcessFuncCall parent_process);
void ChildProcess(ProcessFuncCall child_process);
Status ParentProcess(const ProcessFuncCall &parent_process);
void ChildProcess(const ProcessFuncCall &child_process);
};
} // namespace mindspore
#endif // MINDSPORE_CCSRC_CXXAPI_MULTI_PROCESS_H

4
mindspore/ccsrc/cxx_api/model/model_converter_utils/shared_memory.h
View File

@ -26,9 +26,11 @@ class SharedMemory {
Status Attach();
void Detach();
void Destroy();
uint8_t *GetSharedMemoryAddr() { return shmat_addr_; }
private:
friend class MultiProcess;
uint8_t *GetSharedMemoryAddr() { return shmat_addr_; }
int shm_id_ = -1;
uint8_t *shmat_addr_ = nullptr;
};

14
mindspore/ccsrc/cxx_api/types.cc
View File

@ -360,25 +360,25 @@ bool MSTensor::IsDevice() const {
return impl_->IsDevice();
}
void MSTensor::SetShape(const std::vector<int64_t> &shape) { MS_LOG_EXCEPTION << "Invalid implement."; }
void MSTensor::SetShape(const std::vector<int64_t> &) { MS_LOG_EXCEPTION << "Invalid implement."; }
void MSTensor::SetDataType(enum DataType data_type) { MS_LOG_EXCEPTION << "Invalid implement."; }
void MSTensor::SetDataType(enum DataType) { MS_LOG_EXCEPTION << "Invalid implement."; }
void MSTensor::SetTensorName(const std::string &name) { MS_LOG_EXCEPTION << "Invalid implement."; }
void MSTensor::SetTensorName(const std::string &) { MS_LOG_EXCEPTION << "Invalid implement."; }
void MSTensor::SetAllocator(std::shared_ptr<Allocator> allocator) { MS_LOG_EXCEPTION << "Invalid implement."; }
void MSTensor::SetAllocator(std::shared_ptr<Allocator>) { MS_LOG_EXCEPTION << "Invalid implement."; }
std::shared_ptr<Allocator> MSTensor::allocator() const { MS_LOG_EXCEPTION << "Invalid implement."; }
void MSTensor::SetFormat(mindspore::Format format) { MS_LOG_EXCEPTION << "Invalid implement."; }
void MSTensor::SetFormat(mindspore::Format) { MS_LOG_EXCEPTION << "Invalid implement."; }
mindspore::Format MSTensor::format() const { MS_LOG_EXCEPTION << "Invalid implement."; }
void MSTensor::SetData(void *data) { MS_LOG_EXCEPTION << "Invalid implement."; }
void MSTensor::SetData(void *) { MS_LOG_EXCEPTION << "Invalid implement."; }
std::vector<QuantParam> MSTensor::QuantParams() const { MS_LOG_EXCEPTION << "Invalid implement."; }
void MSTensor::SetQuantParams(std::vector<QuantParam> quant_params) { MS_LOG_EXCEPTION << "Invalid implement."; }
void MSTensor::SetQuantParams(std::vector<QuantParam>) { MS_LOG_EXCEPTION << "Invalid implement."; }
Buffer::Buffer() : impl_(std::make_shared<Impl>()) {}
Buffer::Buffer(const void *data, size_t data_len) : impl_(std::make_shared<Impl>(data, data_len)) {}

4
mindspore/ccsrc/runtime/device/ascend/ascend_bucket.cc
View File

@ -124,8 +124,8 @@ void AscendBucket::LaunchAllReduce() {
MS_LOG(EXCEPTION) << "allreduce input have different dtype";
}
auto iter = CONST_OP_HCOM_DATA_TYPE_MAP.find(type);
if (iter == CONST_OP_HCOM_DATA_TYPE_MAP.end()) {
auto iter = kConstOpHcomDataTypeMap.find(type);
if (iter == kConstOpHcomDataTypeMap.end()) {
MS_LOG(EXCEPTION) << "unknown data type:" << type;
}