Col_lin
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix codecheck and pclint

This commit is contained in:
limingqi107 2021-05-26 11:39:49 +08:00
parent 22d62f20ff
commit c22185d586
11 changed files with 62 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ccsrc/backend/optimizer/common/helper.cc
View File

@ -437,7 +437,7 @@ std::shared_ptr<std::vector<std::pair<AnfNodePtr, int>>> GetRealNodeUsedListByOu
} else if (AnfAlgo::GetCNodeName(node) == prim::kPrimTupleGetItem->name()) {
used_output_index = output_index;
} else {
auto kernel_with_index = AnfAlgo::GetPrevNodeOutput(output_info.first, output_info.second - 1);
auto kernel_with_index = AnfAlgo::GetPrevNodeOutput(output_info.first, IntToSize(output_info.second - 1));
if (kernel_with_index.first.get() != node.get()) {
MS_LOG(EXCEPTION) << "Get used node failed for op[" << AnfAlgo::GetCNodeName(node) << "]";
}

20
mindspore/ccsrc/backend/optimizer/gpu/insert_format_transform_op.cc
View File

@ -28,6 +28,8 @@
namespace mindspore {
namespace opt {
namespace {
const int kFakeTransposeShapeOneNum = 2;
std::vector<int64_t> TransposeAxis(const std::string &src_format, const std::string &dst_format) {
if ((src_format == kOpFormat_NCHW) && (dst_format == kOpFormat_NHWC)) {
return {0, 2, 3, 1};
@ -43,14 +45,14 @@ std::vector<int64_t> TransposeAxis(const std::string &src_format, const std::str
// 2. out_shape [x, y, 1, 1]
// 3. out_shape [x, 1, y, 1]
bool IsFakeTranspose(const std::vector<size_t> &out_shape, const std::vector<int64_t> &transpose_perm) {
if (out_shape.size() != 4) {
if (out_shape.size() != device::gpu::kFormatTransformDimension) {
MS_LOG(EXCEPTION) << "Invalid data shape, 4-D data was needed, but get " << out_shape.size() << "-D.";
}
std::vector<int64_t> perm1 = {0, 2, 3, 1};
std::vector<int64_t> perm2 = {0, 3, 1, 2};
auto num = std::count(out_shape.begin(), out_shape.end(), 1);
if ((transpose_perm == perm1) || (transpose_perm == perm2)) {
if (num >= 2) {
if (num >= kFakeTransposeShapeOneNum) {
return true;
}
}
@ -130,9 +132,9 @@ const AnfNodePtr InsertFormatTransformOp::Process(const FuncGraphPtr &graph, con
if ((inputs_format[i] != kOpFormat_DEFAULT) && (inputs_format[i] != origin_data_format)) {
auto input_node = AnfAlgo::GetInputNode(utils::cast<CNodePtr>(node), i);
MS_EXCEPTION_IF_NULL(input_node);
auto transpose_perm = TransposeAxis(origin_data_format, inputs_format[i]);
auto transpose_op = InsertTransposeOp(graph, input_node, node, i, transpose_perm);
SetTransposeOpBuildInfo(kOpFormat_DEFAULT, inputs_format[i], transpose_op);
auto input_transpose_perm = TransposeAxis(origin_data_format, inputs_format[i]);
auto input_transpose_op = InsertTransposeOp(graph, input_node, node, i, input_transpose_perm);
SetTransposeOpBuildInfo(kOpFormat_DEFAULT, inputs_format[i], input_transpose_op);
}
}
@ -145,15 +147,15 @@ const AnfNodePtr InsertFormatTransformOp::Process(const FuncGraphPtr &graph, con
for (size_t j = 0; j < used_node_list->size(); j++) {
auto used_node = used_node_list->at(j).first;
auto used_node_index = used_node_list->at(j).second - 1;
auto transpose_perm = TransposeAxis(outputs_format[i], origin_data_format);
auto output_transpose_perm = TransposeAxis(outputs_format[i], origin_data_format);
if (AnfAlgo::GetCNodeName(used_node) == prim::kPrimTupleGetItem->name()) {
MS_LOG(DEBUG) << "The used node of [" << node->fullname_with_scope() << "] is tuple item.";
// The tuple item need get next used nodes again.
ProcessForTupleItem(graph, used_node, used_node_index, transpose_perm, outputs_format[i]);
ProcessForTupleItem(graph, used_node, used_node_index, output_transpose_perm, outputs_format[i]);
continue;
}
auto transpose_op = InsertTransposeOp(graph, node, used_node, used_node_index, transpose_perm);
SetTransposeOpBuildInfo(outputs_format[i], kOpFormat_DEFAULT, transpose_op);
auto output_transpose_op = InsertTransposeOp(graph, node, used_node, used_node_index, output_transpose_perm);
SetTransposeOpBuildInfo(outputs_format[i], kOpFormat_DEFAULT, output_transpose_op);
}
}
}

2
mindspore/ccsrc/pipeline/jit/action.cc
View File

@ -611,7 +611,7 @@ bool ExecuteAction(const ResourcePtr &res) {
}
#if (ENABLE_CPU && !_WIN32)
bool StartPSWorkerAction(const ResourcePtr &res) {
bool StartPSWorkerAction(const ResourcePtr &) {
ps::Worker::GetInstance().Run();
return true;
}

48
mindspore/ccsrc/ps/ps_cache/ascend/ascend_ps_cache.cc
View File

@ -214,14 +214,17 @@ bool AscendPsCache::HashSwapOut(void *hash_table_addr, void *swap_out_value_addr
auto hash_swap_out_mod = std::make_shared<kernel::AicpuOpKernelMod>();
MS_ERROR_IF_NULL(hash_swap_out_mod);
hash_swap_out_mod->SetNodeName(kEmbeddingLookupOpName);
std::vector<std::vector<size_t>> input_shape;
std::vector<std::vector<size_t>> output_shape;
std::vector<size_t> hash_table_shape = {cache_vocab_size, embedding_size};
std::vector<size_t> swap_out_index_shape = {swap_out_size};
std::vector<size_t> offset_shape = {1};
std::vector<std::vector<size_t>> input_shape = {hash_table_shape, swap_out_index_shape, offset_shape};
std::vector<size_t> swap_out_value_shape = {swap_out_size, embedding_size};
std::vector<std::vector<size_t>> output_shape = {swap_out_value_shape};
std::vector<TypeId> input_type = {TypeId::kNumberTypeFloat32, TypeId::kNumberTypeInt32, TypeId::kNumberTypeInt32};
std::vector<TypeId> output_type = {TypeId::kNumberTypeFloat32};
input_shape.push_back({cache_vocab_size, embedding_size});
input_shape.push_back({swap_out_size});
input_shape.push_back({1});
output_shape.push_back({swap_out_size, embedding_size});
auto op_info =
std::make_shared<KernelNodeInfo>(kEmbeddingLookupOpName, input_shape, input_type, output_shape, output_type);
RETURN_IF_FALSE(SetNodedefProto(op_info, hash_swap_out_mod));
@ -230,10 +233,10 @@ bool AscendPsCache::HashSwapOut(void *hash_table_addr, void *swap_out_value_addr
AddressPtrList kernel_outputs = {
std::make_shared<Address>(swap_out_value_addr, swap_out_size * embedding_size * sizeof(float))};
AddressPtrList kernel_workspaces;
kernel_inputs.push_back(
kernel_inputs.emplace_back(
std::make_shared<Address>(hash_table_addr, cache_vocab_size * embedding_size * sizeof(float)));
kernel_inputs.push_back(std::make_shared<Address>(swap_out_index_addr, swap_out_size * sizeof(int)));
kernel_inputs.push_back(std::make_shared<Address>(offset_addr_, sizeof(int)));
kernel_inputs.emplace_back(std::make_shared<Address>(swap_out_index_addr, swap_out_size * sizeof(int)));
kernel_inputs.emplace_back(std::make_shared<Address>(offset_addr_, sizeof(int)));
auto ret = hash_swap_out_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, stream_);
if (!ret) {
MS_LOG(ERROR) << "Hash swap out launch failed.";
@ -250,16 +253,18 @@ bool AscendPsCache::HashSwapIn(void *hash_table_addr, void *swap_in_value_addr,
auto hash_swap_in_mod = std::make_shared<kernel::AicpuOpKernelMod>();
MS_ERROR_IF_NULL(hash_swap_in_mod);
hash_swap_in_mod->SetNodeName(kernel::kUpdateCache);
std::vector<std::vector<size_t>> input_shape;
std::vector<std::vector<size_t>> output_shape;
std::vector<size_t> hash_table_shape = {cache_vocab_size, embedding_size};
std::vector<size_t> swap_in_index_shape = {swap_in_size};
std::vector<size_t> swap_in_value_shape = {swap_in_size, embedding_size};
std::vector<size_t> offset_shape = {1};
std::vector<std::vector<size_t>> input_shape = {hash_table_shape, swap_in_index_shape, swap_in_value_shape,
offset_shape};
std::vector<std::vector<size_t>> output_shape = {offset_shape};
std::vector<TypeId> input_type = {TypeId::kNumberTypeFloat32, TypeId::kNumberTypeInt32, TypeId::kNumberTypeFloat32,
TypeId::kNumberTypeInt32};
std::vector<TypeId> output_type = {TypeId::kNumberTypeInt32};
input_shape.push_back({cache_vocab_size, embedding_size});
input_shape.push_back({swap_in_size});
input_shape.push_back({swap_in_size, embedding_size});
input_shape.push_back({1});
output_shape.push_back({1});
auto op_info =
std::make_shared<KernelNodeInfo>(kernel::kUpdateCache, input_shape, input_type, output_shape, output_type);
SetNodedefProto(op_info, hash_swap_in_mod);
@ -267,13 +272,14 @@ bool AscendPsCache::HashSwapIn(void *hash_table_addr, void *swap_in_value_addr,
AddressPtrList kernel_inputs;
AddressPtrList kernel_outputs;
AddressPtrList kernel_workspaces;
kernel_inputs.push_back(
kernel_inputs.emplace_back(
std::make_shared<Address>(hash_table_addr, cache_vocab_size * embedding_size * sizeof(float)));
kernel_inputs.push_back(std::make_shared<Address>(swap_in_index_addr, swap_in_size * sizeof(int)));
kernel_inputs.push_back(std::make_shared<Address>(swap_in_value_addr, swap_in_size * embedding_size * sizeof(float)));
kernel_inputs.push_back(std::make_shared<Address>(cache_vocab_size_addr_, sizeof(int)));
kernel_inputs.emplace_back(std::make_shared<Address>(swap_in_index_addr, swap_in_size * sizeof(int)));
kernel_inputs.emplace_back(
std::make_shared<Address>(swap_in_value_addr, swap_in_size * embedding_size * sizeof(float)));
kernel_inputs.emplace_back(std::make_shared<Address>(cache_vocab_size_addr_, sizeof(int)));
// The output of updateCache kernel is required but not useful, so any address can be assigned.
kernel_outputs.push_back(std::make_shared<Address>(offset_addr_, sizeof(int)));
kernel_outputs.emplace_back(std::make_shared<Address>(offset_addr_, sizeof(int)));
auto ret = hash_swap_in_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, stream_);
if (!ret) {
MS_LOG(ERROR) << "Hash swap in launch failed.";

2
mindspore/ccsrc/ps/ps_cache/embedding_hash_map.cc
View File

@ -48,7 +48,7 @@ int EmbeddingHashMap::ParseData(const int id, int *const swap_out_index, int *co
return hash_index;
}
int EmbeddingHashMap::FindInsertionPos(const size_t data_step, const size_t graph_running_step, bool *const need_swap,
int EmbeddingHashMap::FindInsertionPos(const size_t, const size_t graph_running_step, bool *const need_swap,
bool *const need_wait_graph) {
MS_EXCEPTION_IF_NULL(need_swap);
MS_EXCEPTION_IF_NULL(need_wait_graph);

8
mindspore/ccsrc/ps/ps_cache/ps_cache_manager.cc
View File

@ -1087,7 +1087,7 @@ void PsCacheManager::DumpHashTables(bool dump_device_tables) const {
<< ", device cache address:" << reinterpret_cast<void *>(item.second.device_address.addr)
<< ", host cache address:" << reinterpret_cast<void *>(item.second.host_address.get());
if (dump_device_tables) {
std::unique_ptr<float[]> output = std::make_unique<float[]>(item.second.device_address.size / 4);
std::unique_ptr<float[]> output = std::make_unique<float[]>(item.second.device_address.size / sizeof(float));
embedding_device_cache_->cache_->CopyDeviceMemToHost(output.get(), item.second.device_address.addr,
item.second.device_address.size);
embedding_device_cache_->cache_->SynchronizeStream();
@ -1104,6 +1104,7 @@ void PsCacheManager::DumpHashTables(bool dump_device_tables) const {
void PsCacheManager::DumpStatisticsInfo(size_t each_print_step) {
// Default each 1000 step prints ps cache hit rate.
const size_t kFloatToPercentSign = 100;
if (data_step_ % each_print_step == 0) {
statistics_info_.batch_id_unique_count_ = statistics_info_.hash_hit_count_ + statistics_info_.host_to_device_size_;
auto repeat_rate = SizeToFloat(statistics_info_.batch_id_count_ - statistics_info_.batch_id_unique_count_) /
@ -1117,8 +1118,9 @@ void PsCacheManager::DumpStatisticsInfo(size_t each_print_step) {
<< ", device swap to host num:" << statistics_info_.device_to_host_size_
<< ", host swap to server num:" << statistics_info_.host_to_server_size_
<< ", server swap to host num:" << statistics_info_.server_to_host_size_
<< ", data repeat rate:" << repeat_rate * 100 << "%, device cache hit rate:" << device_hit_rate * 100
<< "%, host cache hit rate:" << host_hit_rate * 100 << ").";
<< ", data repeat rate:" << (repeat_rate * kFloatToPercentSign)
<< "%, device cache hit rate:" << (device_hit_rate * kFloatToPercentSign)
<< "%, host cache hit rate:" << (host_hit_rate * kFloatToPercentSign) << ").";
}
}
} // namespace ps

10
mindspore/ccsrc/ps/ps_cache/ps_data/ps_data_prefetch.cc
View File

@ -19,6 +19,8 @@
namespace mindspore {
namespace ps {
const size_t kTimeoutLoopCount = 10;
void PsDataPrefetch::CreateDataChannel(const std::string &channel_name, size_t step_num) {
if (cache_enable_ == false) {
return;
@ -69,7 +71,8 @@ bool PsDataPrefetch::PrefetchData(const std::string &channel_name, void *data, c
if (!need_wait_) {
return true;
}
for (int i = 0; i < 10; i++) {
for (size_t i = 0; i < kTimeoutLoopCount; ++i) {
if (data_prefetch_.wait_for(locker, std::chrono::seconds(30),
[this] { return data_ready_ == false || need_wait_ == false; })) {
return true;
@ -94,7 +97,8 @@ bool PsDataPrefetch::FinalizeData(const std::string &channel_name) {
if (!need_wait_) {
return true;
}
for (int i = 0; i < 10; i++) {
for (size_t i = 0; i < kTimeoutLoopCount; ++i) {
if (data_process_.wait_for(locker, std::chrono::seconds(30),
[this] { return data_ready_ == true || need_wait_ == false; })) {
return true;
@ -147,7 +151,7 @@ bool PsDataPrefetch::TryWakeChannel(const std::string &channel_name) {
}
void PsDataPrefetch::WakeAllChannel() {
for (auto iter = ps_data_channel_map_.begin(); iter != ps_data_channel_map_.end(); iter++) {
for (auto iter = ps_data_channel_map_.begin(); iter != ps_data_channel_map_.end(); ++iter) {
auto channel = iter->second;
if (channel == nullptr) {
return;

4
mindspore/ccsrc/runtime/device/gpu/blocking_queue.cc
View File

@ -21,6 +21,8 @@
namespace mindspore {
namespace device {
const size_t kTimeout = 100;
GpuQueue::GpuQueue(void *addr, const std::vector<size_t> &shape, const size_t &capacity)
: buffer_(addr),
head_(0),
@ -110,7 +112,7 @@ void BlockingQueue::RegisterRelease(const std::function<void(void *, int32_t)> &
BlockQueueStatus_T BlockingQueue::Push(const std::vector<DataItemGpu> &data, unsigned int) {
std::unique_lock<std::mutex> locker(mutex_);
if (queue_->IsFull()) {
if (not_full_cond_.wait_for(locker, std::chrono::microseconds(100)) == std::cv_status::timeout) {
if (not_full_cond_.wait_for(locker, std::chrono::microseconds(kTimeout)) == std::cv_status::timeout) {
return TIMEOUT;
}
}

4
mindspore/ccsrc/runtime/device/gpu/gpu_memory_allocator.cc
View File

@ -24,13 +24,15 @@
namespace mindspore {
namespace device {
namespace gpu {
const size_t kGBToByte = 1024 << 20;
bool GPUMemoryAllocator::Init() {
size_t total_size = total_mem_size();
size_t free_size = CudaDriver::free_mem_size();
auto context_ptr = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context_ptr);
limited_device_memory_ = context_ptr->get_param<float>(MS_CTX_MAX_DEVICE_MEMORY);
available_device_memory_ = FloatToSize(limited_device_memory_ * 1024 * 1024 * 1024);
available_device_memory_ = FloatToSize(limited_device_memory_ * kGBToByte);
if (total_size > 0 && free_size > 0 && available_device_memory_ > 0) {
MS_LOG(INFO) << "GPU device total memory size " << total_size << ", current free memory size " << free_size
<< ", set max available memory size " << available_device_memory_ << ".";

4
mindspore/ccsrc/runtime/device/gpu/kernel_info_setter.cc
View File

@ -220,7 +220,7 @@ bool IsNeedProcessFormatInfo(const CNodePtr &kernel_node, const std::vector<Type
for (const auto &input_format_position : inputs_format_position) {
auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, input_format_position);
// Only support the transformer between NCHW and NHWC, so need the shape is 4 dimension.
if (input_shape.size() != 4) {
if (input_shape.size() != kFormatTransformDimension) {
return false;
}
}
@ -231,7 +231,7 @@ bool IsNeedProcessFormatInfo(const CNodePtr &kernel_node, const std::vector<Type
for (const auto &output_format_position : outputs_format_position) {
auto output_shape = AnfAlgo::GetOutputInferShape(kernel_node, output_format_position);
// Only support the transformer between NCHW and NHWC, so need the shape is 4 dimension.
if (output_shape.size() != 4) {
if (output_shape.size() != kFormatTransformDimension) {
return false;
}
}

1
mindspore/ccsrc/runtime/device/gpu/kernel_info_setter.h
View File

@ -32,6 +32,7 @@ namespace mindspore {
namespace device {
namespace gpu {
const size_t kAllPositions = SIZE_MAX;
const size_t kFormatTransformDimension = 4;
// Map<opName, (inputFormatPosition, outputFormatPosition)>, used for getting the inserted position of format transform.
// If the inserted position is kAllPositions, then insert all the positions, because the input or output numbers of