ssk015
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Bugfix] get ckpt failed when call init nccl twice

This commit is contained in:
lizhenyu 2022-03-28 22:35:01 +08:00
parent bc070833c7
commit e3b1fb76bc
2 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
mindspore/ccsrc/distributed/collective/collective_manager.h
View File

@ -74,6 +74,9 @@ class BACKEND_EXPORT CollectiveManager {
// Get whether need reinitialize collective communication.
bool need_reinit() const { return need_reinit_.load(); }
// Return collective manager is initialized.
bool initialized() const { return inited_.load(); }
private:
CollectiveManager();

2
mindspore/ccsrc/distributed/init.cc
View File

@ -30,7 +30,7 @@ bool Initialize() {
}
#if ((defined ENABLE_CPU) && (!defined _WIN32))
if (cluster::ClusterContext::instance()->initialized()) {
if (cluster::ClusterContext::instance()->initialized() && !collective::CollectiveManager::instance()->initialized()) {
// Server and Scheduler don't use collective communication library.
auto node = cluster::ClusterContext::instance()->node();
MS_EXCEPTION_IF_NULL(node);