From 0aae542f4fa77a1212b5c029ede651a18aaf048a Mon Sep 17 00:00:00 2001
From: lizhenyu <lizhenyu13@huawei.com>
Date: Thu, 31 Mar 2022 18:44:30 +0800
Subject: [PATCH] bugfix: recovery fail when kill and pull worker twice

---
 mindspore/ccsrc/distributed/collective/collective_manager.cc     | 1 +
 .../plugin/device/gpu/hal/hardware/nvidia_collective_comm_lib.cc | 1 +
 2 files changed, 2 insertions(+)

diff --git a/mindspore/ccsrc/distributed/collective/collective_manager.cc b/mindspore/ccsrc/distributed/collective/collective_manager.cc
index 3f4b882a4a4..7fb3abb7ae5 100644
--- a/mindspore/ccsrc/distributed/collective/collective_manager.cc
+++ b/mindspore/ccsrc/distributed/collective/collective_manager.cc
@@ -277,6 +277,7 @@ bool CollectiveManager::Finalize() {
       MS_LOG(WARNING) << "Failed to finalize device communication library.";
     }
 
+    inited_ = false;
     finalized_ = true;
     return true;
   };
diff --git a/mindspore/ccsrc/plugin/device/gpu/hal/hardware/nvidia_collective_comm_lib.cc b/mindspore/ccsrc/plugin/device/gpu/hal/hardware/nvidia_collective_comm_lib.cc
index 6c26b9e257f..d8ab7d15031 100644
--- a/mindspore/ccsrc/plugin/device/gpu/hal/hardware/nvidia_collective_comm_lib.cc
+++ b/mindspore/ccsrc/plugin/device/gpu/hal/hardware/nvidia_collective_comm_lib.cc
@@ -29,6 +29,7 @@ bool NvidiaCollectiveCommLib::Initialize(uint32_t global_rank, uint32_t global_r
   global_rank_id_ = global_rank;
   global_rank_size_ = global_rank_size;
   initialized_ = true;
+  finalized_ = false;
   return true;
 }