diff --git a/mindspore/ccsrc/distributed/collective/collective_manager.cc b/mindspore/ccsrc/distributed/collective/collective_manager.cc
index 3f4b882a4a4..7fb3abb7ae5 100644
--- a/mindspore/ccsrc/distributed/collective/collective_manager.cc
+++ b/mindspore/ccsrc/distributed/collective/collective_manager.cc
@@ -277,6 +277,7 @@ bool CollectiveManager::Finalize() {
       MS_LOG(WARNING) << "Failed to finalize device communication library.";
     }
 
+    inited_ = false;
     finalized_ = true;
     return true;
   };
diff --git a/mindspore/ccsrc/plugin/device/gpu/hal/hardware/nvidia_collective_comm_lib.cc b/mindspore/ccsrc/plugin/device/gpu/hal/hardware/nvidia_collective_comm_lib.cc
index 6c26b9e257f..d8ab7d15031 100644
--- a/mindspore/ccsrc/plugin/device/gpu/hal/hardware/nvidia_collective_comm_lib.cc
+++ b/mindspore/ccsrc/plugin/device/gpu/hal/hardware/nvidia_collective_comm_lib.cc
@@ -29,6 +29,7 @@ bool NvidiaCollectiveCommLib::Initialize(uint32_t global_rank, uint32_t global_r
   global_rank_id_ = global_rank;
   global_rank_size_ = global_rank_size;
   initialized_ = true;
+  finalized_ = false;
   return true;
 }