!19773 seg fault and existing dump fix

Merge pull request !19773 from john_tzanakakis/1.3

mindspore/ccsrc/debug/data_dump/e2e_dump.cc

@@ -379,7 +379,7 @@ bool E2eDump::DumpData(const session::KernelGraph *graph, uint32_t rank_id, cons
         command = "mv " + zero_dir_dump_path + "/* " + cur_iter_dump_path + "/.";
         MS_LOG(INFO) << "mv command: " << command;
         if (system(command.c_str())) {
-          MS_LOG(EXCEPTION) << "Ascend runtime has changed the dump dir structure!!!";
+          MS_LOG(INFO) << "issue with move command";
         }
       } else {
         MS_LOG(INFO) << "active dump dir, not created yet";

mindspore/ccsrc/debug/debug_services.cc

@@ -168,7 +168,12 @@ void DebugServices::AddWatchPointsToCheck(bool init_dbg_suspend, bool step_end,
     if (wp.change_condition() && !step_end) continue;
     // if recheck, ignore the cache results and reanalyze everything.
     // if not a recheck, check only unanalyzed tensors
-    if (!recheck && wp_id_cache[tensor_name].count(wp.id)) continue;
+    if (!recheck) {
+      wp_lock_.lock();
+      bool wp_cache_hit = wp_id_cache[tensor_name].count(wp.id);
+      wp_lock_.unlock();
+      if (wp_cache_hit) continue;
+    }
     std::string found = wp.FindQualifiedTensorName(tensor_name_no_slot);
     if (!found.empty()) {
       *qualified_tensor_name = found;
@@ -186,7 +191,9 @@ void DebugServices::AddAnalyzedTensorToCache(const bool recheck, const unsigned
                                              const std::string &tensor_name) {
   // add analyzed tensor to cache
   if (!recheck) {
+    wp_lock_.lock();
     wp_id_cache[tensor_name].insert(id);
+    wp_lock_.unlock();
   }
 }
 

mindspore/ccsrc/debug/debug_services.h

@@ -319,6 +319,7 @@ class DebugServices {
 
  private:
   std::mutex lock_;
+  std::mutex wp_lock_;
 
   // to keep track of watchpoints that have been checked already for a tensor in current step
   std::unordered_map<std::string, std::set<int32_t>> wp_id_cache;