1. Add unique id for .dat and .dot file to avoid covering

2. Dump the end graph in gpu session and cu session

mindspore/ccsrc/backend/session/cpu_session.cc

@@ -28,6 +28,8 @@
 #include "backend/optimizer/common/optimizer.h"
 #include "backend/optimizer/common/pass_manager.h"
 #include "backend/optimizer/pass/replace_node_by_proxy.h"
+#include "debug/anf_ir_dump.h"
+#include "debug/dump_proto.h"
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
 #include "ps/util.h"
 #include "ps/ps_context.h"
@@ -98,6 +100,8 @@ GraphId CPUSession::CompileGraphImpl(const AnfNodePtrList &lst, const AnfNodePtr
 
   MS_LOG(INFO) << "Assign kernel address";
   runtime_.AssignKernelAddress(graph.get());
+
+  DumpGraph(graph);
   return graph_id;
 }
 

mindspore/ccsrc/backend/session/gpu_session.cc

@@ -403,6 +403,8 @@ GraphId GPUSession::CompileGraphImpl(KernelGraphPtr graph) {
     AllocateMemory(graph.get());
   }
 
+  DumpGraph(graph);
+
 #ifdef ENABLE_DEBUGGER
   if (debugger_ && debugger_->DebuggerBackendEnabled()) {
     debugger_->LoadGraphs(graph);

mindspore/ccsrc/backend/session/session_basic.cc

@@ -38,6 +38,7 @@
 #include "ir/func_graph_cloner.h"
 #include "utils/utils.h"
 #include "debug/anf_ir_dump.h"
+#include "debug/dump_proto.h"
 #include "debug/common.h"
 #include "utils/trace_base.h"
 #include "frontend/parallel/context.h"
@@ -2451,6 +2452,19 @@ void SessionBasic::ClearAllBucket(const GraphId &graph_id) {
   }
 }
 
+void SessionBasic::DumpGraph(const std::shared_ptr<KernelGraph> &kernel_graph) {
+#ifdef ENABLE_DUMP_IR
+  auto context_ptr = MsContext::GetInstance();
+  MS_EXCEPTION_IF_NULL(context_ptr);
+  bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG);
+  if (save_graphs) {
+    DumpIR("graph_build_" + std::to_string(kernel_graph->graph_id()) + ".ir", kernel_graph, true, kWholeStack);
+    DumpIRProto(kernel_graph, "vm_build_" + std::to_string(kernel_graph->graph_id()));
+    DumpIR("trace_code_graph", kernel_graph, true, kWholeStack);
+  }
+#endif
+}
+
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
 void SessionBasic::InitPsWorker(const KernelGraphPtr &kernel_graph) {
   if (!ps::PSContext::instance()->is_worker()) {

mindspore/ccsrc/backend/session/session_basic.h

@@ -230,6 +230,7 @@ class SessionBasic : public std::enable_shared_from_this<SessionBasic> {
   void ClearAllBucket(const GraphId &graph_id);
   std::vector<uint32_t> GetAllReduceSplitIndex();
   virtual std::string GetCommWorldGroup() { return std::string(); }
+  void DumpGraph(const std::shared_ptr<KernelGraph> &kernel_graph);
 #if (ENABLE_CPU && (ENABLE_D || ENABLE_GPU))
   void CheckPSModeConsistence(const KernelGraphPtr &kernel_graph) const;
   void GetBatchElements(const AnfNodePtr &kernel_node) const;

mindspore/ccsrc/debug/anf_ir_dump.cc

@@ -527,24 +527,6 @@ void DumpSubgraph(const OrderedMap<FuncGraphPtr, std::shared_ptr<SubGraphIRInfo>
   }
 }
 
-std::string AddGlobalId(const std::string &filename) {
-  static size_t g_id = 0;
-  std::ostringstream s;
-  auto i = filename.rfind(".ir");
-  if (i >= filename.size()) {
-    s << filename;
-    s << "_" << std::setfill('0') << std::setw(4) << g_id;
-  } else {
-    s << filename.substr(0, i);
-    s << "_" << std::setfill('0') << std::setw(4) << g_id;
-    if (i + 1 < filename.size()) {
-      s << filename.substr(i);
-    }
-  }
-  ++g_id;
-  return s.str();
-}
-
 void GetEnvDumpIrLineLevel(LocDumpMode *dump_location) {
   static std::unordered_map<std::string, enum LocDumpMode> dump_level_map = {
     {std::to_string(kOff), kOff}, {std::to_string(kTopStack), kTopStack}, {std::to_string(kWholeStack), kWholeStack}};
@@ -564,7 +546,7 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu
   if (graph == nullptr) {
     return;
   }
-  auto path = pipeline::GetSaveGraphsPathName(AddGlobalId(filename));
+  auto path = pipeline::GetSaveGraphsPathName(Common::AddId(filename, ".ir"));
   if (!target_file.empty()) {
     path = target_file;
   }
@@ -609,7 +591,7 @@ void DumpIRForRDR(const std::string &filename, const FuncGraphPtr &graph, bool d
   if (graph == nullptr) {
     return;
   }
-  auto path = AddGlobalId(filename);
+  auto path = Common::AddId(filename, ".ir");
   auto realpath = Common::GetRealPath(path);
   if (!realpath.has_value()) {
     MS_LOG(ERROR) << "Get real path failed. path=" << path;

mindspore/ccsrc/debug/anf_ir_utils.cc

@@ -40,6 +40,7 @@
 #include "utils/ms_context.h"
 #include "frontend/operator/ops.h"
 #include "pipeline/jit/base.h"
+#include "debug/common.h"
 
 using mindspore::tensor::TensorPy;
 
@@ -189,6 +190,14 @@ std::string AnfExporter::GetMultitypeFuncGraphText(const prim::MultitypeFuncGrap
   return oss.str();
 }
 
+inline bool Skip(const MetaFuncGraphPtr &meta_func_graph) {
+  return meta_func_graph->isa<prim::Tail>() || meta_func_graph->isa<prim::MakeTupleGradient>() ||
+         meta_func_graph->isa<prim::MakeListGradient>() || meta_func_graph->isa<prim::TupleAdd>() ||
+         meta_func_graph->isa<prim::TupleSlice>() || meta_func_graph->isa<prim::UnpackCall>() ||
+         meta_func_graph->isa<prim::ZipOperation>() || meta_func_graph->isa<prim::ListAppend>() ||
+         meta_func_graph->isa<prim::DoSignatureMetaFuncGraph>();
+}
+
 /* inherit relation of MetaFuncGraph
  *
  * MetaGraph
@@ -239,23 +248,7 @@ std::string AnfExporter::GetMetaFuncGraphText(const MetaFuncGraphPtr &meta_func_
     prim::GradOperationPtr grad_op = meta_func_graph->cast<prim::GradOperationPtr>();
     oss << "{get_all=" << grad_op->get_all_ << ", get_by_list=" << grad_op->get_by_list_
         << ", sens_param=" << grad_op->sens_param_ << "}";
-  } else if (meta_func_graph->isa<prim::Tail>()) {
-    // do nothing
-  } else if (meta_func_graph->isa<prim::MakeTupleGradient>()) {
-    // do nothing
-  } else if (meta_func_graph->isa<prim::MakeListGradient>()) {
-    // do nothing
-  } else if (meta_func_graph->isa<prim::TupleAdd>()) {
-    // do nothing
-  } else if (meta_func_graph->isa<prim::TupleSlice>()) {
-    // do nothing
-  } else if (meta_func_graph->isa<prim::UnpackCall>()) {
-    // do nothing
-  } else if (meta_func_graph->isa<prim::ZipOperation>()) {
-    // do nothing
-  } else if (meta_func_graph->isa<prim::ListAppend>()) {
-    // do nothing
-  } else if (meta_func_graph->isa<prim::DoSignatureMetaFuncGraph>()) {
+  } else if (Skip(meta_func_graph)) {
     // do nothing
   } else {
     MS_LOG(EXCEPTION) << "Unknown MetaFuncGraph type " << meta_func_graph->type_name();
@@ -711,7 +704,7 @@ void ExportIR(const std::string &filename, const std::string &id, const FuncGrap
     return;
   }
 
-  auto real_filename = pipeline::GetSaveGraphsPathName(filename);
+  auto real_filename = pipeline::GetSaveGraphsPathName(Common::AddId(filename, ".dat"));
   AnfExporter exporter(id);
   ChangeFileMode(real_filename, S_IRWXU);
   exporter.ExportFuncGraph(real_filename, func_graph);
@@ -720,7 +713,7 @@ void ExportIR(const std::string &filename, const std::string &id, const FuncGrap
 }
 
 void ExportIR(const std::string &filename, const std::vector<TaggedGraph> &graphs) {
-  auto real_filename = pipeline::GetSaveGraphsPathName(filename);
+  auto real_filename = pipeline::GetSaveGraphsPathName(Common::AddId(filename, ".dat"));
   AnfExporter exporter("", false);
   ChangeFileMode(real_filename, S_IRWXU);
   exporter.ExportFuncGraph(real_filename, graphs);

mindspore/ccsrc/debug/common.cc

@@ -17,6 +17,7 @@
 #include "debug/common.h"
 
 #include <memory>
+#include <iomanip>
 #include <optional>
 #include "utils/ms_context.h"
 #include "utils/system/env.h"
@@ -271,4 +272,22 @@ bool Common::IsFilenameValid(const std::string &filename, const int &length_limi
   }
   return true;
 }
+
+std::string Common::AddId(const std::string &filename, const std::string &suffix) {
+  static size_t g_id = 0;
+  std::ostringstream s;
+  auto i = filename.rfind(suffix);
+  if (i >= filename.size()) {
+    s << filename;
+    s << "_" << std::setfill('0') << std::setw(4) << g_id;
+  } else {
+    s << filename.substr(0, i);
+    s << "_" << std::setfill('0') << std::setw(4) << g_id;
+    if (i + 1 < filename.size()) {
+      s << filename.substr(i);
+    }
+  }
+  g_id++;
+  return s.str();
+}
 }  // namespace mindspore

mindspore/ccsrc/debug/common.h

@@ -40,6 +40,8 @@ class Common {
                               const std::string &error_message = "");
   static bool CreateNotExistDirs(const std::string &path);
 
+  static std::string AddId(const std::string &filename, const std::string &suffix);
+
  private:
   static bool IsEveryFilenameValid(const std::string &path, const int &length_limit, const std::string &error_message);
 };

mindspore/ccsrc/debug/draw.cc

@@ -30,6 +30,7 @@
 #include "pipeline/jit/parse/resolve.h"
 #include "ir/tensor.h"
 #include "pipeline/jit/base.h"
+#include "debug/common.h"
 
 namespace mindspore {
 
@@ -190,11 +191,11 @@ void Draw(const std::string &filename, const FuncGraphPtr &func_graph) {
   const std::string dot_suffix = ".dot";
   std::string filename_with_suffix =
     (filename.rfind(dot_suffix) != (filename.size() - dot_suffix.size())) ? (filename + dot_suffix) : filename;
-  DrawByOpt(pipeline::GetSaveGraphsPathName(filename_with_suffix), func_graph, false);
+  DrawByOpt(pipeline::GetSaveGraphsPathName(Common::AddId(filename_with_suffix, dot_suffix)), func_graph, false);
 }
 
 void DrawUserFuncGraph(const std::string &filename, const FuncGraphPtr &func_graph) {
-  DrawByOpt(pipeline::GetSaveGraphsPathName(filename), func_graph, true);
+  DrawByOpt(pipeline::GetSaveGraphsPathName(Common::AddId(filename, ".dot")), func_graph, true);
 }
 #else
 void Draw(const std::string &, const FuncGraphPtr &) {