All the cells no need to read the cached graphs when check hash consistency failed

mindspore/ccsrc/pipeline/jit/compile_cache_manager.cc

@@ -123,36 +123,6 @@ std::string GetCompileDepFilesHash(const py::list &dep_files) {
   return files_hash;
 }
 
-bool CheckDepFilesHashConsistency(const std::string &current_dep_files_hash) {
-  if (current_dep_files_hash.empty()) {
-    MS_LOG(ERROR) << "Get current dependency files hash failed.";
-    return false;
-  }
-  std::string dep_files_hash_path = GetDepFilesHashPath();
-  auto realpath = Common::CreatePrefixPath(dep_files_hash_path, true);
-  if (!realpath.has_value()) {
-    MS_LOG(ERROR) << "Get real path of file " << dep_files_hash_path << " failed.";
-    return false;
-  }
-  std::fstream input(realpath.value(), std::ios::in | std::ios::binary);
-  if (!input) {
-    MS_LOG(WARNING) << "Open the hash file " << realpath.value() << " failed. The file may not exist."
-                    << ErrnoToString(errno);
-    return false;
-  }
-  std::string checkpoint_hash;
-  input >> checkpoint_hash;
-  if (checkpoint_hash.empty()) {
-    MS_LOG(ERROR) << "Get the compilation dependency files hash from " << realpath.value() << " failed.";
-    return false;
-  }
-  if (checkpoint_hash != current_dep_files_hash) {
-    MS_LOG(WARNING) << "The compilation dependency files are changed.";
-    return false;
-  }
-  return true;
-}
-
 std::map<string, ValuePtr> GenerateWeightsValueMap(const py::dict &weights) {
   std::map<string, ValuePtr> ret{};
   for (auto weight = weights.begin(); weight != weights.end(); ++weight) {
@@ -230,14 +200,38 @@ void CompileCacheManager::InitCompileCacheHash(const py::list &compile_cache_dep
   compile_cache_dep_files_hash_ = GetCompileDepFilesHash(compile_cache_dep_files);
 }
 
+bool CompileCacheManager::CheckDepFilesHashConsistency() {
+  if (compile_cache_dep_files_hash_.empty()) {
+    MS_LOG(ERROR) << "Get current dependency files hash failed.";
+    return false;
+  }
+  std::string dep_files_hash_path = GetDepFilesHashPath();
+  auto realpath = Common::CreatePrefixPath(dep_files_hash_path, true);
+  if (!realpath.has_value()) {
+    MS_LOG(ERROR) << "Get real path of file " << dep_files_hash_path << " failed.";
+    return false;
+  }
+  std::fstream input(realpath.value(), std::ios::in | std::ios::binary);
+  if (!input) {
+    MS_LOG(WARNING) << "Open the hash file " << realpath.value() << " failed. The file may not exist."
+                    << ErrnoToString(errno);
+    return false;
+  }
+  std::string checkpoint_hash;
+  input >> checkpoint_hash;
+  if (checkpoint_hash.empty()) {
+    MS_LOG(ERROR) << "Get the compilation dependency files hash from " << realpath.value() << " failed.";
+    return false;
+  }
+  if (checkpoint_hash != compile_cache_dep_files_hash_) {
+    MS_LOG(WARNING) << "The compilation dependency files are changed.";
+    return false;
+  }
+  return true;
+}
+
 FuncGraphPtr CompileCacheManager::GetCachedFuncGraph(const FuncGraphManagerPtr &manager, const py::dict &weights,
                                                      const std::string &queue_name) {
-  // Compare the dependency files hash.
-  if (!CheckDepFilesHashConsistency(compile_cache_dep_files_hash_)) {
-    MS_LOG(WARNING) << "Check the consistency of dependency files hash failed. Execute all the compilation actions.";
-    return nullptr;
-  }
-
   // Determine whether to load parallel information.
   std::string parallel_mode = parallel::ParallelContext::GetInstance()->parallel_mode();
   bool has_parallel_info = false;

mindspore/ccsrc/pipeline/jit/compile_cache_manager.h

@@ -35,6 +35,8 @@ class CompileCacheManager {
 
   // Get the hash of dependent files when compiling graph.
   void InitCompileCacheHash(const py::list &compile_cache_dep_files);
+  // Compare the dependency files hash.
+  bool CheckDepFilesHashConsistency();
   // Load the cached func_graph from mindir file.
   FuncGraphPtr GetCachedFuncGraph(const FuncGraphManagerPtr &manager, const py::dict &weights,
                                   const std::string &queue_name);

mindspore/ccsrc/pipeline/jit/pipeline.cc

@@ -768,7 +768,7 @@ void GraphExecutorPy::InitCompileCacheInfo(const ResourcePtr &resource, const st
   double t1 = GetTime();
 #endif
   static size_t idx = 0;
-  resource->GetCompileCacheResource(compile_cache_dep_files_, weights_, queue_name_, idx++);
+  resource->GetCompileCacheResource(compile_cache_dep_files_, weights_, queue_name_, idx++, &compile_cache_consistent_);
 #ifdef ENABLE_PROFILE
   double t2 = GetTime();
   MsProfile::StatTime("LoadCachedFuncGraph", t2 - t1);

mindspore/ccsrc/pipeline/jit/pipeline.h

@@ -162,6 +162,7 @@ class GraphExecutorPy : public std::enable_shared_from_this<GraphExecutorPy> {
   std::string queue_name_;
   bool enable_tuple_broaden_{false};
   py::list compile_cache_dep_files_;
+  bool compile_cache_consistent_{true};
   py::dict weights_;
   std::map<PyObject *, AbstractBasePtr> cur_convert_input_;
 };

mindspore/ccsrc/pipeline/jit/resource.cc

@@ -354,9 +354,20 @@ Any Resource::GetAttrPtr(const TypeId &type, const std::string &name) {
 }
 
 void Resource::GetCompileCacheResource(const py::list &compile_cache_dep_files, const py::dict &weights,
-                                       const std::string &queue_name, size_t compile_cache_id) {
+                                       const std::string &queue_name, size_t compile_cache_id,
+                                       bool *compile_cache_consistent) {
   compile_cache_manager_ = std::make_shared<CompileCacheManager>(compile_cache_id);
+  MS_EXCEPTION_IF_NULL(compile_cache_consistent);
+  if (!*compile_cache_consistent) {
+    MS_LOG(WARNING) << "Check the consistency of dependency files hash failed. Execute all the compilation actions.";
+    return;
+  }
   compile_cache_manager_->InitCompileCacheHash(compile_cache_dep_files);
+  *compile_cache_consistent = compile_cache_manager_->CheckDepFilesHashConsistency();
+  if (!*compile_cache_consistent) {
+    MS_LOG(WARNING) << "Check the consistency of dependency files hash failed. Execute all the compilation actions.";
+    return;
+  }
   func_graph_ = compile_cache_manager_->GetCachedFuncGraph(manager_, weights, queue_name);
   layout_map_ = compile_cache_manager_->layout_map();
 }

mindspore/ccsrc/pipeline/jit/resource.h

@@ -93,7 +93,7 @@ class Resource : public ResourceBase {
 
   // Get the cached func_graph and parameters layout map.
   void GetCompileCacheResource(const py::list &compile_cache_dep_files, const py::dict &weights,
-                               const std::string &queue_name, size_t compile_cache_id);
+                               const std::string &queue_name, size_t compile_cache_id, bool *compile_cache_consistent);
   void CacheFuncGraph() const;
   bool EnableCompileCache() const { return compile_cache_manager_ != nullptr; }
 

tests/st/frontend_compile_cache/run_lenet_two_cells.py

@@ -0,0 +1,78 @@
+# Copyright 2021 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+import sys
+import numpy as np
+
+import mindspore.context as context
+import mindspore.nn as nn
+from mindspore import Tensor
+from mindspore.nn import TrainOneStepCell, WithLossCell
+from mindspore.nn.optim import Momentum
+from mindspore.ops import operations as P
+
+
+class LeNet(nn.Cell):
+    def __init__(self):
+        super(LeNet, self).__init__()
+        self.relu = P.ReLU()
+        self.batch_size = 32
+
+        self.conv1 = nn.Conv2d(1, 6, kernel_size=5, stride=1, padding=0, has_bias=False, pad_mode='valid')
+        self.conv2 = nn.Conv2d(6, 16, kernel_size=5, stride=1, padding=0, has_bias=False, pad_mode='valid')
+        self.pool = nn.MaxPool2d(kernel_size=2, stride=2)
+        self.reshape = P.Reshape()
+        self.fc1 = nn.Dense(400, 120)
+        self.fc2 = nn.Dense(120, 84)
+        self.fc3 = nn.Dense(84, 10)
+
+    def construct(self, input_x):
+        output = self.conv1(input_x)
+        output = self.relu(output)
+        output = self.pool(output)
+        output = self.conv2(output)
+        output = self.relu(output)
+        output = self.pool(output)
+        output = self.reshape(output, (self.batch_size, -1))
+        output = self.fc1(output)
+        output = self.relu(output)
+        output = self.fc2(output)
+        output = self.relu(output)
+        output = self.fc3(output)
+        return output
+
+
+def train(net, data, label):
+    learning_rate = 0.01
+    momentum = 0.9
+
+    optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate, momentum)
+    criterion = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
+    net_with_criterion = WithLossCell(net, criterion)
+    train_network = TrainOneStepCell(net_with_criterion, optimizer)  # optimizer
+    train_network.set_train()
+    res = train_network(data, label)
+    print("{", res, "}")
+    print("{", res.asnumpy().shape, "}")
+
+
+if __name__ == "__main__":
+    context.set_context(enable_compile_cache=True, compile_cache_path=sys.argv[1])
+    input_data = Tensor(np.ones([32, 1, 32, 32]).astype(np.float32) * 0.01)
+    input_label = Tensor(np.ones([32]).astype(np.int32))
+    lenet1 = LeNet()
+    train(lenet1, input_data, input_label)
+    lenet2 = LeNet()
+    train(lenet2, input_data, input_label)
+    context.set_context(enable_compile_cache=False)

tests/st/frontend_compile_cache/test_compile_cache.py

@@ -109,6 +109,27 @@ def run_twice_with_different_networks(file_name_first, file_name_second, cache_p
     shutil.rmtree(cache_path)
 
 
+def run_two_cells_networks_once(file_name, cache_path, log_file_name):
+    # Clear compile cache folder
+    if os.path.exists(cache_path):
+        shutil.rmtree(cache_path)
+    assert not os.path.exists(cache_path)
+
+    # First run without compile cache
+    cmd = f"GLOG_v=2 python " + file_name + " '" + cache_path + "' > " + log_file_name + " 2>&1"
+    subprocess.check_output(cmd, shell=True)
+    assert os.path.exists(log_file_name)
+    assert os.path.exists(cache_path)
+    with open(log_file_name, "r") as f:
+        data = f.read()
+    assert data.count(
+        "Check the consistency of dependency files hash failed. Execute all the compilation actions.") == 2
+
+    # Clean log files
+    os.remove(log_file_name)
+    shutil.rmtree(cache_path)
+
+
 def check_log(role, log_name, str_to_check):
     assert os.path.exists(role + "/" + log_name)
     with open(role + "/" + log_name, "r") as f:
@@ -291,3 +312,16 @@ def test_compile_cache_ms_function():
     """
     run_twice_with_same_network("run_lenet_ms_function.py", "./lenet_ms_function", "lenet_ms_function_first.txt",
                                 "lenet_ms_function_second.txt")
+
+
+@pytest.mark.level0
+@pytest.mark.platform_x86_ascend_training
+@pytest.mark.platform_arm_ascend_training
+@pytest.mark.env_onecard
+def test_compile_cache_run_two_cells_once():
+    """
+    Feature: Compile cache.
+    Description: Test whether all the cells don't read the cached graph when run multiple cells once.
+    Expectation: success.
+    """
+    run_two_cells_networks_once("run_lenet_two_cells.py", "./lenet_two_cells", "lenet_two_cells.txt")