From 1fb716d394f06be9d891e65d1ff5b3da44ddf1af Mon Sep 17 00:00:00 2001 From: yujianfeng Date: Fri, 4 Mar 2022 14:49:36 +0800 Subject: [PATCH] All the cells no need to read the cached graphs when check hash consistency failed --- .../pipeline/jit/compile_cache_manager.cc | 66 +++++++--------- .../pipeline/jit/compile_cache_manager.h | 2 + mindspore/ccsrc/pipeline/jit/pipeline.cc | 2 +- mindspore/ccsrc/pipeline/jit/pipeline.h | 1 + mindspore/ccsrc/pipeline/jit/resource.cc | 13 +++- mindspore/ccsrc/pipeline/jit/resource.h | 2 +- .../run_lenet_two_cells.py | 78 +++++++++++++++++++ .../test_compile_cache.py | 34 ++++++++ 8 files changed, 159 insertions(+), 39 deletions(-) create mode 100644 tests/st/frontend_compile_cache/run_lenet_two_cells.py diff --git a/mindspore/ccsrc/pipeline/jit/compile_cache_manager.cc b/mindspore/ccsrc/pipeline/jit/compile_cache_manager.cc index 03f775bdfcb..1c7b4ee8299 100644 --- a/mindspore/ccsrc/pipeline/jit/compile_cache_manager.cc +++ b/mindspore/ccsrc/pipeline/jit/compile_cache_manager.cc @@ -123,36 +123,6 @@ std::string GetCompileDepFilesHash(const py::list &dep_files) { return files_hash; } -bool CheckDepFilesHashConsistency(const std::string ¤t_dep_files_hash) { - if (current_dep_files_hash.empty()) { - MS_LOG(ERROR) << "Get current dependency files hash failed."; - return false; - } - std::string dep_files_hash_path = GetDepFilesHashPath(); - auto realpath = Common::CreatePrefixPath(dep_files_hash_path, true); - if (!realpath.has_value()) { - MS_LOG(ERROR) << "Get real path of file " << dep_files_hash_path << " failed."; - return false; - } - std::fstream input(realpath.value(), std::ios::in | std::ios::binary); - if (!input) { - MS_LOG(WARNING) << "Open the hash file " << realpath.value() << " failed. The file may not exist." - << ErrnoToString(errno); - return false; - } - std::string checkpoint_hash; - input >> checkpoint_hash; - if (checkpoint_hash.empty()) { - MS_LOG(ERROR) << "Get the compilation dependency files hash from " << realpath.value() << " failed."; - return false; - } - if (checkpoint_hash != current_dep_files_hash) { - MS_LOG(WARNING) << "The compilation dependency files are changed."; - return false; - } - return true; -} - std::map GenerateWeightsValueMap(const py::dict &weights) { std::map ret{}; for (auto weight = weights.begin(); weight != weights.end(); ++weight) { @@ -230,14 +200,38 @@ void CompileCacheManager::InitCompileCacheHash(const py::list &compile_cache_dep compile_cache_dep_files_hash_ = GetCompileDepFilesHash(compile_cache_dep_files); } +bool CompileCacheManager::CheckDepFilesHashConsistency() { + if (compile_cache_dep_files_hash_.empty()) { + MS_LOG(ERROR) << "Get current dependency files hash failed."; + return false; + } + std::string dep_files_hash_path = GetDepFilesHashPath(); + auto realpath = Common::CreatePrefixPath(dep_files_hash_path, true); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path of file " << dep_files_hash_path << " failed."; + return false; + } + std::fstream input(realpath.value(), std::ios::in | std::ios::binary); + if (!input) { + MS_LOG(WARNING) << "Open the hash file " << realpath.value() << " failed. The file may not exist." + << ErrnoToString(errno); + return false; + } + std::string checkpoint_hash; + input >> checkpoint_hash; + if (checkpoint_hash.empty()) { + MS_LOG(ERROR) << "Get the compilation dependency files hash from " << realpath.value() << " failed."; + return false; + } + if (checkpoint_hash != compile_cache_dep_files_hash_) { + MS_LOG(WARNING) << "The compilation dependency files are changed."; + return false; + } + return true; +} + FuncGraphPtr CompileCacheManager::GetCachedFuncGraph(const FuncGraphManagerPtr &manager, const py::dict &weights, const std::string &queue_name) { - // Compare the dependency files hash. - if (!CheckDepFilesHashConsistency(compile_cache_dep_files_hash_)) { - MS_LOG(WARNING) << "Check the consistency of dependency files hash failed. Execute all the compilation actions."; - return nullptr; - } - // Determine whether to load parallel information. std::string parallel_mode = parallel::ParallelContext::GetInstance()->parallel_mode(); bool has_parallel_info = false; diff --git a/mindspore/ccsrc/pipeline/jit/compile_cache_manager.h b/mindspore/ccsrc/pipeline/jit/compile_cache_manager.h index 9ba361d1a1b..ac40b9509af 100644 --- a/mindspore/ccsrc/pipeline/jit/compile_cache_manager.h +++ b/mindspore/ccsrc/pipeline/jit/compile_cache_manager.h @@ -35,6 +35,8 @@ class CompileCacheManager { // Get the hash of dependent files when compiling graph. void InitCompileCacheHash(const py::list &compile_cache_dep_files); + // Compare the dependency files hash. + bool CheckDepFilesHashConsistency(); // Load the cached func_graph from mindir file. FuncGraphPtr GetCachedFuncGraph(const FuncGraphManagerPtr &manager, const py::dict &weights, const std::string &queue_name); diff --git a/mindspore/ccsrc/pipeline/jit/pipeline.cc b/mindspore/ccsrc/pipeline/jit/pipeline.cc index 88d5b2773b0..4e008524cfd 100644 --- a/mindspore/ccsrc/pipeline/jit/pipeline.cc +++ b/mindspore/ccsrc/pipeline/jit/pipeline.cc @@ -768,7 +768,7 @@ void GraphExecutorPy::InitCompileCacheInfo(const ResourcePtr &resource, const st double t1 = GetTime(); #endif static size_t idx = 0; - resource->GetCompileCacheResource(compile_cache_dep_files_, weights_, queue_name_, idx++); + resource->GetCompileCacheResource(compile_cache_dep_files_, weights_, queue_name_, idx++, &compile_cache_consistent_); #ifdef ENABLE_PROFILE double t2 = GetTime(); MsProfile::StatTime("LoadCachedFuncGraph", t2 - t1); diff --git a/mindspore/ccsrc/pipeline/jit/pipeline.h b/mindspore/ccsrc/pipeline/jit/pipeline.h index fc74fdc465a..65ef9335ed6 100644 --- a/mindspore/ccsrc/pipeline/jit/pipeline.h +++ b/mindspore/ccsrc/pipeline/jit/pipeline.h @@ -162,6 +162,7 @@ class GraphExecutorPy : public std::enable_shared_from_this { std::string queue_name_; bool enable_tuple_broaden_{false}; py::list compile_cache_dep_files_; + bool compile_cache_consistent_{true}; py::dict weights_; std::map cur_convert_input_; }; diff --git a/mindspore/ccsrc/pipeline/jit/resource.cc b/mindspore/ccsrc/pipeline/jit/resource.cc index ad4d77ebb88..e527fdaf306 100644 --- a/mindspore/ccsrc/pipeline/jit/resource.cc +++ b/mindspore/ccsrc/pipeline/jit/resource.cc @@ -354,9 +354,20 @@ Any Resource::GetAttrPtr(const TypeId &type, const std::string &name) { } void Resource::GetCompileCacheResource(const py::list &compile_cache_dep_files, const py::dict &weights, - const std::string &queue_name, size_t compile_cache_id) { + const std::string &queue_name, size_t compile_cache_id, + bool *compile_cache_consistent) { compile_cache_manager_ = std::make_shared(compile_cache_id); + MS_EXCEPTION_IF_NULL(compile_cache_consistent); + if (!*compile_cache_consistent) { + MS_LOG(WARNING) << "Check the consistency of dependency files hash failed. Execute all the compilation actions."; + return; + } compile_cache_manager_->InitCompileCacheHash(compile_cache_dep_files); + *compile_cache_consistent = compile_cache_manager_->CheckDepFilesHashConsistency(); + if (!*compile_cache_consistent) { + MS_LOG(WARNING) << "Check the consistency of dependency files hash failed. Execute all the compilation actions."; + return; + } func_graph_ = compile_cache_manager_->GetCachedFuncGraph(manager_, weights, queue_name); layout_map_ = compile_cache_manager_->layout_map(); } diff --git a/mindspore/ccsrc/pipeline/jit/resource.h b/mindspore/ccsrc/pipeline/jit/resource.h index f0c802e4acb..e9644e0f858 100644 --- a/mindspore/ccsrc/pipeline/jit/resource.h +++ b/mindspore/ccsrc/pipeline/jit/resource.h @@ -93,7 +93,7 @@ class Resource : public ResourceBase { // Get the cached func_graph and parameters layout map. void GetCompileCacheResource(const py::list &compile_cache_dep_files, const py::dict &weights, - const std::string &queue_name, size_t compile_cache_id); + const std::string &queue_name, size_t compile_cache_id, bool *compile_cache_consistent); void CacheFuncGraph() const; bool EnableCompileCache() const { return compile_cache_manager_ != nullptr; } diff --git a/tests/st/frontend_compile_cache/run_lenet_two_cells.py b/tests/st/frontend_compile_cache/run_lenet_two_cells.py new file mode 100644 index 00000000000..7799f173879 --- /dev/null +++ b/tests/st/frontend_compile_cache/run_lenet_two_cells.py @@ -0,0 +1,78 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import sys +import numpy as np + +import mindspore.context as context +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.nn.optim import Momentum +from mindspore.ops import operations as P + + +class LeNet(nn.Cell): + def __init__(self): + super(LeNet, self).__init__() + self.relu = P.ReLU() + self.batch_size = 32 + + self.conv1 = nn.Conv2d(1, 6, kernel_size=5, stride=1, padding=0, has_bias=False, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, kernel_size=5, stride=1, padding=0, has_bias=False, pad_mode='valid') + self.pool = nn.MaxPool2d(kernel_size=2, stride=2) + self.reshape = P.Reshape() + self.fc1 = nn.Dense(400, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 10) + + def construct(self, input_x): + output = self.conv1(input_x) + output = self.relu(output) + output = self.pool(output) + output = self.conv2(output) + output = self.relu(output) + output = self.pool(output) + output = self.reshape(output, (self.batch_size, -1)) + output = self.fc1(output) + output = self.relu(output) + output = self.fc2(output) + output = self.relu(output) + output = self.fc3(output) + return output + + +def train(net, data, label): + learning_rate = 0.01 + momentum = 0.9 + + optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate, momentum) + criterion = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + net_with_criterion = WithLossCell(net, criterion) + train_network = TrainOneStepCell(net_with_criterion, optimizer) # optimizer + train_network.set_train() + res = train_network(data, label) + print("{", res, "}") + print("{", res.asnumpy().shape, "}") + + +if __name__ == "__main__": + context.set_context(enable_compile_cache=True, compile_cache_path=sys.argv[1]) + input_data = Tensor(np.ones([32, 1, 32, 32]).astype(np.float32) * 0.01) + input_label = Tensor(np.ones([32]).astype(np.int32)) + lenet1 = LeNet() + train(lenet1, input_data, input_label) + lenet2 = LeNet() + train(lenet2, input_data, input_label) + context.set_context(enable_compile_cache=False) diff --git a/tests/st/frontend_compile_cache/test_compile_cache.py b/tests/st/frontend_compile_cache/test_compile_cache.py index 74fd164deb4..0719d09b37d 100644 --- a/tests/st/frontend_compile_cache/test_compile_cache.py +++ b/tests/st/frontend_compile_cache/test_compile_cache.py @@ -109,6 +109,27 @@ def run_twice_with_different_networks(file_name_first, file_name_second, cache_p shutil.rmtree(cache_path) +def run_two_cells_networks_once(file_name, cache_path, log_file_name): + # Clear compile cache folder + if os.path.exists(cache_path): + shutil.rmtree(cache_path) + assert not os.path.exists(cache_path) + + # First run without compile cache + cmd = f"GLOG_v=2 python " + file_name + " '" + cache_path + "' > " + log_file_name + " 2>&1" + subprocess.check_output(cmd, shell=True) + assert os.path.exists(log_file_name) + assert os.path.exists(cache_path) + with open(log_file_name, "r") as f: + data = f.read() + assert data.count( + "Check the consistency of dependency files hash failed. Execute all the compilation actions.") == 2 + + # Clean log files + os.remove(log_file_name) + shutil.rmtree(cache_path) + + def check_log(role, log_name, str_to_check): assert os.path.exists(role + "/" + log_name) with open(role + "/" + log_name, "r") as f: @@ -291,3 +312,16 @@ def test_compile_cache_ms_function(): """ run_twice_with_same_network("run_lenet_ms_function.py", "./lenet_ms_function", "lenet_ms_function_first.txt", "lenet_ms_function_second.txt") + + +@pytest.mark.level0 +@pytest.mark.platform_x86_ascend_training +@pytest.mark.platform_arm_ascend_training +@pytest.mark.env_onecard +def test_compile_cache_run_two_cells_once(): + """ + Feature: Compile cache. + Description: Test whether all the cells don't read the cached graph when run multiple cells once. + Expectation: success. + """ + run_two_cells_networks_once("run_lenet_two_cells.py", "./lenet_two_cells", "lenet_two_cells.txt")