diff --git a/mindspore/ccsrc/ps/constants.h b/mindspore/ccsrc/ps/constants.h index 45b43ba0f0d..47db975de85 100644 --- a/mindspore/ccsrc/ps/constants.h +++ b/mindspore/ccsrc/ps/constants.h @@ -114,7 +114,7 @@ constexpr uint32_t kCheckRegisteredRetryCount = 30; constexpr uint32_t kCheckRegisteredIntervalInMs = 1000; // The type of persistent storage, currently only supports file storage. -constexpr char kStoreType[] = "storge_type"; +constexpr char kStoreType[] = "storage_type"; // The file used to storage metadata. constexpr char kStoreFilePath[] = "storage_file_path"; // 1 indicates that the persistent storage type is file. diff --git a/mindspore/ccsrc/ps/core/comm_util.cc b/mindspore/ccsrc/ps/core/comm_util.cc index 41ff0d0b429..91ba81edc27 100644 --- a/mindspore/ccsrc/ps/core/comm_util.cc +++ b/mindspore/ccsrc/ps/core/comm_util.cc @@ -187,7 +187,7 @@ std::string CommUtil::ParseConfig(const Configuration &config, const std::string return ""; } - std::string path = config.Get(key, ""); + std::string path = config.GetString(key, ""); return path; } } // namespace core diff --git a/mindspore/ccsrc/ps/core/communicator/tcp_client.cc b/mindspore/ccsrc/ps/core/communicator/tcp_client.cc index bcb66f50916..3388a5a90e7 100644 --- a/mindspore/ccsrc/ps/core/communicator/tcp_client.cc +++ b/mindspore/ccsrc/ps/core/communicator/tcp_client.cc @@ -226,7 +226,7 @@ bool TcpClient::EstablishSSL() { // 2. Parse the client password. std::string client_password = CommUtil::ParseConfig(*config_, kClientPassword); - if (!client_password.empty()) { + if (client_password.empty()) { MS_LOG(WARNING) << "The key:" << kClientPassword << "'s value is empty."; return false; } diff --git a/mindspore/ccsrc/ps/core/communicator/tcp_server.cc b/mindspore/ccsrc/ps/core/communicator/tcp_server.cc index 66085480e5f..1f1e1a2964f 100644 --- a/mindspore/ccsrc/ps/core/communicator/tcp_server.cc +++ b/mindspore/ccsrc/ps/core/communicator/tcp_server.cc @@ -308,7 +308,7 @@ void TcpServer::ListenerCallback(struct evconnlistener *, evutil_socket_t fd, st // 2. Parse the server password. std::string server_password = CommUtil::ParseConfig(*(server->config_), kServerPassword); - if (!server_password.empty()) { + if (server_password.empty()) { MS_LOG(EXCEPTION) << "The key:" << kServerPassword << "'s value is empty."; } diff --git a/mindspore/ccsrc/ps/core/configuration.h b/mindspore/ccsrc/ps/core/configuration.h index 9991be1fc57..6651a88cab9 100644 --- a/mindspore/ccsrc/ps/core/configuration.h +++ b/mindspore/ccsrc/ps/core/configuration.h @@ -45,9 +45,12 @@ class Configuration { // Determine whether the initialization has been completed. virtual bool IsInitialized() const = 0; - // Get configuration data from database or config file. + // Get configuration data from database or config file.The returned string is quoted virtual std::string Get(const std::string &key, const std::string &defaultvalue) const = 0; + // Get configuration data from database or config file.The returned string is not quoted + virtual std::string GetString(const std::string &key, const std::string &defaultvalue) const = 0; + // Put configuration data to database or config file. virtual void Put(const std::string &key, const std::string &defaultvalue) = 0; diff --git a/mindspore/ccsrc/ps/core/file_configuration.cc b/mindspore/ccsrc/ps/core/file_configuration.cc index eaaf4104067..2b813a0edc2 100644 --- a/mindspore/ccsrc/ps/core/file_configuration.cc +++ b/mindspore/ccsrc/ps/core/file_configuration.cc @@ -49,6 +49,15 @@ std::string FileConfiguration::Get(const std::string &key, const std::string &de return res; } +std::string FileConfiguration::GetString(const std::string &key, const std::string &defaultvalue) const { + if (!js.contains(key)) { + MS_LOG(WARNING) << "The key:" << key << " is not exist."; + return defaultvalue; + } + std::string res = js.at(key); + return res; +} + void FileConfiguration::Put(const std::string &key, const std::string &value) { std::ofstream output_file(file_path_); js[key] = value; diff --git a/mindspore/ccsrc/ps/core/file_configuration.h b/mindspore/ccsrc/ps/core/file_configuration.h index d5558967f32..8415a4ce5cb 100644 --- a/mindspore/ccsrc/ps/core/file_configuration.h +++ b/mindspore/ccsrc/ps/core/file_configuration.h @@ -56,6 +56,8 @@ class FileConfiguration : public Configuration { std::string Get(const std::string &key, const std::string &defaultvalue) const override; + std::string GetString(const std::string &key, const std::string &defaultvalue) const override; + void Put(const std::string &key, const std::string &value) override; bool Exists(const std::string &key) override; diff --git a/mindspore/ccsrc/ps/core/scheduler_node.cc b/mindspore/ccsrc/ps/core/scheduler_node.cc index b23e7fb312f..f741f4d1360 100644 --- a/mindspore/ccsrc/ps/core/scheduler_node.cc +++ b/mindspore/ccsrc/ps/core/scheduler_node.cc @@ -172,6 +172,7 @@ void SchedulerNode::ProcessRegister(const std::shared_ptr &server, MS_LOG(INFO) << "The scheduler send metadata to scale in node:" << id; auto scale_in_client = GetOrCreateClient(nodes[id]); SendMetadata(scale_in_client, nodes[id].rank_id_); + node_manager_.UpdateHeartbeat(id); } } node_manager_.UpdateNodesInfo(); @@ -179,6 +180,7 @@ void SchedulerNode::ProcessRegister(const std::shared_ptr &server, for (const auto &kvs : node_infos) { auto client = GetOrCreateClient(kvs.second); SendMetadata(client, kvs.second.rank_id_); + node_manager_.UpdateHeartbeat(kvs.first); } node_manager_.UpdateClusterState(ClusterState::CLUSTER_READY); wait_start_cond_.notify_all(); diff --git a/mindspore/ccsrc/ps/core/server_node.cc b/mindspore/ccsrc/ps/core/server_node.cc index a66f2eeb134..86a619ab627 100644 --- a/mindspore/ccsrc/ps/core/server_node.cc +++ b/mindspore/ccsrc/ps/core/server_node.cc @@ -27,14 +27,14 @@ bool ServerNode::Start(const uint32_t &timeout) { MS_LOG(INFO) << "[Server start]: 4. The node role:" << CommUtil::NodeRoleToString(node_info_.node_role_) << " the node id:" << node_info_.node_id_ << " successfully registered to the scheduler!"; + StartHeartbeatTimer(client_to_scheduler_); + MS_LOG(INFO) << "[Server start]: 5. Server start heartbeat timer!"; + if (!WaitForStart(timeout)) { MS_LOG(ERROR) << "Start server node timeout!"; return false; } - StartHeartbeatTimer(client_to_scheduler_); - MS_LOG(INFO) << "[Server start]: 5. Server start heartbeat timer!"; - MsException::Instance().CheckException(); MS_LOG(INFO) << "[Server start]: 6. Successfully start server node!"; return true; diff --git a/mindspore/ccsrc/ps/core/worker_node.cc b/mindspore/ccsrc/ps/core/worker_node.cc index b01169f2ef0..8e4ff69625f 100644 --- a/mindspore/ccsrc/ps/core/worker_node.cc +++ b/mindspore/ccsrc/ps/core/worker_node.cc @@ -27,14 +27,14 @@ bool WorkerNode::Start(const uint32_t &timeout) { MS_LOG(INFO) << "[Worker start]: 4. The node role:" << CommUtil::NodeRoleToString(node_info_.node_role_) << " the node id:" << node_info_.node_id_ << " successfully registered to the scheduler!"; + StartHeartbeatTimer(client_to_scheduler_); + MS_LOG(INFO) << "[Worker start]: 5. Worker start heartbeat timer!"; + if (!WaitForStart(timeout)) { MS_LOG(ERROR) << "Start Worker node timeout!"; return false; } - StartHeartbeatTimer(client_to_scheduler_); - MS_LOG(INFO) << "[Worker start]: 5. Worker start heartbeat timer!"; - MsException::Instance().CheckException(); MS_LOG(INFO) << "[Worker start]: 6. Successfully start worker node!"; return true; diff --git a/tests/st/fl/hybrid_lenet/config.json b/tests/st/fl/hybrid_lenet/config.json index 3e6d09945ea..3f877d231b4 100644 --- a/tests/st/fl/hybrid_lenet/config.json +++ b/tests/st/fl/hybrid_lenet/config.json @@ -1,6 +1,6 @@ { "recovery": { - "storge_type": 1, + "storage_type": 1, "storage_file_path": "recovery.json" }, "server_cert_path": "server.crt", diff --git a/tests/st/fl/mobile/config.json b/tests/st/fl/mobile/config.json index 500e644ecc3..b693d5cb436 100644 --- a/tests/st/fl/mobile/config.json +++ b/tests/st/fl/mobile/config.json @@ -1,6 +1,6 @@ { "recovery": { - "storge_type": 1, + "storage_type": 1, "storage_file_path": "recovery.json" }, "server_cert_path": "server.crt",